Skip to content

Build AI methods with your coding agent, then run them anywhere: as an MCP for chatbots, as a webapp for people, or via API for your software.

EPIC - pipe builder (#280)

Thomas Hebrard700c153b9d08
diff --git a/.badges/tests.json b/.badges/tests.json
new file mode 100644
index 000000000..00505132c
--- /dev/null
+++ b/.badges/tests.json
@@ -0,0 +1,7 @@
+{
+ "schemaVersion": 1,
+ "label": "tests",
+ "message": "1088",
+ "color": "blue",
+ "cacheSeconds": 300
+ }
\ No newline at end of file
diff --git a/.cursor/rules/coding_standards.mdc b/.cursor/rules/coding_standards.mdc
index c94a3de96..6c6d06f8f 100644
--- a/.cursor/rules/coding_standards.mdc
+++ b/.cursor/rules/coding_standards.mdc
@@ -99,7 +99,7 @@ Always fix any issues reported by these tools before proceeding.
```bash
make tp
```
- Runs tests with markers: `(dry_runnable or not (inference or llm or imgg or ocr)) and not (needs_output or pipelex_api)`
+ Runs tests with markers: `(dry_runnable or not (inference or llm or img_gen or ocr)) and not (needs_output or pipelex_api)`
2. **Specific Tests**:
```bash
@@ -109,7 +109,7 @@ Always fix any issues reported by these tools before proceeding.
```
Note: Matches names starting with the provided string.
-**Important**: Never run `make ti`, `make test-inference`, `make to`, `make test-ocr`, `make tg`, or `make test-imgg` - these use costly inference.
+**Important**: Never run `make ti`, `make test-inference`, `make to`, `make test-ocr`, `make tg`, or `make test-img-gen` - these use costly inference.
## Pipelines
diff --git a/.cursor/rules/pytest.mdc b/.cursor/rules/pytest.mdc
index 52a1c771d..7b41726bb 100644
--- a/.cursor/rules/pytest.mdc
+++ b/.cursor/rules/pytest.mdc
@@ -24,7 +24,7 @@ These rules apply when writing unit tests.
Apply the appropriate markers:
- "llm: uses an LLM to generate text or objects"
-- "imgg: uses an image generation AI"
+- "img_gen: uses an image generation AI"
- "inference: uses either an LLM or an image generation AI"
- "gha_disabled: will not be able to run properly on GitHub Actions"
diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md
index f7e79caf1..b9b72d0fb 100644
--- a/.github/copilot-instructions.md
+++ b/.github/copilot-instructions.md
@@ -95,7 +95,7 @@ Always fix any issues reported by these tools before proceeding.
```bash
make tp
```
- Runs tests with markers: `(dry_runnable or not (inference or llm or imgg or ocr)) and not (needs_output or pipelex_api)`
+ Runs tests with markers: `(dry_runnable or not (inference or llm or img_gen or ocr)) and not (needs_output or pipelex_api)`
2. **Specific Tests**:
```bash
@@ -105,7 +105,7 @@ Always fix any issues reported by these tools before proceeding.
```
Note: Matches names starting with the provided string.
-**Important**: Never run `make ti`, `make test-inference`, `make to`, `make test-ocr`, `make tg`, or `make test-imgg` - these use costly inference.
+**Important**: Never run `make ti`, `make test-inference`, `make to`, `make test-ocr`, `make tg`, or `make test-img-gen` - these use costly inference.
## Pipelines
@@ -757,7 +757,7 @@ These rules apply when writing unit tests.
Apply the appropriate markers:
- "llm: uses an LLM to generate text or objects"
-- "imgg: uses an image generation AI"
+- "img_gen: uses an image generation AI"
- "inference: uses either an LLM or an image generation AI"
- "gha_disabled: will not be able to run properly on GitHub Actions"
diff --git a/.github/workflows/check-test-count-badge.yml b/.github/workflows/check-test-count-badge.yml
new file mode 100644
index 000000000..ee6e243c1
--- /dev/null
+++ b/.github/workflows/check-test-count-badge.yml
@@ -0,0 +1,35 @@
+name: Check test count matches badge
+
+on:
+ pull_request:
+ branches:
+ - main
+ - dev
+ - "release/v[0-9]+.[0-9]+.[0-9]+"
+
+jobs:
+ check:
+ runs-on: ubuntu-latest
+ env:
+ VIRTUAL_ENV: ${{ github.workspace }}/.venv
+ ENV: dev
+ steps:
+ - name: Checkout
+ uses: actions/checkout@v4
+
+ - name: Set up Python
+ uses: actions/setup-python@v5
+ with:
+ python-version: "3.11"
+
+ - name: Check UV installation
+ run: make check-uv
+
+ - name: Verify UV installation
+ run: uv --version
+
+ - name: Install dependencies
+ run: make install
+
+ - name: Check test count equals badge value
+ run: make check-test-badge
diff --git a/.github/workflows/manual-trigger-tests-check.yml b/.github/workflows/manual-trigger-tests-check.yml
index d3e620c60..e11374eea 100644
--- a/.github/workflows/manual-trigger-tests-check.yml
+++ b/.github/workflows/manual-trigger-tests-check.yml
@@ -65,4 +65,4 @@ jobs:
run: make tp TEST=TestFundamentals
- name: Run tests
- run: make run-manual-trigger-gha-tests
+ run: make run-manual-trigger-gha-tests
\ No newline at end of file
diff --git a/.pipelex/inference/backends.toml b/.pipelex/inference/backends.toml
index 957ac68ca..f4a4005b9 100644
--- a/.pipelex/inference/backends.toml
+++ b/.pipelex/inference/backends.toml
@@ -1,49 +1,58 @@
[pipelex_inference]
+enabled = true
endpoint = "https://inference.pipelex.com/v1"
api_key = "${PIPELEX_INFERENCE_API_KEY}"
-[blackboxai]
-enabled = true
-endpoint = "https://api.blackbox.ai/v1"
-api_key = "${BLACKBOX_API_KEY}"
-
-[openai]
-enabled = true
-api_key = "${OPENAI_API_KEY}"
-
[azure_openai]
enabled = true
endpoint = "${AZURE_API_BASE}"
api_key = "${AZURE_API_KEY}"
api_version = "${AZURE_API_VERSION}"
+[bedrock]
+enabled = true
+aws_region = "${AWS_REGION}"
+
+[google]
+enabled = true
+api_key = "${GOOGLE_API_KEY}"
+
+[vertexai]
+enabled = false
+gcp_project_id = "${GCP_PROJECT_ID}"
+gcp_location = "${GCP_LOCATION}"
+gcp_credentials_file_path = "${GCP_CREDENTIALS_FILE_PATH}"
+
+[openai]
+enabled = true
+api_key = "${OPENAI_API_KEY}"
+
[anthropic]
enabled = true
api_key = "${ANTHROPIC_API_KEY}"
claude_4_tokens_limit = 8192
-[ollama]
+[mistral]
enabled = true
-endpoint = "http://localhost:11434/v1"
+api_key = "${MISTRAL_API_KEY}"
[xai]
enabled = true
endpoint = "https://api.x.ai/v1"
api_key = "${XAI_API_KEY}"
-[bedrock]
+[ollama]
enabled = true
-aws_region = "${AWS_REGION}"
+endpoint = "http://localhost:11434/v1"
-[vertexai]
-enabled = false
-gcp_project_id = "${GCP_PROJECT_ID}"
-gcp_location = "${GCP_LOCATION}"
-gcp_credentials_file_path = "${GCP_CREDENTIALS_FILE_PATH}"
+[blackboxai]
+enabled = true
+endpoint = "https://api.blackbox.ai/v1"
+api_key = "${BLACKBOX_API_KEY}"
-[mistral]
+[fal]
enabled = true
-api_key = "${MISTRAL_API_KEY}"
+api_key = "${FAL_API_KEY}"
[internal] # software-only backend, runs internally, without AI
enabled = true
diff --git a/.pipelex/inference/backends/anthropic.toml b/.pipelex/inference/backends/anthropic.toml
index 0dc85517c..a7b98c0c0 100644
--- a/.pipelex/inference/backends/anthropic.toml
+++ b/.pipelex/inference/backends/anthropic.toml
@@ -1,6 +1,32 @@
-default_sdk = "anthropic"
-default_prompting_target = "anthropic"
+################################################################################
+# Anthropic Backend Configuration
+################################################################################
+#
+# This file defines the model specifications for Anthropic Claude models.
+# It contains model definitions for various Claude language models
+# accessible through the Anthropic API.
+#
+# Configuration structure:
+# - Each model is defined in its own section with the model name as the header
+# - Headers with dots must be quoted (e.g., ["claude-3.5-sonnet"])
+# - Model costs are in USD per million tokens (input/output)
+#
+################################################################################
+################################################################################
+# MODEL DEFAULTS
+################################################################################
+
+[defaults]
+model_type = "llm"
+sdk = "anthropic"
+prompting_target = "anthropic"
+
+################################################################################
+# LANGUAGE MODELS
+################################################################################
+
+# --- Claude 3 Series ----------------------------------------------------------
[claude-3-haiku]
model_id = "claude-3-haiku-20240307"
max_tokens = 4096
@@ -17,6 +43,7 @@ outputs = ["text", "structured"]
max_prompt_images = 100
costs = { input = 15.0, output = 75.0 }
+# --- Claude 3.5 Series --------------------------------------------------------
["claude-3.5-sonnet"]
model_id = "claude-3-5-sonnet-20240620"
max_tokens = 8192
@@ -33,6 +60,7 @@ outputs = ["text", "structured"]
max_prompt_images = 100
costs = { input = 3.0, output = 15.0 }
+# --- Claude 3.7 Series --------------------------------------------------------
["claude-3.7-sonnet"]
model_id = "claude-3-7-sonnet-20250219"
max_tokens = 8192
@@ -41,6 +69,7 @@ outputs = ["text", "structured"]
max_prompt_images = 100
costs = { input = 3.0, output = 15.0 }
+# --- Claude 4 Series ----------------------------------------------------------
[claude-4-sonnet]
model_id = "claude-sonnet-4-20250514"
max_tokens = 64000
@@ -57,6 +86,7 @@ outputs = ["text", "structured"]
max_prompt_images = 100
costs = { input = 3.0, output = 15.0 }
+# --- Claude 4.1 Series --------------------------------------------------------
["claude-4.1-opus"]
model_id = "claude-opus-4-1-20250805"
max_tokens = 32000
diff --git a/.pipelex/inference/backends/azure_openai.toml b/.pipelex/inference/backends/azure_openai.toml
index b26178854..26e01b4e1 100644
--- a/.pipelex/inference/backends/azure_openai.toml
+++ b/.pipelex/inference/backends/azure_openai.toml
@@ -1,6 +1,32 @@
-default_sdk = "azure_openai"
-default_prompting_target = "openai"
+################################################################################
+# Azure OpenAI Backend Configuration
+################################################################################
+#
+# This file defines the model specifications for Azure OpenAI models.
+# It contains model definitions for OpenAI models deployed on Azure
+# accessible through the Azure OpenAI API.
+#
+# Configuration structure:
+# - Each model is defined in its own section with the model name as the header
+# - Headers with dots must be quoted (e.g., ["gpt-4.1"])
+# - Model costs are in USD per million tokens (input/output)
+#
+################################################################################
+################################################################################
+# MODEL DEFAULTS
+################################################################################
+
+[defaults]
+model_type = "llm"
+sdk = "azure_openai"
+prompting_target = "openai"
+
+################################################################################
+# LANGUAGE MODELS
+################################################################################
+
+# --- GPT-4o Series ------------------------------------------------------------
[gpt-4o]
model_id = "gpt-4o-2024-11-20"
inputs = ["text", "images"]
@@ -13,6 +39,7 @@ inputs = ["text", "images"]
outputs = ["text", "structured"]
costs = { input = 0.15, output = 0.6 }
+# --- GPT-4.1 Series -----------------------------------------------------------
["gpt-4.1"]
model_id = "gpt-4.1-2025-04-14"
inputs = ["text", "images"]
@@ -31,6 +58,7 @@ inputs = ["text", "images"]
outputs = ["text", "structured"]
costs = { input = 0.1, output = 0.4 }
+# --- o Series ----------------------------------------------------------------
[o1-mini]
model_id = "o1-mini-2024-09-12"
inputs = ["text"]
@@ -49,6 +77,7 @@ inputs = ["text"]
outputs = ["text", "structured"]
costs = { input = 1.1, output = 4.4 }
+# --- GPT-5 Series -------------------------------------------------------------
[gpt-5-mini]
model_id = "gpt-5-mini-2025-08-07"
inputs = ["text", "images"]
diff --git a/.pipelex/inference/backends/bedrock.toml b/.pipelex/inference/backends/bedrock.toml
index 447c76f03..3c354bb0b 100644
--- a/.pipelex/inference/backends/bedrock.toml
+++ b/.pipelex/inference/backends/bedrock.toml
@@ -1,6 +1,32 @@
-default_sdk = "bedrock_aioboto3"
-default_prompting_target = "anthropic"
+################################################################################
+# AWS Bedrock Backend Configuration
+################################################################################
+#
+# This file defines the model specifications for AWS Bedrock models.
+# It contains model definitions for various language models
+# accessible through the AWS Bedrock service.
+#
+# Configuration structure:
+# - Each model is defined in its own section with the model name as the header
+# - Headers with dots must be quoted (e.g., ["claude-3.5-sonnet"])
+# - Model costs are in USD per million tokens (input/output)
+#
+################################################################################
+
+################################################################################
+# MODEL DEFAULTS
+################################################################################
+
+[defaults]
+model_type = "llm"
+sdk = "bedrock_aioboto3"
+prompting_target = "anthropic"
+################################################################################
+# LANGUAGE MODELS
+################################################################################
+
+# --- Mistral Models -----------------------------------------------------------
[bedrock-mistral-large]
model_id = "mistral.mistral-large-2407-v1:0"
max_tokens = 8192
@@ -8,6 +34,7 @@ inputs = ["text"]
outputs = ["text"]
costs = { input = 4.0, output = 12.0 }
+# --- Meta Llama Models --------------------------------------------------------
[bedrock-meta-llama-3-3-70b-instruct]
model_id = "us.meta.llama3-3-70b-instruct-v1:0"
max_tokens = 8192
@@ -16,6 +43,7 @@ outputs = ["text"]
# TODO: find out the actual cost per million tokens for llama3 on bedrock
costs = { input = 3.0, output = 15.0 }
+# --- Amazon Nova Models -------------------------------------------------------
[bedrock-nova-pro]
model_id = "us.amazon.nova-pro-v1:0"
max_tokens = 5120
@@ -24,10 +52,7 @@ outputs = ["text"]
# TODO: find out the actual cost per million tokens for nova on bedrock
costs = { input = 3.0, output = 15.0 }
-################################################################################
-# Anthropic Models
-################################################################################
-
+# --- Claude LLMs --------------------------------------------------------------
["claude-3.5-sonnet"]
sdk = "bedrock_anthropic"
model_id = "us.anthropic.claude-3-5-sonnet-20240620-v1:0"
diff --git a/.pipelex/inference/backends/blackboxai.toml b/.pipelex/inference/backends/blackboxai.toml
index 14ca78283..4202fd79b 100644
--- a/.pipelex/inference/backends/blackboxai.toml
+++ b/.pipelex/inference/backends/blackboxai.toml
@@ -1,8 +1,31 @@
-
-default_sdk = "openai"
-default_prompting_target = "anthropic"
-
-# OpenAI Models
+################################################################################
+# BlackBoxAI Backend Configuration
+################################################################################
+#
+# This file defines the model specifications for BlackBoxAI models.
+# It contains model definitions for various language models from different providers
+# accessible through the BlackBoxAI API.
+#
+# Configuration structure:
+# - Each model is defined in its own section with the model name as the header
+# - Headers with dots must be quoted (e.g., ["gpt-4.5-preview"])
+# - Model costs are in USD per million tokens (input/output)
+#
+################################################################################
+
+################################################################################
+# MODEL DEFAULTS
+################################################################################
+
+[defaults]
+model_type = "llm"
+sdk = "openai"
+
+################################################################################
+# LANGUAGE MODELS
+################################################################################
+
+# --- OpenAI Models ------------------------------------------------------------
[gpt-4o-mini]
model_id = "blackboxai/openai/gpt-4o-mini"
inputs = ["text", "images"]
@@ -27,13 +50,7 @@ inputs = ["text"]
outputs = ["text", "structured"]
costs = { input = 1.10, output = 4.40 }
-["gpt-4.5-preview"]
-model_id = "blackboxai/openai/gpt-4.5-preview"
-inputs = ["text", "images"]
-outputs = ["text", "structured"]
-costs = { input = 75.00, output = 150.00 }
-
-# Anthropic Models
+# --- Claude LLMs --------------------------------------------------------------
["claude-3.5-haiku"]
model_id = "blackboxai/anthropic/claude-3.5-haiku"
inputs = ["text", "images"]
@@ -64,7 +81,7 @@ inputs = ["text", "images"]
outputs = ["text", "structured"]
costs = { input = 3.00, output = 15.00 }
-# Google Models
+# --- Google Models ------------------------------------------------------------
["gemini-2.5-flash"]
model_id = "blackboxai/google/gemini-2.5-flash"
inputs = ["text", "images"]
@@ -83,26 +100,7 @@ inputs = ["text", "images"]
outputs = ["text", "structured"]
costs = { input = 0.04, output = 0.15 }
-# Free Models
-[deepseek-chat-free]
-model_id = "blackboxai/deepseek/deepseek-chat:free"
-inputs = ["text"]
-outputs = ["text", "structured"]
-costs = { input = 0.00, output = 0.00 }
-
-[deepseek-r1-free]
-model_id = "blackboxai/deepseek/deepseek-r1:free"
-inputs = ["text"]
-outputs = ["text", "structured"]
-costs = { input = 0.00, output = 0.00 }
-
-["llama-3.3-70b-instruct-free"]
-model_id = "blackboxai/meta-llama/llama-3.3-70b-instruct:free"
-inputs = ["text"]
-outputs = ["text", "structured"]
-costs = { input = 0.00, output = 0.00 }
-
-# Mistral Models
+# --- Mistral Models -----------------------------------------------------------
[mistral-large]
model_id = "blackboxai/mistralai/mistral-large"
inputs = ["text"]
@@ -115,33 +113,33 @@ inputs = ["text", "images"]
outputs = ["text", "structured"]
costs = { input = 2.00, output = 6.00 }
-# Cost-Effective Models
+# --- Meta Llama Models --------------------------------------------------------
["llama-3.3-70b-instruct"]
model_id = "blackboxai/meta-llama/llama-3.3-70b-instruct"
inputs = ["text"]
outputs = ["text", "structured"]
costs = { input = 0.04, output = 0.12 }
-["qwen-2.5-72b-instruct"]
-model_id = "blackboxai/qwen/qwen-2.5-72b-instruct"
-inputs = ["text"]
-outputs = ["text", "structured"]
-costs = { input = 0.12, output = 0.39 }
-
-# Vision Models
["llama-3.2-11b-vision-instruct"]
model_id = "blackboxai/meta-llama/llama-3.2-11b-vision-instruct"
inputs = ["text", "images"]
outputs = ["text", "structured"]
costs = { input = 0.05, output = 0.05 }
+# --- Qwen Models --------------------------------------------------------------
+["qwen-2.5-72b-instruct"]
+model_id = "blackboxai/qwen/qwen-2.5-72b-instruct"
+inputs = ["text"]
+outputs = ["text", "structured"]
+costs = { input = 0.12, output = 0.39 }
+
["qwen2.5-vl-72b-instruct"]
model_id = "blackboxai/qwen/qwen2.5-vl-72b-instruct"
inputs = ["text", "images"]
outputs = ["text", "structured"]
costs = { input = 0.25, output = 0.75 }
-# Amazon Nova Models
+# --- Amazon Nova Models -------------------------------------------------------
[nova-micro-v1]
model_id = "blackboxai/amazon/nova-micro-v1"
inputs = ["text"]
@@ -154,3 +152,27 @@ inputs = ["text"]
outputs = ["text", "structured"]
costs = { input = 0.06, output = 0.24 }
+################################################################################
+# FREE MODELS
+################################################################################
+
+# --- DeepSeek Free Models -----------------------------------------------------
+[deepseek-chat-free]
+model_id = "blackboxai/deepseek/deepseek-chat:free"
+inputs = ["text"]
+outputs = ["text", "structured"]
+costs = { input = 0.00, output = 0.00 }
+
+[deepseek-r1-free]
+model_id = "blackboxai/deepseek/deepseek-r1:free"
+inputs = ["text"]
+outputs = ["text", "structured"]
+costs = { input = 0.00, output = 0.00 }
+
+# --- Meta Llama Free Models ---------------------------------------------------
+["llama-3.3-70b-instruct-free"]
+model_id = "blackboxai/meta-llama/llama-3.3-70b-instruct:free"
+inputs = ["text"]
+outputs = ["text", "structured"]
+costs = { input = 0.00, output = 0.00 }
+
diff --git a/.pipelex/inference/backends/fal.toml b/.pipelex/inference/backends/fal.toml
new file mode 100644
index 000000000..ed15e6bd3
--- /dev/null
+++ b/.pipelex/inference/backends/fal.toml
@@ -0,0 +1,54 @@
+################################################################################
+# FAL Backend Configuration
+################################################################################
+#
+# This file defines the model specifications for FAL (Fast AI Labs) models.
+# It contains model definitions for various image generation models
+# accessible through the FAL API.
+#
+# Configuration structure:
+# - Each model is defined in its own section with the model name as the header
+# - Headers with dots must be quoted (e.g., ["flux-pro/v1.1"])
+# - Model costs are in USD per million tokens (input/output)
+#
+################################################################################
+
+################################################################################
+# MODEL DEFAULTS
+################################################################################
+
+[defaults]
+model_type = "img_gen"
+sdk = "fal"
+prompting_target = "fal"
+
+################################################################################
+# IMAGE GENERATION MODELS
+################################################################################
+
+# --- Flux Pro Series ----------------------------------------------------------
+[flux-pro]
+model_id = "fal-ai/flux-pro"
+inputs = ["text"]
+outputs = ["image"]
+costs = { input = 0.05, output = 0.0 }
+
+["flux-pro/v1.1"]
+model_id = "fal-ai/flux-pro/v1.1"
+inputs = ["text"]
+outputs = ["image"]
+costs = { input = 0.05, output = 0.0 }
+
+["flux-pro/v1.1-ultra"]
+model_id = "fal-ai/flux-pro/v1.1-ultra"
+inputs = ["text"]
+outputs = ["image"]
+costs = { input = 0.06, output = 0.0 }
+
+# --- SDXL models --------------------------------------------------------------
+[fast-lightning-sdxl]
+model_id = "fal-ai/fast-lightning-sdxl"
+inputs = ["text"]
+outputs = ["image"]
+costs = { input = 0.0003, output = 0.0 }
+
diff --git a/.pipelex/inference/backends/google.toml b/.pipelex/inference/backends/google.toml
new file mode 100644
index 000000000..072b835dd
--- /dev/null
+++ b/.pipelex/inference/backends/google.toml
@@ -0,0 +1,80 @@
+################################################################################
+# Google Gemini API Backend Configuration
+################################################################################
+#
+# This file defines the model specifications for Google Gemini API models.
+# It contains model definitions for Gemini language models
+# accessible through the Google Gemini API (not VertexAI).
+#
+# Configuration structure:
+# - Each model is defined in its own section with the model name as the header
+# - Headers with dots must be quoted (e.g., ["gemini-2.0-flash"])
+# - Model costs are in USD per million tokens (input/output)
+#
+################################################################################
+
+################################################################################
+# MODEL DEFAULTS
+################################################################################
+
+[defaults]
+model_type = "llm"
+sdk = "google"
+prompting_target = "gemini"
+
+################################################################################
+# LANGUAGE MODELS
+################################################################################
+
+# --- Gemini 2.0 Series --------------------------------------------------------
+["gemini-2.0-flash-exp"]
+model_id = "gemini-2.0-flash-exp"
+inputs = ["text", "images"]
+outputs = ["text", "structured"]
+max_prompt_images = 3000
+costs = { input = 0.0, output = 0.0 } # Free tier during experimental phase
+
+# --- Gemini 1.5 Series --------------------------------------------------------
+["gemini-1.5-pro"]
+model_id = "gemini-1.5-pro"
+inputs = ["text", "images"]
+outputs = ["text", "structured"]
+max_prompt_images = 3000
+costs = { input = 1.25, output = 5.0 } # Updated pricing
+
+["gemini-1.5-flash"]
+model_id = "gemini-1.5-flash"
+inputs = ["text", "images"]
+outputs = ["text", "structured"]
+max_prompt_images = 3000
+costs = { input = 0.075, output = 0.30 } # Updated pricing
+
+["gemini-1.5-flash-8b"]
+model_id = "gemini-1.5-flash-8b"
+inputs = ["text", "images"]
+outputs = ["text", "structured"]
+max_prompt_images = 3000
+costs = { input = 0.0375, output = 0.15 } # Updated pricing
+
+# --- Gemini 2.5 Series (when available) ----------------------------------------
+["gemini-2.5-pro"]
+model_id = "gemini-2.5-pro"
+inputs = ["text", "images"]
+outputs = ["text", "structured"]
+max_prompt_images = 3000
+costs = { input = 1.25, output = 10.0 }
+
+["gemini-2.5-flash"]
+model_id = "gemini-2.5-flash"
+inputs = ["text", "images"]
+outputs = ["text", "structured"]
+max_prompt_images = 3000
+costs = { input = 0.30, output = 2.50 }
+
+["gemini-2.5-flash-lite"]
+model_id = "gemini-2.5-flash-lite"
+inputs = ["text", "images"]
+outputs = ["text", "structured"]
+max_prompt_images = 3000
+costs = { input = 0.10, output = 0.40 }
+
diff --git a/.pipelex/inference/backends/internal.toml b/.pipelex/inference/backends/internal.toml
index 54dba5a0a..0dbd6e7be 100644
--- a/.pipelex/inference/backends/internal.toml
+++ b/.pipelex/inference/backends/internal.toml
@@ -1,5 +1,21 @@
-# software-only backend, runs internally, without AI
+################################################################################
+# Internal Backend Configuration
+################################################################################
+#
+# This file defines the model specifications for internal software-only models.
+# These models run internally without external APIs or AI services.
+#
+# Configuration structure:
+# - Each model is defined in its own section with the model name as the header
+# - Model costs are in USD per million tokens (input/output)
+#
+################################################################################
+################################################################################
+# TEXT EXTRACTION MODELS
+################################################################################
+
+# --- PyPDFium2 Text Extractor -------------------------------------------------
[pypdfium2-extract-text]
model_type = "text_extractor"
sdk = "pypdfium2"
diff --git a/.pipelex/inference/backends/mistral.toml b/.pipelex/inference/backends/mistral.toml
index b452cef8b..634736c74 100644
--- a/.pipelex/inference/backends/mistral.toml
+++ b/.pipelex/inference/backends/mistral.toml
@@ -1,6 +1,32 @@
-default_sdk = "mistral"
-default_prompting_target = "mistral"
+################################################################################
+# Mistral Backend Configuration
+################################################################################
+#
+# This file defines the model specifications for Mistral AI models.
+# It contains model definitions for various Mistral language models and specialized models
+# accessible through the Mistral API.
+#
+# Configuration structure:
+# - Each model is defined in its own section with the model name as the header
+# - Headers with dots must be quoted (e.g., ["ministral-3b"])
+# - Model costs are in USD per million tokens (input/output)
+#
+################################################################################
+################################################################################
+# MODEL DEFAULTS
+################################################################################
+
+[defaults]
+model_type = "llm"
+sdk = "mistral"
+prompting_target = "mistral"
+
+################################################################################
+# LANGUAGE MODELS
+################################################################################
+
+# --- Ministral Series ---------------------------------------------------------
[ministral-3b]
model_id = "ministral-3b-latest"
max_tokens = 131072
@@ -15,6 +41,7 @@ inputs = ["text"]
outputs = ["text", "structured"]
costs = { input = 0.1, output = 0.1 }
+# --- Mistral 7B Series --------------------------------------------------------
[mistral-7b-2312]
model_id = "mistral-large-2402"
max_tokens = 32768
@@ -22,6 +49,7 @@ inputs = ["text"]
outputs = ["text", "structured"]
costs = { input = 0.25, output = 0.25 }
+# --- Mistral 8x7B Series ------------------------------------------------------
[mistral-8x7b-2312]
model_id = "open-mixtral-8x7b"
max_tokens = 32768
@@ -29,6 +57,7 @@ inputs = ["text"]
outputs = ["text"]
costs = { input = 0.7, output = 0.7 }
+# --- Mistral Codestral Series -------------------------------------------------
[mistral-codestral-2405]
model_id = "codestral-2405"
max_tokens = 262144
@@ -36,6 +65,7 @@ inputs = ["text"]
outputs = ["text"]
costs = { input = 1.0, output = 3.0 }
+# --- Mistral Large Series -----------------------------------------------------
[mistral-large-2402]
model_id = "mistral-large-2402"
max_tokens = 32768
@@ -50,6 +80,7 @@ inputs = ["text"]
outputs = ["text", "structured"]
costs = { input = 4.0, output = 12.0 }
+# --- Mistral Small Series -----------------------------------------------------
[mistral-small-2402]
model_id = "mistral-small-2402"
max_tokens = 32768
@@ -64,6 +95,7 @@ inputs = ["text"]
outputs = ["text", "structured"]
costs = { input = 1.0, output = 3.0 }
+# --- Pixtral Series -----------------------------------------------------------
[pixtral-12b]
model_id = "pixtral-12b-latest"
max_tokens = 131072
@@ -78,6 +110,7 @@ inputs = ["text", "images"]
outputs = ["text", "structured"]
costs = { input = 2.0, output = 6.0 }
+# --- Mistral Medium Series ----------------------------------------------------
[mistral-medium]
model_id = "mistral-medium-latest"
max_tokens = 128000
@@ -92,6 +125,11 @@ inputs = ["text", "images"]
outputs = ["text", "structured"]
costs = { input = 0.4, output = 2.0 }
+################################################################################
+# OCR MODELS
+################################################################################
+
+# --- OCR Models ---------------------------------------------------------------
[mistral-ocr]
model_type = "text_extractor"
model_id = "mistral-ocr-latest"
diff --git a/.pipelex/inference/backends/ollama.toml b/.pipelex/inference/backends/ollama.toml
index 2b66dee18..7a56b7384 100644
--- a/.pipelex/inference/backends/ollama.toml
+++ b/.pipelex/inference/backends/ollama.toml
@@ -1,6 +1,32 @@
-default_sdk = "openai"
-default_prompting_target = "anthropic"
+################################################################################
+# Ollama Backend Configuration
+################################################################################
+#
+# This file defines the model specifications for Ollama models.
+# It contains model definitions for local language models
+# accessible through the Ollama API.
+#
+# Configuration structure:
+# - Each model is defined in its own section with the model name as the header
+# - Headers with dots must be quoted (e.g., ["mistral-small3.1-24b"])
+# - Model costs are in USD per million tokens (input/output)
+#
+################################################################################
+################################################################################
+# MODEL DEFAULTS
+################################################################################
+
+[defaults]
+model_type = "llm"
+sdk = "openai"
+prompting_target = "anthropic"
+
+################################################################################
+# LANGUAGE MODELS
+################################################################################
+
+# --- Gemma Models -------------------------------------------------------------
[gemma3-4b]
model_id = "gemma3:4b"
inputs = ["text"]
@@ -8,6 +34,7 @@ outputs = ["text"]
max_prompt_images = 3000
costs = { input = 0, output = 0 }
+# --- Llama Models -------------------------------------------------------------
[llama4-scout]
model_id = "llama4:scout"
inputs = ["text"]
@@ -15,6 +42,7 @@ outputs = ["text"]
max_prompt_images = 3000
costs = { input = 0, output = 0 }
+# --- Mistral Models -----------------------------------------------------------
["mistral-small3.1-24b"]
model_id = "mistral-small3.1:24b"
inputs = ["text"]
@@ -22,6 +50,7 @@ outputs = ["text"]
max_prompt_images = 3000
costs = { input = 0, output = 0 }
+# --- Qwen Models --------------------------------------------------------------
[qwen3-8b]
model_id = "qwen3:8b"
inputs = ["text"]
diff --git a/.pipelex/inference/backends/openai.toml b/.pipelex/inference/backends/openai.toml
index 6e3619499..4eabe7491 100644
--- a/.pipelex/inference/backends/openai.toml
+++ b/.pipelex/inference/backends/openai.toml
@@ -1,12 +1,39 @@
-default_sdk = "openai"
-default_prompting_target = "openai"
-
+################################################################################
+# OpenAI Backend Configuration
+################################################################################
+#
+# This file defines the model specifications for OpenAI models.
+# It contains model definitions for various LLM and image generation models
+# accessible through the OpenAI API.
+#
+# Configuration structure:
+# - Each model is defined in its own section with the model name as the header
+# - Headers with dots must be quoted (e.g., ["gpt-4.1"])
+# - Model costs are in USD per million tokens (input/output)
+#
+################################################################################
+
+################################################################################
+# MODEL DEFAULTS
+################################################################################
+
+[defaults]
+model_type = "llm"
+sdk = "openai"
+prompting_target = "openai"
+
+################################################################################
+# LANGUAGE MODELS
+################################################################################
+
+# --- GPT-3.5 Series -----------------------------------------------------------
["gpt-3.5-turbo"]
model_id = "gpt-3.5-turbo-1106"
inputs = ["text"]
outputs = ["text", "structured"]
costs = { input = 0.5, output = 1.5 }
+# --- GPT-4 Series -------------------------------------------------------------
[gpt-4]
model_id = "gpt-4"
costs = { input = 30.0, output = 60.0 }
@@ -17,6 +44,7 @@ inputs = ["text"]
outputs = ["text", "structured"]
costs = { input = 10.0, output = 30.0 }
+# --- GPT-4o Series ------------------------------------------------------------
[gpt-4o-2024-11-20]
model_id = "gpt-4o-2024-11-20"
inputs = ["text", "images"]
@@ -41,6 +69,7 @@ inputs = ["text", "images"]
outputs = ["text", "structured"]
costs = { input = 0.15, output = 0.6 }
+# --- GPT-4.1 Series -----------------------------------------------------------
["gpt-4.1"]
model_id = "gpt-4.1"
inputs = ["text", "images"]
@@ -59,6 +88,7 @@ inputs = ["text", "images"]
outputs = ["text", "structured"]
costs = { input = 0.1, output = 0.4 }
+# --- o Series ----------------------------------------------------------------
[o1-mini]
model_id = "o1-mini"
inputs = ["text"]
@@ -94,6 +124,7 @@ outputs = ["text", "structured"]
costs = { input = 1.1, output = 4.4 }
constraints = ["temperature_must_be_1"]
+# --- GPT-5 Series -------------------------------------------------------------
[gpt-5]
model_id = "gpt-5"
inputs = ["text", "images"]
@@ -122,3 +153,15 @@ outputs = ["text"]
costs = { input = 1.25, output = 10.0 }
constraints = ["temperature_must_be_1"]
+################################################################################
+# IMAGE GENERATION MODELS
+################################################################################
+
+# --- OpenAI Image Generation --------------------------------------------------
+[gpt-image-1]
+model_type = "img_gen"
+model_id = "gpt-image-1"
+inputs = ["text"]
+outputs = ["image"]
+costs = { input = 0.04, output = 0.0 }
+
diff --git a/.pipelex/inference/backends/perplexity.toml b/.pipelex/inference/backends/perplexity.toml
index 4eb004db5..ea905512c 100644
--- a/.pipelex/inference/backends/perplexity.toml
+++ b/.pipelex/inference/backends/perplexity.toml
@@ -1,6 +1,32 @@
-default_sdk = "openai"
-default_prompting_target = "anthropic"
+################################################################################
+# Perplexity Backend Configuration
+################################################################################
+#
+# This file defines the model specifications for Perplexity AI models.
+# It contains model definitions for Sonar and reasoning models
+# accessible through the Perplexity API.
+#
+# Configuration structure:
+# - Each model is defined in its own section with the model name as the header
+# - Headers with dots must be quoted (e.g., ["sonar-pro"])
+# - Model costs are in USD per million tokens (input/output)
+#
+################################################################################
+################################################################################
+# MODEL DEFAULTS
+################################################################################
+
+[defaults]
+model_type = "llm"
+sdk = "openai"
+prompting_target = "anthropic"
+
+################################################################################
+# LANGUAGE MODELS
+################################################################################
+
+# --- Sonar Series -------------------------------------------------------------
[sonar-pro]
model_id = "sonar-pro"
inputs = ["text"]
@@ -16,6 +42,7 @@ model_id = "sonar-deep-research"
inputs = ["text"]
outputs = ["text"]
+# --- Sonar Reasoning Series ---------------------------------------------------
[sonar-reasoning-pro]
model_id = "sonar-reasoning-pro"
inputs = ["text"]
@@ -26,7 +53,8 @@ model_id = "sonar-reasoning"
inputs = ["text"]
outputs = ["text"]
+# --- DeepSeek Models -----------------------------------------------------------
[perplexity-deepseek-r1]
model_id = "r1-1776"
inputs = ["text"]
-outputs = ["text"]
+outputs = ["text"]
\ No newline at end of file
diff --git a/.pipelex/inference/backends/pipelex_inference.toml b/.pipelex/inference/backends/pipelex_inference.toml
index 22016adc7..2c326cfce 100644
--- a/.pipelex/inference/backends/pipelex_inference.toml
+++ b/.pipelex/inference/backends/pipelex_inference.toml
@@ -1,7 +1,38 @@
-default_sdk = "openai"
-default_prompting_target = "anthropic"
+################################################################################
+# Pipelex Inference Backend Configuration
+################################################################################
+#
+# This file defines the model specifications for the Pipelex Inference backend.
+# It contains model definitions for various LLM and image generation models
+# accessible through the Pipelex unified inference API.
+#
+# Configuration structure:
+# - Each model is defined in its own section with the model name as the header
+# - Headers with dots must be quoted (e.g., ["gpt-4.1"])
+# - Model costs are in USD per million tokens (input/output)
+#
+################################################################################
+
+################################################################################
+# MODEL DEFAULTS
+################################################################################
+
+[defaults]
+model_type = "llm"
+sdk = "openai"
+prompting_target = "anthropic"
+
+################################################################################
+# LANGUAGE MODELS
+################################################################################
+
+# --- OpenAI LLMs --------------------------------------------------------------
+[gpt-4o]
+model_id = "pipelex/gpt-4o"
+inputs = ["text", "images"]
+outputs = ["text", "structured"]
+costs = { input = 2.75, output = 11.00 }
-# OpenAI Models
[gpt-4o-mini]
model_id = "pipelex/gpt-4o-mini"
inputs = ["text", "images"]
@@ -44,13 +75,7 @@ inputs = ["text", "images"]
outputs = ["text"]
costs = { input = 1.25, output = 10.00 }
-[gpt-image-1]
-model_id = "pipelex/gpt-image-1"
-inputs = ["text"]
-outputs = ["images"]
-costs = { input = 5, output = 0.01 }
-
-# Anthropic Models
+# --- Claude LLMs --------------------------------------------------------------
["claude-4-sonnet"]
model_id = "pipelex/claude-4-sonnet"
inputs = ["text", "images"]
@@ -63,7 +88,7 @@ inputs = ["text", "images"]
outputs = ["text", "structured"]
costs = { input = 15, output = 75 }
-# Google Models using Google API
+# --- Gemini LLMs --------------------------------------------------------------
["gemini-2.0-flash"]
model_id = "gemini/gemini-2.0-flash"
inputs = ["text", "images"]
@@ -82,7 +107,8 @@ inputs = ["text", "images"]
outputs = ["text", "structured"]
costs = { input = 0.10, output = 0.40 }
-# XAI Models
+# --- XAI LLMs --------------------------------------------------------------
+
[grok-3]
model_id = "grok-3"
inputs = ["text"]
@@ -95,10 +121,10 @@ inputs = ["text"]
outputs = ["text"]
costs = { input = 0.3, output = 0.5 }
-# Flux Models
-[fast-lightning-sdxl]
-model_id = "pipelex/fast-lightning-sdxl"
-inputs = ["text"]
-outputs = ["images"]
-costs = { input = 0.01, output = 0.01 }
+################################################################################
+# OCR and IMAGE GENERATION MODELS
+################################################################################
+
+# We are still working in giving you acces to OCR and image generation models
+# and to the best models from Mistral through the Pipelex Inference backend.
diff --git a/.pipelex/inference/backends/vertexai.toml b/.pipelex/inference/backends/vertexai.toml
index e476cfcb1..cb8edb13e 100644
--- a/.pipelex/inference/backends/vertexai.toml
+++ b/.pipelex/inference/backends/vertexai.toml
@@ -1,6 +1,32 @@
-default_sdk = "openai"
-default_prompting_target = "gemini"
+################################################################################
+# VertexAI Backend Configuration
+################################################################################
+#
+# This file defines the model specifications for Google VertexAI models.
+# It contains model definitions for Gemini language models
+# accessible through the Google VertexAI API.
+#
+# Configuration structure:
+# - Each model is defined in its own section with the model name as the header
+# - Headers with dots must be quoted (e.g., ["gemini-2.0-flash"])
+# - Model costs are in USD per million tokens (input/output)
+#
+################################################################################
+################################################################################
+# MODEL DEFAULTS
+################################################################################
+
+[defaults]
+model_type = "llm"
+sdk = "openai"
+prompting_target = "gemini"
+
+################################################################################
+# LANGUAGE MODELS
+################################################################################
+
+# --- Gemini 2.0 Series --------------------------------------------------------
["gemini-2.0-flash"]
model_id = "google/gemini-2.0-flash"
inputs = ["text", "images"]
@@ -8,6 +34,7 @@ outputs = ["text", "structured"]
max_prompt_images = 3000
costs = { input = 0.1, output = 0.4 }
+# --- Gemini 2.5 Series --------------------------------------------------------
["gemini-2.5-pro"]
model_id = "google/gemini-2.5-pro"
inputs = ["text", "images"]
@@ -20,5 +47,4 @@ model_id = "google/gemini-2.5-flash"
inputs = ["text", "images"]
outputs = ["text", "structured"]
max_prompt_images = 3000
-costs = { input = 0.30, output = 2.50 }
-
+costs = { input = 0.30, output = 2.50 }
\ No newline at end of file
diff --git a/.pipelex/inference/backends/xai.toml b/.pipelex/inference/backends/xai.toml
index ea8fb58f7..545b905f2 100644
--- a/.pipelex/inference/backends/xai.toml
+++ b/.pipelex/inference/backends/xai.toml
@@ -1,6 +1,32 @@
-default_sdk = "openai"
-default_prompting_target = "anthropic"
+################################################################################
+# XAI Backend Configuration
+################################################################################
+#
+# This file defines the model specifications for XAI (formerly Twitter AI) models.
+# It contains model definitions for Grok language models
+# accessible through the XAI API.
+#
+# Configuration structure:
+# - Each model is defined in its own section with the model name as the header
+# - Headers with dots must be quoted (e.g., ["grok-3"])
+# - Model costs are in USD per million tokens (input/output)
+#
+################################################################################
+################################################################################
+# MODEL DEFAULTS
+################################################################################
+
+[defaults]
+model_type = "llm"
+sdk = "openai"
+prompting_target = "anthropic"
+
+################################################################################
+# LANGUAGE MODELS
+################################################################################
+
+# --- Grok 3 Series ------------------------------------------------------------
[grok-3]
model_id = "grok-3"
inputs = ["text"]
diff --git a/.pipelex/inference/deck/base_deck.toml b/.pipelex/inference/deck/base_deck.toml
index 51d646c9a..900235960 100644
--- a/.pipelex/inference/deck/base_deck.toml
+++ b/.pipelex/inference/deck/base_deck.toml
@@ -26,11 +26,16 @@ base_llm_for_object = "gpt-4o-mini"
llm_to_engineer = ["claude-4.1-opus", "gemini-2.5-pro"]
llm_for_large_codebase = ["gemini-2.5-pro", "claude-4-sonnet", "gpt-5"]
+# Image generation aliases
+base-img-gen = "flux-pro/v1.1"
+best-img-gen = "flux-pro/v1.1-ultra"
+fast-img-gen = "fast-lightning-sdxl"
+
####################################################################################################
# LLM Presets
####################################################################################################
-[llm_presets]
+[llm.presets]
####################################################################################################
# LLM Presets — General purpose
@@ -67,10 +72,11 @@ llm_for_question_and_excerpt_reformulation = { llm_handle = "gpt-4o", temperatur
# Engineering skills
llm_to_engineer = { llm_handle = "base-claude", temperature = 0.2 }
+llm_to_write = { llm_handle = "base-gemini", temperature = 0.2 }
llm_to_pipelex = { llm_handle = "base-claude", temperature = 0.2 }
# Image skills
-llm_to_write_imgg_prompt = { llm_handle = "base-claude", temperature = 0.2 }
+llm_to_write_img_gen_prompt = { llm_handle = "base-claude", temperature = 0.2 }
llm_to_describe_img = { llm_handle = "base-claude", temperature = 0.5 }
llm_to_design_fashion = { llm_handle = "base-claude", temperature = 0.7 }
llm_for_img_to_text = { llm_handle = "base-claude", temperature = 0.1 }
@@ -86,7 +92,38 @@ llm_to_extract_tables = { llm_handle = "base-claude", temperature = 0.1 }
# LLM Choices
####################################################################################################
-[llm_choice_defaults]
+[llm.choice_defaults]
for_text = "cheap_llm_for_text"
for_object = "cheap_llm_for_object"
+####################################################################################################
+# OCR Presets
+####################################################################################################
+
+[ocr]
+choice_default = "base_ocr_mistral"
+
+[ocr.presets]
+base_ocr_mistral = { ocr_handle = "mistral-ocr", max_nb_images = 100, image_min_size = 50 }
+base_ocr_pypdfium2 = { ocr_handle = "pypdfium2-extract-text", max_nb_images = 100, image_min_size = 50 }
+
+####################################################################################################
+# Image Generation Presets
+####################################################################################################
+
+[img_gen]
+choice_default = "base_img_gen"
+
+[img_gen.presets]
+# General purpose
+base_img_gen = { img_gen_handle = "base-img-gen", quality = "medium", guidance_scale = 7.5, is_moderated = true, safety_tolerance = 3 }
+fast_img_gen = { img_gen_handle = "fast-img-gen", nb_steps = 4, guidance_scale = 5.0, is_moderated = true, safety_tolerance = 3 }
+high_quality_img_gen = { img_gen_handle = "best-img-gen", quality = "high", guidance_scale = 8.0, is_moderated = true, safety_tolerance = 3 }
+
+# Specific skills
+img_gen_for_art = { img_gen_handle = "best-img-gen", quality = "high", guidance_scale = 9.0, is_moderated = false, safety_tolerance = 5 }
+img_gen_for_diagram = { img_gen_handle = "base-img-gen", quality = "medium", guidance_scale = 7.0, is_moderated = true, safety_tolerance = 2 }
+img_gen_for_mockup = { img_gen_handle = "base-img-gen", quality = "medium", guidance_scale = 6.5, is_moderated = true, safety_tolerance = 3 }
+img_gen_for_product = { img_gen_handle = "best-img-gen", quality = "high", guidance_scale = 8.5, is_moderated = true, safety_tolerance = 2 }
+img_gen_for_testing = { img_gen_handle = "fast-img-gen", nb_steps = 4, guidance_scale = 4.0, is_moderated = true, safety_tolerance = 4 }
+
diff --git a/.pipelex/inference/deck/overrides.toml b/.pipelex/inference/deck/overrides.toml
index 999cc4d1a..d341ec7d2 100644
--- a/.pipelex/inference/deck/overrides.toml
+++ b/.pipelex/inference/deck/overrides.toml
@@ -4,7 +4,7 @@
# LLM Deck overrides
####################################################################################################
-[llm_choice_overrides]
+[llm.choice_overrides]
for_text = "disabled"
for_object = "disabled"
diff --git a/.pipelex/inference/routing_profiles.toml b/.pipelex/inference/routing_profiles.toml
index be9b51d2c..7290382e0 100644
--- a/.pipelex/inference/routing_profiles.toml
+++ b/.pipelex/inference/routing_profiles.toml
@@ -6,12 +6,12 @@
# ================================================
# Which profile to use (change this to switch routing)
-active = "all_pipelex"
+active = "pipelex_first"
-# We recommend using the "all_pipelex" profile to get a head start with all models.
+# We recommend using the "pipelex_first" profile to get a head start with all models.
# The Pipelex Inference backend is currently not recommended for production use,
# but it's great for development and testing.
-# To use the Pipelex Inference backend (all_pipelex profile):
+# To use the Pipelex Inference backend (pipelex_first profile):
# 1. Join our Discord community to get your free API key (no credit card required):
# Visit https://go.pipelex.com/discord and request your key in the appropriate channel
# 2. Set the environment variable: export PIPELEX_INFERENCE_API_KEY="your-api-key"
@@ -22,32 +22,48 @@ active = "all_pipelex"
# Routing Profiles
# ============================================
-[profiles.all_pipelex]
-description = "Use Pipelex Inference backend for all models"
+[profiles.pipelex_first]
+description = "Use Pipelex Inference backend for all its supported models"
default = "pipelex_inference"
+[profiles.pipelex_first.routes]
+# Pattern matching: "model-pattern" = "backend-name"
+"gpt-*" = "pipelex_inference"
+"claude-*" = "pipelex_inference"
+"grok-*" = "pipelex_inference"
+"gemini-*" = "google"
+"*-sdxl" = "fal"
+"flux-*" = "fal"
+"gpt-image-1" = "openai"
+"mistral-ocr" = "mistral"
+
[profiles.all_anthropic]
description = "Use Anthropic backend for all its supported models"
default = "anthropic"
+[profiles.all_gemini]
+description = "Use Google GenAI backend for all its supported models"
+default = "google"
+
[profiles.all_azure_openai]
description = "Use Azure OpenAI backend for all its supported models"
default = "azure_openai"
-
-[profiles.to_test_various_backends]
-description = "Route models to various backends"
-
[profiles.custom_routing]
description = "Custom routing"
default = "pipelex_inference"
[profiles.custom_routing.routes]
# Pattern matching: "model-pattern" = "backend-name"
-"gpt-*" = "pipelex_inference"
-"claude-*" = "pipelex_inference"
-"gemini-*" = "pipelex_inference"
-"grok-*" = "pipelex_inference"
-"*-sdxl" = "pipelex_inference"
+"gpt-*" = "azure_openai"
+"claude-*" = "bedrock"
+"gemini-*" = "google"
+"grok-*" = "xai"
+"*-sdxl" = "fal"
+"flux-*" = "fal"
+"gpt-image-1" = "openai"
+
+[profiles.to_test_various_backends]
+description = "Route models to various backends"
[profiles.to_test_various_backends.routes]
"gpt-5-nano" = "pipelex_inference"
diff --git a/.pipelex/pipelex.toml b/.pipelex/pipelex.toml
index 643a21e68..b8f8b129f 100644
--- a/.pipelex/pipelex.toml
+++ b/.pipelex/pipelex.toml
@@ -18,3 +18,4 @@ page_output_text_file_name = "page_text.md"
is_pipeline_tracking_enabled = false
is_activity_tracking_enabled = false
is_reporting_enabled = true
+
diff --git a/.vscode/launch.json b/.vscode/launch.json
index 35afe5a40..942c9c079 100644
--- a/.vscode/launch.json
+++ b/.vscode/launch.json
@@ -36,7 +36,7 @@
"-s",
"-vv",
"-m inference",
- "-k test_simple_gen"
+ "-k test_pipe_llm_structured"
],
"console": "integratedTerminal"
},
diff --git a/AGENTS.md b/AGENTS.md
index d7446120c..6ea9e9a54 100644
--- a/AGENTS.md
+++ b/AGENTS.md
@@ -95,7 +95,7 @@ Always fix any issues reported by these tools before proceeding.
```bash
make tp
```
- Runs tests with markers: `(dry_runnable or not (inference or llm or imgg or ocr)) and not (needs_output or pipelex_api)`
+ Runs tests with markers: `(dry_runnable or not (inference or llm or img_gen or ocr)) and not (needs_output or pipelex_api)`
2. **Specific Tests**:
```bash
@@ -105,7 +105,7 @@ Always fix any issues reported by these tools before proceeding.
```
Note: Matches names starting with the provided string.
-**Important**: Never run `make ti`, `make test-inference`, `make to`, `make test-ocr`, `make tg`, or `make test-imgg` - these use costly inference.
+**Important**: Never run `make ti`, `make test-inference`, `make to`, `make test-ocr`, `make tg`, or `make test-img-gen` - these use costly inference.
## Pipelines
@@ -756,7 +756,7 @@ These rules apply when writing unit tests.
Apply the appropriate markers:
- "llm: uses an LLM to generate text or objects"
-- "imgg: uses an image generation AI"
+- "img_gen: uses an image generation AI"
- "inference: uses either an LLM or an image generation AI"
- "gha_disabled: will not be able to run properly on GitHub Actions"
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 41f96fd74..f7b508fd9 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,5 +1,38 @@
# Changelog
+## Unreleased
+
+### Highlights
+
+- **New pipe builder** pipeline to generate Pipes based on a brief in natural language: use the cli `pipelex build pipe "Your task"` to build the pipe.
+- **Full refactoring of OCR and Image Generation** to use the same patterns as `LLM` workers and pipes.
+
+### Added
+
+ - Added a badge on the `README.md` to display the number of tests.
+ - Added new placeholder utility module with `make_placeholder_value()` and `value_is_placeholder()` functions
+ - Added comprehensive test suite for placeholder functionality
+ - Added new test cases for environment variable functions
+ - Added new documentation for `PipeFunc` on how to register functions.
+
+### Changed
+ - Renamed `llm_deck` terminology to `model_deck` throughout codebase and documentation
+ - Renamed `get_llm_deck_paths()` method to `get_model_deck_paths()`
+ - Renamed `is_gha_testing` property to `is_ci_testing` in RuntimeManager
+ - Updated environment variable handling to use new placeholder utility functions
+ - Refactored `all_env_vars_are_set()` function to only accept a list of keys, single string support now uses `is_env_var_set()`
+ - Modified `any_env_var_is_placeholder()` to use new placeholder detection logic
+ - Updated test environment setup to use dynamic placeholder generation instead of hardcoded values
+ - Restructured test classes and methods in environment tests
+
+### Fixed
+ - Fixed logic error in `any_env_var_is_placeholder()` function - now correctly returns False when no placeholders are found
+
+### Removed
+ - Removed `ENV_DUMMY_PLACEHOLDER_VALUE` constant
+ - Removed `get_rooted_path()` and `get_env_rooted_path()` utility functions
+ - Removed hardcoded placeholder dictionary in test setup
+
## [v0.10.2] - 2025-09-18
### Added
@@ -157,10 +190,19 @@ For complete details, see the [Inference Backend Configuration](pages/configurat
- Fixed the `structuring_method` behavior in the `PipeLLM` pipe: Putting it to `preliminary_text`, the `PipeLLM` will always generate text before generating the structure -> Reliability increased by a lot.
+### Fixed
+
+- Fixed a bug in the `needed_inputs` method of the `PipeSequence` pipe.
+
### Changed
+- `dry_run_pipe` now returns a `DryRunOutput` object instead of a `str` with additional information.
- Updated `cocode` dependency from version `v0.0.10` to `v0.0.15`.
+### Added
+
+- Added the `FuncRegistryUtils` class to register functions in the library.
+
## [v0.8.1] - 2025-08-27
### Bugfix
diff --git a/CLAUDE.md b/CLAUDE.md
index 315ab1b1c..181e3835b 100644
--- a/CLAUDE.md
+++ b/CLAUDE.md
@@ -95,7 +95,7 @@ Always fix any issues reported by these tools before proceeding.
```bash
make tp
```
- Runs tests with markers: `(dry_runnable or not (inference or llm or imgg or ocr)) and not (needs_output or pipelex_api)`
+ Runs tests with markers: `(dry_runnable or not (inference or llm or img_gen or ocr)) and not (needs_output or pipelex_api)`
2. **Specific Tests**:
```bash
@@ -105,7 +105,7 @@ Always fix any issues reported by these tools before proceeding.
```
Note: Matches names starting with the provided string.
-**Important**: Never run `make ti`, `make test-inference`, `make to`, `make test-ocr`, `make tg`, or `make test-imgg` - these use costly inference.
+**Important**: Never run `make ti`, `make test-inference`, `make to`, `make test-ocr`, `make tg`, or `make test-img-gen` - these use costly inference.
## Pipelines
@@ -757,7 +757,7 @@ These rules apply when writing unit tests.
Apply the appropriate markers:
- "llm: uses an LLM to generate text or objects"
-- "imgg: uses an image generation AI"
+- "img_gen: uses an image generation AI"
- "inference: uses either an LLM or an image generation AI"
- "gha_disabled: will not be able to run properly on GitHub Actions"
diff --git a/Makefile b/Makefile
index 0c3379060..15f51017b 100644
--- a/Makefile
+++ b/Makefile
@@ -17,7 +17,7 @@ VENV_MKDOCS := $(VIRTUAL_ENV)/bin/mkdocs
UV_MIN_VERSION = $(shell grep -m1 'required-version' pyproject.toml | sed -E 's/.*= *"([^<>=, ]+).*/\1/')
-USUAL_PYTEST_MARKERS := "(dry_runnable or not (inference or llm or imgg or ocr)) and not (needs_output or pipelex_api)"
+USUAL_PYTEST_MARKERS := "(dry_runnable or not (inference or llm or img_gen or ocr)) and not (needs_output or pipelex_api)"
define PRINT_TITLE
$(eval PROJECT_PART := [$(PROJECT_NAME)])
@@ -79,8 +79,8 @@ make ti - Shorthand -> test-inference
make tip - Shorthand -> test-inference-with-prints (parallelized inference tests)
make test-ocr - Run unit tests only for ocr (with prints)
make to - Shorthand -> test-ocr
-make test-imgg - Run unit tests only for imgg (with prints)
-make test-g - Shorthand -> test-imgg
+make test-img-gen - Run unit tests only for img_gen (with prints)
+make test-g - Shorthand -> test-img-gen
make check-unused-imports - Check for unused imports without fixing
make fix-unused-imports - Fix unused imports with ruff
@@ -104,12 +104,13 @@ export HELP
format lint pyright mypy \
cleanderived cleanenv cleanlibraries cleanall \
test test-xdist t test-quiet tq test-with-prints tp test-inference ti \
- test-imgg tg test-ocr to codex-tests gha-tests \
+ test-img-gen tg test-ocr to codex-tests gha-tests \
run-all-tests run-manual-trigger-gha-tests run-gha_disabled-tests \
validate v check c cc \
merge-check-ruff-lint merge-check-ruff-format merge-check-mypy merge-check-pyright \
li check-unused-imports fix-unused-imports check-uv check-TODOs docs docs-check docs-deploy \
- config-template cft
+ config-template cft \
+ test-count check-test-badge
all help:
@echo "$$HELP"
@@ -165,8 +166,7 @@ build: env
config-template:
$(call PRINT_TITLE,"Updating config template from .pipelex/")
- @rm -rf pipelex/config_template/*
- @cp -r .pipelex/* pipelex/config_template/
+ @rsync -av --exclude='inference/backends.toml' --delete .pipelex/ pipelex/config_template/
cft: config-template
@echo "> done: cft = config-template"
@@ -285,17 +285,17 @@ tp: test-with-prints
test-inference-with-prints: env
$(call PRINT_TITLE,"Unit testing")
@if [ -n "$(TEST)" ]; then \
- $(VENV_PYTEST) --pipe-run-mode live -m "inference and not imgg" -s -k "$(TEST)" $(if $(filter 1,$(VERBOSE)),-v,$(if $(filter 2,$(VERBOSE)),-vv,$(if $(filter 3,$(VERBOSE)),-vvv,))); \
+ $(VENV_PYTEST) --pipe-run-mode live -m "inference and not img_gen" -s -k "$(TEST)" $(if $(filter 1,$(VERBOSE)),-v,$(if $(filter 2,$(VERBOSE)),-vv,$(if $(filter 3,$(VERBOSE)),-vvv,))); \
else \
- $(VENV_PYTEST) --pipe-run-mode live -m "inference and not imgg" -s $(if $(filter 1,$(VERBOSE)),-v,$(if $(filter 2,$(VERBOSE)),-vv,$(if $(filter 3,$(VERBOSE)),-vvv,))); \
+ $(VENV_PYTEST) --pipe-run-mode live -m "inference and not img_gen" -s $(if $(filter 1,$(VERBOSE)),-v,$(if $(filter 2,$(VERBOSE)),-vv,$(if $(filter 3,$(VERBOSE)),-vvv,))); \
fi
test-inference-fast: env
$(call PRINT_TITLE,"Unit testing")
@if [ -n "$(TEST)" ]; then \
- $(VENV_PYTEST) -n auto --pipe-run-mode live -m "inference and not imgg" -s -k "$(TEST)" $(if $(filter 1,$(VERBOSE)),-v,$(if $(filter 2,$(VERBOSE)),-vv,$(if $(filter 3,$(VERBOSE)),-vvv,))); \
+ $(VENV_PYTEST) -n auto --pipe-run-mode live -m "inference and not img_gen" -s -k "$(TEST)" $(if $(filter 1,$(VERBOSE)),-v,$(if $(filter 2,$(VERBOSE)),-vv,$(if $(filter 3,$(VERBOSE)),-vvv,))); \
else \
- $(VENV_PYTEST) -n auto --pipe-run-mode live -m "inference and not imgg" -s $(if $(filter 1,$(VERBOSE)),-v,$(if $(filter 2,$(VERBOSE)),-vv,$(if $(filter 3,$(VERBOSE)),-vvv,))); \
+ $(VENV_PYTEST) -n auto --pipe-run-mode live -m "inference and not img_gen" -s $(if $(filter 1,$(VERBOSE)),-v,$(if $(filter 2,$(VERBOSE)),-vv,$(if $(filter 3,$(VERBOSE)),-vvv,))); \
fi
tip: test-inference-with-prints
@@ -307,9 +307,9 @@ ti: test-inference-fast
ti-dry: env
$(call PRINT_TITLE,"Unit testing")
@if [ -n "$(TEST)" ]; then \
- $(VENV_PYTEST) --pipe-run-mode dry --exitfirst -m "inference and not imgg" -s -k "$(TEST)" $(if $(filter 1,$(VERBOSE)),-v,$(if $(filter 2,$(VERBOSE)),-vv,$(if $(filter 3,$(VERBOSE)),-vvv,))); \
+ $(VENV_PYTEST) --pipe-run-mode dry --exitfirst -m "inference and not img_gen" -s -k "$(TEST)" $(if $(filter 1,$(VERBOSE)),-v,$(if $(filter 2,$(VERBOSE)),-vv,$(if $(filter 3,$(VERBOSE)),-vvv,))); \
else \
- $(VENV_PYTEST) --pipe-run-mode dry --exitfirst -m "inference and not imgg" -s $(if $(filter 1,$(VERBOSE)),-v,$(if $(filter 2,$(VERBOSE)),-vv,$(if $(filter 3,$(VERBOSE)),-vvv,))); \
+ $(VENV_PYTEST) --pipe-run-mode dry --exitfirst -m "inference and not img_gen" -s $(if $(filter 1,$(VERBOSE)),-v,$(if $(filter 2,$(VERBOSE)),-vv,$(if $(filter 3,$(VERBOSE)),-vvv,))); \
fi
test-ocr: env
@@ -323,16 +323,16 @@ test-ocr: env
to: test-ocr
@echo "> done: to = test-ocr"
-test-imgg: env
+test-img-gen: env
$(call PRINT_TITLE,"Unit testing")
@if [ -n "$(TEST)" ]; then \
- $(VENV_PYTEST) --pipe-run-mode live --exitfirst -m "imgg" -s -k "$(TEST)" $(if $(filter 1,$(VERBOSE)),-v,$(if $(filter 2,$(VERBOSE)),-vv,$(if $(filter 3,$(VERBOSE)),-vvv,))); \
+ $(VENV_PYTEST) --pipe-run-mode live --exitfirst -m "img_gen" -s -k "$(TEST)" $(if $(filter 1,$(VERBOSE)),-v,$(if $(filter 2,$(VERBOSE)),-vv,$(if $(filter 3,$(VERBOSE)),-vvv,))); \
else \
- $(VENV_PYTEST) --pipe-run-mode live --exitfirst -m "imgg" -s $(if $(filter 1,$(VERBOSE)),-v,$(if $(filter 2,$(VERBOSE)),-vv,$(if $(filter 3,$(VERBOSE)),-vvv,))); \
+ $(VENV_PYTEST) --pipe-run-mode live --exitfirst -m "img_gen" -s $(if $(filter 1,$(VERBOSE)),-v,$(if $(filter 2,$(VERBOSE)),-vv,$(if $(filter 3,$(VERBOSE)),-vvv,))); \
fi
-tg: test-imgg
- @echo "> done: tg = test-imgg"
+tg: test-img-gen
+ @echo "> done: tg = test-img-gen"
test-pipelex-api: env
$(call PRINT_TITLE,"Unit testing")
@@ -460,4 +460,26 @@ v: validate
@echo "> done: v = validate"
li: lock install
- @echo "> done: lock install"
\ No newline at end of file
+ @echo "> done: lock install"
+
+
+##########################################################################################
+### TEST BADGE
+##########################################################################################
+
+## Print the number of collected pytest tests (just the integer)
+test-count: env
+ @COUNT=$$($(VENV_PYTEST) --collect-only --disable-warnings -q | awk 'NF' | wc -l | tr -d ' '); \
+ echo $$COUNT
+
+## Compare current test count vs .badges/tests.json -> .message; fail if mismatch
+check-test-badge: env
+ @EXPECTED=$$($(VENV_PYTHON) -c 'import json;print(int(json.load(open(".badges/tests.json"))["message"]))'); \
+ ACTUAL=$$($(VENV_PYTEST) --collect-only --disable-warnings -q | awk 'NF' | wc -l | tr -d ' '); \
+ if [ "$$EXPECTED" != "$$ACTUAL" ]; then \
+ echo "❌ Test count mismatch: badge=$$EXPECTED, actual=$$ACTUAL"; \
+ exit 1; \
+ else \
+ echo "✅ Test count matches: $$ACTUAL"; \
+ fi
+
diff --git a/NEW_INFERENCE_BACKEND_CONFIG_SYSTEM.md b/NEW_INFERENCE_BACKEND_CONFIG_SYSTEM.md
index eed439a74..9005b72f6 100644
--- a/NEW_INFERENCE_BACKEND_CONFIG_SYSTEM.md
+++ b/NEW_INFERENCE_BACKEND_CONFIG_SYSTEM.md
@@ -9,11 +9,11 @@ The core components of this system reside in the following directories:
## Core Concepts
-The new system is built around a few key concepts: Inference Backends, Model Specs, Routing Profiles, and the Model Deck.
+The new system is built around a few key concepts: Inference Backends, Model Specs, Routing Profiles, and the Model Deck. This unified system now supports all types of AI models: LLMs for text generation, OCR models for text extraction, and Image Generation models.
### 1. Inference Backends
-An **Inference Backend** represents a provider of LLM services, such as OpenAI, Anthropic, Google Vertex AI, etc. Each backend is defined by its name, API endpoint, and authentication details (e.g., API key).
+An **Inference Backend** represents a provider of AI services, such as OpenAI, Anthropic, Google Vertex AI, FAL, etc. Each backend is defined by its name, API endpoint, and authentication details (e.g., API key). Backends can provide different types of models including LLMs, OCR models, and Image Generation models.
- **Implementation**: `pipelex.cogt.model_backends.backend.InferenceBackend`
- **Configuration**: Backends are defined in a central TOML file (e.g., `backends.toml`), specified by `cogt.inference_config.backends_library_path` in the main configuration.
@@ -22,10 +22,11 @@ An **Inference Backend** represents a provider of LLM services, such as OpenAI,
An **Inference Model Spec** (`InferenceModelSpec`) provides detailed information about a specific model available through a backend. This includes:
+- `model_type`: The type of model (`llm`, `text_extractor`, or `img_gen`).
- `model_id`: The identifier used by the backend's API.
-- `sdk`: The Pipelex SDK to use for communication (e.g., `openai`, `anthropic`).
-- Capabilities: Supported `inputs` (e.g., `text`, `images`) and `outputs` (e.g., `text`, `structured`).
-- Pricing: `costs` for input and output tokens.
+- `sdk`: The Pipelex SDK to use for communication (e.g., `openai`, `anthropic`, `fal`, `pypdfium2`).
+- Capabilities: Supported `inputs` (e.g., `text`, `images`, `pdf`) and `outputs` (e.g., `text`, `structured`, `pages`, `image`).
+- Pricing: `costs` for input and output tokens or operations.
- Constraints: `max_tokens`, `max_prompt_images`, and other model-specific limitations.
- **Implementation**: `pipelex.cogt.model_backends.model_spec.InferenceModelSpec`
@@ -47,27 +48,69 @@ The routing logic supports exact matches, as well as prefix, suffix, and "contai
The **Model Deck** (`ModelDeck`) is the final, unified collection of all configured and available models in the system. It is constructed at runtime by the `ModelManager` and serves as the single source of truth for the rest of the application. The Model Deck contains:
-- **Inference Models**: A resolved map of model names (`llm_handle`) to their corresponding `InferenceModelSpec`.
-- **Aliases**: User-friendly names that map to one or more model names, creating a fallback chain (e.g., `best-claude` -> `claude-3-opus-20240229`).
+- **Inference Models**: A resolved map of model names (`model_handle`) to their corresponding `InferenceModelSpec`.
+- **Aliases**: User-friendly names that map to one or more model names, creating a fallback chain (e.g., `best-claude` -> `claude-3-opus-20240229`, `base-img-gen` -> `flux-pro/v1.1`).
- **LLM Presets**: Pre-defined configurations (`LLMSetting`) for specific tasks, combining an `llm_handle` with parameters like `temperature` and `max_tokens` (e.g., `llm_to_reason`).
-- **Default Choices**: Default model presets to use for text generation (`for_text`) and structured data extraction (`for_object`).
+- **OCR Presets**: Pre-defined configurations (`OcrSetting`) for OCR tasks, combining an `ocr_handle` with parameters like `max_nb_images` and `image_min_size` (e.g., `base_ocr_mistral`).
+- **Image Generation Presets**: Pre-defined configurations (`ImgGenSetting`) for image generation tasks, combining an `img_gen_handle` with parameters like `quality`, `guidance_scale`, and `safety_tolerance` (e.g., `base_img_gen`, `high_quality_img_gen`).
+- **Default Choices**: Default model presets to use for text generation (`for_text`), structured data extraction (`for_object`), OCR operations (`ocr.choice_default`), and image generation (`img_gen.choice_default`).
- **Implementation**: `pipelex.cogt.models.model_deck.ModelDeck`
-- **Configuration**: Aliases, presets, and defaults are loaded from one or more `llm_deck.toml` files, specified by `cogt.inference_config.get_llm_deck_paths()`.
+- **Configuration**: Aliases, presets, and defaults are loaded from one or more `model_deck.toml` files, specified by `cogt.inference_config.get_model_deck_paths()`.
+
+### 5. OCR Integration
+
+**OCR (Optical Character Recognition)** models are now fully integrated into the inference backend system. OCR models extract text and images from PDFs and images, producing structured page data.
+
+- **Model Type**: `text_extractor`
+- **Common SDKs**: `mistral` (for Mistral OCR models), `pypdfium2` (for local PDF text extraction)
+- **Inputs**: `pdf`, `image`
+- **Outputs**: `pages` (structured page data with text and images)
+- **Usage**: OCR models are referenced by their `ocr_handle` in `PipeOcr` operations and can use OCR presets for common configurations
+
+Example OCR model configuration:
+```toml
+[mistral-ocr]
+model_type = "text_extractor"
+model_id = "mistral-ocr-latest"
+inputs = ["pdf", "image"]
+outputs = ["pages"]
+costs = { input = 0.4, output = 2.0 }
+```
+
+### 6. Image Generation Integration
+
+**Image Generation** models are integrated into the same system, allowing for consistent configuration and routing of image generation requests.
+
+- **Model Type**: `img_gen`
+- **Common SDKs**: `fal` (for FAL models), `openai` (for DALL-E models)
+- **Inputs**: `text` (prompts)
+- **Outputs**: `image`
+- **Usage**: Image generation models are referenced by their `img_gen_handle` in `PipeImgGen` operations and support presets for different quality levels and use cases
+
+Example Image Generation model configuration:
+```toml
+["flux-pro/v1.1"]
+model_type = "img_gen"
+model_id = "fal-ai/flux-pro/v1.1"
+inputs = ["text"]
+outputs = ["image"]
+costs = { input = 0.05, output = 0.0 }
+```
## How It All Works Together: The Loading Process
The entire system is orchestrated by the `ModelManager` during application startup. Here is a step-by-step breakdown of the process:
1. **Load Backends**: The `InferenceBackendLibrary` reads the `backends.toml` file to get a list of all defined backends. For each enabled backend, it proceeds to load its associated models.
-2. **Load Model Specs**: For each backend, the library reads the corresponding `<backend_name>.toml` file to load the `InferenceModelSpec` for every model that backend provides.
+2. **Load Model Specs**: For each backend, the library reads the corresponding `<backend_name>.toml` file to load the `InferenceModelSpec` for every model that backend provides. This includes LLM models, OCR models (`text_extractor` type), and Image Generation models (`img_gen` type).
3. **Load Routing Profiles**: The `RoutingProfileLibrary` reads the `routing.toml` file to load all defined routing profiles and identifies the `active` profile.
4. **Build the Deck**: The `ModelManager` begins to build the `ModelDeck`.
- a. It gets the list of all models from all loaded backends.
+ a. It gets the list of all models from all loaded backends (LLMs, OCR models, and Image Generation models).
b. For each model, it consults the `active` `RoutingProfile` to determine which backend should be used.
c. It retrieves the `InferenceModelSpec` for that model from the chosen backend.
d. This resolved model spec is added to the `ModelDeck`'s list of `inference_models`.
-5. **Load Deck Configuration**: The `ModelManager` reads the `llm_deck.toml` files and loads the `aliases`, `llm_presets`, and default choices into the `ModelDeck`.
-6. **Finalization**: The `ModelDeck` is now fully constructed and ready to be used by the rest of Pipelex to get model specifications and settings via `get_models_manager().get_llm_deck()`.
+5. **Load Deck Configuration**: The `ModelManager` reads the `base_deck.toml` and other model deck files from the `deck` directory and loads the `aliases`, `llm_presets`, `ocr_presets`, `img_gen_presets`, and default choices into the `ModelDeck`.
+6. **Finalization**: The `ModelDeck` is now fully constructed and ready to be used by the rest of Pipelex to get model specifications and settings for LLMs, OCR operations, and Image Generation via `get_model_deck()`.
-This layered configuration system provides a powerful way to manage models from various providers, route them according to specific needs, and define reusable presets for common tasks, all through a set of clear and manageable TOML files.
+This layered configuration system provides a powerful way to manage models from various providers, route them according to specific needs, and define reusable presets for common tasks, all through a set of clear and manageable TOML files. The unified approach ensures that LLMs, OCR models, and Image Generation models are all managed consistently within the same infrastructure.
diff --git a/README.md b/README.md
index 663b0d6d0..5b2c1632f 100644
--- a/README.md
+++ b/README.md
@@ -18,6 +18,7 @@ Stop reinventing AI workflows from scratch. With Pipelex, your proven methods be
<p align="center">
<a href="LICENSE"><img src="https://img.shields.io/badge/License-MIT-blue.svg" alt="MIT License"></a>
+ <a href="https://github.com/Pipelex/pipelex/actions/workflows/check-test-count-badge.yml"><img src="https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/Pipelex/pipelex/feature/pipe-builder/.badges/tests.json" alt="Tests"></a>
<img src="https://img.shields.io/pypi/v/pipelex?logo=pypi&logoColor=white&color=blue&style=flat-square"
alt="PyPI – latest release">
<br/>
@@ -61,7 +62,7 @@ Stop reinventing AI workflows from scratch. With Pipelex, your proven methods be
Pipelex makes it easy for developers to define and run repeatable AI workflows. At its core is a clear, declarative pipeline language specifically crafted for knowledge-processing tasks.
-Build **pipelines** from modular pipes that snap together. Each pipe can use a different language model (LLM) or software to process knowledge. Pipes consistently deliver **structured, predictable outputs** at each stage.
+Build **pipelines** from modular pipes that snap together. Each pipe can use different AI models - language models (LLMs) for text generation, OCR models for document processing, or image generation models for creating visuals. Pipes consistently deliver **structured, predictable outputs** at each stage.
Pipelex uses its own syntax PLX, based on TOML, making workflows readable and shareable. Business professionals, developers, and AI coding agents can all understand and modify the same pipeline definitions.
@@ -124,10 +125,10 @@ uv pip install pipelex
The package supports the following additional features:
-- `anthropic`: Anthropic/Claude support+
-- `google`: Google models (Vertex) support
-- `mistralai`: Mistral AI support√
-- `bedrock`: AWS Bedrock support
+- `anthropic`: Anthropic/Claude support for text generation
+- `google`: Google models (Vertex) support for text generation
+- `mistralai`: Mistral AI support for text generation and OCR
+- `bedrock`: AWS Bedrock support for text generation
- `fal`: Image generation with Black Forest Labs "FAL" service
Install all extras:
@@ -139,7 +140,7 @@ pip install "pipelex[anthropic,google,mistralai,bedrock,fal]"
Using `poetry`:
```bash
-poetry add "pipelex[anthropic,google,mistralai,bedrock,fal]"
+poetry add "pipelex[anthropic,google,mistralai,bedrock,fal,pypdfium2]"
```
Using `uv`:
diff --git a/docs/pages/build-reliable-ai-workflows-with-pipelex/pipe-controllers/PipeSequence.md b/docs/pages/build-reliable-ai-workflows-with-pipelex/pipe-controllers/PipeSequence.md
index ba06aa3e1..c148a631c 100644
--- a/docs/pages/build-reliable-ai-workflows-with-pipelex/pipe-controllers/PipeSequence.md
+++ b/docs/pages/build-reliable-ai-workflows-with-pipelex/pipe-controllers/PipeSequence.md
@@ -44,7 +44,7 @@ Let's imagine a pipeline that first extracts text from an image, then summarizes
type = "PipeOcr"
definition = "Extract text from an image"
output = "Text"
-ocr_model = "mistral-ocr"
+ocr = "mistral-ocr"
[pipe.summarize_text]
type = "PipeLLM"
diff --git a/docs/pages/build-reliable-ai-workflows-with-pipelex/pipe-operators/PipeFunc.md b/docs/pages/build-reliable-ai-workflows-with-pipelex/pipe-operators/PipeFunc.md
index aadb71b8e..8db2b8eac 100644
--- a/docs/pages/build-reliable-ai-workflows-with-pipelex/pipe-operators/PipeFunc.md
+++ b/docs/pages/build-reliable-ai-workflows-with-pipelex/pipe-operators/PipeFunc.md
@@ -4,34 +4,43 @@ The `PipeFunc` operator provides an essential escape hatch, allowing you to exec
## How it works
-`PipeFunc` operates by calling a Python function that has been registered with Pipelex's central function registry.
+`PipeFunc` operates by calling a Python function that has been automatically registered with Pipelex's central function registry.
-1. **Function Registration**: First, you must define a standard Python function and register it with a unique name. This is done once when your application starts up.
-2. **Function Signature**: The registered function **must** accept a single argument: `working_memory: WorkingMemory`. This object provides access to all the data currently available in the pipeline.
-3. **Execution**: When the `PipeFunc` pipe is executed, it looks up your function by its registered name and calls it, passing in the current `working_memory`.
-4. **Returning Data**: The function can return data, which `PipeFunc` will then place back into the working memory, associated with the pipe's `output` concept.
+1. **Automatic Registration**: Functions are automatically discovered and registered from Python files in the `pipelex/libraries/` directory when Pipelex starts up.
+2. **Function Signature**: Eligible functions are automatically registered using their function name as the registry key.
+3. **Execution**: When the `PipeFunc` pipe is executed, it looks up your function by name and calls it, passing in the current `working_memory`.
+4. **Returning Data**: The function returns data, which `PipeFunc` places back into the working memory, associated with the pipe's `output` concept.
+
+## Function Eligibility Requirements
+
+For a function to be automatically registered and available to `PipeFunc`, it **must** meet all of the following criteria:
+
+!!! warning "Function Eligibility Requirements"
+ - **Must be an async function** (defined with `async def`)
+ - **Must have exactly 1 parameter** named `working_memory`
+ - **Parameter type must be** `WorkingMemory`
+ - **Return type must be** a subclass of `StuffContent` (or a generic type like `ListContent[SomeType]`)
+ - **Must be defined in a Pipelex library file** within the `pipelines/` directory
### Return values
-Your Python function can return one of the following:
+Your async Python function can return:
- A `StuffContent` object (e.g., `TextContent`, `ImageContent`, or a custom `StructuredContent` model).
-- A `list` of `StuffContent` objects.
-- A simple Python `str`, which will be automatically converted to a `TextContent`.
+- A `ListContent` containing `StuffContent` objects.
-## How to Register a Function
+## How to Create a Function
-To make a Python function available to `PipeFunc`, you must register it using the global `func_registry`.
+To make a Python function available to `PipeFunc`, simply create it in any Python file within the `pipelex/libraries/` directory structure.
-Here is an example of a function and its registration:
+Here is an example of an eligible function:
```python
-# in a file like my_custom_functions.py
+# in a file like pipelex/libraries/my_custom_functions.py
from pipelex.core.memory.working_memory import WorkingMemory
from pipelex.core.stuffs.stuff_content import TextContent
-from pipelex.tools.func_registry import func_registry
-def concatenate_texts(working_memory: WorkingMemory) -> TextContent:
+async def concatenate_texts(working_memory: WorkingMemory) -> TextContent:
"""
Retrieves two text stuffs, concatenates them, and returns a new text stuff.
"""
@@ -42,14 +51,9 @@ def concatenate_texts(working_memory: WorkingMemory) -> TextContent:
concatenated = f"{text1} -- {text2}"
return TextContent(text=concatenated)
-
-def register_my_functions():
- """This function should be called at application startup."""
- func_registry.register_function(concatenate_texts, name="combine_two_texts")
-
```
-You would then call `register_my_functions()` when your Pipelex application initializes.
+The function will be automatically registered with the name `concatenate_texts` (the function name) when Pipelex starts up.
## Configuration
@@ -66,12 +70,12 @@ Once the function is registered, you can use it in your `.plx` file.
### Example
-This PLX snippet shows how to use the `combine_two_texts` function defined above. It assumes two previous pipes have produced outputs named `text_a` and `text_b`.
+This PLX snippet shows how to use the `concatenate_texts` function defined above. It assumes two previous pipes have produced outputs named `text_a` and `text_b`.
```plx
[pipe.combine_them]
type = "PipeFunc"
definition = "Combine two text inputs using a custom Python function"
-function_name = "combine_two_texts"
+function_name = "concatenate_texts"
output = "ConcatenatedText"
```
diff --git a/docs/pages/build-reliable-ai-workflows-with-pipelex/pipe-operators/PipeImgGen.md b/docs/pages/build-reliable-ai-workflows-with-pipelex/pipe-operators/PipeImgGen.md
index a6ad8bb3c..abb570b90 100644
--- a/docs/pages/build-reliable-ai-workflows-with-pipelex/pipe-operators/PipeImgGen.md
+++ b/docs/pages/build-reliable-ai-workflows-with-pipelex/pipe-operators/PipeImgGen.md
@@ -4,10 +4,10 @@ The `PipeImgGen` operator is used to generate images from a text prompt using a
## How it works
-`PipeImgGen` takes a text prompt and a set of parameters, then calls an underlying image generation service (like DALL-E 3 or Stable Diffusion) to create one or more images.
+`PipeImgGen` takes a text prompt and a set of parameters, then calls an underlying image generation service (like GPT Image or Flux) to create one or more images.
The prompt can be provided in two ways:
-1. As a static string directly in the pipe's PLX definition using the `imgg_prompt` parameter.
+1. As a static string directly in the pipe's PLX definition using the `img_gen_prompt` parameter.
2. As a dynamic input by connecting a concept that is or refines `native.Text`.
The pipe can be configured to generate a single image or a list of images.
@@ -16,6 +16,22 @@ The pipe can be configured to generate a single image or a list of images.
`PipeImgGen` is configured in your pipeline's `.plx` file.
+### Image Generation Models and Backend System
+
+PipeImgGen uses the unified inference backend system to manage image generation models. This means you can:
+
+- Use different image generation providers (OpenAI GPT Image, FAL models like Flux, etc.)
+- Configure image generation models through the same backend system as LLMs and OCR models
+- Use image generation presets for consistent configurations across your pipelines
+- Route image generation requests to different backends based on your routing profile
+
+Common image generation model handles:
+- `base-img-gen`: Base image generation model (alias for flux-pro/v1.1)
+- `best-img-gen`: Best quality image generation model (alias for flux-pro/v1.1-ultra)
+- `fast-img-gen`: Fast image generation model (alias for fast-lightning-sdxl)
+
+Image generation presets are defined in your model deck configuration and can include parameters like `quality`, `guidance_scale`, and `safety_tolerance`.
+
### PLX Parameters
| Parameter | Type | Description | Required |
@@ -24,9 +40,9 @@ The pipe can be configured to generate a single image or a list of images.
| `definition` | string | A description of the image generation operation. | Yes |
| `inputs` | dictionary | The input concept(s) for the image generation operation, as a dictionary mapping input names to concept codes. | Yes |
| `output` | string | The output concept produced by the image generation operation. | Yes |
-| `imgg_prompt` | string | A static text prompt for image generation. Use this *or* `input`. | No |
+| `img_gen_prompt` | string | A static text prompt for image generation. Use this *or* `input`. | No |
| `nb_output` | integer | The number of images to generate. If omitted, a single image is generated. | No |
-| `imgg_handle` | string | The handle for the image generation model to use (e.g., `"dall-e-3"`). Defaults to the model specified in the global config. | No |
+| `img_gen` | string | The choice of image generation model handle preset or setting to use (e.g., `"gpt-image-1"`). Defaults to the model specified in the global config. | No |
| `aspect_ratio` | string | The desired aspect ratio of the image (e.g., `"16:9"`, `"1:1"`). | No |
| `quality` | string | The quality of the generated image (e.g., `"standard"`, `"hd"`). | No |
| `seed` | integer or "auto" | A seed for the random number generator to ensure reproducibility. `"auto"` uses a random seed. | No |
@@ -44,7 +60,7 @@ type = "PipeImgGen"
definition = "Generate a futuristic car image"
output = "Image"
img_gen_prompt = "A sleek, futuristic sports car driving on a neon-lit highway at night."
-imgg_handle = "dall-e-3"
+img_gen = "base_img_gen"
aspect_ratio = "16:9"
quality = "hd"
```
@@ -63,7 +79,7 @@ definition = "Generate three logo variations from a prompt"
inputs = { prompt = "images.ImgGenPrompt" }
output = "Image"
nb_output = 3
-imgg_handle = "stable-diffusion"
+img_gen = "base_img_gen"
aspect_ratio = "1:1"
```
diff --git a/docs/pages/build-reliable-ai-workflows-with-pipelex/pipe-operators/PipeOcr.md b/docs/pages/build-reliable-ai-workflows-with-pipelex/pipe-operators/PipeOcr.md
index 4c9fcc692..130a8cc69 100644
--- a/docs/pages/build-reliable-ai-workflows-with-pipelex/pipe-operators/PipeOcr.md
+++ b/docs/pages/build-reliable-ai-workflows-with-pipelex/pipe-operators/PipeOcr.md
@@ -21,6 +21,21 @@ The `PageContent` object has the following structure:
`PipeOcr` is configured in your pipeline's `.plx` file.
+### OCR Models and Backend System
+
+PipeOcr uses the unified inference backend system to manage OCR models. This means you can:
+
+- Use different OCR providers (Mistral OCR, local PDF extraction via internal backend, etc.)
+- Configure OCR models through the same backend system as LLMs and image generation models
+- Use OCR presets for consistent configurations across your pipelines
+- Route OCR requests to different backends based on your routing profile
+
+Common OCR model handles:
+- `mistral-ocr`: Mistral's OCR model for high-quality text and image extraction
+- `pypdfium2-extract-text`: Local PDF text extraction (no API calls required)
+
+OCR presets are defined in your model deck configuration and can include parameters like `max_nb_images` and `image_min_size`.
+
### PLX Parameters
| Parameter | Type | Description | Required |
@@ -33,6 +48,7 @@ The `PageContent` object has the following structure:
| `page_views` | boolean | If `true`, a high-fidelity image of each page will be included in the `page_view` field. Defaults to `false`. | No |
| `page_views_dpi` | integer | The resolution (in Dots Per Inch) for the generated page views when processing a PDF. Defaults to `150`. | No |
| `page_image_captions` | boolean | If `true`, the OCR service may attempt to generate captions for the images found. *Note: This feature depends on the OCR provider.* | No |
+| `ocr` | string | The OCR model handle, preset, or setting to use (e.g., `"mistral-ocr"`, `"base_ocr_mistral"`). Defaults to the model specified in the global config. | No |
### Example: Processing a PDF
@@ -53,7 +69,6 @@ inputs = { ocr_input = "ScannedDocument" }
output = "Page"
page_views = true
page_views_dpi = 200
-ocr_model = "mistral-ocr"
```
The output of the PipeOcr must be exactly the native `Page` concept.
diff --git a/docs/pages/build-reliable-ai-workflows-with-pipelex/pipe-operators/index.md b/docs/pages/build-reliable-ai-workflows-with-pipelex/pipe-operators/index.md
index fac87270f..8b209edf5 100644
--- a/docs/pages/build-reliable-ai-workflows-with-pipelex/pipe-operators/index.md
+++ b/docs/pages/build-reliable-ai-workflows-with-pipelex/pipe-operators/index.md
@@ -10,7 +10,7 @@ Here are the primary pipe operators available in Pipelex:
- [**`PipeLLM`**](./PipeLLM.md): The core operator for all interactions with Large Language Models (LLMs), including text generation, structured data extraction, and vision tasks.
- [**`PipeOcr`**](./PipeOcr.md): Performs Optical Character Recognition (OCR) on images and PDF documents to extract text and embedded images.
-- [**`PipeImgGen`**](./PipeImgGen.md): Generates images from a text prompt using models like DALL-E 3 or Stable Diffusion.
+- [**`PipeImgGen`**](./PipeImgGen.md): Generates images from a text prompt using models like GPT Image, Flux, or other image generation models.
- [**`PipeFunc`**](./PipeFunc.md): An escape hatch that allows you to execute any custom Python function, giving you maximum flexibility.
- [**`PipeJinja2`**](./PipeJinja2.md): Renders a Jinja2 template using data from the working memory, perfect for creating formatted reports or complex prompts.
@@ -21,7 +21,7 @@ Pipelex provides the following pipe operators:
- `PipeLLM`: For LLM-based text generation and processing
- `PipeOcr`: For optical character recognition and document processing
- `PipeFunc`: For executing custom functions
-- `PipeImgGen`: For image generation and manipulation
+- `PipeImgGen`: For AI-powered image generation
## PipeLLM
diff --git a/docs/pages/build-reliable-ai-workflows-with-pipelex/pipeline-creation.md b/docs/pages/build-reliable-ai-workflows-with-pipelex/pipeline-creation.md
index 7a0099869..a6404d251 100644
--- a/docs/pages/build-reliable-ai-workflows-with-pipelex/pipeline-creation.md
+++ b/docs/pages/build-reliable-ai-workflows-with-pipelex/pipeline-creation.md
@@ -1,15 +1,13 @@
# Pipeline Creation
-Pipelex provides powerful tools to automatically generate pipeline blueprints from natural language requirements. This feature leverages AI to translate your ideas into working pipeline code, dramatically speeding up development.
+Pipelex provides powerful tools to automatically generate complete, working pipelines from natural language requirements. This feature leverages AI to translate your ideas into fully functional pipeline code, dramatically speeding up development.
## Overview
-The pipeline creation system works in several stages:
+The pipeline creation system creates a fully working pipeline that has been both statically and dynamically validated. The system automatically handles all aspects of pipeline generation, from understanding requirements to producing executable code.
-1. **Draft Generation**: Convert natural language requirements into structured markdown drafts
-2. **Blueprint Creation**: Transform drafts into formal pipeline blueprints (PLX format)
-3. **Validation**: Automatically validate and fix common issues
-4. **Implementation**: Generate executable pipeline code
+!!! warning "Execution Time"
+ Pipeline generation can take 4 to 7 minutes to complete, as the system performs comprehensive validation and testing to ensure the pipeline works correctly.
## Core Commands
@@ -18,55 +16,19 @@ The pipeline creation system works in several stages:
Generate a complete pipeline blueprint from requirements:
```bash
-pipelex build blueprint PIPELINE_NAME --requirements "REQUIREMENTS_TEXT" [OPTIONS]
+pipelex build blueprint "BRIEF IN NATURAL LANGUAGE" [OPTIONS]
```
**Example:**
```bash
-pipelex build blueprint gen_photopposite \
- --requirements "Take a photo as input, analyze its content and its most important feature, imagine the opposite to that feature, render it as a photo" \
- -c pipelex/libraries \
- -o pipelex/libraries/pipelines/temp/result_4/photopposite \
- --validate
+pipelex build blueprint "Take a photo as input, and render the opposite of the photo" \
+ -c your/path/to/pipelex/libraries \
+ -o output/pipeline/file/path
```
**Options:**
-- `--domain, -d`: Domain for the pipeline (default: "wip_domain")
-- `--requirements, -r`: Requirements text directly
-- `--file, -f`: Path to file containing requirements
- `--output, -o`: Output path for generated files
-- `--validate`: Validate the generated blueprint
-- `--config-folder-path, -c`: Path to libraries folder
-
-### Build Draft
-
-Generate just a pipeline draft (intermediate step):
-
-```bash
-pipelex build draft PIPELINE_NAME --requirements "REQUIREMENTS_TEXT" [OPTIONS]
-```
-
-**Options:**
-- `--raw`: Generate raw text draft instead of structured draft
-- All other options same as blueprint command
-
-### Validate Blueprint
-
-Validate an existing blueprint file:
-
-```bash
-pipelex validate blueprint BLUEPRINT_PATH [OPTIONS]
-```
-
-**Example:**
-```bash
-pipelex validate blueprint pipelex/libraries/pipelines/temp/result_3/photopposite.plx \
- -c pipelex/libraries
-```
-
-**Options:**
-- `--config-folder-path, -c`: Path to libraries folder
-- `--fix/--no-fix, -f/-F`: Enable/disable automatic error fixing (default: enabled)
+- `--config, -c`: Path to libraries folder
## Complete Workflow
@@ -75,219 +37,6 @@ pipelex validate blueprint pipelex/libraries/pipelines/temp/result_3/photopposit
Start with clear, specific requirements:
```text
-Take a photo as input, analyze its content and its most important feature,
-imagine the opposite to that feature, render it as a photo
-```
-
-### 2. Draft Generation
-
-The system first creates a structured markdown draft:
-
-```markdown
-# Pipeline: gen_photopposite
-Domain: image_processing
-Purpose: Generate opposite version of photo's main feature
-
-## Inputs
-- `photo` : Input photo to analyze and transform
-
-## Flow
-
-WorkingMemory starts with: {photo}
-
-### Step 1: Analyze photo content
-LLM[analyze]: (photo) → analysis
-
-WorkingMemory now contains: {photo, analysis}
-
-### Step 2: Generate opposite concept
-LLM[conceptualize]: (analysis) → opposite_concept
-
-WorkingMemory now contains: {photo, analysis, opposite_concept}
-
-### Step 3: Render opposite photo
-ImgGen: (opposite_concept) → opposite_photo
-
-WorkingMemory now contains: {photo, analysis, opposite_concept, opposite_photo}
-
-## Output
-Returns: `opposite_photo` - Photo showing opposite of main feature
-```
-
-### 3. Blueprint Creation
-
-The draft is converted into a formal PLX blueprint:
-
-```plx
-domain = "image_processing"
-definition = "Generate opposite version of photo's main feature"
-
-[concept]
-PhotoAnalysis = "Analysis of photo content and main features"
-OppositePhotoPrompt = "Prompt for generating opposite version of photo"
-
-[pipe.gen_photopposite]
-type = "PipeSequence"
-definition = "Generate opposite version of photo's main feature"
-output = "Image"
-steps = [
- { pipe = "analyze_photo", result = "analysis" },
- { pipe = "create_opposite_concept", result = "opposite_concept" },
- { pipe = "render_opposite", result = "opposite_photo" }
-]
-
-[pipe.analyze_photo]
-type = "PipeLLM"
-definition = "Analyze photo content and identify main feature"
-inputs = { photo = "Image" }
-output = "PhotoAnalysis"
-prompt_template = """
-Analyze this photo and identify its most important visual feature.
-
-@photo
-
-Focus on the dominant element, color, mood, or characteristic that defines this image.
-"""
-
-[pipe.create_opposite_concept]
-type = "PipeLLM"
-definition = "Create concept for opposite version"
-inputs = { analysis = "PhotoAnalysis" }
-output = "OppositePhotoPrompt"
-prompt_template = """
-Based on this photo analysis, create a detailed prompt for generating the opposite version.
-
-@analysis
-
-Describe what the opposite would look like, focusing on reversing the main feature while maintaining photo realism.
-"""
-
-[pipe.render_opposite]
-type = "PipeImgGen"
-definition = "Generate the opposite photo"
-inputs = { opposite_concept = "OppositePhotoPrompt" }
-output = "Image"
-```
-
-### 4. Validation and Fixing
-
-The system automatically:
-
-- **Validates syntax**: Ensures PLX structure is correct
-- **Checks concepts**: Verifies all concepts are properly defined
-- **Validates inputs**: Ensures all pipe inputs are available
-- **Fixes errors**: Automatically adds missing inputs or corrects common issues
-- **Dry runs**: Tests pipeline execution without making API calls
-
-## Generated Files
-
-When you run the build command, several files are created:
-
-```
-output_path_base_draft.md # Original markdown draft
-output_path_base_rough.plx # Initial blueprint
-output_path_base_rough.json # Blueprint in JSON format
-output_path_base_fixed.plx # Validated and fixed blueprint (if --validate used)
-output_path_base_fixed.json # Fixed blueprint in JSON format
-```
-
-## Best Practices
-
-### Writing Requirements
-
-**Good requirements:**
-```text
-Analyze a PDF document, extract key financial data, and generate a summary report with charts
-```
-
-**Better requirements:**
-```text
-Take a PDF financial statement as input, use OCR to extract text and tables,
-identify revenue, expenses, and profit figures, then generate a structured
-summary report with visual charts showing trends
-```
-
-### Domain Selection
-
-Choose descriptive domain names:
-- `financial_analysis` instead of `finance`
-- `document_processing` instead of `docs`
-- `image_generation` instead of `images`
-
-### Output Path Organization
-
-Organize generated pipelines:
+Take a photo as input, and render the opposite of the photo
```
-pipelex/libraries/pipelines/
- temp/ # Temporary/experimental pipelines
- result_1/
- result_2/
- domain_name/ # Production pipelines
- pipeline_name.plx
- structures.py
-```
-
-## Advanced Features
-
-### Custom Templates
-
-The system uses built-in templates from:
-- `.pipelex/create/draft_pipelines.md` - Drafting guidelines
-- `.pipelex/create/build_pipelines.md` - Blueprint creation rules
-
-### Error Fixing
-
-The validation system can automatically fix:
-- Missing input variables
-- Incorrect concept references
-- Malformed pipe definitions
-- Input/output mismatches
-
-### Integration with Existing Libraries
-
-Generated blueprints automatically integrate with:
-- Native concepts (`Text`, `Image`, `PDF`, etc.)
-- Library concepts (`documents.TextAndImages`, etc.)
-- Custom domain concepts
-
-## Troubleshooting
-
-### Common Issues
-
-**"Missing input variable" error:**
-- Usually fixed automatically with `--validate`
-- Check that all pipe inputs are available in WorkingMemory
-
-**"Invalid concept" error:**
-- Ensure concept names use PascalCase
-- Don't redefine native concepts
-- Avoid plurals in concept names
-
-**"Pipe not found" error:**
-- Check that all referenced pipes exist in the blueprint
-- Verify pipe names match exactly
-
-### Getting Help
-
-```bash
-# Validate your entire setup
-pipelex validate all -c path/to/your/pipelex/config/folder
-
-# Test a specific pipe
-pipelex validate pipe your_pipe_name -c path/to/your/pipelex/config/folder
-
-# Check blueprint structure
-pipelex validate blueprint your_blueprint.plx -c path/to/your/pipelex/config/folder --no-fix
-```
-
-## Next Steps
-
-After creating your blueprint:
-
-1. **Review the generated code** - Check that it matches your requirements
-2. **Create structure classes** - Add Python classes for custom concepts if needed
-3. **Test the pipeline** - Run validation and dry runs
-4. **Execute the pipeline** - Use `execute_pipeline()` to run it with real data
-5. **Iterate and improve** - Refine based on results
-The pipeline creation system makes it easy to go from idea to working code in minutes, while maintaining the full power and flexibility of Pipelex pipelines.
diff --git a/docs/pages/configuration/config-technical/cogt-config.md b/docs/pages/configuration/config-technical/cogt-config.md
index 8c71605a3..ea0d9e368 100644
--- a/docs/pages/configuration/config-technical/cogt-config.md
+++ b/docs/pages/configuration/config-technical/cogt-config.md
@@ -1,6 +1,6 @@
# Cognitive Tools (Cogt) Configuration
-The Cogt configuration manages all cognitive tools in Pipelex, including LLM (Language Models), IMGG (Image Generation), and OCR (Optical Character Recognition) capabilities.
+The Cogt configuration manages all cognitive tools in Pipelex, including LLM (Language Models), Image Generation, and OCR (Optical Character Recognition) capabilities.
## Overview
@@ -9,7 +9,7 @@ The Cogt configuration manages all cognitive tools in Pipelex, including LLM (La
# Main Cogt configuration sections
[cogt.inference_manager_config]
[cogt.llm_config]
-[cogt.imgg_config]
+[cogt.img_gen_config]
[cogt.ocr_config]
```
@@ -20,7 +20,7 @@ Controls automatic setup of various cognitive tools:
```toml
[cogt.inference_manager_config]
is_auto_setup_preset_llm = true
-is_auto_setup_preset_imgg = true
+is_auto_setup_preset_img_gen = true
is_auto_setup_preset_ocr = true
```
@@ -50,20 +50,16 @@ When configuring LLM jobs, you can set:
- `max_tokens` (optional int): Maximum tokens in response
- `seed` (optional int): For reproducible outputs
-## Image Generation (IMGG) Configuration
+## Image Generation Configuration
Configuration for image generation capabilities:
```toml
-[cogt.imgg_config]
-default_imgg_handle = "fal-ai/flux-pro"
-imgg_handles = ["fal-ai/flux-pro", "fal-ai/fast-lightning-sdxl"]
-
-[cogt.imgg_config.imgg_job_config]
+[cogt.img_gen_config.img_gen_job_config]
is_sync_mode = false
# Default parameters for image generation
-[cogt.imgg_config.imgg_param_defaults]
+[cogt.img_gen_config.img_gen_param_defaults]
aspect_ratio = "square" # Options: square, landscape_4_3, landscape_3_2, landscape_16_9, landscape_21_9,
# portrait_3_4, portrait_2_3, portrait_9_16, portrait_9_21
background = "auto" # Options: transparent, opaque, auto
@@ -77,7 +73,7 @@ output_format = "jpg" # Options: png, jpg, webp
seed = "auto" # "auto" or specific integer
```
-### IMGG Job Parameters
+### ImageGen Job Parameters
Image generation jobs support these parameters:
@@ -111,7 +107,25 @@ Configuration for Optical Character Recognition:
[cogt.ocr_config]
page_output_text_file_name = "page_text.md"
default_page_views_dpi = 72
-```
+
+## Unified Backend Integration
+
+All cognitive tools (LLMs, OCR, and Image Generation) now use the same unified inference backend system:
+
+### Benefits of Unified System
+
+- **Consistent Configuration**: Same configuration patterns across all AI capabilities
+- **Unified Routing**: All models routed through the same routing profiles
+- **Shared Presets**: Presets for LLMs, OCR, and image generation in the same deck
+- **Single API Management**: Manage all AI provider credentials in one place
+
+### Backend Integration Details
+
+- **LLM Models**: `model_type = "llm"` for text generation and structured outputs
+- **OCR Models**: `model_type = "text_extractor"` for document processing
+- **Image Generation Models**: `model_type = "img_gen"` for image creation
+
+All model types support the same routing, aliasing, and preset systems.
## Validation Rules
@@ -121,7 +135,7 @@ default_page_views_dpi = 72
- Max retries must be between 1 and 10
- Seeds must be non-negative
-### IMGG Configuration
+### ImageGen Configuration
- Guidance scale must be positive
- Safety tolerance must be between 1 and 6
- Number of steps must be positive
@@ -135,7 +149,7 @@ default_page_views_dpi = 72
- Use streaming for better user experience
- Set appropriate retry limits based on your use case
-2. **IMGG Settings**:
+2. **ImageGen Settings**:
- Enable moderation for production use
- Use appropriate aspect ratios for your use case
@@ -153,7 +167,7 @@ default_page_views_dpi = 72
[cogt]
[cogt.inference_manager_config]
is_auto_setup_preset_llm = true
-is_auto_setup_preset_imgg = true
+is_auto_setup_preset_img_gen = true
is_auto_setup_preset_ocr = true
[cogt.llm_config]
@@ -166,14 +180,10 @@ max_retries = 3
[cogt.llm_config.instructor_config]
is_openai_structured_output_enabled = false
-[cogt.imgg_config]
-default_imgg_handle = "fal-ai/flux-pro/v1.1-ultra"
-imgg_handles = ["fal-ai/flux-pro", "fal-ai/fast-lightning-sdxl"]
-
-[cogt.imgg_config.imgg_job_config]
+[cogt.img_gen_config.img_gen_job_config]
is_sync_mode = false
-[cogt.imgg_config.imgg_param_defaults]
+[cogt.img_gen_config.img_gen_param_defaults]
aspect_ratio = "square"
background = "auto"
quality = "low"
diff --git a/docs/pages/configuration/config-technical/inference-backend-config.md b/docs/pages/configuration/config-technical/inference-backend-config.md
index 092b73271..83eb572ea 100644
--- a/docs/pages/configuration/config-technical/inference-backend-config.md
+++ b/docs/pages/configuration/config-technical/inference-backend-config.md
@@ -1,15 +1,15 @@
# Inference Backend Configuration
-The Inference Backend Configuration System manages how Pipelex handles LLM providers, model routing, and inference settings. This system provides a flexible and scalable way to configure multiple inference backends and route models to the appropriate providers.
+The Inference Backend Configuration System manages how Pipelex handles AI model providers, model routing, and inference settings across LLMs, OCR, and image generation. This unified system provides a flexible and scalable way to configure multiple inference backends and route different types of AI models to the appropriate providers.
## Overview
The inference backend system is built around four key concepts:
-1. **Inference Backends**: Providers of LLM services (OpenAI, Anthropic, Google Vertex AI, etc.)
-2. **Model Specs**: Detailed information about specific models available through backends
-3. **Routing Profiles**: Rules for selecting which backend should handle specific models
-4. **Model Deck**: Unified collection of configured models, aliases, and presets
+1. **Inference Backends**: Providers of AI services (OpenAI, Anthropic, Google Vertex AI, FAL, etc.) for LLMs, OCR, and image generation
+2. **Model Specs**: Detailed information about specific models available through backends (text generation, text extraction, image generation)
+3. **Routing Profiles**: Rules for selecting which backend should handle specific models across all AI capabilities
+4. **Model Deck**: Unified collection of configured models, aliases, and presets for LLMs, OCR, and image generation
## Directory Structure
@@ -21,11 +21,13 @@ All inference backend configurations are stored in the `.pipelex/inference/` dir
├── backends.toml # Backend provider configurations
├── routing_profiles.toml # Model routing rules
├── backends/ # Individual backend model specifications
- │ ├── openai.toml
- │ ├── anthropic.toml
- │ ├── bedrock.toml
- │ ├── mistral.toml
- │ ├── vertexai.toml
+ │ ├── openai.toml # OpenAI models (LLMs, image generation)
+ │ ├── anthropic.toml # Anthropic models (LLMs)
+ │ ├── bedrock.toml # AWS Bedrock models (LLMs)
+ │ ├── mistral.toml # Mistral models (LLMs, OCR)
+ │ ├── vertexai.toml # Google Vertex AI models (LLMs)
+ │ ├── fal.toml # FAL models (image generation)
+ │ ├── internal.toml # Internal/local models (OCR)
│ └── ...
└── deck/ # Model deck configurations
├── base_deck.toml # Core aliases and presets
@@ -38,9 +40,10 @@ Pipelex Inference is a unified inference backend that provides access to all maj
### Benefits
-- **Single API Key**: Access OpenAI, Anthropic, Google, Mistral, and more with one key
+- **Single API Key**: Access OpenAI, Anthropic, Google, Mistral, FAL, and more with one key
- **Simplified Configuration**: No need to manage multiple provider credentials
-- **Automatic Routing**: Models are automatically routed to their respective providers
+- **Automatic Routing**: All AI models (LLMs, OCR, image generation) are automatically routed to their respective providers
+- **Unified Interface**: Same configuration system for text generation, OCR, and image generation
### Setup
@@ -71,7 +74,7 @@ llm = { llm_handle = "claude-4-sonnet", temperature = 0.7 }
## Inference Backends
-Backends represent LLM service providers. Each backend is configured with its endpoint and authentication details.
+Backends represent AI service providers that can offer LLMs, OCR models, or image generation models. Each backend is configured with its endpoint and authentication details.
### Backend Configuration
@@ -86,11 +89,17 @@ api_key = "${OPENAI_API_KEY}"
enabled = true
api_key = "${ANTHROPIC_API_KEY}"
-[azure_openai]
+[mistral]
enabled = true
-endpoint = "${AZURE_API_BASE}"
-api_key = "${AZURE_API_KEY}"
-api_version = "${AZURE_API_VERSION}"
+api_key = "${MISTRAL_API_KEY}"
+
+[fal]
+enabled = true
+api_key = "${FAL_KEY}"
+
+[internal]
+enabled = true
+# No API key needed for internal/local processing
```
### Model Specifications
@@ -99,22 +108,26 @@ Each backend has its own model specification file in `.pipelex/inference/backend
```toml
# openai.toml
-[models.gpt-4o-mini]
+default_sdk = "openai"
+default_prompting_target = "openai"
+
+[gpt-4o-mini]
model_id = "gpt-4o-mini"
-sdk = "openai"
inputs = ["text", "images"]
outputs = ["text", "structured"]
-costs = { input = 0.00015, output = 0.0006 }
-max_tokens = 128000
-max_prompt_images = 100
+costs = { input = 0.15, output = 0.6 }
-[models.gpt-4-turbo]
+[gpt-4-turbo]
model_id = "gpt-4-turbo"
-sdk = "openai"
-inputs = ["text", "images"]
+inputs = ["text"]
outputs = ["text", "structured"]
-costs = { input = 0.01, output = 0.03 }
-max_tokens = 128000
+costs = { input = 10.0, output = 30.0 }
+
+[gpt-image-1]
+model_id = "gpt-image-1"
+inputs = ["text"]
+outputs = ["image"]
+costs = { input = 0.04, output = 0.0 }
```
## Routing Profiles
@@ -138,6 +151,8 @@ default = "openai"
"gemini-*" = "vertexai"
"mistral-*" = "mistral"
"gpt-*" = "openai"
+"gpt-image-*" = "openai"
+"flux-*" = "fal"
```
The routing system supports:
@@ -147,7 +162,7 @@ The routing system supports:
## Model Deck
-The Model Deck is the unified configuration hub for all LLM-related settings.
+The Model Deck is the unified configuration hub for all AI model-related settings, including LLMs, OCR models, and image generation models.
### Aliases
@@ -155,10 +170,20 @@ Define user-friendly names that map to model names in `.pipelex/inference/deck/b
```toml
[aliases]
-best-claude = "claude-4.1-opus"
-best-gemini = "gemini-2.5-pro"
-best-mistral = "mistral-large"
+# LLM aliases
+base-claude = "claude-4-sonnet"
base-gpt = "gpt-5"
+base-gemini = "gemini-2.5-flash"
+base-mistral = "mistral-medium"
+
+# OCR aliases
+best-ocr = "mistral-ocr"
+local-ocr = "pypdfium2-extract-text"
+
+# Image generation aliases
+base-img-gen = "flux-pro/v1.1"
+best-img-gen = "flux-pro/v1.1-ultra"
+fast-img-gen = "fast-lightning-sdxl"
# Aliases can also define fallback chains
llm_to_engineer = ["claude-4.1-opus", "gemini-2.5-pro"]
@@ -169,25 +194,53 @@ llm_to_engineer = ["claude-4.1-opus", "gemini-2.5-pro"]
Presets combine model selection with optimized parameters for specific tasks:
```toml
-[llm_presets]
+[llm.presets]
# General purpose presets
-cheap_llm_for_text = { llm_handle = "gpt-4o-mini", temperature = 0.5 }
-cheap_llm_for_object = { llm_handle = "gpt-4o-mini", temperature = 0.5 }
+cheap_llm_for_text = { llm_handle = "cheap_llm_for_text", temperature = 0.5 }
+cheap_llm_for_object = { llm_handle = "cheap_llm_for_object", temperature = 0.5 }
# Task-specific presets
llm_for_creative_writing = { llm_handle = "claude-4-sonnet", temperature = 0.9 }
-llm_to_extract = { llm_handle = "claude-4-sonnet", temperature = 0.1 }
-llm_to_reason = { llm_handle = "o4-mini", temperature = 1, max_tokens = "auto" }
+llm_to_extract_invoice = { llm_handle = "claude-4-sonnet", temperature = 0.1 }
+llm_to_reason = { llm_handle = "base-claude", temperature = 1 }
+
+### OCR Presets
+
+OCR presets combine OCR model selection with optimized parameters:
+
+```toml
+[ocr.presets]
+# General purpose OCR
+base_ocr_mistral = { ocr_handle = "mistral-ocr", max_nb_images = 100, image_min_size = 50 }
+base_ocr_pypdfium2 = { ocr_handle = "pypdfium2-extract-text", max_nb_images = 100, image_min_size = 50 }
+```
+
+### Image Generation Presets
+
+Image generation presets combine model selection with generation parameters:
+
+```toml
+[img_gen.presets]
+# General purpose image generation
+base_img_gen = { img_gen_handle = "base-img-gen", quality = "medium", guidance_scale = 7.5, is_moderated = true, safety_tolerance = 3 }
+fast_img_gen = { img_gen_handle = "fast-img-gen", nb_steps = 4, guidance_scale = 5.0, is_moderated = true, safety_tolerance = 3 }
+high_quality_img_gen = { img_gen_handle = "best-img-gen", quality = "high", guidance_scale = 8.0, is_moderated = true, safety_tolerance = 3 }
```
### Default Choices
-Set default models for different types of generation:
+Set default models for different types of AI operations:
```toml
-[llm_choice_defaults]
+[llm.choice_defaults]
for_text = "cheap_llm_for_text"
for_object = "cheap_llm_for_object"
+
+[ocr]
+choice_default = "base_ocr_mistral"
+
+[img_gen]
+choice_default = "base_img_gen"
```
## Customization
@@ -198,12 +251,20 @@ Use `.pipelex/inference/deck/overrides.toml` for project-specific customizations
```toml
# Override specific presets
-[llm_presets]
-llm_to_extract = { llm_handle = "gpt-4o-mini", temperature = 0.2 }
+[llm.presets]
+llm_to_extract_invoice = { llm_handle = "gpt-4o-mini", temperature = 0.2 }
+
+[ocr.presets]
+my_custom_ocr = { ocr_handle = "mistral-ocr", max_nb_images = 5 }
+
+[img_gen.presets]
+my_custom_img_gen = { img_gen_handle = "flux-dev", quality = "medium" }
# Add custom aliases
[aliases]
-my_custom_model = "claude-3-sonnet"
+my_custom_llm = "claude-3-sonnet"
+my_custom_ocr = "pypdfium2-extract-text"
+my_custom_img_gen = "base-img-gen"
```
### Adding New Backends
@@ -219,11 +280,11 @@ To add a new backend:
The system loads configurations in this order:
1. **Load Backends**: Read `backends.toml` to get enabled backends
-2. **Load Model Specs**: For each backend, load model specifications
+2. **Load Model Specs**: For each backend, load model specifications (LLMs, OCR models, image generation models)
3. **Load Routing Profiles**: Read routing rules and identify active profile
4. **Build Model Deck**:
- - Apply routing rules to determine backend for each model
- - Load aliases and presets from deck files
+ - Apply routing rules to determine backend for each model across all AI capabilities
+ - Load aliases and presets from deck files for LLMs, OCR, and image generation
- Apply overrides
5. **Finalization**: Validate complete configuration
diff --git a/docs/pages/tools/cli.md b/docs/pages/tools/cli.md
index 37b27f9f7..ce7b65f53 100644
--- a/docs/pages/tools/cli.md
+++ b/docs/pages/tools/cli.md
@@ -32,14 +32,6 @@ pipelex show pipes [-c/--config-folder-path PATH]
pipelex show pipe PIPE_CODE [-c/--config-folder-path PATH]
```
-### Migrate group
-
-Migrate TOML files to the new syntax.
-
-```bash
-pipelex migrate run [-p/--path PATH] [--dry-run] [--backups/--no-backups]
-```
-
## Usage Tips
1. Always run `pipelex validate all -c path/to/your/pipelex/config/folder` after making changes to your configuration or pipelines
diff --git a/pipelex/__init__.py b/pipelex/__init__.py
index 6e391cf23..98ce4968c 100644
--- a/pipelex/__init__.py
+++ b/pipelex/__init__.py
@@ -1,10 +1,11 @@
from pipelex._bootstrap_user_libs import activate as _px_bootstrap_user_libs
from pipelex.tools.log.log import log
-from pipelex.tools.misc.pretty import pretty_print
+from pipelex.tools.misc.pretty import pretty_print, pretty_print_md
__all__ = [
"log",
"pretty_print",
+ "pretty_print_md",
]
# ------------------------------------------------------------
diff --git a/pipelex/cli/commands/build_cmd.py b/pipelex/cli/commands/build_cmd.py
index 7f8c5f87e..cdd4ea3bf 100644
--- a/pipelex/cli/commands/build_cmd.py
+++ b/pipelex/cli/commands/build_cmd.py
@@ -1,145 +1,64 @@
-from __future__ import annotations
-
import asyncio
+import time
from typing import Annotated, Optional
import typer
-from pipelex.create.build_blueprint import do_build_blueprint
-from pipelex.create.draft_pipeline import do_draft_pipeline, do_draft_pipeline_text
-from pipelex.exceptions import PipelexCLIError
+from pipelex import pretty_print
+from pipelex.core.interpreter import PipelexInterpreter
+from pipelex.hub import get_report_delegate
+from pipelex.libraries.pipelines.builder.builder import PipelexBundleSpec
from pipelex.pipelex import Pipelex
-from pipelex.tools.misc.file_utils import load_text_from_path
+from pipelex.pipeline.execute import execute_pipeline
+
+build_app = typer.Typer(help="Build artifacts like pipelines", no_args_is_help=True)
+
+"""
+Today's example:
+pipelex build pipe "Given a scanned invoice, extract employee and articles"
-# Typer group for build commands
-build_app = typer.Typer(help="Build artifacts like pipeline blueprints", no_args_is_help=True)
+Other ideas:
+pipelex build pipe "Take a photo as input, and render the opposite of the photo"
+pipelex build pipe "Given an RDFP PDF, build a compliance matrix"
+"""
-@build_app.command("draft")
-def build_draft_pipeline_cmd(
- pipeline_name: Annotated[
+@build_app.command("pipe")
+def build_pipe_cmd(
+ brief: Annotated[
str,
- typer.Argument(help="Name/code of the pipeline to generate"),
+ typer.Argument(help="Brief description of what the pipeline should do"),
],
- domain: Annotated[
- str,
- typer.Option("--domain", "-d", help="Domain of the pipeline to generate"),
- ] = "wip_domain",
- requirements: Annotated[
- Optional[str],
- typer.Option("--requirements", "-r", help="Requirements text to generate the pipeline blueprint from"),
- ] = None,
- requirements_file: Annotated[
- Optional[str],
- typer.Option("--file", "-f", help="Path to a file containing the requirements text"),
- ] = None,
output_path: Annotated[
Optional[str],
- typer.Option("--output", "-o", help="Path to save the generated PLX blueprint (optional)"),
- ] = None,
- raw: Annotated[
- bool,
- typer.Option("--raw", help="Raw text draft"),
- ] = False,
- relative_config_folder_path: Annotated[
- str,
- typer.Option(
- "--config-folder-path",
- "-c",
- help="Relative path to the config folder path (libraries)",
- ),
- ] = "./pipelex_libraries",
+ typer.Option("--output", "-o", help="Path to save the generated PLX file (use --output='' to skip saving)"),
+ ] = "./generated_pipeline.plx",
) -> None:
- # Initialize Pipelex (loads libraries and pipes)
- Pipelex.make(relative_config_folder_path=relative_config_folder_path, from_file=False)
-
- # Get requirements text
- try:
- requirements_text: str
- if requirements_file:
- requirements_text = load_text_from_path(requirements_file)
- else:
- if not requirements:
- raise PipelexCLIError("You must provide requirements text via --requirements or a file via --file")
- requirements_text = requirements
- except Exception as exc:
- raise PipelexCLIError(f"Failed to load requirements: {exc}") from exc
+ Pipelex.make(relative_config_folder_path="../../../pipelex/libraries", from_file=True)
+ typer.echo("=" * 70)
+ typer.echo(typer.style("🔥 Starting pipe builder... 🚀", fg=typer.colors.GREEN))
+ typer.echo("")
- if raw:
- asyncio.run(
- do_draft_pipeline_text(
- domain=domain,
- pipeline_name=pipeline_name,
- requirements=requirements_text,
- output_path=output_path,
- )
+ async def run_pipeline():
+ pipe_output = await execute_pipeline(
+ pipe_code="pipe_builder",
+ input_memory={"brief": brief},
)
- else:
- asyncio.run(
- do_draft_pipeline(
- pipeline_name=pipeline_name,
- requirements=requirements_text,
- output_path=output_path,
- )
- )
-
+ pretty_print(pipe_output, title="Pipe Output")
+ pipelex_bundle_spec = pipe_output.working_memory.get_stuff_as(name="pipelex_bundle_spec", content_type=PipelexBundleSpec)
+ plx_content = PipelexInterpreter.make_plx_content(blueprint=pipelex_bundle_spec.to_blueprint())
-@build_app.command("blueprint")
-def build_blueprint_cmd(
- pipeline_name: Annotated[
- str,
- typer.Argument(help="Name/code of the pipeline to generate"),
- ],
- domain: Annotated[
- str,
- typer.Option("--domain", "-d", help="Domain of the pipeline to generate"),
- ] = "wip_domain",
- requirements: Annotated[
- Optional[str],
- typer.Option("--requirements", "-r", help="Requirements text to generate the pipeline blueprint from"),
- ] = None,
- requirements_file: Annotated[
- Optional[str],
- typer.Option("--file", "-f", help="Path to a file containing the requirements text"),
- ] = None,
- output_path: Annotated[
- Optional[str],
- typer.Option("--output", "-o", help="Path to save the generated PLX blueprint (optional)"),
- ] = None,
- validate: Annotated[
- bool,
- typer.Option("--validate", help="Dry-run the first generated pipe from the blueprint"),
- ] = False,
- relative_config_folder_path: Annotated[
- str,
- typer.Option(
- "--config-folder-path",
- "-c",
- help="Relative path to the config folder path (libraries)",
- ),
- ] = "./pipelex_libraries",
-) -> None:
- # Initialize Pipelex (loads libraries and pipes)
- Pipelex.make(relative_config_folder_path=relative_config_folder_path, from_file=False)
+ # Save to file unless explicitly disabled with empty string
+ if output_path and output_path != "":
+ with open(output_path, "w") as f:
+ f.write(plx_content)
+ typer.echo(typer.style(f"\n✅ Pipeline saved to: {output_path}", fg=typer.colors.GREEN))
+ elif output_path == "":
+ typer.echo(typer.style("\n⚠️ Pipeline not saved to file (--output='' specified)", fg=typer.colors.YELLOW))
- # Get requirements text
- try:
- requirements_text: str
- if requirements_file:
- requirements_text = load_text_from_path(requirements_file)
- else:
- if not requirements:
- raise PipelexCLIError("You must provide requirements text via --requirements or a file via --file")
- requirements_text = requirements
- except Exception as exc:
- raise PipelexCLIError(f"Failed to load requirements: {exc}") from exc
+ start_time = time.time()
+ asyncio.run(run_pipeline())
+ end_time = time.time()
+ typer.echo(typer.style(f"\n✅ Pipeline built in {end_time - start_time:.2f} seconds", fg=typer.colors.GREEN))
- asyncio.run(
- do_build_blueprint(
- domain=domain,
- pipeline_name=pipeline_name,
- requirements=requirements_text,
- output_path=output_path,
- validate=validate,
- )
- )
+ get_report_delegate().generate_report()
diff --git a/pipelex/cli/commands/init_cmd.py b/pipelex/cli/commands/init_cmd.py
index 84b6fbfbb..b1b8b68f0 100644
--- a/pipelex/cli/commands/init_cmd.py
+++ b/pipelex/cli/commands/init_cmd.py
@@ -1,7 +1,6 @@
-from __future__ import annotations
-
import os
import shutil
+from importlib.metadata import metadata
from typing import Annotated, List
import typer
@@ -10,9 +9,11 @@ from pipelex.exceptions import PipelexCLIError
from pipelex.libraries.library_config import LibraryConfig
from pipelex.tools.config.manager import config_manager
+PACKAGE_NAME = __name__.split(".", maxsplit=1)[0]
+PACKAGE_VERSION = metadata(PACKAGE_NAME)["Version"]
+
def do_init_libraries(directory: str = ".", overwrite: bool = False) -> None:
- """Initialize pipelex libraries in a pipelex_libraries folder in the specified directory."""
try:
target_dir = os.path.join(directory, "pipelex_libraries")
os.makedirs(directory, exist_ok=True)
diff --git a/pipelex/cli/commands/validate_cmd.py b/pipelex/cli/commands/validate_cmd.py
index 91f76084e..bf99797ce 100644
--- a/pipelex/cli/commands/validate_cmd.py
+++ b/pipelex/cli/commands/validate_cmd.py
@@ -7,8 +7,8 @@ import typer
from pipelex import log
from pipelex.cli.commands.common import is_pipelex_libraries_folder
-from pipelex.hub import get_pipeline_tracker
-from pipelex.pipe_works.pipe_dry import dry_run_all_pipes, dry_run_single_pipe
+from pipelex.hub import get_pipe_provider, get_pipeline_tracker
+from pipelex.pipe_works.pipe_dry import dry_run_pipe, dry_run_pipes
from pipelex.pipelex import Pipelex
@@ -21,7 +21,7 @@ def do_validate_all_libraries_and_dry_run(relative_config_folder_path: str = "./
pipelex_instance = Pipelex.make(relative_config_folder_path=relative_config_folder_path, from_file=False)
pipelex_instance.validate_libraries()
- asyncio.run(dry_run_all_pipes())
+ asyncio.run(dry_run_pipes(pipes=get_pipe_provider().get_pipes(), raise_on_failure=True))
log.info("Setup sequence passed OK, config and pipelines are validated.")
@@ -34,9 +34,13 @@ def do_dry_run_pipe(pipe_code: str, relative_config_folder_path: str = "./pipele
pipelex_instance = Pipelex.make(relative_config_folder_path=relative_config_folder_path, from_file=False)
pipelex_instance.validate_libraries()
- asyncio.run(dry_run_single_pipe(pipe_code))
- typer.echo(f"✅ Pipe '{pipe_code}' validation successful")
+ asyncio.run(
+ dry_run_pipe(
+ get_pipe_provider().get_required_pipe(pipe_code=pipe_code),
+ raise_on_failure=True,
+ )
+ )
get_pipeline_tracker().output_flowchart()
diff --git a/pipelex/client/api_serializer.py b/pipelex/client/api_serializer.py
index ffe0543c4..7d8ef22ee 100644
--- a/pipelex/client/api_serializer.py
+++ b/pipelex/client/api_serializer.py
@@ -34,7 +34,7 @@ class ApiSerializer:
return compact_memory
for stuff_name, stuff in working_memory.root.items():
- if stuff.concept.code == NativeConceptEnum.TEXT.value:
+ if NativeConceptEnum.is_text(concept_code=stuff.concept.code):
stuff_content = cast(TextContent, stuff.content)
item_dict: Dict[str, Any] = {
"concept_code": stuff.concept.code,
diff --git a/pipelex/cogt/config_cogt.py b/pipelex/cogt/config_cogt.py
index 4f11f37b3..b29539574 100644
--- a/pipelex/cogt/config_cogt.py
+++ b/pipelex/cogt/config_cogt.py
@@ -1,9 +1,6 @@
from typing import List
-from pydantic import Field
-
-from pipelex.cogt.imgg.imgg_handle import ImggHandle
-from pipelex.cogt.imgg.imgg_job_components import ImggJobConfig, ImggJobParams, ImggJobParamsDefaults
+from pipelex.cogt.img_gen.img_gen_job_components import ImgGenJobConfig, ImgGenJobParams, ImgGenJobParamsDefaults
from pipelex.cogt.llm.llm_job_components import LLMJobConfig
from pipelex.plugins.fal.fal_config import FalConfig
from pipelex.tools.config.config_model import ConfigModel
@@ -15,15 +12,13 @@ class OcrConfig(ConfigModel):
default_page_views_dpi: int
-class ImggConfig(ConfigModel):
- default_imgg_handle: ImggHandle = Field(strict=False)
- imgg_job_config: ImggJobConfig
- imgg_param_defaults: ImggJobParamsDefaults
- imgg_handles: List[str]
+class ImgGenConfig(ConfigModel):
+ img_gen_job_config: ImgGenJobConfig
+ img_gen_param_defaults: ImgGenJobParamsDefaults
fal_config: FalConfig
- def make_default_imgg_job_params(self) -> ImggJobParams:
- return self.imgg_param_defaults.make_imgg_job_params()
+ def make_default_img_gen_job_params(self) -> ImgGenJobParams:
+ return self.img_gen_param_defaults.make_img_gen_job_params()
class InstructorConfig(ConfigModel):
@@ -39,7 +34,7 @@ class LLMConfig(ConfigModel):
class InferenceManagerConfig(ConfigModel):
is_auto_setup_preset_llm: bool
- is_auto_setup_preset_imgg: bool
+ is_auto_setup_preset_img_gen: bool
is_auto_setup_preset_ocr: bool
@@ -57,9 +52,9 @@ class InferenceConfig(ConfigModel):
def model_specs_path(self, backend_name: str) -> str:
return f"{self.inference_config_path}/backends/{backend_name}.toml"
- def get_llm_deck_paths(self) -> List[str]:
+ def get_model_deck_paths(self) -> List[str]:
"""Get all LLM deck TOML file paths sorted alphabetically."""
- llm_deck_paths = [
+ model_deck_paths = [
str(path)
for path in find_files_in_dir(
dir_path=f"{self.inference_config_path}/deck",
@@ -67,13 +62,13 @@ class InferenceConfig(ConfigModel):
is_recursive=True,
)
]
- llm_deck_paths.sort()
- return llm_deck_paths
+ model_deck_paths.sort()
+ return model_deck_paths
class Cogt(ConfigModel):
inference_config: InferenceConfig
inference_manager_config: InferenceManagerConfig
llm_config: LLMConfig
- imgg_config: ImggConfig
+ img_gen_config: ImgGenConfig
ocr_config: OcrConfig
diff --git a/pipelex/cogt/content_generation/assignment_models.py b/pipelex/cogt/content_generation/assignment_models.py
index 2ae27021f..3a3e5e0e7 100644
--- a/pipelex/cogt/content_generation/assignment_models.py
+++ b/pipelex/cogt/content_generation/assignment_models.py
@@ -3,11 +3,9 @@ from typing import Any, Dict, Optional, Type
from pydantic import BaseModel
from typing_extensions import override
-from pipelex import log
from pipelex.cogt.exceptions import LLMAssignmentError
-from pipelex.cogt.imgg.imgg_handle import ImggHandle
-from pipelex.cogt.imgg.imgg_job_components import ImggJobConfig, ImggJobParams
-from pipelex.cogt.imgg.imgg_prompt import ImggPrompt
+from pipelex.cogt.img_gen.img_gen_job_components import ImgGenJobConfig, ImgGenJobParams
+from pipelex.cogt.img_gen.img_gen_prompt import ImgGenPrompt
from pipelex.cogt.llm.llm_job_components import LLMJobParams
from pipelex.cogt.llm.llm_prompt import LLMPrompt
from pipelex.cogt.llm.llm_prompt_factory_abstract import LLMPromptFactoryAbstract
@@ -31,7 +29,6 @@ class LLMAssignmentFactory(BaseModel):
llm_setting: Optional[LLMSetting] = None,
**prompt_arguments: Any,
) -> "LLMAssignment":
- log.verbose(f"Making LLMAssignment with prompt arguments: {prompt_arguments}, using {self.llm_prompt_factory.desc}")
llm_prompt = await self.llm_prompt_factory.make_llm_prompt_from_args(**prompt_arguments)
return LLMAssignment(
job_metadata=job_metadata or self.job_metadata,
@@ -79,7 +76,6 @@ class LLMAssignment(BaseModel):
@property
def llm_handle(self) -> str:
- log.dev(f"Using llm_setting {self.llm_setting}")
return self.llm_setting.llm_handle
@property
@@ -121,12 +117,12 @@ class TextThenObjectAssignment(BaseModel):
llm_assignment_factory_to_object: LLMAssignmentFactory
-class ImggAssignment(BaseModel):
+class ImgGenAssignment(BaseModel):
job_metadata: JobMetadata
- imgg_handle: ImggHandle
- imgg_prompt: ImggPrompt
- imgg_job_params: ImggJobParams
- imgg_job_config: ImggJobConfig
+ img_gen_handle: str
+ img_gen_prompt: ImgGenPrompt
+ img_gen_job_params: ImgGenJobParams
+ img_gen_job_config: ImgGenJobConfig
nb_images: int
diff --git a/pipelex/cogt/content_generation/content_generator.py b/pipelex/cogt/content_generation/content_generator.py
index fabff4d6e..5da23a8e2 100644
--- a/pipelex/cogt/content_generation/content_generator.py
+++ b/pipelex/cogt/content_generation/content_generator.py
@@ -4,7 +4,7 @@ from typing_extensions import override
from pipelex import log
from pipelex.cogt.content_generation.assignment_models import (
- ImggAssignment,
+ ImgGenAssignment,
Jinja2Assignment,
LLMAssignment,
LLMAssignmentFactory,
@@ -13,14 +13,13 @@ from pipelex.cogt.content_generation.assignment_models import (
TextThenObjectAssignment,
)
from pipelex.cogt.content_generation.content_generator_protocol import ContentGeneratorProtocol, update_job_metadata
-from pipelex.cogt.content_generation.imgg_generate import imgg_gen_image_list, imgg_gen_single_image
+from pipelex.cogt.content_generation.img_gen_generate import img_gen_image_list, img_gen_single_image
from pipelex.cogt.content_generation.jinja2_generate import jinja2_gen_text
from pipelex.cogt.content_generation.llm_generate import llm_gen_object, llm_gen_object_list, llm_gen_text
from pipelex.cogt.content_generation.ocr_generate import ocr_gen_extract_pages
from pipelex.cogt.image.generated_image import GeneratedImage
-from pipelex.cogt.imgg.imgg_handle import ImggHandle
-from pipelex.cogt.imgg.imgg_job_components import ImggJobConfig, ImggJobParams
-from pipelex.cogt.imgg.imgg_prompt import ImggPrompt
+from pipelex.cogt.img_gen.img_gen_job_components import ImgGenJobConfig, ImgGenJobParams
+from pipelex.cogt.img_gen.img_gen_prompt import ImgGenPrompt
from pipelex.cogt.llm.llm_prompt import LLMPrompt
from pipelex.cogt.llm.llm_prompt_factory_abstract import LLMPromptFactoryAbstract
from pipelex.cogt.llm.llm_prompt_template import LLMPromptTemplate
@@ -90,6 +89,7 @@ class ContentGenerator(ContentGeneratorProtocol):
llm_prompt_for_text: LLMPrompt,
llm_prompt_factory_for_object: Optional[LLMPromptFactoryAbstract] = None,
) -> BaseModelTypeVar:
+ log.verbose(llm_prompt_for_text.user_text, title="llm_prompt_for_text")
llm_assignment_for_text = LLMAssignment.make_from_prompt(
job_metadata=job_metadata,
llm_setting=llm_setting_main,
@@ -101,6 +101,7 @@ class ContentGenerator(ContentGeneratorProtocol):
llm_setting=llm_setting_for_object,
llm_prompt_factory=llm_prompt_factory_for_object or LLMPromptTemplate.for_structure_from_preliminary_text(),
)
+
workflow_arg = TextThenObjectAssignment(
object_class_name=object_class.__name__,
llm_assignment_for_text=llm_assignment_for_text,
@@ -109,7 +110,7 @@ class ContentGenerator(ContentGeneratorProtocol):
preliminary_text = await llm_gen_text(llm_assignment=llm_assignment_for_text)
- log.dev(f"preliminary_text: {preliminary_text}")
+ log.verbose(f"preliminary_text: {preliminary_text}")
fup_llm_assignment = await workflow_arg.llm_assignment_factory_to_object.make_llm_assignment(
preliminary_text=preliminary_text,
@@ -198,21 +199,21 @@ class ContentGenerator(ContentGeneratorProtocol):
async def make_single_image( # pyright: ignore[reportIncompatibleMethodOverride]
self,
job_metadata: JobMetadata,
- imgg_handle: ImggHandle,
- imgg_prompt: ImggPrompt,
- imgg_job_params: Optional[ImggJobParams] = None,
- imgg_job_config: Optional[ImggJobConfig] = None,
+ img_gen_handle: str,
+ img_gen_prompt: ImgGenPrompt,
+ img_gen_job_params: Optional[ImgGenJobParams] = None,
+ img_gen_job_config: Optional[ImgGenJobConfig] = None,
) -> GeneratedImage:
- imgg_config = get_config().cogt.imgg_config
- imgg_assignment = ImggAssignment(
+ img_gen_config = get_config().cogt.img_gen_config
+ img_gen_assignment = ImgGenAssignment(
job_metadata=job_metadata,
- imgg_handle=imgg_handle,
- imgg_prompt=imgg_prompt,
- imgg_job_params=imgg_job_params or imgg_config.make_default_imgg_job_params(),
- imgg_job_config=imgg_job_config or imgg_config.imgg_job_config,
+ img_gen_handle=img_gen_handle,
+ img_gen_prompt=img_gen_prompt,
+ img_gen_job_params=img_gen_job_params or img_gen_config.make_default_img_gen_job_params(),
+ img_gen_job_config=img_gen_job_config or img_gen_config.img_gen_job_config,
nb_images=1,
)
- generated_image = await imgg_gen_single_image(imgg_assignment=imgg_assignment)
+ generated_image = await img_gen_single_image(img_gen_assignment=img_gen_assignment)
log.dev(f"{self.__class__.__name__} generated image: {generated_image}")
return generated_image
@@ -221,22 +222,22 @@ class ContentGenerator(ContentGeneratorProtocol):
async def make_image_list( # pyright: ignore[reportIncompatibleMethodOverride]
self,
job_metadata: JobMetadata,
- imgg_handle: ImggHandle,
- imgg_prompt: ImggPrompt,
+ img_gen_handle: str,
+ img_gen_prompt: ImgGenPrompt,
nb_images: int,
- imgg_job_params: Optional[ImggJobParams] = None,
- imgg_job_config: Optional[ImggJobConfig] = None,
+ img_gen_job_params: Optional[ImgGenJobParams] = None,
+ img_gen_job_config: Optional[ImgGenJobConfig] = None,
) -> List[GeneratedImage]:
- imgg_config = get_config().cogt.imgg_config
- imgg_assignment = ImggAssignment(
+ img_gen_config = get_config().cogt.img_gen_config
+ img_gen_assignment = ImgGenAssignment(
job_metadata=job_metadata,
- imgg_handle=imgg_handle,
- imgg_prompt=imgg_prompt,
- imgg_job_params=imgg_job_params or imgg_config.make_default_imgg_job_params(),
- imgg_job_config=imgg_job_config or imgg_config.imgg_job_config,
+ img_gen_handle=img_gen_handle,
+ img_gen_prompt=img_gen_prompt,
+ img_gen_job_params=img_gen_job_params or img_gen_config.make_default_img_gen_job_params(),
+ img_gen_job_config=img_gen_job_config or img_gen_config.img_gen_job_config,
nb_images=nb_images,
)
- generated_image_list = await imgg_gen_image_list(imgg_assignment=imgg_assignment)
+ generated_image_list = await img_gen_image_list(img_gen_assignment=img_gen_assignment)
log.dev(f"{self.__class__.__name__} generated image list: {generated_image_list}")
return generated_image_list
diff --git a/pipelex/cogt/content_generation/content_generator_dry.py b/pipelex/cogt/content_generation/content_generator_dry.py
index 0eac3b24f..d7df93572 100644
--- a/pipelex/cogt/content_generation/content_generator_dry.py
+++ b/pipelex/cogt/content_generation/content_generator_dry.py
@@ -6,9 +6,8 @@ from typing_extensions import override
from pipelex import log
from pipelex.cogt.content_generation.content_generator_protocol import ContentGeneratorProtocol, update_job_metadata
from pipelex.cogt.image.generated_image import GeneratedImage
-from pipelex.cogt.imgg.imgg_handle import ImggHandle
-from pipelex.cogt.imgg.imgg_job_components import ImggJobConfig, ImggJobParams
-from pipelex.cogt.imgg.imgg_prompt import ImggPrompt
+from pipelex.cogt.img_gen.img_gen_job_components import ImgGenJobConfig, ImgGenJobParams
+from pipelex.cogt.img_gen.img_gen_prompt import ImgGenPrompt
from pipelex.cogt.llm.llm_prompt import LLMPrompt
from pipelex.cogt.llm.llm_prompt_factory_abstract import LLMPromptFactoryAbstract
from pipelex.cogt.llm.llm_setting import LLMSetting
@@ -64,7 +63,11 @@ class ContentGeneratorDry(ContentGeneratorProtocol):
__use_examples__ = True
__allow_none_optionals__ = False # Ensure Optional fields always get values
- obj = ObjectFactory.build()
+ # `factory_use_contruct=True` prevents from running the model_validator/field_validator.
+ # It is that way because the dry run was failing a lot of pipes that had validation test on the
+ # field values. For example, if a string requires to be a snake_case, the ObjectFactory would
+ # generate something like `DOIJZjoDoIJDZOjDZJo` which is... not a snake_case.
+ obj = ObjectFactory.build(factory_use_construct=True)
return obj
@override
@@ -139,10 +142,10 @@ class ContentGeneratorDry(ContentGeneratorProtocol):
async def make_single_image( # pyright: ignore[reportIncompatibleMethodOverride]
self,
job_metadata: JobMetadata,
- imgg_handle: ImggHandle,
- imgg_prompt: ImggPrompt,
- imgg_job_params: Optional[ImggJobParams] = None,
- imgg_job_config: Optional[ImggJobConfig] = None,
+ img_gen_handle: str,
+ img_gen_prompt: ImgGenPrompt,
+ img_gen_job_params: Optional[ImgGenJobParams] = None,
+ img_gen_job_config: Optional[ImgGenJobConfig] = None,
) -> GeneratedImage:
func_name = "make_single_image"
log.dev(f"🤡 DRY RUN: {self.__class__.__name__}.{func_name}")
@@ -160,11 +163,11 @@ class ContentGeneratorDry(ContentGeneratorProtocol):
async def make_image_list( # pyright: ignore[reportIncompatibleMethodOverride]
self,
job_metadata: JobMetadata,
- imgg_handle: ImggHandle,
- imgg_prompt: ImggPrompt,
+ img_gen_handle: str,
+ img_gen_prompt: ImgGenPrompt,
nb_images: int,
- imgg_job_params: Optional[ImggJobParams] = None,
- imgg_job_config: Optional[ImggJobConfig] = None,
+ img_gen_job_params: Optional[ImgGenJobParams] = None,
+ img_gen_job_config: Optional[ImgGenJobConfig] = None,
) -> List[GeneratedImage]:
func_name = "make_image_list"
log.dev(f"🤡 DRY RUN: {self.__class__.__name__}.{func_name}")
diff --git a/pipelex/cogt/content_generation/content_generator_protocol.py b/pipelex/cogt/content_generation/content_generator_protocol.py
index be11adba0..c52c139a7 100644
--- a/pipelex/cogt/content_generation/content_generator_protocol.py
+++ b/pipelex/cogt/content_generation/content_generator_protocol.py
@@ -2,9 +2,8 @@ from functools import wraps
from typing import Any, Awaitable, Callable, Coroutine, Dict, List, Optional, ParamSpec, Protocol, Type, TypeVar
from pipelex.cogt.image.generated_image import GeneratedImage
-from pipelex.cogt.imgg.imgg_handle import ImggHandle
-from pipelex.cogt.imgg.imgg_job_components import ImggJobConfig, ImggJobParams
-from pipelex.cogt.imgg.imgg_prompt import ImggPrompt
+from pipelex.cogt.img_gen.img_gen_job_components import ImgGenJobConfig, ImgGenJobParams
+from pipelex.cogt.img_gen.img_gen_prompt import ImgGenPrompt
from pipelex.cogt.llm.llm_prompt import LLMPrompt
from pipelex.cogt.llm.llm_prompt_factory_abstract import LLMPromptFactoryAbstract
from pipelex.cogt.llm.llm_setting import LLMSetting
@@ -91,20 +90,20 @@ class ContentGeneratorProtocol(Protocol):
async def make_single_image(
self,
job_metadata: JobMetadata,
- imgg_handle: ImggHandle,
- imgg_prompt: ImggPrompt,
- imgg_job_params: Optional[ImggJobParams] = None,
- imgg_job_config: Optional[ImggJobConfig] = None,
+ img_gen_handle: str,
+ img_gen_prompt: ImgGenPrompt,
+ img_gen_job_params: Optional[ImgGenJobParams] = None,
+ img_gen_job_config: Optional[ImgGenJobConfig] = None,
) -> GeneratedImage: ...
async def make_image_list(
self,
job_metadata: JobMetadata,
- imgg_handle: ImggHandle,
- imgg_prompt: ImggPrompt,
+ img_gen_handle: str,
+ img_gen_prompt: ImgGenPrompt,
nb_images: int,
- imgg_job_params: Optional[ImggJobParams] = None,
- imgg_job_config: Optional[ImggJobConfig] = None,
+ img_gen_job_params: Optional[ImgGenJobParams] = None,
+ img_gen_job_config: Optional[ImgGenJobConfig] = None,
) -> List[GeneratedImage]: ...
async def make_jinja2_text(
diff --git a/pipelex/cogt/content_generation/img_gen_generate.py b/pipelex/cogt/content_generation/img_gen_generate.py
new file mode 100644
index 000000000..b7ae6a7e5
--- /dev/null
+++ b/pipelex/cogt/content_generation/img_gen_generate.py
@@ -0,0 +1,43 @@
+from typing import List, Union
+
+from pipelex import log
+from pipelex.cogt.content_generation.assignment_models import ImgGenAssignment
+from pipelex.cogt.image.generated_image import GeneratedImage
+from pipelex.cogt.img_gen.img_gen_job_factory import ImgGenJobFactory
+from pipelex.hub import get_img_gen_worker
+
+
+async def img_gen_single_image(img_gen_assignment: ImgGenAssignment) -> GeneratedImage:
+ img_gen_worker = get_img_gen_worker(img_gen_handle=img_gen_assignment.img_gen_handle)
+ img_gen_job = ImgGenJobFactory.make_img_gen_job_from_prompt(
+ img_gen_prompt=img_gen_assignment.img_gen_prompt,
+ img_gen_job_params=img_gen_assignment.img_gen_job_params,
+ img_gen_job_config=img_gen_assignment.img_gen_job_config,
+ job_metadata=img_gen_assignment.job_metadata,
+ )
+ generated_image = await img_gen_worker.gen_image(img_gen_job=img_gen_job)
+ log.verbose(f"generated_image:\n{generated_image}")
+ return generated_image
+
+
+async def img_gen_image_list(img_gen_assignment: ImgGenAssignment) -> List[GeneratedImage]:
+ img_gen_worker = get_img_gen_worker(img_gen_handle=img_gen_assignment.img_gen_handle)
+ img_gen_job = ImgGenJobFactory.make_img_gen_job_from_prompt(
+ img_gen_prompt=img_gen_assignment.img_gen_prompt,
+ img_gen_job_params=img_gen_assignment.img_gen_job_params,
+ img_gen_job_config=img_gen_assignment.img_gen_job_config,
+ job_metadata=img_gen_assignment.job_metadata,
+ )
+ generated_image_list = await img_gen_worker.gen_image_list(
+ img_gen_job=img_gen_job,
+ nb_images=img_gen_assignment.nb_images,
+ )
+ log.verbose(f"generated_image_list:\n{generated_image_list}")
+ return generated_image_list
+
+
+async def img_gen_image(img_gen_assignment: ImgGenAssignment) -> Union[GeneratedImage, List[GeneratedImage]]:
+ if img_gen_assignment.nb_images > 1:
+ return await img_gen_image_list(img_gen_assignment)
+ else:
+ return await img_gen_single_image(img_gen_assignment)
diff --git a/pipelex/cogt/content_generation/imgg_generate.py b/pipelex/cogt/content_generation/imgg_generate.py
deleted file mode 100644
index 870a4f35c..000000000
--- a/pipelex/cogt/content_generation/imgg_generate.py
+++ /dev/null
@@ -1,43 +0,0 @@
-from typing import List, Union
-
-from pipelex import log
-from pipelex.cogt.content_generation.assignment_models import ImggAssignment
-from pipelex.cogt.image.generated_image import GeneratedImage
-from pipelex.cogt.imgg.imgg_job_factory import ImggJobFactory
-from pipelex.hub import get_imgg_worker
-
-
-async def imgg_gen_single_image(imgg_assignment: ImggAssignment) -> GeneratedImage:
- imgg_worker = get_imgg_worker(imgg_handle=imgg_assignment.imgg_handle)
- imgg_job = ImggJobFactory.make_imgg_job_from_prompt(
- imgg_prompt=imgg_assignment.imgg_prompt,
- imgg_job_params=imgg_assignment.imgg_job_params,
- imgg_job_config=imgg_assignment.imgg_job_config,
- job_metadata=imgg_assignment.job_metadata,
- )
- generated_image = await imgg_worker.gen_image(imgg_job=imgg_job)
- log.verbose(f"generated_image:\n{generated_image}")
- return generated_image
-
-
-async def imgg_gen_image_list(imgg_assignment: ImggAssignment) -> List[GeneratedImage]:
- imgg_worker = get_imgg_worker(imgg_handle=imgg_assignment.imgg_handle)
- imgg_job = ImggJobFactory.make_imgg_job_from_prompt(
- imgg_prompt=imgg_assignment.imgg_prompt,
- imgg_job_params=imgg_assignment.imgg_job_params,
- imgg_job_config=imgg_assignment.imgg_job_config,
- job_metadata=imgg_assignment.job_metadata,
- )
- generated_image_list = await imgg_worker.gen_image_list(
- imgg_job=imgg_job,
- nb_images=imgg_assignment.nb_images,
- )
- log.verbose(f"generated_image_list:\n{generated_image_list}")
- return generated_image_list
-
-
-async def imgg_gen_image(imgg_assignment: ImggAssignment) -> Union[GeneratedImage, List[GeneratedImage]]:
- if imgg_assignment.nb_images > 1:
- return await imgg_gen_image_list(imgg_assignment)
- else:
- return await imgg_gen_single_image(imgg_assignment)
diff --git a/pipelex/cogt/content_generation/llm_generate.py b/pipelex/cogt/content_generation/llm_generate.py
index 95a169781..21ddbce2c 100644
--- a/pipelex/cogt/content_generation/llm_generate.py
+++ b/pipelex/cogt/content_generation/llm_generate.py
@@ -1,4 +1,4 @@
-from typing import List
+from typing import List, cast
from pydantic import BaseModel
@@ -22,7 +22,6 @@ async def llm_gen_text(llm_assignment: LLMAssignment) -> str:
async def llm_gen_object(object_assignment: ObjectAssignment) -> BaseModel:
llm_assignment = object_assignment.llm_assignment_for_object
- log.verbose(f"llm_gen_object to generate a: '{object_assignment.object_class_name}'")
llm_worker = get_llm_worker(llm_handle=llm_assignment.llm_handle)
llm_job = LLMJobFactory.make_llm_job(
job_metadata=llm_assignment.job_metadata,
@@ -57,5 +56,5 @@ async def llm_gen_object_list(object_assignment: ObjectAssignment) -> List[BaseM
llm_job=llm_job,
schema=ListSchema,
)
- generated_list: List[BaseModel] = wrapped_list.items # pyright: ignore[reportUnknownMemberType]
+ generated_list: List[BaseModel] = cast(List[BaseModel], wrapped_list.items) # pyright: ignore[reportUnknownMemberType]
return generated_list
diff --git a/pipelex/cogt/exceptions.py b/pipelex/cogt/exceptions.py
index e70b17602..a8c893aa6 100644
--- a/pipelex/cogt/exceptions.py
+++ b/pipelex/cogt/exceptions.py
@@ -32,7 +32,15 @@ class LLMWorkerError(CogtError):
pass
-class LLMPresetNotFoundError(CogtError):
+class LLMChoiceNotFoundError(CogtError):
+ pass
+
+
+class OcrChoiceNotFoundError(CogtError):
+ pass
+
+
+class ImgGenChoiceNotFoundError(CogtError):
pass
@@ -40,6 +48,10 @@ class LLMSettingsValidationError(CogtError):
pass
+class ImgGenSettingsValidationError(CogtError):
+ pass
+
+
class ModelDeckValidatonError(CogtError):
pass
@@ -96,19 +108,19 @@ class PromptImageFormatError(CogtError):
pass
-class ImggPromptError(CogtError):
+class ImgGenPromptError(CogtError):
pass
-class ImggParameterError(CogtError):
+class ImgGenParameterError(CogtError):
pass
-class ImggGenerationError(CogtError):
+class ImgGenGenerationError(CogtError):
pass
-class ImggGeneratedTypeError(ImggGenerationError):
+class ImgGenGeneratedTypeError(ImgGenGenerationError):
pass
diff --git a/pipelex/cogt/image/prompt_image.py b/pipelex/cogt/image/prompt_image.py
index df7f3e544..433712d3b 100644
--- a/pipelex/cogt/image/prompt_image.py
+++ b/pipelex/cogt/image/prompt_image.py
@@ -1,3 +1,4 @@
+import base64
from abc import ABC
from typing import Union
@@ -5,16 +6,16 @@ from pydantic import BaseModel
from typing_extensions import override
from pipelex.tools.misc.attribute_utils import AttributePolisher
-from pipelex.tools.misc.filetype_utils import FileType, detect_file_type_from_base64, detect_file_type_from_path
+from pipelex.tools.misc.filetype_utils import FileType, detect_file_type_from_base64, detect_file_type_from_bytes, detect_file_type_from_path
from pipelex.tools.typing.pydantic_utils import CustomBaseModel
-class PromptImageTypedBytes(CustomBaseModel):
+class PromptImageTypedBase64(CustomBaseModel):
base_64: bytes
file_type: FileType
-PromptImageTypedBytesOrUrl = Union[PromptImageTypedBytes, str]
+PromptImageTypedUrlOrBase64 = Union[str, PromptImageTypedBase64]
class PromptImage(BaseModel, ABC):
@@ -27,6 +28,9 @@ class PromptImagePath(PromptImage):
def get_file_type(self) -> FileType:
return detect_file_type_from_path(self.file_path)
+ def get_mime_type(self) -> str:
+ return self.get_file_type().mime
+
@override
def __str__(self) -> str:
return f"PromptImagePath(file_path='{self.file_path}')"
@@ -45,17 +49,23 @@ class PromptImageUrl(PromptImage):
return self.__str__()
-class PromptImageBytes(PromptImage):
+class PromptImageBase64(PromptImage):
base_64: bytes
def get_file_type(self) -> FileType:
return detect_file_type_from_base64(self.base_64)
+ def get_mime_type(self) -> str:
+ return self.get_file_type().mime
+
+ def get_decoded_bytes(self) -> bytes:
+ return base64.b64decode(self.base_64)
+
@override
def __str__(self) -> str:
base_64_str = str(self.base_64)
truncated_base_64 = AttributePolisher.get_truncated_value(name="base_64", value=base_64_str)
- return f"PromptImageBytes(image_bytes={truncated_base_64!r})"
+ return f"PromptImageBase64(base_64={truncated_base_64!r})"
@override
def __repr__(self) -> str:
@@ -65,5 +75,23 @@ class PromptImageBytes(PromptImage):
def __format__(self, format_spec: str) -> str:
return self.__str__()
- def make_prompt_image_typed_bytes(self) -> PromptImageTypedBytes:
- return PromptImageTypedBytes(base_64=self.base_64, file_type=self.get_file_type())
+ def make_prompt_image_typed_base64(self) -> PromptImageTypedBase64:
+ return PromptImageTypedBase64(base_64=self.base_64, file_type=self.get_file_type())
+
+
+class PromptImageBinary(PromptImage):
+ binary: bytes
+
+ def get_file_type(self) -> FileType:
+ return detect_file_type_from_bytes(self.binary)
+
+ def get_mime_type(self) -> str:
+ return self.get_file_type().mime
+
+ @override
+ def __str__(self) -> str:
+ return "PromptImageBinary(binary=...)"
+
+ @override
+ def __repr__(self) -> str:
+ return self.__str__()
diff --git a/pipelex/cogt/image/prompt_image_factory.py b/pipelex/cogt/image/prompt_image_factory.py
index 17f99fd06..e5b4a4df9 100644
--- a/pipelex/cogt/image/prompt_image_factory.py
+++ b/pipelex/cogt/image/prompt_image_factory.py
@@ -1,10 +1,11 @@
from typing import Optional
from pipelex.cogt.exceptions import PromptImageFactoryError
-from pipelex.cogt.image.prompt_image import PromptImage, PromptImageBytes, PromptImagePath, PromptImageUrl
+from pipelex.cogt.image.prompt_image import PromptImage, PromptImageBase64, PromptImageBinary, PromptImagePath, PromptImageUrl
from pipelex.tools.misc.base_64_utils import (
encode_to_base64_async,
load_binary_as_base64_async,
+ load_binary_async,
)
from pipelex.tools.misc.file_fetch_utils import fetch_file_from_url_httpx_async
from pipelex.tools.misc.path_utils import clarify_path_or_url
@@ -23,7 +24,7 @@ class PromptImageFactory:
elif url:
return PromptImageUrl(url=url)
elif base_64:
- return PromptImageBytes(base_64=base_64)
+ return PromptImageBase64(base_64=base_64)
else:
raise PromptImageFactoryError("PromptImageFactory requires one of file_path, url, or image_bytes")
@@ -39,22 +40,42 @@ class PromptImageFactory:
)
@classmethod
- async def make_promptimagebytes_from_url_async(
+ async def make_promptimagebase64_from_url_async(
cls,
prompt_image_url: PromptImageUrl,
- ) -> PromptImageBytes:
+ ) -> PromptImageBase64:
raw_image_bytes = await fetch_file_from_url_httpx_async(prompt_image_url.url)
base_64 = await encode_to_base64_async(raw_image_bytes)
- return PromptImageBytes(base_64=base_64)
+ return PromptImageBase64(base_64=base_64)
@classmethod
- async def promptimage_to_b64_async(cls, image_prompt: PromptImage) -> bytes:
- if isinstance(image_prompt, PromptImagePath):
- return await load_binary_as_base64_async(image_prompt.file_path)
- elif isinstance(image_prompt, PromptImageBytes):
- return image_prompt.base_64
- elif isinstance(image_prompt, PromptImageUrl):
- image_bytes = await cls.make_promptimagebytes_from_url_async(image_prompt)
+ async def make_promptimagebinary_from_url_async(
+ cls,
+ prompt_image_url: PromptImageUrl,
+ ) -> PromptImageBinary:
+ raw_image_bytes = await fetch_file_from_url_httpx_async(prompt_image_url.url)
+ return PromptImageBinary(binary=raw_image_bytes)
+
+ @classmethod
+ async def promptimage_to_b64_async(cls, prompt_image: PromptImage) -> bytes:
+ if isinstance(prompt_image, PromptImagePath):
+ return await load_binary_as_base64_async(prompt_image.file_path)
+ elif isinstance(prompt_image, PromptImageBase64):
+ return prompt_image.base_64
+ elif isinstance(prompt_image, PromptImageUrl):
+ image_bytes = await cls.make_promptimagebase64_from_url_async(prompt_image)
return image_bytes.base_64
else:
- raise PromptImageFactoryError(f"Unknown PromptImage type: {image_prompt}")
+ raise PromptImageFactoryError(f"Unknown PromptImage type: {prompt_image}")
+
+ @classmethod
+ async def promptimage_to_bytes_async(cls, prompt_image: PromptImage) -> bytes:
+ if isinstance(prompt_image, PromptImagePath):
+ return await load_binary_async(prompt_image.file_path)
+ elif isinstance(prompt_image, PromptImageBase64):
+ return prompt_image.get_decoded_bytes()
+ elif isinstance(prompt_image, PromptImageUrl):
+ image_bytes = await cls.make_promptimagebinary_from_url_async(prompt_image)
+ return image_bytes.binary
+ else:
+ raise PromptImageFactoryError(f"Unknown PromptImage type: {prompt_image}")
diff --git a/pipelex/cogt/imgg/__init__.py b/pipelex/cogt/img_gen/__init__.py
similarity index 100%
rename from pipelex/cogt/imgg/__init__.py
rename to pipelex/cogt/img_gen/__init__.py
diff --git a/pipelex/cogt/img_gen/img_gen_job.py b/pipelex/cogt/img_gen/img_gen_job.py
new file mode 100644
index 000000000..a8a8cf81c
--- /dev/null
+++ b/pipelex/cogt/img_gen/img_gen_job.py
@@ -0,0 +1,28 @@
+from datetime import datetime
+
+from typing_extensions import override
+
+from pipelex.cogt.img_gen.img_gen_job_components import ImgGenJobConfig, ImgGenJobParams, ImgGenJobReport
+from pipelex.cogt.img_gen.img_gen_prompt import ImgGenPrompt
+from pipelex.cogt.inference.inference_job_abstract import InferenceJobAbstract
+
+
+class ImgGenJob(InferenceJobAbstract):
+ img_gen_prompt: ImgGenPrompt
+ job_params: ImgGenJobParams
+ job_config: ImgGenJobConfig
+ job_report: ImgGenJobReport
+
+ @override
+ def validate_before_execution(self):
+ self.img_gen_prompt.validate_before_execution()
+
+ def img_gen_job_before_start(self):
+ # Reset metadata
+ self.job_metadata.started_at = datetime.now()
+
+ # Reset outputs
+ self.job_report = ImgGenJobReport()
+
+ def img_gen_job_after_complete(self):
+ self.job_metadata.completed_at = datetime.now()
diff --git a/pipelex/cogt/imgg/imgg_job_components.py b/pipelex/cogt/img_gen/img_gen_job_components.py
similarity index 85%
rename from pipelex/cogt/imgg/imgg_job_components.py
rename to pipelex/cogt/img_gen/img_gen_job_components.py
index b5f23e7ae..a272e8aef 100644
--- a/pipelex/cogt/imgg/imgg_job_components.py
+++ b/pipelex/cogt/img_gen/img_gen_job_components.py
@@ -36,20 +36,20 @@ class Background(StrEnum):
AUTO = "auto"
-class ImggJobParams(BaseModel):
+class ImgGenJobParams(BaseModel):
aspect_ratio: AspectRatio = Field(strict=False)
background: Background = Field(strict=False)
quality: Optional[Quality] = Field(default=None, strict=False)
nb_steps: Optional[int] = Field(default=None, gt=0)
- guidance_scale: float = Field(..., gt=0)
- is_moderated: bool
- safety_tolerance: int = Field(..., ge=1, le=6)
+ guidance_scale: Optional[float] = Field(default=None, gt=0)
+ is_moderated: bool = False
+ safety_tolerance: Optional[int] = Field(default=None, ge=1, le=6)
is_raw: bool
output_format: OutputFormat = Field(strict=False)
seed: Optional[int] = Field(None, ge=0)
-class ImggJobParamsDefaults(ConfigModel):
+class ImgGenJobParamsDefaults(ConfigModel):
aspect_ratio: AspectRatio = Field(strict=False)
background: Background = Field(strict=False)
quality: Optional[Quality] = Field(default=None, strict=False)
@@ -61,13 +61,13 @@ class ImggJobParamsDefaults(ConfigModel):
output_format: OutputFormat = Field(strict=False)
seed: Union[int, Literal["auto"]]
- def make_imgg_job_params(self) -> ImggJobParams:
+ def make_img_gen_job_params(self) -> ImgGenJobParams:
seed: Optional[int]
if isinstance(self.seed, str) and self.seed == "auto":
seed = None
else:
seed = self.seed
- return ImggJobParams(
+ return ImgGenJobParams(
aspect_ratio=self.aspect_ratio,
background=self.background,
quality=self.quality,
@@ -81,7 +81,7 @@ class ImggJobParamsDefaults(ConfigModel):
)
-class ImggJobConfig(ConfigModel):
+class ImgGenJobConfig(ConfigModel):
is_sync_mode: bool
@@ -90,5 +90,5 @@ class ImggJobConfig(ConfigModel):
########################################################################
-class ImggJobReport(ConfigModel):
+class ImgGenJobReport(ConfigModel):
pass
diff --git a/pipelex/cogt/img_gen/img_gen_job_factory.py b/pipelex/cogt/img_gen/img_gen_job_factory.py
new file mode 100644
index 000000000..166c55521
--- /dev/null
+++ b/pipelex/cogt/img_gen/img_gen_job_factory.py
@@ -0,0 +1,58 @@
+from typing import Optional
+
+from pipelex.cogt.img_gen.img_gen_job import ImgGenJob
+from pipelex.cogt.img_gen.img_gen_job_components import ImgGenJobConfig, ImgGenJobParams, ImgGenJobReport
+from pipelex.cogt.img_gen.img_gen_prompt import ImgGenPrompt
+from pipelex.config import get_config
+from pipelex.pipeline.job_metadata import JobCategory, JobMetadata
+
+
+class ImgGenJobFactory:
+ @classmethod
+ def make_img_gen_job_from_prompt(
+ cls,
+ img_gen_prompt: ImgGenPrompt,
+ img_gen_job_params: Optional[ImgGenJobParams] = None,
+ img_gen_job_config: Optional[ImgGenJobConfig] = None,
+ job_metadata: Optional[JobMetadata] = None,
+ ) -> ImgGenJob:
+ img_gen_config = get_config().cogt.img_gen_config
+ job_metadata = job_metadata or JobMetadata(
+ job_category=JobCategory.IMG_GEN_JOB,
+ )
+ job_params = img_gen_job_params or img_gen_config.make_default_img_gen_job_params()
+ job_config = img_gen_job_config or img_gen_config.img_gen_job_config
+ job_report = ImgGenJobReport()
+
+ return ImgGenJob(
+ job_metadata=job_metadata,
+ img_gen_prompt=img_gen_prompt,
+ job_params=job_params,
+ job_config=job_config,
+ job_report=job_report,
+ )
+
+ @classmethod
+ def make_img_gen_job_from_prompt_contents(
+ cls,
+ positive_text: str,
+ img_gen_job_params: Optional[ImgGenJobParams] = None,
+ img_gen_job_config: Optional[ImgGenJobConfig] = None,
+ job_metadata: Optional[JobMetadata] = None,
+ ) -> ImgGenJob:
+ img_gen_config = get_config().cogt.img_gen_config
+ job_metadata = job_metadata or JobMetadata(
+ job_category=JobCategory.IMG_GEN_JOB,
+ )
+ img_gen_prompt = ImgGenPrompt(positive_text=positive_text)
+ job_params = img_gen_job_params or img_gen_config.make_default_img_gen_job_params()
+ job_config = img_gen_job_config or img_gen_config.img_gen_job_config
+ job_report = ImgGenJobReport()
+
+ return ImgGenJob(
+ job_metadata=job_metadata,
+ img_gen_prompt=img_gen_prompt,
+ job_params=job_params,
+ job_config=job_config,
+ job_report=job_report,
+ )
diff --git a/pipelex/cogt/imgg/imgg_prompt.py b/pipelex/cogt/img_gen/img_gen_prompt.py
similarity index 64%
rename from pipelex/cogt/imgg/imgg_prompt.py
rename to pipelex/cogt/img_gen/img_gen_prompt.py
index dbff961af..83e3558e5 100644
--- a/pipelex/cogt/imgg/imgg_prompt.py
+++ b/pipelex/cogt/img_gen/img_gen_prompt.py
@@ -2,12 +2,12 @@ from pydantic import BaseModel
from typing_extensions import override
from pipelex import log
-from pipelex.cogt.exceptions import ImggPromptError
+from pipelex.cogt.exceptions import ImgGenPromptError
from pipelex.tools.misc.json_utils import json_str
from pipelex.tools.runtime_manager import ProblemReaction, runtime_manager
-class ImggPrompt(BaseModel):
+class ImgGenPrompt(BaseModel):
positive_text: str
def validate_before_execution(self):
@@ -17,11 +17,11 @@ class ImggPrompt(BaseModel):
pass
case ProblemReaction.RAISE:
if self.positive_text == "":
- raise ImggPromptError("Imgg prompt positive_text must not be an empty string")
+ raise ImgGenPromptError("ImgGen prompt positive_text must not be an empty string")
case ProblemReaction.LOG:
if self.positive_text == "":
- log.warning("Imgg prompt positive_text should not be an empty string")
+ log.warning("ImgGen prompt positive_text should not be an empty string")
@override
def __str__(self) -> str:
- return json_str(self, title="imgg_prompt", is_spaced=True)
+ return json_str(self, title="img_gen_prompt", is_spaced=True)
diff --git a/pipelex/cogt/img_gen/img_gen_setting.py b/pipelex/cogt/img_gen/img_gen_setting.py
new file mode 100644
index 000000000..5245dc66c
--- /dev/null
+++ b/pipelex/cogt/img_gen/img_gen_setting.py
@@ -0,0 +1,33 @@
+from typing import Optional, Union
+
+from pydantic import Field, model_validator
+from typing_extensions import Self
+
+from pipelex.cogt.exceptions import ImgGenSettingsValidationError
+from pipelex.cogt.img_gen.img_gen_job_components import Quality
+from pipelex.tools.config.config_model import ConfigModel
+
+
+class ImgGenSetting(ConfigModel):
+ img_gen_handle: str
+ quality: Optional[Quality] = Field(default=None, strict=False)
+ nb_steps: Optional[int] = Field(default=None, gt=0)
+ guidance_scale: Optional[float] = Field(default=None, gt=0)
+ is_moderated: bool = False
+ safety_tolerance: Optional[int] = Field(default=None, ge=1, le=6)
+
+ @model_validator(mode="after")
+ def validate_quality_or_nb_steps(self) -> Self:
+ if self.quality is not None and self.nb_steps is not None:
+ raise ImgGenSettingsValidationError("ImgGenSetting cannot have both 'quality' and 'nb_steps' specified. Use one or the other.")
+ return self
+
+ def desc(self) -> str:
+ return (
+ f"ImgGenSetting(img_gen_handle={self.img_gen_handle}, quality={self.quality}, "
+ f"nb_steps={self.nb_steps}, guidance_scale={self.guidance_scale}, "
+ f"is_moderated={self.is_moderated}, safety_tolerance={self.safety_tolerance})"
+ )
+
+
+ImgGenChoice = Union[ImgGenSetting, str]
diff --git a/pipelex/cogt/imgg/imgg_worker_abstract.py b/pipelex/cogt/img_gen/img_gen_worker_abstract.py
similarity index 56%
rename from pipelex/cogt/imgg/imgg_worker_abstract.py
rename to pipelex/cogt/img_gen/img_gen_worker_abstract.py
index c8df26736..1e4432ae9 100644
--- a/pipelex/cogt/imgg/imgg_worker_abstract.py
+++ b/pipelex/cogt/img_gen/img_gen_worker_abstract.py
@@ -5,21 +5,21 @@ from typing_extensions import override
from pipelex import log
from pipelex.cogt.image.generated_image import GeneratedImage
-from pipelex.cogt.imgg.imgg_engine import ImggEngine
-from pipelex.cogt.imgg.imgg_job import ImggJob
+from pipelex.cogt.img_gen.img_gen_job import ImgGenJob
from pipelex.cogt.inference.inference_worker_abstract import InferenceWorkerAbstract
+from pipelex.cogt.model_backends.model_spec import InferenceModelSpec
from pipelex.pipeline.job_metadata import UnitJobId
from pipelex.reporting.reporting_protocol import ReportingProtocol
-class ImggWorkerAbstract(InferenceWorkerAbstract):
+class ImgGenWorkerAbstract(InferenceWorkerAbstract):
def __init__(
self,
- imgg_engine: ImggEngine,
+ inference_model: InferenceModelSpec,
reporting_delegate: Optional[ReportingProtocol] = None,
):
InferenceWorkerAbstract.__init__(self, reporting_delegate=reporting_delegate)
- self.imgg_engine = imgg_engine
+ self.inference_model = inference_model
#########################################################
# Instance methods
@@ -28,80 +28,80 @@ class ImggWorkerAbstract(InferenceWorkerAbstract):
@property
@override
def desc(self) -> str:
- return f"Img Worker using:\n{self.imgg_engine.desc}"
+ return f"ImgGen-Worker:{self.inference_model.tag}"
- def _check_can_perform_job(self, imgg_job: ImggJob):
+ def _check_can_perform_job(self, img_gen_job: ImgGenJob):
# This can be overridden by subclasses for specific checks
pass
async def gen_image(
self,
- imgg_job: ImggJob,
+ img_gen_job: ImgGenJob,
) -> GeneratedImage:
- log.debug(f"Image gen worker gen_image:\n{self.imgg_engine.desc}")
+ log.debug(f"Image gen worker gen_image using {self.desc}")
# Verify that the job is valid
- imgg_job.validate_before_execution()
+ img_gen_job.validate_before_execution()
# Verify feasibility
- self._check_can_perform_job(imgg_job=imgg_job)
+ self._check_can_perform_job(img_gen_job=img_gen_job)
# metadata
- imgg_job.job_metadata.unit_job_id = UnitJobId.IMGG_TEXT_TO_IMAGE
+ img_gen_job.job_metadata.unit_job_id = UnitJobId.IMG_GEN_TEXT_TO_IMAGE
# Prepare job
- imgg_job.imgg_job_before_start(imgg_engine=self.imgg_engine)
+ img_gen_job.img_gen_job_before_start()
# Execute job
- result = await self._gen_image(imgg_job=imgg_job)
+ result = await self._gen_image(img_gen_job=img_gen_job)
# Report job
- imgg_job.imgg_job_after_complete()
+ img_gen_job.img_gen_job_after_complete()
if self.reporting_delegate:
- self.reporting_delegate.report_inference_job(inference_job=imgg_job)
+ self.reporting_delegate.report_inference_job(inference_job=img_gen_job)
return result
@abstractmethod
async def _gen_image(
self,
- imgg_job: ImggJob,
+ img_gen_job: ImgGenJob,
) -> GeneratedImage:
pass
async def gen_image_list(
self,
- imgg_job: ImggJob,
+ img_gen_job: ImgGenJob,
nb_images: int,
) -> List[GeneratedImage]:
- log.debug(f"Image gen worker gen_image_list:\n{self.imgg_engine.desc}")
+ log.debug(f"Image gen worker gen_image_list using {self.desc}")
# Verify that the job is valid
- imgg_job.validate_before_execution()
+ img_gen_job.validate_before_execution()
# Verify feasibility
- self._check_can_perform_job(imgg_job=imgg_job)
+ self._check_can_perform_job(img_gen_job=img_gen_job)
# metadata
- imgg_job.job_metadata.unit_job_id = UnitJobId.IMGG_TEXT_TO_IMAGE
+ img_gen_job.job_metadata.unit_job_id = UnitJobId.IMG_GEN_TEXT_TO_IMAGE
# Prepare job
- imgg_job.imgg_job_before_start(imgg_engine=self.imgg_engine)
+ img_gen_job.img_gen_job_before_start()
# Execute job
- result = await self._gen_image_list(imgg_job=imgg_job, nb_images=nb_images)
+ result = await self._gen_image_list(img_gen_job=img_gen_job, nb_images=nb_images)
# Report job
- imgg_job.imgg_job_after_complete()
+ img_gen_job.img_gen_job_after_complete()
if self.reporting_delegate:
- self.reporting_delegate.report_inference_job(inference_job=imgg_job)
+ self.reporting_delegate.report_inference_job(inference_job=img_gen_job)
return result
@abstractmethod
async def _gen_image_list(
self,
- imgg_job: ImggJob,
+ img_gen_job: ImgGenJob,
nb_images: int,
) -> List[GeneratedImage]:
pass
diff --git a/pipelex/cogt/imgg/imgg_worker_factory.py b/pipelex/cogt/img_gen/img_gen_worker_factory.py
similarity index 51%
rename from pipelex/cogt/imgg/imgg_worker_factory.py
rename to pipelex/cogt/img_gen/img_gen_worker_factory.py
index f78f2a2a7..97e7c9ed9 100644
--- a/pipelex/cogt/imgg/imgg_worker_factory.py
+++ b/pipelex/cogt/img_gen/img_gen_worker_factory.py
@@ -1,11 +1,9 @@
from typing import Optional
from pipelex.cogt.exceptions import CogtError, MissingDependencyError
-from pipelex.cogt.imgg.imgg_engine import ImggEngine
-from pipelex.cogt.imgg.imgg_platform import ImggPlatform
-from pipelex.cogt.imgg.imgg_worker_abstract import ImggWorkerAbstract
+from pipelex.cogt.img_gen.img_gen_worker_abstract import ImgGenWorkerAbstract
+from pipelex.cogt.model_backends.model_spec import InferenceModelSpec
from pipelex.hub import get_models_manager, get_plugin_manager, get_secret
-from pipelex.plugins.openai.openai_imgg_worker import OpenAIImggWorker
from pipelex.plugins.plugin_sdk_registry import Plugin
from pipelex.reporting.reporting_protocol import ReportingProtocol
from pipelex.tools.secrets.secrets_errors import SecretNotFoundError
@@ -15,18 +13,18 @@ class FalCredentialsError(CogtError):
pass
-class ImggWorkerFactory:
- def make_imgg_worker(
+class ImgGenWorkerFactory:
+ def make_img_gen_worker(
self,
- imgg_engine: ImggEngine,
+ inference_model: InferenceModelSpec,
reporting_delegate: Optional[ReportingProtocol] = None,
- ) -> ImggWorkerAbstract:
- img_gen_plugin = Plugin.make_for_imgg_engine(imgg_platform=imgg_engine.imgg_platform)
- backend = get_models_manager().get_required_inference_backend("openai")
+ ) -> ImgGenWorkerAbstract:
+ plugin = Plugin.make_for_inference_model(inference_model=inference_model)
+ backend = get_models_manager().get_required_inference_backend(inference_model.backend_name)
plugin_sdk_registry = get_plugin_manager().plugin_sdk_registry
- imgg_worker: ImggWorkerAbstract
- match imgg_engine.imgg_platform:
- case ImggPlatform.FAL_AI:
+ img_gen_worker: ImgGenWorkerAbstract
+ match plugin.sdk:
+ case "fal":
try:
fal_api_key = get_secret(secret_id="FAL_API_KEY")
except SecretNotFoundError as exc:
@@ -39,33 +37,36 @@ class ImggWorkerFactory:
"fal-client", "fal", "The fal-client SDK is required to use FAL models (generation of images)."
) from exc
- from pipelex.plugins.fal.fal_imgg_worker import FalImggWorker
+ from pipelex.plugins.fal.fal_img_gen_worker import FalImgGenWorker
- imgg_sdk_instance = plugin_sdk_registry.get_sdk_instance(plugin=img_gen_plugin) or plugin_sdk_registry.set_sdk_instance(
- plugin=img_gen_plugin,
+ img_gen_sdk_instance = plugin_sdk_registry.get_sdk_instance(plugin=plugin) or plugin_sdk_registry.set_sdk_instance(
+ plugin=plugin,
sdk_instance=FalAsyncClient(key=fal_api_key),
)
- imgg_worker = FalImggWorker(
- sdk_instance=imgg_sdk_instance,
- imgg_engine=imgg_engine,
+ img_gen_worker = FalImgGenWorker(
+ sdk_instance=img_gen_sdk_instance,
+ inference_model=inference_model,
reporting_delegate=reporting_delegate,
)
- case ImggPlatform.OPENAI:
+ case "openai":
from pipelex.plugins.openai.openai_factory import OpenAIFactory
+ from pipelex.plugins.openai.openai_img_gen_worker import OpenAIImgGenWorker
- imgg_sdk_instance = plugin_sdk_registry.get_sdk_instance(plugin=img_gen_plugin) or plugin_sdk_registry.set_sdk_instance(
- plugin=img_gen_plugin,
+ img_gen_sdk_instance = plugin_sdk_registry.get_sdk_instance(plugin=plugin) or plugin_sdk_registry.set_sdk_instance(
+ plugin=plugin,
sdk_instance=OpenAIFactory.make_openai_client(
- plugin=img_gen_plugin,
+ plugin=plugin,
backend=backend,
),
)
- imgg_worker = OpenAIImggWorker(
- sdk_instance=imgg_sdk_instance,
- imgg_engine=imgg_engine,
+ img_gen_worker = OpenAIImgGenWorker(
+ sdk_instance=img_gen_sdk_instance,
+ inference_model=inference_model,
reporting_delegate=reporting_delegate,
)
+ case _:
+ raise NotImplementedError(f"Plugin '{plugin}' is not supported for image generation")
- return imgg_worker
+ return img_gen_worker
diff --git a/pipelex/cogt/imgg/imgg_engine.py b/pipelex/cogt/imgg/imgg_engine.py
deleted file mode 100644
index 0fa0efd51..000000000
--- a/pipelex/cogt/imgg/imgg_engine.py
+++ /dev/null
@@ -1,12 +0,0 @@
-from pydantic import BaseModel
-
-from pipelex.cogt.imgg.imgg_platform import ImggPlatform
-
-
-class ImggEngine(BaseModel):
- imgg_platform: ImggPlatform
- imgg_model_name: str
-
- @property
- def desc(self) -> str:
- return f"Imgg Engine '{self.imgg_platform}/{self.imgg_model_name}'"
diff --git a/pipelex/cogt/imgg/imgg_engine_factory.py b/pipelex/cogt/imgg/imgg_engine_factory.py
deleted file mode 100644
index 325565b4e..000000000
--- a/pipelex/cogt/imgg/imgg_engine_factory.py
+++ /dev/null
@@ -1,29 +0,0 @@
-from pipelex.cogt.exceptions import CogtError
-from pipelex.cogt.imgg.imgg_engine import ImggEngine
-from pipelex.cogt.imgg.imgg_platform import ImggPlatform
-
-
-class ImggEngineFactoryError(CogtError):
- pass
-
-
-class ImggEngineFactory:
- @classmethod
- def make_imgg_engine(
- cls,
- imgg_handle: str,
- ) -> ImggEngine:
- if "/" not in imgg_handle:
- raise ImggEngineFactoryError(f"Invalid Imgg handle: {imgg_handle}. Expected format: platform/model_name")
-
- # split at the first "/"
- parts = imgg_handle.split("/", 1)
-
- try:
- imgg_platform = ImggPlatform(parts[0])
- except ValueError:
- raise ImggEngineFactoryError(f"Unknown Imgg platform:' {parts[0]}' extracted from handle '{imgg_handle}'")
-
- imgg_model_name = parts[1]
-
- return ImggEngine(imgg_platform=imgg_platform, imgg_model_name=imgg_model_name)
diff --git a/pipelex/cogt/imgg/imgg_handle.py b/pipelex/cogt/imgg/imgg_handle.py
deleted file mode 100644
index a6ba93736..000000000
--- a/pipelex/cogt/imgg/imgg_handle.py
+++ /dev/null
@@ -1,10 +0,0 @@
-from pipelex.types import StrEnum
-
-
-# List of classic Img generation preset handles, for convenience
-class ImggHandle(StrEnum):
- FLUX_1_PRO_LEGACY = "fal-ai/flux-pro"
- FLUX_1_1_PRO = "fal-ai/flux-pro/v1.1"
- FLUX_1_1_ULTRA = "fal-ai/flux-pro/v1.1-ultra"
- SDXL_LIGHTNING = "fal-ai/fast-lightning-sdxl"
- OPENAI_GPT_IMAGE_1 = "openai/gpt-image-1"
diff --git a/pipelex/cogt/imgg/imgg_job.py b/pipelex/cogt/imgg/imgg_job.py
deleted file mode 100644
index b6bf70906..000000000
--- a/pipelex/cogt/imgg/imgg_job.py
+++ /dev/null
@@ -1,29 +0,0 @@
-from datetime import datetime
-
-from typing_extensions import override
-
-from pipelex.cogt.imgg.imgg_engine import ImggEngine
-from pipelex.cogt.imgg.imgg_job_components import ImggJobConfig, ImggJobParams, ImggJobReport
-from pipelex.cogt.imgg.imgg_prompt import ImggPrompt
-from pipelex.cogt.inference.inference_job_abstract import InferenceJobAbstract
-
-
-class ImggJob(InferenceJobAbstract):
- imgg_prompt: ImggPrompt
- job_params: ImggJobParams
- job_config: ImggJobConfig
- job_report: ImggJobReport
-
- @override
- def validate_before_execution(self):
- self.imgg_prompt.validate_before_execution()
-
- def imgg_job_before_start(self, imgg_engine: ImggEngine):
- # Reset metadata
- self.job_metadata.started_at = datetime.now()
-
- # Reset outputs
- self.job_report = ImggJobReport()
-
- def imgg_job_after_complete(self):
- self.job_metadata.completed_at = datetime.now()
diff --git a/pipelex/cogt/imgg/imgg_job_factory.py b/pipelex/cogt/imgg/imgg_job_factory.py
deleted file mode 100644
index e5e7c1fc7..000000000
--- a/pipelex/cogt/imgg/imgg_job_factory.py
+++ /dev/null
@@ -1,60 +0,0 @@
-from typing import Optional
-
-from pipelex.cogt.imgg.imgg_job import ImggJob
-from pipelex.cogt.imgg.imgg_job_components import ImggJobConfig, ImggJobParams, ImggJobReport
-from pipelex.cogt.imgg.imgg_prompt import ImggPrompt
-from pipelex.config import get_config
-from pipelex.pipeline.job_metadata import JobCategory, JobMetadata
-
-
-class ImggJobFactory:
- # straightforward: the prompt is provided
-
- @classmethod
- def make_imgg_job_from_prompt(
- cls,
- imgg_prompt: ImggPrompt,
- imgg_job_params: Optional[ImggJobParams] = None,
- imgg_job_config: Optional[ImggJobConfig] = None,
- job_metadata: Optional[JobMetadata] = None,
- ) -> ImggJob:
- imgg_config = get_config().cogt.imgg_config
- job_metadata = job_metadata or JobMetadata(
- job_category=JobCategory.IMGG_JOB,
- )
- job_params = imgg_job_params or imgg_config.make_default_imgg_job_params()
- job_config = imgg_job_config or imgg_config.imgg_job_config
- job_report = ImggJobReport()
-
- return ImggJob(
- job_metadata=job_metadata,
- imgg_prompt=imgg_prompt,
- job_params=job_params,
- job_config=job_config,
- job_report=job_report,
- )
-
- @classmethod
- def make_imgg_job_from_prompt_contents(
- cls,
- positive_text: str,
- imgg_job_params: Optional[ImggJobParams] = None,
- imgg_job_config: Optional[ImggJobConfig] = None,
- job_metadata: Optional[JobMetadata] = None,
- ) -> ImggJob:
- imgg_config = get_config().cogt.imgg_config
- job_metadata = job_metadata or JobMetadata(
- job_category=JobCategory.IMGG_JOB,
- )
- imgg_prompt = ImggPrompt(positive_text=positive_text)
- job_params = imgg_job_params or imgg_config.make_default_imgg_job_params()
- job_config = imgg_job_config or imgg_config.imgg_job_config
- job_report = ImggJobReport()
-
- return ImggJob(
- job_metadata=job_metadata,
- imgg_prompt=imgg_prompt,
- job_params=job_params,
- job_config=job_config,
- job_report=job_report,
- )
diff --git a/pipelex/cogt/imgg/imgg_platform.py b/pipelex/cogt/imgg/imgg_platform.py
deleted file mode 100644
index 82535d6b0..000000000
--- a/pipelex/cogt/imgg/imgg_platform.py
+++ /dev/null
@@ -1,6 +0,0 @@
-from pipelex.types import StrEnum
-
-
-class ImggPlatform(StrEnum):
- FAL_AI = "fal-ai"
- OPENAI = "openai"
diff --git a/pipelex/cogt/inference/inference_manager.py b/pipelex/cogt/inference/inference_manager.py
index cabc2a26e..fec461379 100644
--- a/pipelex/cogt/inference/inference_manager.py
+++ b/pipelex/cogt/inference/inference_manager.py
@@ -4,9 +4,8 @@ from typing_extensions import override
from pipelex import log
from pipelex.cogt.exceptions import InferenceManagerWorkerSetupError
-from pipelex.cogt.imgg.imgg_engine_factory import ImggEngineFactory
-from pipelex.cogt.imgg.imgg_worker_abstract import ImggWorkerAbstract
-from pipelex.cogt.imgg.imgg_worker_factory import ImggWorkerFactory
+from pipelex.cogt.img_gen.img_gen_worker_abstract import ImgGenWorkerAbstract
+from pipelex.cogt.img_gen.img_gen_worker_factory import ImgGenWorkerFactory
from pipelex.cogt.inference.inference_manager_protocol import InferenceManagerProtocol
from pipelex.cogt.llm.llm_worker_abstract import LLMWorkerAbstract
from pipelex.cogt.llm.llm_worker_factory import LLMWorkerFactory
@@ -20,22 +19,22 @@ from pipelex.hub import get_models_manager, get_report_delegate
class InferenceManager(InferenceManagerProtocol):
def __init__(self):
- self.imgg_worker_factory = ImggWorkerFactory()
+ self.img_gen_worker_factory = ImgGenWorkerFactory()
self.ocr_worker_factory = OcrWorkerFactory()
self.llm_workers: Dict[str, LLMWorkerAbstract] = {}
- self.imgg_workers: Dict[str, ImggWorkerAbstract] = {}
+ self.img_gen_workers: Dict[str, ImgGenWorkerAbstract] = {}
self.ocr_workers: Dict[str, OcrWorkerAbstract] = {}
@override
def teardown(self):
- self.imgg_worker_factory = ImggWorkerFactory()
+ self.img_gen_worker_factory = ImgGenWorkerFactory()
self.ocr_worker_factory = OcrWorkerFactory()
for llm_worker in self.llm_workers.values():
llm_worker.teardown()
self.llm_workers = {}
- for imgg_worker in self.imgg_workers.values():
- imgg_worker.teardown()
- self.imgg_workers = {}
+ for img_gen_worker in self.img_gen_workers.values():
+ img_gen_worker.teardown()
+ self.img_gen_workers = {}
for ocr_worker in self.ocr_workers.values():
ocr_worker.teardown()
self.ocr_workers = {}
@@ -47,9 +46,9 @@ class InferenceManager(InferenceManagerProtocol):
log.debug(f" {handle}:")
log.debug(llm_worker.desc)
log.debug("Image Workers:")
- for handle, imgg_worker_async in self.imgg_workers.items():
+ for handle, img_gen_worker_async in self.img_gen_workers.items():
log.debug(f" {handle}:")
- log.debug(imgg_worker_async.desc)
+ log.debug(img_gen_worker_async.desc)
log.debug("OCR Workers:")
for handle, ocr_worker_async in self.ocr_workers.items():
log.debug(f" {handle}:")
@@ -101,30 +100,30 @@ class InferenceManager(InferenceManagerProtocol):
self.llm_workers[llm_handle] = llm_worker_class(reporting_delegate=get_report_delegate())
####################################################################################################
- # Manage IMGG Workers
+ # Manage ImageGen Workers
####################################################################################################
- def _setup_one_imgg_worker(self, imgg_handle: str) -> ImggWorkerAbstract:
- imgg_engine = ImggEngineFactory.make_imgg_engine(imgg_handle=imgg_handle)
- log.verbose(imgg_engine.desc, title=f"Setting up ImgEngine for '{imgg_handle}'")
- imgg_worker = self.imgg_worker_factory.make_imgg_worker(
- imgg_engine=imgg_engine,
+ def _setup_one_img_gen_worker(self, img_gen_handle: str) -> ImgGenWorkerAbstract:
+ inference_model = get_models_manager().get_inference_model(model_handle=img_gen_handle)
+ log.verbose(f"Setting up Image Generation Worker for '{img_gen_handle}'")
+ img_gen_worker = self.img_gen_worker_factory.make_img_gen_worker(
+ inference_model=inference_model,
reporting_delegate=get_report_delegate(),
)
- self.imgg_workers[imgg_handle] = imgg_worker
- return imgg_worker
+ self.img_gen_workers[img_gen_handle] = img_gen_worker
+ return img_gen_worker
@override
- def get_imgg_worker(self, imgg_handle: str) -> ImggWorkerAbstract:
- imgg_worker = self.imgg_workers.get(imgg_handle)
- if imgg_worker is None:
- if not get_config().cogt.inference_manager_config.is_auto_setup_preset_imgg:
+ def get_img_gen_worker(self, img_gen_handle: str) -> ImgGenWorkerAbstract:
+ img_gen_worker = self.img_gen_workers.get(img_gen_handle)
+ if img_gen_worker is None:
+ if not get_config().cogt.inference_manager_config.is_auto_setup_preset_img_gen:
raise InferenceManagerWorkerSetupError(
- f"Found no Imgg worker for '{imgg_handle}', set it up or enable cogt.inference_manager_config.is_auto_setup_preset_imgg"
+ f"Found no Imgg worker for '{img_gen_handle}', set it up or enable cogt.inference_manager_config.is_auto_setup_preset_img_gen"
)
- imgg_worker = self._setup_one_imgg_worker(imgg_handle=imgg_handle)
- return imgg_worker
+ img_gen_worker = self._setup_one_img_gen_worker(img_gen_handle=img_gen_handle)
+ return img_gen_worker
####################################################################################################
# Manage OCR Workers
diff --git a/pipelex/cogt/inference/inference_manager_protocol.py b/pipelex/cogt/inference/inference_manager_protocol.py
index 53fcfd2ec..30d4b82c9 100644
--- a/pipelex/cogt/inference/inference_manager_protocol.py
+++ b/pipelex/cogt/inference/inference_manager_protocol.py
@@ -1,6 +1,6 @@
from typing import Protocol, Type
-from pipelex.cogt.imgg.imgg_worker_abstract import ImggWorkerAbstract
+from pipelex.cogt.img_gen.img_gen_worker_abstract import ImgGenWorkerAbstract
from pipelex.cogt.llm.llm_worker_abstract import LLMWorkerAbstract
from pipelex.cogt.ocr.ocr_worker_abstract import OcrWorkerAbstract
@@ -30,7 +30,7 @@ class InferenceManagerProtocol(Protocol):
# IMG Generation Workers
####################################################################################################
- def get_imgg_worker(self, imgg_handle: str) -> ImggWorkerAbstract: ...
+ def get_img_gen_worker(self, img_gen_handle: str) -> ImgGenWorkerAbstract: ...
####################################################################################################
# OCR Workers
diff --git a/pipelex/cogt/llm/llm_prompt_template.py b/pipelex/cogt/llm/llm_prompt_template.py
index 791b8ba0f..d5fe01206 100644
--- a/pipelex/cogt/llm/llm_prompt_template.py
+++ b/pipelex/cogt/llm/llm_prompt_template.py
@@ -70,7 +70,6 @@ class LLMPromptTemplate(LLMPromptFactoryAbstract):
is_user_images_append: Optional[bool] = None,
template_inputs: Optional[LLMPromptTemplateInputs] = None,
) -> LLMPrompt:
- log.verbose("_make_llm_prompt called for LLMPromptTemplate")
if not is_none_or_has_text(system_text):
if system_text == "":
log.warning(f"Prompt template system_text should be None or contain text. system_text = '{system_text}'")
diff --git a/pipelex/cogt/llm/llm_setting.py b/pipelex/cogt/llm/llm_setting.py
index 9384de8b3..495499b4b 100644
--- a/pipelex/cogt/llm/llm_setting.py
+++ b/pipelex/cogt/llm/llm_setting.py
@@ -1,9 +1,8 @@
from typing import Literal, Optional, Set, Union
-from pydantic import Field, field_validator, model_validator
+from pydantic import Field, field_validator
from typing_extensions import Self
-from pipelex.cogt.exceptions import LLMSettingsValidationError
from pipelex.cogt.llm.llm_job_components import LLMJobParams
from pipelex.cogt.model_backends.prompting_target import PromptingTarget
from pipelex.tools.config.config_model import ConfigModel
@@ -28,16 +27,6 @@ class LLMSetting(ConfigModel):
else:
raise ConfigValidationError(f'Invalid max_tokens shoubd be an int or "auto" but it is a {type(value)}: {value}')
- @model_validator(mode="after")
- def validate_temperature(self) -> Self:
- if self.llm_handle.startswith("gemini") and self.temperature > 1:
- error_msg = (
- f"Gemini LLMs such as '{self.llm_handle}' support temperatures up to 2 but we normalize between 0 and 1, "
- f"so you can't set a temperature of {self.temperature}"
- )
- raise LLMSettingsValidationError(error_msg)
- return self
-
def make_llm_job_params(self) -> LLMJobParams:
return LLMJobParams(
temperature=self.temperature,
@@ -52,35 +41,35 @@ class LLMSetting(ConfigModel):
)
-LLMSettingOrPresetId = Union[LLMSetting, str]
+LLMChoice = Union[LLMSetting, str]
class LLMSettingChoicesDefaults(ConfigModel):
- for_text: LLMSettingOrPresetId
- for_object: LLMSettingOrPresetId
+ for_text: LLMChoice
+ for_object: LLMChoice
class LLMSettingChoices(ConfigModel):
- for_text: Optional[LLMSettingOrPresetId]
- for_object: Optional[LLMSettingOrPresetId]
+ for_text: Optional[LLMChoice]
+ for_object: Optional[LLMChoice]
- def list_used_presets(self) -> Set[str]:
+ def list_choices(self) -> Set[str]:
return set(
[
- setting
- for setting in [
+ choice
+ for choice in [
self.for_text,
self.for_object,
]
- if isinstance(setting, str)
+ if isinstance(choice, str)
]
)
@classmethod
def make_completed_with_defaults(
cls,
- for_text: Optional[LLMSettingOrPresetId] = None,
- for_object: Optional[LLMSettingOrPresetId] = None,
+ for_text: Optional[LLMChoice] = None,
+ for_object: Optional[LLMChoice] = None,
) -> Self:
return cls(
for_text=for_text,
diff --git a/pipelex/cogt/llm/llm_worker_abstract.py b/pipelex/cogt/llm/llm_worker_abstract.py
index 1d95cc3c0..d8569aea1 100644
--- a/pipelex/cogt/llm/llm_worker_abstract.py
+++ b/pipelex/cogt/llm/llm_worker_abstract.py
@@ -66,7 +66,6 @@ class LLMWorkerAbstract(InferenceWorkerAbstract, ABC):
llm_job: LLMJob,
) -> str:
log.debug("LLM Worker gen_text")
- log.verbose(llm_job.params_desc)
log.verbose(llm_job.llm_prompt.desc(), title="llm_prompt")
# metadata
@@ -93,7 +92,6 @@ class LLMWorkerAbstract(InferenceWorkerAbstract, ABC):
schema: Type[BaseModelTypeVar],
) -> BaseModelTypeVar:
log.debug(f"LLM Worker gen_object using {self.desc}")
- log.verbose(llm_job.params_desc)
log.verbose(llm_job.llm_prompt.desc(), title="llm_prompt")
# metadata
diff --git a/pipelex/cogt/llm/llm_worker_factory.py b/pipelex/cogt/llm/llm_worker_factory.py
index bb1065f2c..bd7ac90ef 100644
--- a/pipelex/cogt/llm/llm_worker_factory.py
+++ b/pipelex/cogt/llm/llm_worker_factory.py
@@ -123,6 +123,30 @@ class LLMWorkerFactory:
inference_model=inference_model,
reporting_delegate=reporting_delegate,
)
+ case "google":
+ try:
+ import google.genai # noqa: F401
+ except ImportError as exc:
+ raise MissingDependencyError(
+ "google-genai",
+ "google",
+ ("The google-genai SDK is required to use Google Gemini API directly. You can install it with 'pip install google-genai'."),
+ ) from exc
+
+ from pipelex.plugins.google.google_factory import GoogleFactory
+ from pipelex.plugins.google.google_llm_worker import GoogleLLMWorker
+
+ sdk_instance = plugin_sdk_registry.get_sdk_instance(plugin=plugin) or plugin_sdk_registry.set_sdk_instance(
+ plugin=plugin,
+ sdk_instance=GoogleFactory.make_google_client(backend=backend),
+ )
+
+ llm_worker = GoogleLLMWorker(
+ sdk_instance=sdk_instance,
+ inference_model=inference_model,
+ structure_method=StructureMethod.INSTRUCTOR_GENAI_STRUCTURED_OUTPUTS,
+ reporting_delegate=reporting_delegate,
+ )
case _:
raise NotImplementedError(f"Plugin '{plugin}' is not supported")
return llm_worker
diff --git a/pipelex/cogt/llm/llm_worker_internal_abstract.py b/pipelex/cogt/llm/llm_worker_internal_abstract.py
index 38a8cf696..4ac2b935b 100644
--- a/pipelex/cogt/llm/llm_worker_internal_abstract.py
+++ b/pipelex/cogt/llm/llm_worker_internal_abstract.py
@@ -2,6 +2,7 @@ from typing import Optional
from typing_extensions import override
+from pipelex import log
from pipelex.cogt.exceptions import LLMCapabilityError
from pipelex.cogt.llm.llm_job import LLMJob
from pipelex.cogt.llm.llm_worker_abstract import LLMWorkerAbstract
@@ -36,7 +37,7 @@ class LLMWorkerInternalAbstract(LLMWorkerAbstract):
@property
@override
def desc(self) -> str:
- return f"LLM-Worker:{self.inference_model.tag}"
+ return f"LLM-Worker: {self.inference_model.tag}"
@property
@override
@@ -48,6 +49,7 @@ class LLMWorkerInternalAbstract(LLMWorkerAbstract):
self,
llm_job: LLMJob,
):
+ log.info(f"LLM Worker: {self.desc} --------------------------------")
await super()._before_job(llm_job=llm_job)
llm_job.llm_job_before_start(inference_model=self.inference_model)
diff --git a/pipelex/cogt/llm/structured_output.py b/pipelex/cogt/llm/structured_output.py
index c44c2c833..7283aa9e7 100644
--- a/pipelex/cogt/llm/structured_output.py
+++ b/pipelex/cogt/llm/structured_output.py
@@ -1,4 +1,4 @@
-from instructor.mode import Mode as InstructorMode
+from instructor import Mode as InstructorMode
from pipelex.types import StrEnum
@@ -8,6 +8,8 @@ class StructureMethod(StrEnum):
INSTRUCTOR_ANTHROPIC_TOOLS = "anthropic_tools"
INSTRUCTOR_MISTRAL_TOOLS = "mistral_tools"
INSTRUCTOR_VERTEX_JSON = "vertex_json"
+ INSTRUCTOR_GENAI_TOOLS = "genai_tools"
+ INSTRUCTOR_GENAI_STRUCTURED_OUTPUTS = "genai_structured_outputs"
def as_instructor_mode(self) -> InstructorMode:
match self:
@@ -18,4 +20,8 @@ class StructureMethod(StrEnum):
case StructureMethod.INSTRUCTOR_MISTRAL_TOOLS:
return InstructorMode.MISTRAL_TOOLS
case StructureMethod.INSTRUCTOR_VERTEX_JSON:
- return InstructorMode.JSON
+ return InstructorMode.ANTHROPIC_JSON
+ case StructureMethod.INSTRUCTOR_GENAI_TOOLS:
+ return InstructorMode.GENAI_TOOLS
+ case StructureMethod.INSTRUCTOR_GENAI_STRUCTURED_OUTPUTS:
+ return InstructorMode.GENAI_STRUCTURED_OUTPUTS
diff --git a/pipelex/cogt/model_backends/backend_factory.py b/pipelex/cogt/model_backends/backend_factory.py
index 8c26a9d34..17c943fbf 100644
--- a/pipelex/cogt/model_backends/backend_factory.py
+++ b/pipelex/cogt/model_backends/backend_factory.py
@@ -26,7 +26,6 @@ class InferenceBackendFactory:
) -> InferenceBackend:
endpoint = blueprint.endpoint
api_key = blueprint.api_key
- # api_version = blueprint.api_version
# Deal with special authentication for some backends
match name:
case "vertexai":
diff --git a/pipelex/cogt/model_backends/backend_library.py b/pipelex/cogt/model_backends/backend_library.py
index dfdea0c6b..6d52df12f 100644
--- a/pipelex/cogt/model_backends/backend_library.py
+++ b/pipelex/cogt/model_backends/backend_library.py
@@ -1,4 +1,4 @@
-from typing import Any, Dict, List, Optional, Set
+from typing import Any, Dict, List, Optional, Set, cast
from pydantic import Field, RootModel, ValidationError
from typing_extensions import Self
@@ -14,7 +14,6 @@ from pipelex.cogt.model_backends.backend import InferenceBackend
from pipelex.cogt.model_backends.backend_factory import InferenceBackendBlueprint, InferenceBackendFactory
from pipelex.cogt.model_backends.model_spec import InferenceModelSpec
from pipelex.cogt.model_backends.model_spec_factory import InferenceModelSpecBlueprint, InferenceModelSpecFactory
-from pipelex.cogt.model_backends.prompting_target import PromptingTarget
from pipelex.config import get_config
from pipelex.tools.misc.dict_utils import apply_to_strings_recursive
from pipelex.tools.misc.toml_utils import load_toml_from_path
@@ -47,7 +46,7 @@ class InferenceBackendLibrary(RootModel[InferenceBackendLibraryRoot]):
inference_backend_blueprint_dict_raw = backend_dict.copy()
if not inference_backend_blueprint_dict_raw.get("enabled", True):
continue
- if runtime_manager.is_gha_testing and backend_name == "vertexai":
+ if runtime_manager.is_ci_testing and backend_name == "vertexai":
continue
try:
inference_backend_blueprint_dict = apply_to_strings_recursive(inference_backend_blueprint_dict_raw, substitute_vars)
@@ -62,7 +61,10 @@ class InferenceBackendLibrary(RootModel[InferenceBackendLibraryRoot]):
raise InferenceBackendCredentialsError(
error_type=InferenceBackendCredentialsErrorType.VAR_NOT_FOUND,
backend_name=backend_name,
- message=f"Variable substitution failed due to a variable not found error in file '{backends_library_path}':\n{var_not_found_exc}",
+ message=(
+ f"Variable substitution failed due to a variable not found error in file '{backends_library_path}':\n"
+ f"{var_not_found_exc}\nRun mode: '{runtime_manager.run_mode}'"
+ ),
key_name=var_not_found_exc.var_name,
) from var_not_found_exc
except UnknownVarPrefixError as unknown_var_prefix_exc:
@@ -92,18 +94,24 @@ class InferenceBackendLibrary(RootModel[InferenceBackendLibraryRoot]):
raise InferenceModelSpecError(f"Variable substitution failed in file '{path_to_model_specs_toml}': {exc}") from exc
except (FileNotFoundError, InferenceModelSpecError) as exc:
raise InferenceBackendLibraryError(f"Failed to load inference model specs from file '{path_to_model_specs_toml}': {exc}") from exc
- default_sdk: Optional[str] = model_specs_dict.pop("default_sdk", None)
- default_prompting_target: Optional[PromptingTarget] = model_specs_dict.pop("default_prompting_target", None)
+ defaults_dict: Dict[str, Any] = model_specs_dict.pop("defaults", {})
backend_model_specs: Dict[str, InferenceModelSpec] = {}
- for model_spec_name, model_spec_dict in model_specs_dict.items():
+ for model_spec_name, value in model_specs_dict.items():
+ if not isinstance(value, dict):
+ raise InferenceModelSpecError(
+ f"Model spec '{model_spec_name}' for backend '{backend_name}' at path '{path_to_model_specs_toml}' is not a dictionary"
+ )
+ model_spec_dict: Dict[str, Any] = cast(Dict[str, Any], value)
try:
- model_spec_blueprint = InferenceModelSpecBlueprint.model_validate(model_spec_dict)
+ # Start from the defaults
+ model_spec_blueprint_dict = defaults_dict.copy()
+ # Override with the attributes from the model spec dict
+ model_spec_blueprint_dict.update(model_spec_dict)
+ model_spec_blueprint = InferenceModelSpecBlueprint.model_validate(model_spec_blueprint_dict)
model_spec = InferenceModelSpecFactory.make_inference_model_spec(
backend_name=backend_name,
name=model_spec_name,
blueprint=model_spec_blueprint,
- default_prompting_target=default_prompting_target,
- fallback_sdk=default_sdk,
)
backend_model_specs[model_spec_name] = model_spec
except (InferenceModelSpecError, ValidationError) as exc:
diff --git a/pipelex/cogt/model_backends/model_spec.py b/pipelex/cogt/model_backends/model_spec.py
index 891fcb641..6a70a0cad 100644
--- a/pipelex/cogt/model_backends/model_spec.py
+++ b/pipelex/cogt/model_backends/model_spec.py
@@ -23,9 +23,6 @@ class InferenceModelSpec(ConfigModel):
prompting_target: Optional[PromptingTarget] = Field(default=None, strict=False)
constraints: List[ModelConstraints] = Field(default_factory=list)
- # TODO: investigate if this is needed
- is_system_prompt_supported: bool = True
-
@property
def tag(self) -> str:
return f"[{self.sdk}@{self.backend_name}]({self.model_id})"
diff --git a/pipelex/cogt/model_backends/model_spec_factory.py b/pipelex/cogt/model_backends/model_spec_factory.py
index 7566885b9..ed74d5ea9 100644
--- a/pipelex/cogt/model_backends/model_spec_factory.py
+++ b/pipelex/cogt/model_backends/model_spec_factory.py
@@ -2,7 +2,6 @@ from typing import Dict, List, Optional
from pydantic import BaseModel, Field, field_validator
-from pipelex.cogt.exceptions import InferenceModelSpecError
from pipelex.cogt.model_backends.model_constraints import ModelConstraints
from pipelex.cogt.model_backends.model_spec import InferenceModelSpec
from pipelex.cogt.model_backends.model_type import ModelType
@@ -13,7 +12,7 @@ from pipelex.tools.config.config_model import ConfigModel
class InferenceModelSpecBlueprint(ConfigModel):
enabled: bool = True
- sdk: Optional[str] = None
+ sdk: str
model_type: ModelType = Field(default=ModelType.LLM, strict=False)
model_id: str
inputs: List[str] = Field(default_factory=list)
@@ -46,17 +45,11 @@ class InferenceModelSpecFactory(BaseModel):
backend_name: str,
name: str,
blueprint: InferenceModelSpecBlueprint,
- default_prompting_target: Optional[PromptingTarget],
- fallback_sdk: Optional[str],
) -> InferenceModelSpec:
- sdk = blueprint.sdk or fallback_sdk
- if not sdk:
- raise InferenceModelSpecError("No sdk choice provided")
- prompting_target = blueprint.prompting_target or default_prompting_target
return InferenceModelSpec(
backend_name=backend_name,
name=name,
- sdk=sdk,
+ sdk=blueprint.sdk,
model_type=blueprint.model_type,
model_id=blueprint.model_id,
inputs=blueprint.inputs,
@@ -64,6 +57,6 @@ class InferenceModelSpecFactory(BaseModel):
costs=blueprint.costs,
max_tokens=blueprint.max_tokens,
max_prompt_images=blueprint.max_prompt_images,
- prompting_target=prompting_target,
+ prompting_target=blueprint.prompting_target,
constraints=blueprint.constraints,
)
diff --git a/pipelex/cogt/model_backends/model_type.py b/pipelex/cogt/model_backends/model_type.py
index 02e01ecf0..bd1ad873e 100644
--- a/pipelex/cogt/model_backends/model_type.py
+++ b/pipelex/cogt/model_backends/model_type.py
@@ -4,3 +4,4 @@ from pipelex.types import StrEnum
class ModelType(StrEnum):
LLM = "llm"
TEXT_EXTRACTOR = "text_extractor"
+ IMG_GEN = "img_gen"
diff --git a/pipelex/cogt/model_backends/prompting_target.py b/pipelex/cogt/model_backends/prompting_target.py
index a53a68a26..f05783d05 100644
--- a/pipelex/cogt/model_backends/prompting_target.py
+++ b/pipelex/cogt/model_backends/prompting_target.py
@@ -6,3 +6,4 @@ class PromptingTarget(StrEnum):
ANTHROPIC = "anthropic"
MISTRAL = "mistral"
GEMINI = "gemini"
+ FAL = "fal"
diff --git a/pipelex/cogt/models/model_deck.py b/pipelex/cogt/models/model_deck.py
index a8fd66b62..45ac09e87 100644
--- a/pipelex/cogt/models/model_deck.py
+++ b/pipelex/cogt/models/model_deck.py
@@ -4,10 +4,19 @@ from pydantic import Field, field_validator, model_validator
from typing_extensions import Self
from pipelex import log
-from pipelex.cogt.exceptions import LLMHandleNotFoundError, LLMPresetNotFoundError, LLMSettingsValidationError, ModelDeckValidatonError
-from pipelex.cogt.llm.llm_setting import LLMSetting, LLMSettingChoices, LLMSettingChoicesDefaults, LLMSettingOrPresetId
+from pipelex.cogt.exceptions import (
+ ImgGenChoiceNotFoundError,
+ LLMChoiceNotFoundError,
+ LLMHandleNotFoundError,
+ LLMSettingsValidationError,
+ ModelDeckValidatonError,
+ OcrChoiceNotFoundError,
+)
+from pipelex.cogt.img_gen.img_gen_setting import ImgGenChoice, ImgGenSetting
+from pipelex.cogt.llm.llm_setting import LLMChoice, LLMSetting, LLMSettingChoices, LLMSettingChoicesDefaults
from pipelex.cogt.model_backends.model_constraints import ModelConstraints
from pipelex.cogt.model_backends.model_spec import InferenceModelSpec
+from pipelex.cogt.ocr.ocr_setting import OcrChoice, OcrSetting
from pipelex.tools.config.config_model import ConfigModel
from pipelex.tools.exceptions import ConfigValidationError
@@ -16,16 +25,33 @@ LLM_PRESET_DISABLED = "disabled"
Waterfall = Union[str, List[str]]
-class ModelDeckBlueprint(ConfigModel):
- aliases: Dict[str, Waterfall] = Field(default_factory=dict)
- llm_presets: Dict[str, LLMSetting] = Field(default_factory=dict)
- llm_choice_defaults: LLMSettingChoicesDefaults
- llm_choice_overrides: LLMSettingChoices = LLMSettingChoices(
+class LLMDeckBlueprint(ConfigModel):
+ presets: Dict[str, LLMSetting] = Field(default_factory=dict)
+ choice_defaults: LLMSettingChoicesDefaults
+ choice_overrides: LLMSettingChoices = LLMSettingChoices(
for_text=None,
for_object=None,
)
+class OCRDeckBlueprint(ConfigModel):
+ presets: Dict[str, OcrSetting] = Field(default_factory=dict)
+ choice_default: OcrChoice
+
+
+class ImgGenDeckBlueprint(ConfigModel):
+ presets: Dict[str, ImgGenSetting] = Field(default_factory=dict)
+ choice_default: ImgGenChoice
+
+
+class ModelDeckBlueprint(ConfigModel):
+ aliases: Dict[str, Waterfall] = Field(default_factory=dict)
+
+ llm: LLMDeckBlueprint
+ ocr: OCRDeckBlueprint
+ img_gen: ImgGenDeckBlueprint
+
+
class ModelDeck(ConfigModel):
inference_models: Dict[str, InferenceModelSpec] = Field(default_factory=dict)
aliases: Dict[str, Waterfall] = Field(default_factory=dict)
@@ -37,7 +63,13 @@ class ModelDeck(ConfigModel):
for_object=None,
)
- def check_llm_setting(self, llm_setting_or_preset_id: LLMSettingOrPresetId, is_disabled_allowed: bool = False):
+ ocr_presets: Dict[str, OcrSetting] = Field(default_factory=dict)
+ ocr_choice_default: OcrChoice
+
+ img_gen_presets: Dict[str, ImgGenSetting] = Field(default_factory=dict)
+ img_gen_choice_default: ImgGenChoice
+
+ def check_llm_setting(self, llm_setting_or_preset_id: LLMChoice, is_disabled_allowed: bool = False):
if isinstance(llm_setting_or_preset_id, LLMSetting):
return
preset_id: str = llm_setting_or_preset_id
@@ -45,17 +77,43 @@ class ModelDeck(ConfigModel):
return
if preset_id == LLM_PRESET_DISABLED and is_disabled_allowed:
return
- raise LLMPresetNotFoundError(f"llm preset id '{preset_id}' not found in deck")
+ raise LLMChoiceNotFoundError(f"llm preset id '{preset_id}' not found in deck")
- def get_llm_setting(self, llm_setting_or_preset_id: LLMSettingOrPresetId) -> LLMSetting:
- if isinstance(llm_setting_or_preset_id, LLMSetting):
- return llm_setting_or_preset_id
+ def get_llm_setting(self, llm_choice: LLMChoice) -> LLMSetting:
+ if isinstance(llm_choice, LLMSetting):
+ return llm_choice
+ else:
+ # it's a string, so either an llm preset id or an llm handle
+ if llm_preset := self.llm_presets.get(llm_choice):
+ return llm_preset
+ elif self.is_handle_defined(model_handle=llm_choice):
+ return LLMSetting(llm_handle=llm_choice, temperature=0.7, max_tokens=None)
+ else:
+ raise LLMChoiceNotFoundError(f"LLM choice '{llm_choice}' not found in deck")
+
+ def get_ocr_setting(self, ocr_choice: OcrChoice) -> OcrSetting:
+ if isinstance(ocr_choice, OcrSetting):
+ return ocr_choice
+ else:
+ # it's a string, so either an ocr preset id or an ocr handle
+ if ocr_preset := self.ocr_presets.get(ocr_choice):
+ return ocr_preset
+ elif self.is_handle_defined(model_handle=ocr_choice):
+ return OcrSetting(ocr_handle=ocr_choice)
+ else:
+ raise OcrChoiceNotFoundError(f"OCR choice '{ocr_choice}' not found in deck")
+
+ def get_img_gen_setting(self, img_gen_choice: ImgGenChoice) -> ImgGenSetting:
+ if isinstance(img_gen_choice, ImgGenSetting):
+ return img_gen_choice
else:
- # it's a preset id
- the_llm_preset = self.llm_presets.get(llm_setting_or_preset_id)
- if not the_llm_preset:
- raise LLMPresetNotFoundError(f"LLM preset '{llm_setting_or_preset_id}' not found in deck")
- return the_llm_preset
+ # it's a string, so either an img gen preset id or an img gen handle
+ if img_gen_preset := self.img_gen_presets.get(img_gen_choice):
+ return img_gen_preset
+ elif self.is_handle_defined(model_handle=img_gen_choice):
+ return ImgGenSetting(img_gen_handle=img_gen_choice)
+ else:
+ raise ImgGenChoiceNotFoundError(f"Image generation choice '{img_gen_choice}' not found in deck")
@classmethod
def final_validate(cls, deck: Self): # pyright: ignore[reportIncompatibleMethodOverride]
@@ -116,7 +174,7 @@ class ModelDeck(ConfigModel):
return self
def _validate_llm_choices(self, llm_choices: LLMSettingChoices):
- for llm_setting in llm_choices.list_used_presets():
+ for llm_setting in llm_choices.list_choices():
self.check_llm_setting(llm_setting_or_preset_id=llm_setting)
return
@@ -135,6 +193,9 @@ class ModelDeck(ConfigModel):
log.warning(f"Skipping model handle '{model_handle}' because it's not found in deck")
return None
+ def is_handle_defined(self, model_handle: str) -> bool:
+ return model_handle in self.inference_models or model_handle in self.aliases
+
def get_required_inference_model(self, model_handle: str) -> InferenceModelSpec:
inference_model = self.get_optional_inference_model(model_handle=model_handle)
if inference_model is None:
diff --git a/pipelex/cogt/models/model_deck_check.py b/pipelex/cogt/models/model_deck_check.py
index ce9990940..74891c158 100644
--- a/pipelex/cogt/models/model_deck_check.py
+++ b/pipelex/cogt/models/model_deck_check.py
@@ -1,13 +1,38 @@
-from pipelex.cogt.exceptions import LLMPresetNotFoundError
-from pipelex.cogt.llm.llm_setting import LLMSetting, LLMSettingOrPresetId
-from pipelex.hub import get_models_manager
+from pipelex.cogt.exceptions import ImgGenChoiceNotFoundError, LLMChoiceNotFoundError, OcrChoiceNotFoundError
+from pipelex.cogt.img_gen.img_gen_setting import ImgGenChoice, ImgGenSetting
+from pipelex.cogt.llm.llm_setting import LLMChoice, LLMSetting
+from pipelex.cogt.ocr.ocr_setting import OcrChoice, OcrSetting
+from pipelex.hub import get_model_deck
-def check_llm_setting_with_deck(llm_setting_or_preset_id: LLMSettingOrPresetId):
- if isinstance(llm_setting_or_preset_id, LLMSetting):
+def check_llm_choice_with_deck(llm_choice: LLMChoice):
+ if isinstance(llm_choice, LLMSetting):
return
- preset_id: str = llm_setting_or_preset_id
- llm_deck = get_models_manager().get_model_deck()
- if preset_id in llm_deck.llm_presets:
+ llm_deck = get_model_deck()
+ if llm_choice in llm_deck.llm_presets:
return
- raise LLMPresetNotFoundError(f"llm preset id '{preset_id}' not found in deck")
+ elif llm_deck.is_handle_defined(model_handle=llm_choice):
+ return
+ raise LLMChoiceNotFoundError(f"LLM choice '{llm_choice}' not found in deck")
+
+
+def check_ocr_choice_with_deck(ocr_choice: OcrChoice):
+ if isinstance(ocr_choice, OcrSetting):
+ return
+ ocr_deck = get_model_deck()
+ if ocr_choice in ocr_deck.ocr_presets:
+ return
+ elif ocr_deck.is_handle_defined(model_handle=ocr_choice):
+ return
+ raise OcrChoiceNotFoundError(f"OCR choice '{ocr_choice}' not found in deck")
+
+
+def check_img_gen_choice_with_deck(img_gen_choice: ImgGenChoice):
+ if isinstance(img_gen_choice, ImgGenSetting):
+ return
+ img_gen_deck = get_model_deck()
+ if img_gen_choice in img_gen_deck.img_gen_presets:
+ return
+ elif img_gen_deck.is_handle_defined(model_handle=img_gen_choice):
+ return
+ raise ImgGenChoiceNotFoundError(f"Image generation choice '{img_gen_choice}' not found in deck")
diff --git a/pipelex/cogt/models/model_manager.py b/pipelex/cogt/models/model_manager.py
index 8beb112b9..ba7966f29 100644
--- a/pipelex/cogt/models/model_manager.py
+++ b/pipelex/cogt/models/model_manager.py
@@ -47,46 +47,45 @@ class ModelManager(ModelManagerAbstract):
log.debug("LibraryManager deck TOML file formatting")
# Validation of LLM deck paths
- llm_deck_paths = get_config().cogt.inference_config.get_llm_deck_paths()
- for llm_deck_path in llm_deck_paths:
- if os.path.exists(llm_deck_path):
+ deck_paths = get_config().cogt.inference_config.get_model_deck_paths()
+ for deck_path in deck_paths:
+ if os.path.exists(deck_path):
try:
- validate_toml_file(llm_deck_path)
+ validate_toml_file(deck_path)
except TOMLValidationError as exc:
- log.error(f"TOML formatting issues in LLM deck file '{llm_deck_path}': {exc}")
- raise LibraryError(f"TOML validation failed for LLM deck file '{llm_deck_path}': {exc}") from exc
+ log.error(f"TOML formatting issues in LLM deck file '{deck_path}': {exc}")
+ raise LibraryError(f"TOML validation failed for LLM deck file '{deck_path}': {exc}") from exc
@classmethod
def load_deck_blueprint(cls) -> ModelDeckBlueprint:
- llm_deck_paths = get_config().cogt.inference_config.get_llm_deck_paths()
- full_llm_deck_dict: Dict[str, Any] = {}
- if not llm_deck_paths:
+ deck_paths = get_config().cogt.inference_config.get_model_deck_paths()
+ full_deck_dict: Dict[str, Any] = {}
+ if not deck_paths:
raise ModelDeckNotFoundError("No LLM deck paths found. Please run `pipelex init-libraries` to create it.")
- for llm_deck_path in llm_deck_paths:
- if not os.path.exists(llm_deck_path):
- raise ModelDeckNotFoundError(f"LLM deck path `{llm_deck_path}` not found. Please run `pipelex init-libraries` to create it.")
+ for deck_path in deck_paths:
+ if not os.path.exists(deck_path):
+ raise ModelDeckNotFoundError(f"LLM deck path `{deck_path}` not found. Please run `pipelex init-libraries` to create it.")
try:
- llm_deck_dict = load_toml_from_path(path=llm_deck_path)
- log.debug(f"Loaded LLM deck from {llm_deck_path}")
- deep_update(full_llm_deck_dict, llm_deck_dict)
+ deck_dict = load_toml_from_path(path=deck_path)
+ log.debug(f"Loaded LLM deck from {deck_path}")
+ deep_update(full_deck_dict, deck_dict)
except Exception as exc:
- log.error(f"Failed to load LLM deck file '{llm_deck_path}': {exc}")
+ log.error(f"Failed to load LLM deck file '{deck_path}': {exc}")
raise
- llm_deck_blueprint = ModelDeckBlueprint.model_validate(full_llm_deck_dict)
- return llm_deck_blueprint
+ model_deck_blueprint = ModelDeckBlueprint.model_validate(full_deck_dict)
+ return model_deck_blueprint
def build_deck(self, model_deck_blueprint: ModelDeckBlueprint) -> ModelDeck:
all_models_and_possible_backends = self.inference_backend_library.get_all_models_and_possible_backends()
- model_handles: Dict[str, InferenceModelSpec] = {}
+ inference_models: Dict[str, InferenceModelSpec] = {}
for model_name, available_backends in all_models_and_possible_backends.items():
backend_match_for_model = self.routing_profile_library.get_backend_match_for_model_from_active_routing_profile(
model_name=model_name,
)
if backend_match_for_model is None:
- # raise ModelsManagerError(f"No backend match found for model '{model_name}'")
log.verbose(f"No backend match found for model '{model_name}'")
continue
matched_backend_name = backend_match_for_model.backend_name
@@ -115,6 +114,7 @@ class ModelManager(ModelManagerAbstract):
# TODO: enable to set the order or priority of the available backends
for available_backend in available_backends:
if available_backend == matched_backend_name:
+ # we've already checked the matched_backend_name and it didn't have the model spec, that's why we're here
continue
backend = self.inference_backend_library.get_inference_backend(backend_name=available_backend)
if backend is None:
@@ -127,14 +127,18 @@ class ModelManager(ModelManagerAbstract):
f"Model spec '{model_name}' not found in any of the available backends '{available_backends}' "
f"which was set as default in routing profile '{backend_match_for_model.routing_profile_name}'"
)
- model_handles[model_name] = model_spec
+ inference_models[model_name] = model_spec
model_deck = ModelDeck(
- inference_models=model_handles,
+ inference_models=inference_models,
aliases=model_deck_blueprint.aliases,
- llm_presets=model_deck_blueprint.llm_presets,
- llm_choice_defaults=model_deck_blueprint.llm_choice_defaults,
- llm_choice_overrides=model_deck_blueprint.llm_choice_overrides,
+ llm_presets=model_deck_blueprint.llm.presets,
+ llm_choice_defaults=model_deck_blueprint.llm.choice_defaults,
+ llm_choice_overrides=model_deck_blueprint.llm.choice_overrides,
+ ocr_presets=model_deck_blueprint.ocr.presets,
+ ocr_choice_default=model_deck_blueprint.ocr.choice_default,
+ img_gen_presets=model_deck_blueprint.img_gen.presets,
+ img_gen_choice_default=model_deck_blueprint.img_gen.choice_default,
)
return model_deck
diff --git a/pipelex/cogt/ocr/ocr_job_components.py b/pipelex/cogt/ocr/ocr_job_components.py
index d7b0cd0d5..788258065 100644
--- a/pipelex/cogt/ocr/ocr_job_components.py
+++ b/pipelex/cogt/ocr/ocr_job_components.py
@@ -7,14 +7,18 @@ from pipelex.tools.config.config_model import ConfigModel
class OcrJobParams(BaseModel):
should_include_images: bool
+ max_nb_images: Optional[int]
+ image_min_size: Optional[int]
should_caption_images: bool
should_include_page_views: bool
- page_views_dpi: Optional[int] = None
+ page_views_dpi: Optional[int]
@classmethod
def make_default_ocr_job_params(cls) -> "OcrJobParams":
return OcrJobParams(
should_caption_images=False,
+ max_nb_images=None,
+ image_min_size=None,
should_include_page_views=False,
should_include_images=True,
page_views_dpi=None,
diff --git a/pipelex/cogt/ocr/ocr_setting.py b/pipelex/cogt/ocr/ocr_setting.py
new file mode 100644
index 000000000..942320362
--- /dev/null
+++ b/pipelex/cogt/ocr/ocr_setting.py
@@ -0,0 +1,17 @@
+from typing import Optional, Union
+
+from pydantic import Field
+
+from pipelex.tools.config.config_model import ConfigModel
+
+
+class OcrSetting(ConfigModel):
+ ocr_handle: str
+ max_nb_images: Optional[int] = Field(default=None, ge=0)
+ image_min_size: Optional[int] = Field(default=None, ge=0)
+
+ def desc(self) -> str:
+ return f"OcrSetting(ocr_handle={self.ocr_handle}, max_nb_images={self.max_nb_images}, image_min_size={self.image_min_size})"
+
+
+OcrChoice = Union[OcrSetting, str]
diff --git a/pipelex/cogt/usage/token_category.py b/pipelex/cogt/usage/token_category.py
index e5bac8964..7e93b9082 100644
--- a/pipelex/cogt/usage/token_category.py
+++ b/pipelex/cogt/usage/token_category.py
@@ -18,7 +18,7 @@ class TokenCategory(StrEnum):
@property
def to_cost_category(self) -> CostCategory:
- return CostCategory(self.value)
+ return CostCategory(self)
NbTokensByCategoryDict = Dict[TokenCategory, int]
diff --git a/pipelex/config_template/inference/backends.toml b/pipelex/config_template/inference/backends.toml
index 7b68e6d4d..459b4692a 100644
--- a/pipelex/config_template/inference/backends.toml
+++ b/pipelex/config_template/inference/backends.toml
@@ -2,48 +2,57 @@
endpoint = "https://inference.pipelex.com/v1"
api_key = "${PIPELEX_INFERENCE_API_KEY}"
-[blackboxai]
-enabled = false
-endpoint = "https://api.blackbox.ai/v1"
-api_key = "${BLACKBOX_API_KEY}"
-
-[openai]
-enabled = false
-api_key = "${OPENAI_API_KEY}"
-
[azure_openai]
enabled = false
endpoint = "${AZURE_API_BASE}"
api_key = "${AZURE_API_KEY}"
api_version = "${AZURE_API_VERSION}"
+[bedrock]
+enabled = false
+aws_region = "${AWS_REGION}"
+
+[google]
+enabled = false
+api_key = "${GOOGLE_API_KEY}"
+
+[vertexai]
+enabled = false
+gcp_project_id = "${GCP_PROJECT_ID}"
+gcp_location = "${GCP_LOCATION}"
+gcp_credentials_file_path = "${GCP_CREDENTIALS_FILE_PATH}"
+
+[openai]
+enabled = false
+api_key = "${OPENAI_API_KEY}"
+
[anthropic]
enabled = false
api_key = "${ANTHROPIC_API_KEY}"
claude_4_tokens_limit = 8192
-[ollama]
+[mistral]
enabled = false
-endpoint = "http://localhost:11434/v1"
+api_key = "${MISTRAL_API_KEY}"
[xai]
enabled = false
endpoint = "https://api.x.ai/v1"
api_key = "${XAI_API_KEY}"
-[bedrock]
+[ollama]
enabled = false
-aws_region = "${AWS_REGION}"
+endpoint = "http://localhost:11434/v1"
-[vertexai]
+[blackboxai]
enabled = false
-gcp_project_id = "${GCP_PROJECT_ID}"
-gcp_location = "${GCP_LOCATION}"
-gcp_credentials_file_path = "${GCP_CREDENTIALS_FILE_PATH}"
+endpoint = "https://api.blackbox.ai/v1"
+api_key = "${BLACKBOX_API_KEY}"
-[mistral]
+[fal]
enabled = false
-api_key = "${MISTRAL_API_KEY}"
+api_key = "${FAL_API_KEY}"
[internal] # software-only backend, runs internally, without AI
enabled = true
+
diff --git a/pipelex/config_template/inference/backends/anthropic.toml b/pipelex/config_template/inference/backends/anthropic.toml
index 0dc85517c..a7b98c0c0 100644
--- a/pipelex/config_template/inference/backends/anthropic.toml
+++ b/pipelex/config_template/inference/backends/anthropic.toml
@@ -1,6 +1,32 @@
-default_sdk = "anthropic"
-default_prompting_target = "anthropic"
+################################################################################
+# Anthropic Backend Configuration
+################################################################################
+#
+# This file defines the model specifications for Anthropic Claude models.
+# It contains model definitions for various Claude language models
+# accessible through the Anthropic API.
+#
+# Configuration structure:
+# - Each model is defined in its own section with the model name as the header
+# - Headers with dots must be quoted (e.g., ["claude-3.5-sonnet"])
+# - Model costs are in USD per million tokens (input/output)
+#
+################################################################################
+################################################################################
+# MODEL DEFAULTS
+################################################################################
+
+[defaults]
+model_type = "llm"
+sdk = "anthropic"
+prompting_target = "anthropic"
+
+################################################################################
+# LANGUAGE MODELS
+################################################################################
+
+# --- Claude 3 Series ----------------------------------------------------------
[claude-3-haiku]
model_id = "claude-3-haiku-20240307"
max_tokens = 4096
@@ -17,6 +43,7 @@ outputs = ["text", "structured"]
max_prompt_images = 100
costs = { input = 15.0, output = 75.0 }
+# --- Claude 3.5 Series --------------------------------------------------------
["claude-3.5-sonnet"]
model_id = "claude-3-5-sonnet-20240620"
max_tokens = 8192
@@ -33,6 +60,7 @@ outputs = ["text", "structured"]
max_prompt_images = 100
costs = { input = 3.0, output = 15.0 }
+# --- Claude 3.7 Series --------------------------------------------------------
["claude-3.7-sonnet"]
model_id = "claude-3-7-sonnet-20250219"
max_tokens = 8192
@@ -41,6 +69,7 @@ outputs = ["text", "structured"]
max_prompt_images = 100
costs = { input = 3.0, output = 15.0 }
+# --- Claude 4 Series ----------------------------------------------------------
[claude-4-sonnet]
model_id = "claude-sonnet-4-20250514"
max_tokens = 64000
@@ -57,6 +86,7 @@ outputs = ["text", "structured"]
max_prompt_images = 100
costs = { input = 3.0, output = 15.0 }
+# --- Claude 4.1 Series --------------------------------------------------------
["claude-4.1-opus"]
model_id = "claude-opus-4-1-20250805"
max_tokens = 32000
diff --git a/pipelex/config_template/inference/backends/azure_openai.toml b/pipelex/config_template/inference/backends/azure_openai.toml
index b26178854..26e01b4e1 100644
--- a/pipelex/config_template/inference/backends/azure_openai.toml
+++ b/pipelex/config_template/inference/backends/azure_openai.toml
@@ -1,6 +1,32 @@
-default_sdk = "azure_openai"
-default_prompting_target = "openai"
+################################################################################
+# Azure OpenAI Backend Configuration
+################################################################################
+#
+# This file defines the model specifications for Azure OpenAI models.
+# It contains model definitions for OpenAI models deployed on Azure
+# accessible through the Azure OpenAI API.
+#
+# Configuration structure:
+# - Each model is defined in its own section with the model name as the header
+# - Headers with dots must be quoted (e.g., ["gpt-4.1"])
+# - Model costs are in USD per million tokens (input/output)
+#
+################################################################################
+################################################################################
+# MODEL DEFAULTS
+################################################################################
+
+[defaults]
+model_type = "llm"
+sdk = "azure_openai"
+prompting_target = "openai"
+
+################################################################################
+# LANGUAGE MODELS
+################################################################################
+
+# --- GPT-4o Series ------------------------------------------------------------
[gpt-4o]
model_id = "gpt-4o-2024-11-20"
inputs = ["text", "images"]
@@ -13,6 +39,7 @@ inputs = ["text", "images"]
outputs = ["text", "structured"]
costs = { input = 0.15, output = 0.6 }
+# --- GPT-4.1 Series -----------------------------------------------------------
["gpt-4.1"]
model_id = "gpt-4.1-2025-04-14"
inputs = ["text", "images"]
@@ -31,6 +58,7 @@ inputs = ["text", "images"]
outputs = ["text", "structured"]
costs = { input = 0.1, output = 0.4 }
+# --- o Series ----------------------------------------------------------------
[o1-mini]
model_id = "o1-mini-2024-09-12"
inputs = ["text"]
@@ -49,6 +77,7 @@ inputs = ["text"]
outputs = ["text", "structured"]
costs = { input = 1.1, output = 4.4 }
+# --- GPT-5 Series -------------------------------------------------------------
[gpt-5-mini]
model_id = "gpt-5-mini-2025-08-07"
inputs = ["text", "images"]
diff --git a/pipelex/config_template/inference/backends/bedrock.toml b/pipelex/config_template/inference/backends/bedrock.toml
index 447c76f03..3c354bb0b 100644
--- a/pipelex/config_template/inference/backends/bedrock.toml
+++ b/pipelex/config_template/inference/backends/bedrock.toml
@@ -1,6 +1,32 @@
-default_sdk = "bedrock_aioboto3"
-default_prompting_target = "anthropic"
+################################################################################
+# AWS Bedrock Backend Configuration
+################################################################################
+#
+# This file defines the model specifications for AWS Bedrock models.
+# It contains model definitions for various language models
+# accessible through the AWS Bedrock service.
+#
+# Configuration structure:
+# - Each model is defined in its own section with the model name as the header
+# - Headers with dots must be quoted (e.g., ["claude-3.5-sonnet"])
+# - Model costs are in USD per million tokens (input/output)
+#
+################################################################################
+
+################################################################################
+# MODEL DEFAULTS
+################################################################################
+
+[defaults]
+model_type = "llm"
+sdk = "bedrock_aioboto3"
+prompting_target = "anthropic"
+################################################################################
+# LANGUAGE MODELS
+################################################################################
+
+# --- Mistral Models -----------------------------------------------------------
[bedrock-mistral-large]
model_id = "mistral.mistral-large-2407-v1:0"
max_tokens = 8192
@@ -8,6 +34,7 @@ inputs = ["text"]
outputs = ["text"]
costs = { input = 4.0, output = 12.0 }
+# --- Meta Llama Models --------------------------------------------------------
[bedrock-meta-llama-3-3-70b-instruct]
model_id = "us.meta.llama3-3-70b-instruct-v1:0"
max_tokens = 8192
@@ -16,6 +43,7 @@ outputs = ["text"]
# TODO: find out the actual cost per million tokens for llama3 on bedrock
costs = { input = 3.0, output = 15.0 }
+# --- Amazon Nova Models -------------------------------------------------------
[bedrock-nova-pro]
model_id = "us.amazon.nova-pro-v1:0"
max_tokens = 5120
@@ -24,10 +52,7 @@ outputs = ["text"]
# TODO: find out the actual cost per million tokens for nova on bedrock
costs = { input = 3.0, output = 15.0 }
-################################################################################
-# Anthropic Models
-################################################################################
-
+# --- Claude LLMs --------------------------------------------------------------
["claude-3.5-sonnet"]
sdk = "bedrock_anthropic"
model_id = "us.anthropic.claude-3-5-sonnet-20240620-v1:0"
diff --git a/pipelex/config_template/inference/backends/blackboxai.toml b/pipelex/config_template/inference/backends/blackboxai.toml
index 14ca78283..4202fd79b 100644
--- a/pipelex/config_template/inference/backends/blackboxai.toml
+++ b/pipelex/config_template/inference/backends/blackboxai.toml
@@ -1,8 +1,31 @@
-
-default_sdk = "openai"
-default_prompting_target = "anthropic"
-
-# OpenAI Models
+################################################################################
+# BlackBoxAI Backend Configuration
+################################################################################
+#
+# This file defines the model specifications for BlackBoxAI models.
+# It contains model definitions for various language models from different providers
+# accessible through the BlackBoxAI API.
+#
+# Configuration structure:
+# - Each model is defined in its own section with the model name as the header
+# - Headers with dots must be quoted (e.g., ["gpt-4.5-preview"])
+# - Model costs are in USD per million tokens (input/output)
+#
+################################################################################
+
+################################################################################
+# MODEL DEFAULTS
+################################################################################
+
+[defaults]
+model_type = "llm"
+sdk = "openai"
+
+################################################################################
+# LANGUAGE MODELS
+################################################################################
+
+# --- OpenAI Models ------------------------------------------------------------
[gpt-4o-mini]
model_id = "blackboxai/openai/gpt-4o-mini"
inputs = ["text", "images"]
@@ -27,13 +50,7 @@ inputs = ["text"]
outputs = ["text", "structured"]
costs = { input = 1.10, output = 4.40 }
-["gpt-4.5-preview"]
-model_id = "blackboxai/openai/gpt-4.5-preview"
-inputs = ["text", "images"]
-outputs = ["text", "structured"]
-costs = { input = 75.00, output = 150.00 }
-
-# Anthropic Models
+# --- Claude LLMs --------------------------------------------------------------
["claude-3.5-haiku"]
model_id = "blackboxai/anthropic/claude-3.5-haiku"
inputs = ["text", "images"]
@@ -64,7 +81,7 @@ inputs = ["text", "images"]
outputs = ["text", "structured"]
costs = { input = 3.00, output = 15.00 }
-# Google Models
+# --- Google Models ------------------------------------------------------------
["gemini-2.5-flash"]
model_id = "blackboxai/google/gemini-2.5-flash"
inputs = ["text", "images"]
@@ -83,26 +100,7 @@ inputs = ["text", "images"]
outputs = ["text", "structured"]
costs = { input = 0.04, output = 0.15 }
-# Free Models
-[deepseek-chat-free]
-model_id = "blackboxai/deepseek/deepseek-chat:free"
-inputs = ["text"]
-outputs = ["text", "structured"]
-costs = { input = 0.00, output = 0.00 }
-
-[deepseek-r1-free]
-model_id = "blackboxai/deepseek/deepseek-r1:free"
-inputs = ["text"]
-outputs = ["text", "structured"]
-costs = { input = 0.00, output = 0.00 }
-
-["llama-3.3-70b-instruct-free"]
-model_id = "blackboxai/meta-llama/llama-3.3-70b-instruct:free"
-inputs = ["text"]
-outputs = ["text", "structured"]
-costs = { input = 0.00, output = 0.00 }
-
-# Mistral Models
+# --- Mistral Models -----------------------------------------------------------
[mistral-large]
model_id = "blackboxai/mistralai/mistral-large"
inputs = ["text"]
@@ -115,33 +113,33 @@ inputs = ["text", "images"]
outputs = ["text", "structured"]
costs = { input = 2.00, output = 6.00 }
-# Cost-Effective Models
+# --- Meta Llama Models --------------------------------------------------------
["llama-3.3-70b-instruct"]
model_id = "blackboxai/meta-llama/llama-3.3-70b-instruct"
inputs = ["text"]
outputs = ["text", "structured"]
costs = { input = 0.04, output = 0.12 }
-["qwen-2.5-72b-instruct"]
-model_id = "blackboxai/qwen/qwen-2.5-72b-instruct"
-inputs = ["text"]
-outputs = ["text", "structured"]
-costs = { input = 0.12, output = 0.39 }
-
-# Vision Models
["llama-3.2-11b-vision-instruct"]
model_id = "blackboxai/meta-llama/llama-3.2-11b-vision-instruct"
inputs = ["text", "images"]
outputs = ["text", "structured"]
costs = { input = 0.05, output = 0.05 }
+# --- Qwen Models --------------------------------------------------------------
+["qwen-2.5-72b-instruct"]
+model_id = "blackboxai/qwen/qwen-2.5-72b-instruct"
+inputs = ["text"]
+outputs = ["text", "structured"]
+costs = { input = 0.12, output = 0.39 }
+
["qwen2.5-vl-72b-instruct"]
model_id = "blackboxai/qwen/qwen2.5-vl-72b-instruct"
inputs = ["text", "images"]
outputs = ["text", "structured"]
costs = { input = 0.25, output = 0.75 }
-# Amazon Nova Models
+# --- Amazon Nova Models -------------------------------------------------------
[nova-micro-v1]
model_id = "blackboxai/amazon/nova-micro-v1"
inputs = ["text"]
@@ -154,3 +152,27 @@ inputs = ["text"]
outputs = ["text", "structured"]
costs = { input = 0.06, output = 0.24 }
+################################################################################
+# FREE MODELS
+################################################################################
+
+# --- DeepSeek Free Models -----------------------------------------------------
+[deepseek-chat-free]
+model_id = "blackboxai/deepseek/deepseek-chat:free"
+inputs = ["text"]
+outputs = ["text", "structured"]
+costs = { input = 0.00, output = 0.00 }
+
+[deepseek-r1-free]
+model_id = "blackboxai/deepseek/deepseek-r1:free"
+inputs = ["text"]
+outputs = ["text", "structured"]
+costs = { input = 0.00, output = 0.00 }
+
+# --- Meta Llama Free Models ---------------------------------------------------
+["llama-3.3-70b-instruct-free"]
+model_id = "blackboxai/meta-llama/llama-3.3-70b-instruct:free"
+inputs = ["text"]
+outputs = ["text", "structured"]
+costs = { input = 0.00, output = 0.00 }
+
diff --git a/pipelex/config_template/inference/backends/fal.toml b/pipelex/config_template/inference/backends/fal.toml
new file mode 100644
index 000000000..ed15e6bd3
--- /dev/null
+++ b/pipelex/config_template/inference/backends/fal.toml
@@ -0,0 +1,54 @@
+################################################################################
+# FAL Backend Configuration
+################################################################################
+#
+# This file defines the model specifications for FAL (Fast AI Labs) models.
+# It contains model definitions for various image generation models
+# accessible through the FAL API.
+#
+# Configuration structure:
+# - Each model is defined in its own section with the model name as the header
+# - Headers with dots must be quoted (e.g., ["flux-pro/v1.1"])
+# - Model costs are in USD per million tokens (input/output)
+#
+################################################################################
+
+################################################################################
+# MODEL DEFAULTS
+################################################################################
+
+[defaults]
+model_type = "img_gen"
+sdk = "fal"
+prompting_target = "fal"
+
+################################################################################
+# IMAGE GENERATION MODELS
+################################################################################
+
+# --- Flux Pro Series ----------------------------------------------------------
+[flux-pro]
+model_id = "fal-ai/flux-pro"
+inputs = ["text"]
+outputs = ["image"]
+costs = { input = 0.05, output = 0.0 }
+
+["flux-pro/v1.1"]
+model_id = "fal-ai/flux-pro/v1.1"
+inputs = ["text"]
+outputs = ["image"]
+costs = { input = 0.05, output = 0.0 }
+
+["flux-pro/v1.1-ultra"]
+model_id = "fal-ai/flux-pro/v1.1-ultra"
+inputs = ["text"]
+outputs = ["image"]
+costs = { input = 0.06, output = 0.0 }
+
+# --- SDXL models --------------------------------------------------------------
+[fast-lightning-sdxl]
+model_id = "fal-ai/fast-lightning-sdxl"
+inputs = ["text"]
+outputs = ["image"]
+costs = { input = 0.0003, output = 0.0 }
+
diff --git a/pipelex/config_template/inference/backends/google.toml b/pipelex/config_template/inference/backends/google.toml
new file mode 100644
index 000000000..072b835dd
--- /dev/null
+++ b/pipelex/config_template/inference/backends/google.toml
@@ -0,0 +1,80 @@
+################################################################################
+# Google Gemini API Backend Configuration
+################################################################################
+#
+# This file defines the model specifications for Google Gemini API models.
+# It contains model definitions for Gemini language models
+# accessible through the Google Gemini API (not VertexAI).
+#
+# Configuration structure:
+# - Each model is defined in its own section with the model name as the header
+# - Headers with dots must be quoted (e.g., ["gemini-2.0-flash"])
+# - Model costs are in USD per million tokens (input/output)
+#
+################################################################################
+
+################################################################################
+# MODEL DEFAULTS
+################################################################################
+
+[defaults]
+model_type = "llm"
+sdk = "google"
+prompting_target = "gemini"
+
+################################################################################
+# LANGUAGE MODELS
+################################################################################
+
+# --- Gemini 2.0 Series --------------------------------------------------------
+["gemini-2.0-flash-exp"]
+model_id = "gemini-2.0-flash-exp"
+inputs = ["text", "images"]
+outputs = ["text", "structured"]
+max_prompt_images = 3000
+costs = { input = 0.0, output = 0.0 } # Free tier during experimental phase
+
+# --- Gemini 1.5 Series --------------------------------------------------------
+["gemini-1.5-pro"]
+model_id = "gemini-1.5-pro"
+inputs = ["text", "images"]
+outputs = ["text", "structured"]
+max_prompt_images = 3000
+costs = { input = 1.25, output = 5.0 } # Updated pricing
+
+["gemini-1.5-flash"]
+model_id = "gemini-1.5-flash"
+inputs = ["text", "images"]
+outputs = ["text", "structured"]
+max_prompt_images = 3000
+costs = { input = 0.075, output = 0.30 } # Updated pricing
+
+["gemini-1.5-flash-8b"]
+model_id = "gemini-1.5-flash-8b"
+inputs = ["text", "images"]
+outputs = ["text", "structured"]
+max_prompt_images = 3000
+costs = { input = 0.0375, output = 0.15 } # Updated pricing
+
+# --- Gemini 2.5 Series (when available) ----------------------------------------
+["gemini-2.5-pro"]
+model_id = "gemini-2.5-pro"
+inputs = ["text", "images"]
+outputs = ["text", "structured"]
+max_prompt_images = 3000
+costs = { input = 1.25, output = 10.0 }
+
+["gemini-2.5-flash"]
+model_id = "gemini-2.5-flash"
+inputs = ["text", "images"]
+outputs = ["text", "structured"]
+max_prompt_images = 3000
+costs = { input = 0.30, output = 2.50 }
+
+["gemini-2.5-flash-lite"]
+model_id = "gemini-2.5-flash-lite"
+inputs = ["text", "images"]
+outputs = ["text", "structured"]
+max_prompt_images = 3000
+costs = { input = 0.10, output = 0.40 }
+
diff --git a/pipelex/config_template/inference/backends/internal.toml b/pipelex/config_template/inference/backends/internal.toml
index da42bf7d1..0dbd6e7be 100644
--- a/pipelex/config_template/inference/backends/internal.toml
+++ b/pipelex/config_template/inference/backends/internal.toml
@@ -1,4 +1,21 @@
+################################################################################
+# Internal Backend Configuration
+################################################################################
+#
+# This file defines the model specifications for internal software-only models.
+# These models run internally without external APIs or AI services.
+#
+# Configuration structure:
+# - Each model is defined in its own section with the model name as the header
+# - Model costs are in USD per million tokens (input/output)
+#
+################################################################################
+################################################################################
+# TEXT EXTRACTION MODELS
+################################################################################
+
+# --- PyPDFium2 Text Extractor -------------------------------------------------
[pypdfium2-extract-text]
model_type = "text_extractor"
sdk = "pypdfium2"
diff --git a/pipelex/config_template/inference/backends/mistral.toml b/pipelex/config_template/inference/backends/mistral.toml
index b452cef8b..634736c74 100644
--- a/pipelex/config_template/inference/backends/mistral.toml
+++ b/pipelex/config_template/inference/backends/mistral.toml
@@ -1,6 +1,32 @@
-default_sdk = "mistral"
-default_prompting_target = "mistral"
+################################################################################
+# Mistral Backend Configuration
+################################################################################
+#
+# This file defines the model specifications for Mistral AI models.
+# It contains model definitions for various Mistral language models and specialized models
+# accessible through the Mistral API.
+#
+# Configuration structure:
+# - Each model is defined in its own section with the model name as the header
+# - Headers with dots must be quoted (e.g., ["ministral-3b"])
+# - Model costs are in USD per million tokens (input/output)
+#
+################################################################################
+################################################################################
+# MODEL DEFAULTS
+################################################################################
+
+[defaults]
+model_type = "llm"
+sdk = "mistral"
+prompting_target = "mistral"
+
+################################################################################
+# LANGUAGE MODELS
+################################################################################
+
+# --- Ministral Series ---------------------------------------------------------
[ministral-3b]
model_id = "ministral-3b-latest"
max_tokens = 131072
@@ -15,6 +41,7 @@ inputs = ["text"]
outputs = ["text", "structured"]
costs = { input = 0.1, output = 0.1 }
+# --- Mistral 7B Series --------------------------------------------------------
[mistral-7b-2312]
model_id = "mistral-large-2402"
max_tokens = 32768
@@ -22,6 +49,7 @@ inputs = ["text"]
outputs = ["text", "structured"]
costs = { input = 0.25, output = 0.25 }
+# --- Mistral 8x7B Series ------------------------------------------------------
[mistral-8x7b-2312]
model_id = "open-mixtral-8x7b"
max_tokens = 32768
@@ -29,6 +57,7 @@ inputs = ["text"]
outputs = ["text"]
costs = { input = 0.7, output = 0.7 }
+# --- Mistral Codestral Series -------------------------------------------------
[mistral-codestral-2405]
model_id = "codestral-2405"
max_tokens = 262144
@@ -36,6 +65,7 @@ inputs = ["text"]
outputs = ["text"]
costs = { input = 1.0, output = 3.0 }
+# --- Mistral Large Series -----------------------------------------------------
[mistral-large-2402]
model_id = "mistral-large-2402"
max_tokens = 32768
@@ -50,6 +80,7 @@ inputs = ["text"]
outputs = ["text", "structured"]
costs = { input = 4.0, output = 12.0 }
+# --- Mistral Small Series -----------------------------------------------------
[mistral-small-2402]
model_id = "mistral-small-2402"
max_tokens = 32768
@@ -64,6 +95,7 @@ inputs = ["text"]
outputs = ["text", "structured"]
costs = { input = 1.0, output = 3.0 }
+# --- Pixtral Series -----------------------------------------------------------
[pixtral-12b]
model_id = "pixtral-12b-latest"
max_tokens = 131072
@@ -78,6 +110,7 @@ inputs = ["text", "images"]
outputs = ["text", "structured"]
costs = { input = 2.0, output = 6.0 }
+# --- Mistral Medium Series ----------------------------------------------------
[mistral-medium]
model_id = "mistral-medium-latest"
max_tokens = 128000
@@ -92,6 +125,11 @@ inputs = ["text", "images"]
outputs = ["text", "structured"]
costs = { input = 0.4, output = 2.0 }
+################################################################################
+# OCR MODELS
+################################################################################
+
+# --- OCR Models ---------------------------------------------------------------
[mistral-ocr]
model_type = "text_extractor"
model_id = "mistral-ocr-latest"
diff --git a/pipelex/config_template/inference/backends/ollama.toml b/pipelex/config_template/inference/backends/ollama.toml
index 2b66dee18..7a56b7384 100644
--- a/pipelex/config_template/inference/backends/ollama.toml
+++ b/pipelex/config_template/inference/backends/ollama.toml
@@ -1,6 +1,32 @@
-default_sdk = "openai"
-default_prompting_target = "anthropic"
+################################################################################
+# Ollama Backend Configuration
+################################################################################
+#
+# This file defines the model specifications for Ollama models.
+# It contains model definitions for local language models
+# accessible through the Ollama API.
+#
+# Configuration structure:
+# - Each model is defined in its own section with the model name as the header
+# - Headers with dots must be quoted (e.g., ["mistral-small3.1-24b"])
+# - Model costs are in USD per million tokens (input/output)
+#
+################################################################################
+################################################################################
+# MODEL DEFAULTS
+################################################################################
+
+[defaults]
+model_type = "llm"
+sdk = "openai"
+prompting_target = "anthropic"
+
+################################################################################
+# LANGUAGE MODELS
+################################################################################
+
+# --- Gemma Models -------------------------------------------------------------
[gemma3-4b]
model_id = "gemma3:4b"
inputs = ["text"]
@@ -8,6 +34,7 @@ outputs = ["text"]
max_prompt_images = 3000
costs = { input = 0, output = 0 }
+# --- Llama Models -------------------------------------------------------------
[llama4-scout]
model_id = "llama4:scout"
inputs = ["text"]
@@ -15,6 +42,7 @@ outputs = ["text"]
max_prompt_images = 3000
costs = { input = 0, output = 0 }
+# --- Mistral Models -----------------------------------------------------------
["mistral-small3.1-24b"]
model_id = "mistral-small3.1:24b"
inputs = ["text"]
@@ -22,6 +50,7 @@ outputs = ["text"]
max_prompt_images = 3000
costs = { input = 0, output = 0 }
+# --- Qwen Models --------------------------------------------------------------
[qwen3-8b]
model_id = "qwen3:8b"
inputs = ["text"]
diff --git a/pipelex/config_template/inference/backends/openai.toml b/pipelex/config_template/inference/backends/openai.toml
index 6e3619499..4eabe7491 100644
--- a/pipelex/config_template/inference/backends/openai.toml
+++ b/pipelex/config_template/inference/backends/openai.toml
@@ -1,12 +1,39 @@
-default_sdk = "openai"
-default_prompting_target = "openai"
-
+################################################################################
+# OpenAI Backend Configuration
+################################################################################
+#
+# This file defines the model specifications for OpenAI models.
+# It contains model definitions for various LLM and image generation models
+# accessible through the OpenAI API.
+#
+# Configuration structure:
+# - Each model is defined in its own section with the model name as the header
+# - Headers with dots must be quoted (e.g., ["gpt-4.1"])
+# - Model costs are in USD per million tokens (input/output)
+#
+################################################################################
+
+################################################################################
+# MODEL DEFAULTS
+################################################################################
+
+[defaults]
+model_type = "llm"
+sdk = "openai"
+prompting_target = "openai"
+
+################################################################################
+# LANGUAGE MODELS
+################################################################################
+
+# --- GPT-3.5 Series -----------------------------------------------------------
["gpt-3.5-turbo"]
model_id = "gpt-3.5-turbo-1106"
inputs = ["text"]
outputs = ["text", "structured"]
costs = { input = 0.5, output = 1.5 }
+# --- GPT-4 Series -------------------------------------------------------------
[gpt-4]
model_id = "gpt-4"
costs = { input = 30.0, output = 60.0 }
@@ -17,6 +44,7 @@ inputs = ["text"]
outputs = ["text", "structured"]
costs = { input = 10.0, output = 30.0 }
+# --- GPT-4o Series ------------------------------------------------------------
[gpt-4o-2024-11-20]
model_id = "gpt-4o-2024-11-20"
inputs = ["text", "images"]
@@ -41,6 +69,7 @@ inputs = ["text", "images"]
outputs = ["text", "structured"]
costs = { input = 0.15, output = 0.6 }
+# --- GPT-4.1 Series -----------------------------------------------------------
["gpt-4.1"]
model_id = "gpt-4.1"
inputs = ["text", "images"]
@@ -59,6 +88,7 @@ inputs = ["text", "images"]
outputs = ["text", "structured"]
costs = { input = 0.1, output = 0.4 }
+# --- o Series ----------------------------------------------------------------
[o1-mini]
model_id = "o1-mini"
inputs = ["text"]
@@ -94,6 +124,7 @@ outputs = ["text", "structured"]
costs = { input = 1.1, output = 4.4 }
constraints = ["temperature_must_be_1"]
+# --- GPT-5 Series -------------------------------------------------------------
[gpt-5]
model_id = "gpt-5"
inputs = ["text", "images"]
@@ -122,3 +153,15 @@ outputs = ["text"]
costs = { input = 1.25, output = 10.0 }
constraints = ["temperature_must_be_1"]
+################################################################################
+# IMAGE GENERATION MODELS
+################################################################################
+
+# --- OpenAI Image Generation --------------------------------------------------
+[gpt-image-1]
+model_type = "img_gen"
+model_id = "gpt-image-1"
+inputs = ["text"]
+outputs = ["image"]
+costs = { input = 0.04, output = 0.0 }
+
diff --git a/pipelex/config_template/inference/backends/perplexity.toml b/pipelex/config_template/inference/backends/perplexity.toml
index 4eb004db5..ea905512c 100644
--- a/pipelex/config_template/inference/backends/perplexity.toml
+++ b/pipelex/config_template/inference/backends/perplexity.toml
@@ -1,6 +1,32 @@
-default_sdk = "openai"
-default_prompting_target = "anthropic"
+################################################################################
+# Perplexity Backend Configuration
+################################################################################
+#
+# This file defines the model specifications for Perplexity AI models.
+# It contains model definitions for Sonar and reasoning models
+# accessible through the Perplexity API.
+#
+# Configuration structure:
+# - Each model is defined in its own section with the model name as the header
+# - Headers with dots must be quoted (e.g., ["sonar-pro"])
+# - Model costs are in USD per million tokens (input/output)
+#
+################################################################################
+################################################################################
+# MODEL DEFAULTS
+################################################################################
+
+[defaults]
+model_type = "llm"
+sdk = "openai"
+prompting_target = "anthropic"
+
+################################################################################
+# LANGUAGE MODELS
+################################################################################
+
+# --- Sonar Series -------------------------------------------------------------
[sonar-pro]
model_id = "sonar-pro"
inputs = ["text"]
@@ -16,6 +42,7 @@ model_id = "sonar-deep-research"
inputs = ["text"]
outputs = ["text"]
+# --- Sonar Reasoning Series ---------------------------------------------------
[sonar-reasoning-pro]
model_id = "sonar-reasoning-pro"
inputs = ["text"]
@@ -26,7 +53,8 @@ model_id = "sonar-reasoning"
inputs = ["text"]
outputs = ["text"]
+# --- DeepSeek Models -----------------------------------------------------------
[perplexity-deepseek-r1]
model_id = "r1-1776"
inputs = ["text"]
-outputs = ["text"]
+outputs = ["text"]
\ No newline at end of file
diff --git a/pipelex/config_template/inference/backends/pipelex_inference.toml b/pipelex/config_template/inference/backends/pipelex_inference.toml
index 22016adc7..2c326cfce 100644
--- a/pipelex/config_template/inference/backends/pipelex_inference.toml
+++ b/pipelex/config_template/inference/backends/pipelex_inference.toml
@@ -1,7 +1,38 @@
-default_sdk = "openai"
-default_prompting_target = "anthropic"
+################################################################################
+# Pipelex Inference Backend Configuration
+################################################################################
+#
+# This file defines the model specifications for the Pipelex Inference backend.
+# It contains model definitions for various LLM and image generation models
+# accessible through the Pipelex unified inference API.
+#
+# Configuration structure:
+# - Each model is defined in its own section with the model name as the header
+# - Headers with dots must be quoted (e.g., ["gpt-4.1"])
+# - Model costs are in USD per million tokens (input/output)
+#
+################################################################################
+
+################################################################################
+# MODEL DEFAULTS
+################################################################################
+
+[defaults]
+model_type = "llm"
+sdk = "openai"
+prompting_target = "anthropic"
+
+################################################################################
+# LANGUAGE MODELS
+################################################################################
+
+# --- OpenAI LLMs --------------------------------------------------------------
+[gpt-4o]
+model_id = "pipelex/gpt-4o"
+inputs = ["text", "images"]
+outputs = ["text", "structured"]
+costs = { input = 2.75, output = 11.00 }
-# OpenAI Models
[gpt-4o-mini]
model_id = "pipelex/gpt-4o-mini"
inputs = ["text", "images"]
@@ -44,13 +75,7 @@ inputs = ["text", "images"]
outputs = ["text"]
costs = { input = 1.25, output = 10.00 }
-[gpt-image-1]
-model_id = "pipelex/gpt-image-1"
-inputs = ["text"]
-outputs = ["images"]
-costs = { input = 5, output = 0.01 }
-
-# Anthropic Models
+# --- Claude LLMs --------------------------------------------------------------
["claude-4-sonnet"]
model_id = "pipelex/claude-4-sonnet"
inputs = ["text", "images"]
@@ -63,7 +88,7 @@ inputs = ["text", "images"]
outputs = ["text", "structured"]
costs = { input = 15, output = 75 }
-# Google Models using Google API
+# --- Gemini LLMs --------------------------------------------------------------
["gemini-2.0-flash"]
model_id = "gemini/gemini-2.0-flash"
inputs = ["text", "images"]
@@ -82,7 +107,8 @@ inputs = ["text", "images"]
outputs = ["text", "structured"]
costs = { input = 0.10, output = 0.40 }
-# XAI Models
+# --- XAI LLMs --------------------------------------------------------------
+
[grok-3]
model_id = "grok-3"
inputs = ["text"]
@@ -95,10 +121,10 @@ inputs = ["text"]
outputs = ["text"]
costs = { input = 0.3, output = 0.5 }
-# Flux Models
-[fast-lightning-sdxl]
-model_id = "pipelex/fast-lightning-sdxl"
-inputs = ["text"]
-outputs = ["images"]
-costs = { input = 0.01, output = 0.01 }
+################################################################################
+# OCR and IMAGE GENERATION MODELS
+################################################################################
+
+# We are still working in giving you acces to OCR and image generation models
+# and to the best models from Mistral through the Pipelex Inference backend.
diff --git a/pipelex/config_template/inference/backends/vertexai.toml b/pipelex/config_template/inference/backends/vertexai.toml
index e476cfcb1..cb8edb13e 100644
--- a/pipelex/config_template/inference/backends/vertexai.toml
+++ b/pipelex/config_template/inference/backends/vertexai.toml
@@ -1,6 +1,32 @@
-default_sdk = "openai"
-default_prompting_target = "gemini"
+################################################################################
+# VertexAI Backend Configuration
+################################################################################
+#
+# This file defines the model specifications for Google VertexAI models.
+# It contains model definitions for Gemini language models
+# accessible through the Google VertexAI API.
+#
+# Configuration structure:
+# - Each model is defined in its own section with the model name as the header
+# - Headers with dots must be quoted (e.g., ["gemini-2.0-flash"])
+# - Model costs are in USD per million tokens (input/output)
+#
+################################################################################
+################################################################################
+# MODEL DEFAULTS
+################################################################################
+
+[defaults]
+model_type = "llm"
+sdk = "openai"
+prompting_target = "gemini"
+
+################################################################################
+# LANGUAGE MODELS
+################################################################################
+
+# --- Gemini 2.0 Series --------------------------------------------------------
["gemini-2.0-flash"]
model_id = "google/gemini-2.0-flash"
inputs = ["text", "images"]
@@ -8,6 +34,7 @@ outputs = ["text", "structured"]
max_prompt_images = 3000
costs = { input = 0.1, output = 0.4 }
+# --- Gemini 2.5 Series --------------------------------------------------------
["gemini-2.5-pro"]
model_id = "google/gemini-2.5-pro"
inputs = ["text", "images"]
@@ -20,5 +47,4 @@ model_id = "google/gemini-2.5-flash"
inputs = ["text", "images"]
outputs = ["text", "structured"]
max_prompt_images = 3000
-costs = { input = 0.30, output = 2.50 }
-
+costs = { input = 0.30, output = 2.50 }
\ No newline at end of file
diff --git a/pipelex/config_template/inference/backends/xai.toml b/pipelex/config_template/inference/backends/xai.toml
index ea8fb58f7..545b905f2 100644
--- a/pipelex/config_template/inference/backends/xai.toml
+++ b/pipelex/config_template/inference/backends/xai.toml
@@ -1,6 +1,32 @@
-default_sdk = "openai"
-default_prompting_target = "anthropic"
+################################################################################
+# XAI Backend Configuration
+################################################################################
+#
+# This file defines the model specifications for XAI (formerly Twitter AI) models.
+# It contains model definitions for Grok language models
+# accessible through the XAI API.
+#
+# Configuration structure:
+# - Each model is defined in its own section with the model name as the header
+# - Headers with dots must be quoted (e.g., ["grok-3"])
+# - Model costs are in USD per million tokens (input/output)
+#
+################################################################################
+################################################################################
+# MODEL DEFAULTS
+################################################################################
+
+[defaults]
+model_type = "llm"
+sdk = "openai"
+prompting_target = "anthropic"
+
+################################################################################
+# LANGUAGE MODELS
+################################################################################
+
+# --- Grok 3 Series ------------------------------------------------------------
[grok-3]
model_id = "grok-3"
inputs = ["text"]
diff --git a/pipelex/config_template/inference/deck/base_deck.toml b/pipelex/config_template/inference/deck/base_deck.toml
index 51d646c9a..900235960 100644
--- a/pipelex/config_template/inference/deck/base_deck.toml
+++ b/pipelex/config_template/inference/deck/base_deck.toml
@@ -26,11 +26,16 @@ base_llm_for_object = "gpt-4o-mini"
llm_to_engineer = ["claude-4.1-opus", "gemini-2.5-pro"]
llm_for_large_codebase = ["gemini-2.5-pro", "claude-4-sonnet", "gpt-5"]
+# Image generation aliases
+base-img-gen = "flux-pro/v1.1"
+best-img-gen = "flux-pro/v1.1-ultra"
+fast-img-gen = "fast-lightning-sdxl"
+
####################################################################################################
# LLM Presets
####################################################################################################
-[llm_presets]
+[llm.presets]
####################################################################################################
# LLM Presets — General purpose
@@ -67,10 +72,11 @@ llm_for_question_and_excerpt_reformulation = { llm_handle = "gpt-4o", temperatur
# Engineering skills
llm_to_engineer = { llm_handle = "base-claude", temperature = 0.2 }
+llm_to_write = { llm_handle = "base-gemini", temperature = 0.2 }
llm_to_pipelex = { llm_handle = "base-claude", temperature = 0.2 }
# Image skills
-llm_to_write_imgg_prompt = { llm_handle = "base-claude", temperature = 0.2 }
+llm_to_write_img_gen_prompt = { llm_handle = "base-claude", temperature = 0.2 }
llm_to_describe_img = { llm_handle = "base-claude", temperature = 0.5 }
llm_to_design_fashion = { llm_handle = "base-claude", temperature = 0.7 }
llm_for_img_to_text = { llm_handle = "base-claude", temperature = 0.1 }
@@ -86,7 +92,38 @@ llm_to_extract_tables = { llm_handle = "base-claude", temperature = 0.1 }
# LLM Choices
####################################################################################################
-[llm_choice_defaults]
+[llm.choice_defaults]
for_text = "cheap_llm_for_text"
for_object = "cheap_llm_for_object"
+####################################################################################################
+# OCR Presets
+####################################################################################################
+
+[ocr]
+choice_default = "base_ocr_mistral"
+
+[ocr.presets]
+base_ocr_mistral = { ocr_handle = "mistral-ocr", max_nb_images = 100, image_min_size = 50 }
+base_ocr_pypdfium2 = { ocr_handle = "pypdfium2-extract-text", max_nb_images = 100, image_min_size = 50 }
+
+####################################################################################################
+# Image Generation Presets
+####################################################################################################
+
+[img_gen]
+choice_default = "base_img_gen"
+
+[img_gen.presets]
+# General purpose
+base_img_gen = { img_gen_handle = "base-img-gen", quality = "medium", guidance_scale = 7.5, is_moderated = true, safety_tolerance = 3 }
+fast_img_gen = { img_gen_handle = "fast-img-gen", nb_steps = 4, guidance_scale = 5.0, is_moderated = true, safety_tolerance = 3 }
+high_quality_img_gen = { img_gen_handle = "best-img-gen", quality = "high", guidance_scale = 8.0, is_moderated = true, safety_tolerance = 3 }
+
+# Specific skills
+img_gen_for_art = { img_gen_handle = "best-img-gen", quality = "high", guidance_scale = 9.0, is_moderated = false, safety_tolerance = 5 }
+img_gen_for_diagram = { img_gen_handle = "base-img-gen", quality = "medium", guidance_scale = 7.0, is_moderated = true, safety_tolerance = 2 }
+img_gen_for_mockup = { img_gen_handle = "base-img-gen", quality = "medium", guidance_scale = 6.5, is_moderated = true, safety_tolerance = 3 }
+img_gen_for_product = { img_gen_handle = "best-img-gen", quality = "high", guidance_scale = 8.5, is_moderated = true, safety_tolerance = 2 }
+img_gen_for_testing = { img_gen_handle = "fast-img-gen", nb_steps = 4, guidance_scale = 4.0, is_moderated = true, safety_tolerance = 4 }
+
diff --git a/pipelex/config_template/inference/deck/overrides.toml b/pipelex/config_template/inference/deck/overrides.toml
index 999cc4d1a..d341ec7d2 100644
--- a/pipelex/config_template/inference/deck/overrides.toml
+++ b/pipelex/config_template/inference/deck/overrides.toml
@@ -4,7 +4,7 @@
# LLM Deck overrides
####################################################################################################
-[llm_choice_overrides]
+[llm.choice_overrides]
for_text = "disabled"
for_object = "disabled"
diff --git a/pipelex/config_template/inference/routing_profiles.toml b/pipelex/config_template/inference/routing_profiles.toml
index be9b51d2c..cd264a177 100644
--- a/pipelex/config_template/inference/routing_profiles.toml
+++ b/pipelex/config_template/inference/routing_profiles.toml
@@ -6,12 +6,12 @@
# ================================================
# Which profile to use (change this to switch routing)
-active = "all_pipelex"
+active = "pipelex_first"
-# We recommend using the "all_pipelex" profile to get a head start with all models.
+# We recommend using the "pipelex_first" profile to get a head start with all models.
# The Pipelex Inference backend is currently not recommended for production use,
# but it's great for development and testing.
-# To use the Pipelex Inference backend (all_pipelex profile):
+# To use the Pipelex Inference backend (pipelex_first profile):
# 1. Join our Discord community to get your free API key (no credit card required):
# Visit https://go.pipelex.com/discord and request your key in the appropriate channel
# 2. Set the environment variable: export PIPELEX_INFERENCE_API_KEY="your-api-key"
@@ -22,32 +22,48 @@ active = "all_pipelex"
# Routing Profiles
# ============================================
-[profiles.all_pipelex]
-description = "Use Pipelex Inference backend for all models"
+[profiles.pipelex_first]
+description = "Use Pipelex Inference backend for all its supported models"
default = "pipelex_inference"
+[profiles.pipelex_first.routes]
+# Pattern matching: "model-pattern" = "backend-name"
+"gpt-*" = "pipelex_inference"
+"claude-*" = "pipelex_inference"
+"grok-*" = "pipelex_inference"
+"gemini-*" = "google"
+"*-sdxl" = "fal"
+"flux-*" = "fal"
+"gpt-image-1" = "openai"
+"mistral-ocr" = "mistral"
+
[profiles.all_anthropic]
description = "Use Anthropic backend for all its supported models"
default = "anthropic"
+[profiles.all_gemini]
+description = "Use Google GenAI backend for all its supported models"
+default = "google"
+
[profiles.all_azure_openai]
description = "Use Azure OpenAI backend for all its supported models"
default = "azure_openai"
-
-[profiles.to_test_various_backends]
-description = "Route models to various backends"
-
[profiles.custom_routing]
description = "Custom routing"
default = "pipelex_inference"
[profiles.custom_routing.routes]
# Pattern matching: "model-pattern" = "backend-name"
-"gpt-*" = "pipelex_inference"
-"claude-*" = "pipelex_inference"
-"gemini-*" = "pipelex_inference"
-"grok-*" = "pipelex_inference"
-"*-sdxl" = "pipelex_inference"
+"gpt-*" = "azure_openai"
+"claude-*" = "bedrock"
+"gemini-*" = "google"
+"grok-*" = "xai"
+"*-sdxl" = "fal"
+"flux-*" = "fal"
+"gpt-image-1" = "openai"
+
+[profiles.to_test_various_backends]
+description = "Route models to various backends"
[profiles.to_test_various_backends.routes]
"gpt-5-nano" = "pipelex_inference"
@@ -82,4 +98,3 @@ default = "pipelex_inference"
# - Exact names: "gpt-4o-mini"
# - Wildcards: "claude-*" (matches all models starting with claude-)
# - Partial wildcards: "*-sonnet" (matches all sonnet variants)
-
diff --git a/pipelex/core/bundles/pipelex_bundle_blueprint.py b/pipelex/core/bundles/pipelex_bundle_blueprint.py
index 2c30c07c2..d649b3826 100644
--- a/pipelex/core/bundles/pipelex_bundle_blueprint.py
+++ b/pipelex/core/bundles/pipelex_bundle_blueprint.py
@@ -33,8 +33,6 @@ PipeBlueprintUnion = Annotated[
class PipelexBundleBlueprint(BaseModel):
- """Complete blueprint of a pipelex bundle PLX definition."""
-
model_config = ConfigDict(extra="forbid")
domain: str
@@ -43,7 +41,7 @@ class PipelexBundleBlueprint(BaseModel):
system_prompt_to_structure: Optional[str] = None
prompt_template_to_structure: Optional[str] = None
- concept: Optional[Dict[str, ConceptBlueprint | str]] = Field(default_factory=dict)
+ concept: Optional[Dict[str, Union[ConceptBlueprint, str]]] = Field(default_factory=dict)
pipe: Optional[Dict[str, PipeBlueprintUnion]] = Field(default_factory=dict)
diff --git a/pipelex/core/concepts/concept.py b/pipelex/core/concepts/concept.py
index 6b5bbec5b..e88f925be 100644
--- a/pipelex/core/concepts/concept.py
+++ b/pipelex/core/concepts/concept.py
@@ -5,6 +5,7 @@ from pydantic import BaseModel, ConfigDict, field_validator
from pipelex import log
from pipelex.core.concepts.concept_blueprint import ConceptBlueprint
+from pipelex.core.concepts.concept_native import NativeConceptManager
from pipelex.core.domains.domain import SpecialDomain
from pipelex.core.domains.domain_blueprint import DomainBlueprint
from pipelex.core.stuffs.stuff_content import StuffContent
@@ -28,7 +29,7 @@ class Concept(BaseModel):
@classmethod
def is_implicit_concept(cls, concept_string: str) -> bool:
ConceptBlueprint.validate_concept_string(concept_string=concept_string)
- return concept_string.startswith(SpecialDomain.IMPLICIT.value)
+ return concept_string.startswith(SpecialDomain.IMPLICIT)
@field_validator("code")
def validate_code(cls, code: str) -> str:
@@ -53,7 +54,7 @@ class Concept(BaseModel):
@classmethod
def is_native_concept(cls, concept: "Concept") -> bool:
- return ConceptBlueprint.is_native_concept_string_or_concept_code(concept_string_or_concept_code=concept.concept_string)
+ return NativeConceptManager.is_native_concept(concept_string_or_code=concept.concept_string)
@classmethod
def are_concept_compatible(cls, concept_1: "Concept", concept_2: "Concept", strict: bool = False) -> bool:
diff --git a/pipelex/core/concepts/concept_blueprint.py b/pipelex/core/concepts/concept_blueprint.py
index 378bbdadf..46459165c 100644
--- a/pipelex/core/concepts/concept_blueprint.py
+++ b/pipelex/core/concepts/concept_blueprint.py
@@ -1,14 +1,14 @@
+from datetime import datetime
from typing import Any, Dict, List, Optional, Union
from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
from typing_extensions import Self
-from pipelex.core.concepts.concept_native import NativeConceptEnum, is_native_concept
+from pipelex.core.concepts.concept_native import NativeConceptEnum, NativeConceptManager
from pipelex.core.concepts.exceptions import ConceptCodeError, ConceptStringError, ConceptStringOrConceptCodeError
from pipelex.core.domains.domain import SpecialDomain
from pipelex.core.domains.domain_blueprint import DomainBlueprint
from pipelex.tools.misc.string_utils import is_pascal_case
-from pipelex.tools.typing.validation_utils import has_more_than_one_among_attributes_from_list
from pipelex.types import StrEnum
@@ -32,7 +32,7 @@ class ConceptStructureBlueprintFieldType(StrEnum):
class ConceptStructureBlueprint(BaseModel):
definition: str
- type: ConceptStructureBlueprintFieldType | None = None
+ type: Optional[ConceptStructureBlueprintFieldType] = None
item_type: Optional[str] = None
key_type: Optional[str] = None
value_type: Optional[str] = None
@@ -106,8 +106,9 @@ class ConceptStructureBlueprint(BaseModel):
case ConceptStructureBlueprintFieldType.DICT:
if not isinstance(self.default_value, dict):
self._raise_type_mismatch_error("dict", type(self.default_value).__name__)
- case _:
- raise ConceptStructureBlueprintError(f"Unknown type: {self.type} in structure blueprint with definition: {self.definition}")
+ case ConceptStructureBlueprintFieldType.DATE:
+ if not isinstance(self.default_value, datetime):
+ self._raise_type_mismatch_error("date", type(self.default_value).__name__)
def _raise_type_mismatch_error(self, expected_type_name: str, actual_type_name: str) -> None:
"""Raise a type mismatch error with consistent formatting."""
@@ -116,32 +117,19 @@ class ConceptStructureBlueprint(BaseModel):
)
-ConceptStructureBlueprintType = Union[str, ConceptStructureBlueprint]
-
-
class ConceptBlueprint(BaseModel):
model_config = ConfigDict(extra="forbid")
definition: str
- structure: Optional[Union[str, Dict[str, ConceptStructureBlueprintType]]] = None
+ # TODO (non-blockiing): define a type for Union[str, ConceptStructureBlueprint] (ConceptChoice to be consistent with LLMChoice)
+ structure: Optional[Union[str, Dict[str, Union[str, ConceptStructureBlueprint]]]] = None
# TODO: restore possibility of multiple refiles
refines: Optional[str] = None
@classmethod
def is_native_concept_code(cls, concept_code: str) -> bool:
ConceptBlueprint.validate_concept_code(concept_code=concept_code)
- return concept_code in [native_concept.value for native_concept in NativeConceptEnum]
-
- @classmethod
- def is_native_concept_string_or_concept_code(cls, concept_string_or_concept_code: str) -> bool:
- if "." in concept_string_or_concept_code:
- domain, concept_code = concept_string_or_concept_code.split(".", 1)
- if domain == SpecialDomain.NATIVE.value and concept_code in [native_concept.value for native_concept in NativeConceptEnum]:
- return True
- else:
- if is_native_concept(concept_string_or_concept_code):
- return True
- return False
+ return concept_code in [native_concept for native_concept in NativeConceptEnum]
@classmethod
def validate_concept_code(cls, concept_code: str) -> None:
@@ -151,20 +139,20 @@ class ConceptBlueprint(BaseModel):
)
@classmethod
- def validate_concept_string_or_concept_code(cls, concept_string_or_concept_code: str) -> None:
- if concept_string_or_concept_code.count(".") > 1:
+ def validate_concept_string_or_concept_code(cls, concept_string_or_code: str) -> None:
+ if concept_string_or_code.count(".") > 1:
raise ConceptStringOrConceptCodeError(
- f"concept_string_or_concept_code '{concept_string_or_concept_code}' is invalid. "
+ f"concept_string_or_code '{concept_string_or_code}' is invalid. "
"It should either contain a domain in snake_case and a concept code in PascalCase separated by one dot, "
"or be a concept code in PascalCase."
)
- elif concept_string_or_concept_code.count(".") == 1:
- domain, concept_code = concept_string_or_concept_code.split(".")
+ elif concept_string_or_code.count(".") == 1:
+ domain, concept_code = concept_string_or_code.split(".")
DomainBlueprint.validate_domain_code(code=domain)
cls.validate_concept_code(concept_code=concept_code)
else:
- cls.validate_concept_code(concept_code=concept_string_or_concept_code)
+ cls.validate_concept_code(concept_code=concept_string_or_code)
@staticmethod
def validate_concept_string(concept_string: str) -> None:
@@ -192,43 +180,36 @@ class ConceptBlueprint(BaseModel):
)
# Validate that if the concept code is among the native concepts, the domain MUST be native.
- if concept_code in [native_concept.value for native_concept in NativeConceptEnum]:
- if domain != SpecialDomain.NATIVE.value:
+ if concept_code in [concept.value for concept in [native_concept for native_concept in NativeConceptEnum]]:
+ if not SpecialDomain.is_native(domain=domain):
raise ConceptStringError(
f"Concept string '{concept_string}' is invalid. "
- f"Concept code '{concept_code}' is a native concept, so the domain must be '{SpecialDomain.NATIVE.value}', "
+ f"Concept code '{concept_code}' is a native concept, so the domain must be '{SpecialDomain.NATIVE}', "
f"or nothing, but not '{domain}'"
)
# Validate that if the domain is native, the concept code is a native concept
- if domain == SpecialDomain.NATIVE.value:
- if concept_code not in [native_concept.value for native_concept in NativeConceptEnum]:
+ if SpecialDomain.is_native(domain=domain):
+ if concept_code not in [native_concept for native_concept in NativeConceptEnum]:
raise ConceptStringError(
f"Concept string '{concept_string}' is invalid. "
- f"Concept code '{concept_code}' is not a native concept, so the domain must not be '{SpecialDomain.NATIVE.value}'."
+ f"Concept code '{concept_code}' is not a native concept, so the domain must not be '{SpecialDomain.NATIVE}'."
)
@field_validator("refines", mode="before")
@classmethod
def validate_refines(cls, refines: Optional[str] = None) -> Optional[str]:
if refines is not None:
- if not is_native_concept(refines):
+ if not NativeConceptManager.is_native_concept(refines):
raise ConceptBlueprintError(f"Forbidden to refine a non-native concept: '{refines}'. Refining non-native concepts will come soon.")
- cls.validate_concept_string_or_concept_code(concept_string_or_concept_code=refines)
+ cls.validate_concept_string_or_concept_code(concept_string_or_code=refines)
return refines
@model_validator(mode="before")
- def model_validate_blueprint(cls, values: Union[Dict[str, Any], "ConceptBlueprint"]) -> Union[Dict[str, Any], "ConceptBlueprint"]:
- if isinstance(values, dict):
- if values.get("refines") and values.get("structure"):
- raise ConceptBlueprintError(
- f"Forbidden to have refines and structure at the same time: `{values.get('refines')}` "
- f"and `{values.get('structure')}` for concept that has the definition `{values.get('definition')}`"
- )
- elif hasattr(values, "refines") and hasattr(values, "structure"):
- if has_more_than_one_among_attributes_from_list(obj=values, attributes_list=["refines", "structure"]):
- raise ConceptBlueprintError(
- f"Forbidden to have refines and structure at the same time: `{values.refines}` "
- f"and `{values.structure}` for concept that has the definition `{values.definition}`"
- )
+ def model_validate_blueprint(cls, values: Union[Dict[str, Any], str]) -> Union[Dict[str, Any], str]:
+ if isinstance(values, dict) and values.get("refines") and values.get("structure"):
+ raise ConceptBlueprintError(
+ f"Forbidden to have refines and structure at the same time: `{values.get('refines')}` "
+ f"and `{values.get('structure')}` for concept that has the definition `{values.get('definition')}`"
+ )
return values
diff --git a/pipelex/core/concepts/concept_factory.py b/pipelex/core/concepts/concept_factory.py
index 28b5ae33c..eb60fa1f2 100644
--- a/pipelex/core/concepts/concept_factory.py
+++ b/pipelex/core/concepts/concept_factory.py
@@ -1,4 +1,4 @@
-from typing import Any, Dict, List, Optional
+from typing import Any, Dict, List, Optional, Union
from kajson.kajson_manager import KajsonManager
from pydantic import BaseModel
@@ -8,12 +8,11 @@ from pipelex.core.concepts.concept_blueprint import (
ConceptBlueprint,
ConceptStructureBlueprint,
ConceptStructureBlueprintFieldType,
- ConceptStructureBlueprintType,
)
-from pipelex.core.concepts.concept_native import NativeConceptEnumData, is_native_concept
+from pipelex.core.concepts.concept_native import NativeConceptEnumData, NativeConceptManager
+from pipelex.core.concepts.structure_generator import StructureGenerator
from pipelex.core.domains.domain import SpecialDomain
from pipelex.core.stuffs.stuff_content import TextContent
-from pipelex.create.structured_output_generator import StructureGenerator
from pipelex.exceptions import ConceptFactoryError, StructureClassError
@@ -26,7 +25,7 @@ class DomainAndConceptCode(BaseModel):
class ConceptFactory:
@classmethod
- def normalize_structure_blueprint(cls, structure_dict: Dict[str, ConceptStructureBlueprintType]) -> Dict[str, ConceptStructureBlueprint]:
+ def normalize_structure_blueprint(cls, structure_dict: Dict[str, Union[str, ConceptStructureBlueprint]]) -> Dict[str, ConceptStructureBlueprint]:
"""Convert a mixed structure dictionary to a proper ConceptStructureBlueprint dictionary.
Args:
@@ -85,33 +84,29 @@ class ConceptFactory:
@classmethod
def make_domain_and_concept_code_from_concept_string_or_concept_code(
- cls, domain: str, concept_string_or_concept_code: str, concept_codes_from_the_same_domain: Optional[List[str]] = None
+ cls, domain: str, concept_string_or_code: str, concept_codes_from_the_same_domain: Optional[List[str]] = None
) -> DomainAndConceptCode:
- # At this point, the concept_string_or_concept_code is already validated
- if "." in concept_string_or_concept_code:
+ # At this point, the concept_string_or_code is already validated
+ if "." in concept_string_or_code:
# Is a concept string.
- parts = concept_string_or_concept_code.rsplit(".")
+ parts = concept_string_or_code.rsplit(".")
return DomainAndConceptCode(domain=parts[0], concept_code=parts[1])
else:
- if is_native_concept(concept_string_or_concept_code=concept_string_or_concept_code):
- return DomainAndConceptCode(domain=SpecialDomain.NATIVE, concept_code=concept_string_or_concept_code)
+ if NativeConceptManager.is_native_concept(concept_string_or_code=concept_string_or_code):
+ return DomainAndConceptCode(domain=SpecialDomain.NATIVE, concept_code=concept_string_or_code)
elif (
- concept_codes_from_the_same_domain and concept_string_or_concept_code in concept_codes_from_the_same_domain
+ concept_codes_from_the_same_domain and concept_string_or_code in concept_codes_from_the_same_domain
): # Is a concept code from the same domain
- return DomainAndConceptCode(domain=domain, concept_code=concept_string_or_concept_code)
+ return DomainAndConceptCode(domain=domain, concept_code=concept_string_or_code)
else:
- return DomainAndConceptCode(domain=SpecialDomain.IMPLICIT.value, concept_code=concept_string_or_concept_code)
+ return DomainAndConceptCode(domain=SpecialDomain.IMPLICIT, concept_code=concept_string_or_code)
@classmethod
def make_refine(cls, refine: str) -> str:
- if ConceptBlueprint.is_native_concept_string_or_concept_code(concept_string_or_concept_code=refine):
- if "." in refine:
- return refine
- else:
- return f"{SpecialDomain.NATIVE}.{refine}"
- else:
- raise ConceptFactoryError(f"Refine '{refine}' is not a native concept")
+ if NativeConceptManager.is_native_concept(concept_string_or_code=refine):
+ return NativeConceptManager.get_native_concept_string(concept_string_or_code=refine)
+ raise ConceptFactoryError(f"Refine '{refine}' is not a native concept")
@classmethod
def make_refines(cls, blueprint: ConceptBlueprint) -> Optional[str]:
@@ -186,7 +181,7 @@ class ConceptFactory:
domain_and_concept_code = cls.make_domain_and_concept_code_from_concept_string_or_concept_code(
domain=domain,
- concept_string_or_concept_code=concept_code,
+ concept_string_or_code=concept_code,
concept_codes_from_the_same_domain=concept_codes_from_the_same_domain,
)
diff --git a/pipelex/core/concepts/concept_library.py b/pipelex/core/concepts/concept_library.py
index 9748c480e..716c44adb 100644
--- a/pipelex/core/concepts/concept_library.py
+++ b/pipelex/core/concepts/concept_library.py
@@ -30,7 +30,8 @@ class ConceptLibrary(RootModel[ConceptLibraryRoot], ConceptProviderAbstract):
@override
def setup(self):
native_concepts = [
- ConceptFactory.make_native_concept(native_concept_data=NATIVE_CONCEPTS_DATA[native_concept]) for native_concept in NativeConceptEnum
+ ConceptFactory.make_native_concept(native_concept_data=NATIVE_CONCEPTS_DATA[native_concept])
+ for native_concept in [native_concept for native_concept in NativeConceptEnum]
]
self.add_concepts(native_concepts)
@@ -50,13 +51,13 @@ class ConceptLibrary(RootModel[ConceptLibraryRoot], ConceptProviderAbstract):
@override
def get_native_concept(self, native_concept: NativeConceptEnum) -> Concept:
try:
- return self.root[f"{SpecialDomain.NATIVE.value}.{native_concept.value}"]
+ return self.root[f"{SpecialDomain.NATIVE}.{native_concept}"]
except KeyError:
- raise ConceptLibraryConceptNotFoundError(f"Native concept '{native_concept.value}' not found in the library")
+ raise ConceptLibraryConceptNotFoundError(f"Native concept '{native_concept}' not found in the library")
def get_native_concepts(self) -> List[Concept]:
"""Create all native concepts from the hardcoded data"""
- return [self.get_native_concept(native_concept=native_concept) for native_concept in NativeConceptEnum]
+ return [self.get_native_concept(native_concept=native_concept) for native_concept in [native_concept for native_concept in NativeConceptEnum]]
@override
def list_concepts(self) -> List[Concept]:
@@ -77,6 +78,11 @@ class ConceptLibrary(RootModel[ConceptLibraryRoot], ConceptProviderAbstract):
for concept in concepts:
self.add_new_concept(concept=concept)
+ def remove_concepts_by_codes(self, concept_codes: List[str]) -> None:
+ for concept_code in concept_codes:
+ if concept_code in self.root:
+ del self.root[concept_code]
+
@override
def is_compatible(self, tested_concept: Concept, wanted_concept: Concept, strict: bool = False) -> bool:
if Concept.are_concept_compatible(concept_1=tested_concept, concept_2=wanted_concept, strict=strict):
diff --git a/pipelex/core/concepts/concept_native.py b/pipelex/core/concepts/concept_native.py
index de8e2030b..7732bac45 100644
--- a/pipelex/core/concepts/concept_native.py
+++ b/pipelex/core/concepts/concept_native.py
@@ -25,6 +25,28 @@ class NativeConceptEnum(StrEnum):
PAGE = "Page"
ANYTHING = "Anything"
+ @classmethod
+ def is_text(cls, concept_code: str) -> bool:
+ try:
+ enum_value = NativeConceptEnum(concept_code)
+ except ValueError:
+ return False
+
+ match enum_value:
+ case NativeConceptEnum.TEXT:
+ return True
+ case (
+ NativeConceptEnum.DYNAMIC
+ | NativeConceptEnum.IMAGE
+ | NativeConceptEnum.PDF
+ | NativeConceptEnum.TEXT_AND_IMAGES
+ | NativeConceptEnum.NUMBER
+ | NativeConceptEnum.LLM_PROMPT
+ | NativeConceptEnum.PAGE
+ | NativeConceptEnum.ANYTHING
+ ):
+ return False
+
NATIVE_CONCEPTS_DATA: Dict[NativeConceptEnum, NativeConceptEnumData] = {
NativeConceptEnum.DYNAMIC: NativeConceptEnumData(
@@ -59,18 +81,46 @@ NATIVE_CONCEPTS_DATA: Dict[NativeConceptEnum, NativeConceptEnumData] = {
}
-def is_native_concept(concept_string_or_concept_code: str) -> bool:
- """Check if a concept reference is a native concept (short or fully qualified form)."""
- native_concept_values = [native_concept.value for native_concept in NativeConceptEnum]
+class NativeConceptManager:
+ @classmethod
+ def is_native_concept(cls, concept_string_or_code: str) -> bool:
+ native_concept_values = [concept.value for concept in NativeConceptEnum]
- # Check short form (e.g., "Text")
- if concept_string_or_concept_code in native_concept_values:
- return True
+ if "." in concept_string_or_code:
+ domain, concept_code = concept_string_or_code.split(".", 1)
+ if SpecialDomain.is_native(domain=domain) and concept_code in native_concept_values:
+ return True
- # Check fully qualified form (e.g., "native.Text")
- if "." in concept_string_or_concept_code:
- domain, concept_code = concept_string_or_concept_code.split(".", 1)
- if domain == SpecialDomain.NATIVE.value and concept_code in native_concept_values:
+ if concept_string_or_code in native_concept_values:
return True
- return False
+ return False
+
+ @classmethod
+ def get_native_concept_string(cls, concept_string_or_code: str) -> str:
+ if not cls.is_native_concept(concept_string_or_code):
+ msg = f"Trying to get a native concept with code '{concept_string_or_code}' that is not a native concept"
+ raise NativeConceptEnumError(msg)
+
+ if "." in concept_string_or_code and SpecialDomain.is_native(domain=concept_string_or_code.split(".")[0]):
+ return concept_string_or_code
+
+ return f"{SpecialDomain.NATIVE}.{concept_string_or_code}"
+
+ @classmethod
+ def get_native_concept_enum(cls, concept_string_or_code: str) -> NativeConceptEnum:
+ if not cls.is_native_concept(concept_string_or_code):
+ msg = f"Trying to get a native concept with string or code '{concept_string_or_code}' that is not a native concept"
+ raise NativeConceptEnumError(msg)
+
+ if "." in concept_string_or_code:
+ _, concept_code = concept_string_or_code.split(".", 1)
+ else:
+ concept_code = concept_string_or_code
+
+ return NativeConceptEnum(concept_code)
+
+ @classmethod
+ def get_native_concept_data(cls, concept_string_or_code: str) -> NativeConceptEnumData:
+ enum_value = cls.get_native_concept_enum(concept_string_or_code)
+ return NATIVE_CONCEPTS_DATA[enum_value]
diff --git a/pipelex/create/structured_output_generator.py b/pipelex/core/concepts/structure_generator.py
similarity index 100%
rename from pipelex/create/structured_output_generator.py
rename to pipelex/core/concepts/structure_generator.py
diff --git a/pipelex/core/domains/domain.py b/pipelex/core/domains/domain.py
index c3a0b9936..95f15f13d 100644
--- a/pipelex/core/domains/domain.py
+++ b/pipelex/core/domains/domain.py
@@ -10,6 +10,19 @@ class SpecialDomain(StrEnum):
IMPLICIT = "implicit"
NATIVE = "native"
+ @classmethod
+ def is_native(cls, domain: str) -> bool:
+ try:
+ enum_value = SpecialDomain(domain)
+ except ValueError:
+ return False
+
+ match enum_value:
+ case SpecialDomain.NATIVE:
+ return True
+ case SpecialDomain.IMPLICIT:
+ return False
+
class Domain(BaseModel):
code: str
diff --git a/pipelex/core/domains/domain_library.py b/pipelex/core/domains/domain_library.py
index 52ac5501d..4e863de48 100644
--- a/pipelex/core/domains/domain_library.py
+++ b/pipelex/core/domains/domain_library.py
@@ -29,6 +29,10 @@ class DomainLibrary(RootModel[DomainLibraryRoot], DomainProviderAbstract):
else:
self.root[domain_code] = domain
+ def remove_domain_by_code(self, domain_code: str) -> None:
+ if domain_code in self.root:
+ del self.root[domain_code]
+
@override
def get_domain(self, domain: str) -> Optional[Domain]:
return self.root.get(domain)
diff --git a/pipelex/core/interpreter.py b/pipelex/core/interpreter.py
index 2d2993351..9413cd7a3 100644
--- a/pipelex/core/interpreter.py
+++ b/pipelex/core/interpreter.py
@@ -11,7 +11,7 @@ from pipelex.core.exceptions import (
PipelexConfigurationError,
PipelexUnknownPipeError,
)
-from pipelex.core.pipes.pipe_input_spec_blueprint import InputRequirementBlueprint
+from pipelex.core.pipes.pipe_input_blueprint import InputRequirementBlueprint
from pipelex.pipe_controllers.batch.pipe_batch_blueprint import PipeBatchBlueprint
from pipelex.pipe_controllers.condition.pipe_condition_blueprint import (
PipeConditionBlueprint,
@@ -311,7 +311,7 @@ class PipelexInterpreter(BaseModel):
PipelexInterpreter.add_inputs_to_lines_if_exist(lines, pipe.inputs)
# Add output after inputs (or immediately if no inputs)
- lines.append(f'output = "{PipelexInterpreter.escape_plx_string(pipe.output_concept_string_or_concept_code)}"')
+ lines.append(f'output = "{PipelexInterpreter.escape_plx_string(pipe.output)}"')
# Add optional fields
if pipe.nb_output is not None:
@@ -343,8 +343,8 @@ class PipelexInterpreter(BaseModel):
# Add inputs if they exist
PipelexInterpreter.add_inputs_to_lines_if_exist(lines, pipe.inputs)
- lines.append(f'output = "{PipelexInterpreter.escape_plx_string(pipe.output_concept_string_or_concept_code)}"')
- lines.append(f'ocr_model = "{pipe.ocr_model}"')
+ lines.append(f'output = "{PipelexInterpreter.escape_plx_string(pipe.output)}"')
+ lines.append(f'ocr = "{pipe.ocr}"')
return "\n".join(lines)
@@ -360,7 +360,7 @@ class PipelexInterpreter(BaseModel):
# Add inputs if they exist
PipelexInterpreter.add_inputs_to_lines_if_exist(lines, pipe.inputs)
- lines.append(f'output = "{PipelexInterpreter.escape_plx_string(pipe.output_concept_string_or_concept_code)}"')
+ lines.append(f'output = "{PipelexInterpreter.escape_plx_string(pipe.output)}"')
lines.append(f'function_name = "{PipelexInterpreter.escape_plx_string(pipe.function_name)}"')
return "\n".join(lines)
@@ -377,25 +377,13 @@ class PipelexInterpreter(BaseModel):
# Add inputs if they exist
PipelexInterpreter.add_inputs_to_lines_if_exist(lines, pipe.inputs)
- lines.append(f'output = "{PipelexInterpreter.escape_plx_string(pipe.output_concept_string_or_concept_code)}"')
+ lines.append(f'output = "{PipelexInterpreter.escape_plx_string(pipe.output)}"')
# Add optional fields
if pipe.img_gen_prompt:
lines.append(f'img_gen_prompt = "{PipelexInterpreter.escape_plx_string(pipe.img_gen_prompt)}"')
- if pipe.imgg_handle:
- lines.append(f'imgg_handle = "{PipelexInterpreter.escape_plx_string(pipe.imgg_handle)}"')
if pipe.aspect_ratio:
lines.append(f'aspect_ratio = "{pipe.aspect_ratio}"')
- if pipe.quality:
- lines.append(f'quality = "{pipe.quality}"')
- if pipe.nb_steps:
- lines.append(f"nb_steps = {pipe.nb_steps}")
- if pipe.guidance_scale:
- lines.append(f"guidance_scale = {pipe.guidance_scale}")
- if pipe.is_moderated is not None:
- lines.append(f"is_moderated = {str(pipe.is_moderated).lower()}")
- if pipe.safety_tolerance:
- lines.append(f'safety_tolerance = "{pipe.safety_tolerance}"')
if pipe.is_raw is not None:
lines.append(f"is_raw = {str(pipe.is_raw).lower()}")
if pipe.seed:
@@ -417,7 +405,7 @@ class PipelexInterpreter(BaseModel):
# Add inputs if they exist
PipelexInterpreter.add_inputs_to_lines_if_exist(lines, pipe.inputs)
- lines.append(f'output = "{PipelexInterpreter.escape_plx_string(pipe.output_concept_string_or_concept_code)}"')
+ lines.append(f'output = "{PipelexInterpreter.escape_plx_string(pipe.output)}"')
# Add jinja2 template
if pipe.jinja2:
@@ -439,7 +427,7 @@ class PipelexInterpreter(BaseModel):
# Add inputs if they exist
PipelexInterpreter.add_inputs_to_lines_if_exist(lines, pipe.inputs)
- lines.append(f'output = "{PipelexInterpreter.escape_plx_string(pipe.output_concept_string_or_concept_code)}"')
+ lines.append(f'output = "{PipelexInterpreter.escape_plx_string(pipe.output)}"')
# Add pipe_map
if pipe.pipe_map:
@@ -473,7 +461,7 @@ class PipelexInterpreter(BaseModel):
# Add inputs if they exist
PipelexInterpreter.add_inputs_to_lines_if_exist(lines, pipe.inputs)
- lines.append(f'output = "{PipelexInterpreter.escape_plx_string(pipe.output_concept_string_or_concept_code)}"')
+ lines.append(f'output = "{PipelexInterpreter.escape_plx_string(pipe.output)}"')
# Add parallels array
if pipe.parallels:
@@ -503,7 +491,7 @@ class PipelexInterpreter(BaseModel):
# Add inputs if they exist
PipelexInterpreter.add_inputs_to_lines_if_exist(lines, pipe.inputs)
- lines.append(f'output = "{PipelexInterpreter.escape_plx_string(pipe.output_concept_string_or_concept_code)}"')
+ lines.append(f'output = "{PipelexInterpreter.escape_plx_string(pipe.output)}"')
lines.append(f'branch_pipe_code = "{pipe.branch_pipe_code}"')
# Add optional fields
@@ -639,7 +627,7 @@ class PipelexInterpreter(BaseModel):
result["inputs"] = PipelexInterpreter.serialize_inputs(pipe.inputs)
# Then output
- result["output"] = pipe.output_concept_string_or_concept_code
+ result["output"] = pipe.output
# Add optional fields in expected order based on test cases
if pipe.nb_output is not None:
@@ -691,7 +679,7 @@ class PipelexInterpreter(BaseModel):
result: Dict[str, Any] = {
"type": pipe.type,
"definition": pipe.definition,
- "output": pipe.output_concept_string_or_concept_code,
+ "output": pipe.output,
}
PipelexInterpreter.add_common_pipe_fields(result, pipe)
@@ -703,7 +691,7 @@ class PipelexInterpreter(BaseModel):
result: Dict[str, Any] = {
"type": pipe.type,
"definition": pipe.definition,
- "output": pipe.output_concept_string_or_concept_code,
+ "output": pipe.output,
"function_name": pipe.function_name,
}
@@ -716,7 +704,7 @@ class PipelexInterpreter(BaseModel):
result: Dict[str, Any] = {
"type": pipe.type,
"definition": pipe.definition,
- "output": pipe.output_concept_string_or_concept_code,
+ "output": pipe.output,
}
PipelexInterpreter.add_common_pipe_fields(result, pipe)
@@ -724,20 +712,8 @@ class PipelexInterpreter(BaseModel):
# Add optional fields only if they have values
if pipe.img_gen_prompt:
result["img_gen_prompt"] = pipe.img_gen_prompt
- if pipe.imgg_handle:
- result["imgg_handle"] = pipe.imgg_handle
if pipe.aspect_ratio:
result["aspect_ratio"] = pipe.aspect_ratio
- if pipe.quality:
- result["quality"] = pipe.quality
- if pipe.nb_steps:
- result["nb_steps"] = pipe.nb_steps
- if pipe.guidance_scale:
- result["guidance_scale"] = pipe.guidance_scale
- if pipe.is_moderated is not None:
- result["is_moderated"] = pipe.is_moderated
- if pipe.safety_tolerance:
- result["safety_tolerance"] = pipe.safety_tolerance
if pipe.is_raw is not None:
result["is_raw"] = pipe.is_raw
if pipe.seed:
@@ -759,10 +735,10 @@ class PipelexInterpreter(BaseModel):
PipelexInterpreter.add_common_pipe_fields(result, pipe)
# Add output
- result["output"] = pipe.output_concept_string_or_concept_code
+ result["output"] = pipe.output
# Add required fields
- result["ocr_model"] = pipe.ocr_model
+ result["ocr"] = pipe.ocr
# Add optional fields
if pipe.page_images:
@@ -788,7 +764,7 @@ class PipelexInterpreter(BaseModel):
PipelexInterpreter.add_common_pipe_fields(result, pipe)
# Add output
- result["output"] = pipe.output_concept_string_or_concept_code
+ result["output"] = pipe.output
# Add steps
if pipe.steps:
@@ -802,7 +778,7 @@ class PipelexInterpreter(BaseModel):
result: Dict[str, Any] = {
"type": pipe.type,
"definition": pipe.definition,
- "output": pipe.output_concept_string_or_concept_code,
+ "output": pipe.output,
}
PipelexInterpreter.add_common_pipe_fields(result, pipe)
@@ -814,9 +790,7 @@ class PipelexInterpreter(BaseModel):
result["jinja2"] = pipe.jinja2
if pipe.prompting_style:
result["prompting_style"] = pipe.prompting_style
- # Only include template_category if it's not the default value
- if pipe.template_category and pipe.template_category.value != "llm_prompt":
- result["template_category"] = pipe.template_category
+ result["template_category"] = pipe.template_category
return result
@@ -864,7 +838,7 @@ class PipelexInterpreter(BaseModel):
result: Dict[str, Any] = {
"type": pipe.type,
"definition": pipe.definition,
- "output": pipe.output_concept_string_or_concept_code,
+ "output": pipe.output,
}
PipelexInterpreter.add_common_pipe_fields(result, pipe)
@@ -888,7 +862,7 @@ class PipelexInterpreter(BaseModel):
PipelexInterpreter.add_inputs_to_lines_if_exist(lines, pipe.inputs)
# Add output after inputs
- lines.append(f'output = "{pipe.output_concept_string_or_concept_code}"')
+ lines.append(f'output = "{pipe.output}"')
# Add steps array with proper spacing
if pipe.steps:
@@ -906,7 +880,7 @@ class PipelexInterpreter(BaseModel):
result: Dict[str, Any] = {
"type": pipe.type,
"definition": pipe.definition,
- "output": pipe.output_concept_string_or_concept_code,
+ "output": pipe.output,
"pipe_map": pipe.pipe_map,
}
@@ -930,7 +904,7 @@ class PipelexInterpreter(BaseModel):
result: Dict[str, Any] = {
"type": pipe.type,
"definition": pipe.definition,
- "output": pipe.output_concept_string_or_concept_code,
+ "output": pipe.output,
}
PipelexInterpreter.add_common_pipe_fields(result, pipe)
@@ -950,7 +924,7 @@ class PipelexInterpreter(BaseModel):
result: Dict[str, Any] = {
"type": pipe.type,
"definition": pipe.definition,
- "output": pipe.output_concept_string_or_concept_code,
+ "output": pipe.output,
"branch_pipe_code": pipe.branch_pipe_code,
}
diff --git a/pipelex/core/memory/working_memory.py b/pipelex/core/memory/working_memory.py
index a2efebb03..a430f6855 100644
--- a/pipelex/core/memory/working_memory.py
+++ b/pipelex/core/memory/working_memory.py
@@ -149,7 +149,6 @@ class WorkingMemory(BaseModel, ContextProviderAbstract):
self.remove_main_stuff()
self.add_new_stuff(name=name, stuff=stuff, aliases=[MAIN_STUFF_NAME])
log.verbose(f"Setting new main stuff {name}: {stuff.concept.code} = '{stuff.short_desc}'")
- log.verbose(stuff.content.rendered_plain())
else:
self.remove_alias_to_main_stuff()
self.set_stuff(name=MAIN_STUFF_NAME, stuff=stuff)
@@ -209,6 +208,7 @@ class WorkingMemory(BaseModel, ContextProviderAbstract):
@override
def get_typed_object_or_attribute(self, name: str, wanted_type: Optional[Type[Any]] = None) -> Any:
# TODO: Add unit tests for this method
+ # TODO: Refactor this method. In the python paradigm, we should not have those ".", but arrays with field names.
if "." in name:
parts = name.split(".", 1) # Split only at the first dot
base_name = parts[0]
@@ -228,7 +228,7 @@ class WorkingMemory(BaseModel, ContextProviderAbstract):
if stuff_content is not None and wanted_type is not None and not isinstance(stuff_content, wanted_type):
raise WorkingMemoryTypeError(
variable_name=name,
- message=f"Content at '{name}' is of type {type(stuff_content).__name__}, it should be {wanted_type.__name__}",
+ message=f"Content at '{name}' is of type '{type(stuff_content).__name__}', it should be '{wanted_type.__name__}'",
)
return stuff_content
@@ -238,7 +238,7 @@ class WorkingMemory(BaseModel, ContextProviderAbstract):
if wanted_type is not None and not isinstance(content, wanted_type):
raise WorkingMemoryTypeError(
variable_name=name,
- message=f"Content of '{name}' is of type {type(content).__name__}, it should be {wanted_type.__name__}",
+ message=f"Content of '{name}' is of type '{type(content).__name__}', it should be '{wanted_type.__name__}'",
)
return content
diff --git a/pipelex/core/memory/working_memory_factory.py b/pipelex/core/memory/working_memory_factory.py
index 78af56e79..a0f8f80f2 100644
--- a/pipelex/core/memory/working_memory_factory.py
+++ b/pipelex/core/memory/working_memory_factory.py
@@ -9,7 +9,7 @@ from pipelex.client.protocol import CompactMemory, ImplicitMemory
from pipelex.core.concepts.concept import ConceptBlueprint, SpecialDomain
from pipelex.core.concepts.concept_native import NativeConceptEnum
from pipelex.core.memory.working_memory import MAIN_STUFF_NAME, StuffDict, WorkingMemory
-from pipelex.core.pipes.pipe_input_spec import TypedNamedInputRequirement
+from pipelex.core.pipes.pipe_input import TypedNamedInputRequirement
from pipelex.core.stuffs.stuff import Stuff
from pipelex.core.stuffs.stuff_content import ImageContent, ListContent, PDFContent, StuffContent, TextContent
from pipelex.core.stuffs.stuff_factory import StuffFactory
@@ -22,7 +22,7 @@ class WorkingMemoryFactory(BaseModel):
def make_from_text(
cls,
text: str,
- concept_string: str = SpecialDomain.NATIVE.value + "." + NativeConceptEnum.TEXT.value,
+ concept_string: str = SpecialDomain.NATIVE + "." + NativeConceptEnum.TEXT,
name: Optional[str] = "text",
) -> WorkingMemory:
ConceptBlueprint.validate_concept_string(concept_string=concept_string)
@@ -38,7 +38,7 @@ class WorkingMemoryFactory(BaseModel):
def make_from_image(
cls,
image_url: str,
- concept_string: str = SpecialDomain.NATIVE.value + "." + NativeConceptEnum.IMAGE.value,
+ concept_string: str = SpecialDomain.NATIVE + "." + NativeConceptEnum.IMAGE,
name: Optional[str] = "image",
) -> WorkingMemory:
# TODO: validate that the concept is compatible with an image concept
@@ -54,7 +54,7 @@ class WorkingMemoryFactory(BaseModel):
def make_from_pdf(
cls,
pdf_url: str,
- concept_string: str = SpecialDomain.NATIVE.value + "." + NativeConceptEnum.PDF.value,
+ concept_string: str = SpecialDomain.NATIVE + "." + NativeConceptEnum.PDF,
name: Optional[str] = "pdf",
) -> WorkingMemory:
ConceptBlueprint.validate_concept_string(concept_string=concept_string)
@@ -106,7 +106,7 @@ class WorkingMemoryFactory(BaseModel):
continue
text_content = TextContent(text=content)
stuff_dict[name] = StuffFactory.make_stuff(
- concept=get_required_concept(concept_string=SpecialDomain.NATIVE.value + "." + NativeConceptEnum.TEXT.value),
+ concept=get_required_concept(concept_string=SpecialDomain.NATIVE + "." + NativeConceptEnum.TEXT),
content=text_content,
name=name,
code="",
@@ -168,7 +168,7 @@ class WorkingMemoryFactory(BaseModel):
__use_examples__ = True
__allow_none_optionals__ = False # Ensure Optional fields always get values
- return MockFactory.build() # type: ignore
+ return MockFactory.build(factory_use_construct=True) # type: ignore
else:
# Fallback to text content
return TextContent(text=f"DRY RUN: Mock content for '{requirement.variable_name}' ({requirement.concept.code})")
diff --git a/pipelex/core/pipes/pipe_abstract.py b/pipelex/core/pipes/pipe_abstract.py
index 31823d362..3a7948bea 100644
--- a/pipelex/core/pipes/pipe_abstract.py
+++ b/pipelex/core/pipes/pipe_abstract.py
@@ -1,12 +1,12 @@
from abc import ABC, abstractmethod
-from typing import List, Optional, Set, Type
+from typing import Any, List, Optional, Set, Type
from pydantic import BaseModel, ConfigDict, Field, field_validator
from pipelex.core.concepts.concept import Concept
from pipelex.core.memory.working_memory import WorkingMemory
from pipelex.core.pipes.pipe_blueprint import PipeBlueprint
-from pipelex.core.pipes.pipe_input_spec import PipeInputSpec
+from pipelex.core.pipes.pipe_input import PipeInputSpec
from pipelex.core.pipes.pipe_output import PipeOutput
from pipelex.core.pipes.pipe_run_params import PipeRunParams
from pipelex.exceptions import PipeStackOverflowError
@@ -16,6 +16,8 @@ from pipelex.pipeline.job_metadata import JobMetadata
class PipeAbstract(ABC, BaseModel):
model_config = ConfigDict(strict=True, extra="forbid")
+ category: Any # Any so that subclasses can put a Literal
+ type: Any # Any so that subclasses can put a Literal
code: str
domain: str
definition: Optional[str] = None
@@ -58,9 +60,16 @@ class PipeAbstract(ABC, BaseModel):
pass
@abstractmethod
- def needed_inputs(self) -> PipeInputSpec:
+ def needed_inputs(self, visited_pipes: Optional[Set[str]] = None) -> PipeInputSpec:
"""
- Return the inputs that are needed for the pipe to run. (Mostly the inputs of the pipe themselves)
+ Return the inputs that are needed for the pipe to run.
+
+ Args:
+ visited_pipes: Set of pipe codes currently being processed to prevent infinite recursion.
+ If None, starts recursion detection with an empty set.
+
+ Returns:
+ PipeInputSpec containing all needed inputs for this pipe
"""
pass
@@ -87,6 +96,7 @@ class PipeAbstract(ABC, BaseModel):
working_memory: WorkingMemory,
pipe_run_params: PipeRunParams,
output_name: Optional[str] = None,
+ print_intermediate_outputs: Optional[bool] = False,
) -> PipeOutput:
pass
diff --git a/pipelex/core/pipes/pipe_blueprint.py b/pipelex/core/pipes/pipe_blueprint.py
index 9179ad350..f86a22a1f 100644
--- a/pipelex/core/pipes/pipe_blueprint.py
+++ b/pipelex/core/pipes/pipe_blueprint.py
@@ -1,14 +1,23 @@
-from typing import Any, Dict, Optional, Union
+from typing import Any, Dict, List, Optional, Union
-from pydantic import BaseModel, Field, field_validator
+from pydantic import BaseModel, field_validator
from pipelex.core.concepts.concept_blueprint import ConceptBlueprint
from pipelex.core.pipes.exceptions import PipeBlueprintError
-from pipelex.core.pipes.pipe_input_spec_blueprint import InputRequirementBlueprint
+from pipelex.core.pipes.pipe_input_blueprint import InputRequirementBlueprint
from pipelex.tools.misc.string_utils import is_snake_case
from pipelex.types import StrEnum
+class AllowedPipeCategories(StrEnum):
+ PIPE_OPERATOR = "PipeOperator"
+ PIPE_CONTROLLER = "PipeController"
+
+ @classmethod
+ def value_list(cls) -> List[str]:
+ return [value for value in cls]
+
+
class AllowedPipeTypes(StrEnum):
# Pipe Operators
PIPE_FUNC = "PipeFunc"
@@ -22,30 +31,35 @@ class AllowedPipeTypes(StrEnum):
PIPE_PARALLEL = "PipeParallel"
PIPE_SEQUENCE = "PipeSequence"
+ @classmethod
+ def value_list(cls) -> List[str]:
+ return [value for value in cls]
-class PipeBlueprint(BaseModel):
- """Simple data container for pipe blueprint information.
-
- The 'type' field uses Any to avoid type override conflicts but is validated
- at runtime to ensure only valid pipe type values are allowed.
- """
+class PipeBlueprint(BaseModel):
+ category: Any
type: Any # TODO: Find a better way to handle this.
definition: Optional[str] = None
inputs: Optional[Dict[str, Union[str, InputRequirementBlueprint]]] = None
- output_concept_string_or_concept_code: str = Field(alias="output")
+ output: str
@field_validator("type", mode="after")
def validate_pipe_type(cls, value: Any) -> Any:
"""Validate that the pipe type is one of the allowed values."""
- allowed_types = [_type.value for _type in AllowedPipeTypes]
- if value not in allowed_types:
- raise PipeBlueprintError(f"Invalid pipe type '{value}'. Must be one of: {allowed_types}")
+ if value not in AllowedPipeTypes.value_list():
+ raise PipeBlueprintError(f"Invalid pipe type '{value}'. Must be one of: {AllowedPipeTypes.value_list()}")
+ return value
+
+ @field_validator("category", mode="after")
+ def validate_pipe_category(cls, value: Any) -> Any:
+ """Validate that the pipe category is one of the allowed values."""
+ if value not in AllowedPipeCategories.value_list():
+ raise PipeBlueprintError(f"Invalid pipe category '{value}'. Must be one of: {AllowedPipeCategories.value_list()}")
return value
- @field_validator("output_concept_string_or_concept_code", mode="before")
+ @field_validator("output", mode="before")
def validate_concept_string_or_concept_code(cls, output: str) -> str:
- ConceptBlueprint.validate_concept_string_or_concept_code(concept_string_or_concept_code=output)
+ ConceptBlueprint.validate_concept_string_or_concept_code(concept_string_or_code=output)
return output
@classmethod
diff --git a/pipelex/core/pipes/pipe_input_spec.py b/pipelex/core/pipes/pipe_input.py
similarity index 96%
rename from pipelex/core/pipes/pipe_input_spec.py
rename to pipelex/core/pipes/pipe_input.py
index 212254d37..db2ca2071 100644
--- a/pipelex/core/pipes/pipe_input_spec.py
+++ b/pipelex/core/pipes/pipe_input.py
@@ -35,11 +35,6 @@ PipeInputSpecRoot = Dict[str, InputRequirement]
class PipeInputSpec(RootModel[PipeInputSpecRoot]):
- """
- A PipeInputSpec is a dictionary of variable names and their corresponding concept codes.
- It's meant to hold the required input variables declared by a pipe.
- """
-
root: PipeInputSpecRoot = Field(default_factory=dict)
@field_validator("root", mode="wrap")
diff --git a/pipelex/core/pipes/pipe_input_spec_blueprint.py b/pipelex/core/pipes/pipe_input_blueprint.py
similarity index 92%
rename from pipelex/core/pipes/pipe_input_spec_blueprint.py
rename to pipelex/core/pipes/pipe_input_blueprint.py
index c66e417d8..bb1093194 100644
--- a/pipelex/core/pipes/pipe_input_spec_blueprint.py
+++ b/pipelex/core/pipes/pipe_input_blueprint.py
@@ -13,5 +13,5 @@ class InputRequirementBlueprint(BaseModel):
@field_validator("concept", mode="before")
@classmethod
def validate_concept_string(cls, concept_string: str) -> str:
- ConceptBlueprint.validate_concept_string_or_concept_code(concept_string_or_concept_code=concept_string)
+ ConceptBlueprint.validate_concept_string_or_concept_code(concept_string_or_code=concept_string)
return concept_string
diff --git a/pipelex/core/pipes/pipe_input_spec_factory.py b/pipelex/core/pipes/pipe_input_factory.py
similarity index 83%
rename from pipelex/core/pipes/pipe_input_spec_factory.py
rename to pipelex/core/pipes/pipe_input_factory.py
index 0caa6f8fa..c3f0b78e8 100644
--- a/pipelex/core/pipes/pipe_input_spec_factory.py
+++ b/pipelex/core/pipes/pipe_input_factory.py
@@ -2,14 +2,12 @@ from typing import Dict, List, Optional, Union
from pipelex.core.concepts.concept_blueprint import ConceptBlueprint
from pipelex.core.concepts.concept_factory import ConceptFactory
-from pipelex.core.pipes.pipe_input_spec import InputRequirement, PipeInputSpec
-from pipelex.core.pipes.pipe_input_spec_blueprint import InputRequirementBlueprint
+from pipelex.core.pipes.pipe_input import InputRequirement, PipeInputSpec
+from pipelex.core.pipes.pipe_input_blueprint import InputRequirementBlueprint
from pipelex.hub import get_concept_provider
class PipeInputSpecFactory:
- """Factory for creating PipeInputSpec instances with dependencies."""
-
@classmethod
def make_empty(cls) -> PipeInputSpec:
return PipeInputSpec(root={})
@@ -24,10 +22,10 @@ class PipeInputSpecFactory:
input_requirement_blueprint = InputRequirementBlueprint(concept=input_requirement_blueprint)
concept_string = input_requirement_blueprint.concept
- ConceptBlueprint.validate_concept_string_or_concept_code(concept_string_or_concept_code=concept_string)
+ ConceptBlueprint.validate_concept_string_or_concept_code(concept_string_or_code=concept_string)
input_domain_and_code = ConceptFactory.make_domain_and_concept_code_from_concept_string_or_concept_code(
domain=domain,
- concept_string_or_concept_code=concept_string,
+ concept_string_or_code=concept_string,
concept_codes_from_the_same_domain=concept_codes_from_the_same_domain,
)
diff --git a/pipelex/core/pipes/pipe_library.py b/pipelex/core/pipes/pipe_library.py
index cccb95632..9ece942fb 100644
--- a/pipelex/core/pipes/pipe_library.py
+++ b/pipelex/core/pipes/pipe_library.py
@@ -78,6 +78,14 @@ class PipeLibrary(RootModel[PipeLibraryRoot], PipeProviderAbstract):
def get_pipes_dict(self) -> Dict[str, PipeAbstract]:
return self.root
+ def remove_pipes_by_codes(self, pipe_codes: List[str]) -> None:
+ # TODO: We should create a separate library, that copies the original one, and then removes the pipes from it
+ # Then run the dry run + validation to see if removing those pipe has not broken any other pipe.
+ # If validated, it should update the real library.
+ for pipe_code in pipe_codes:
+ if pipe_code in self.root:
+ del self.root[pipe_code]
+
@override
def teardown(self) -> None:
self.root = {}
diff --git a/pipelex/core/pipes/specific_pipe.py b/pipelex/core/pipes/specific_pipe.py
new file mode 100644
index 000000000..b556ebe66
--- /dev/null
+++ b/pipelex/core/pipes/specific_pipe.py
@@ -0,0 +1,25 @@
+from pipelex.types import StrEnum
+
+
+class SpecificPipeCodesEnum(StrEnum):
+ CONTINUE = "continue"
+ # TODO: Implement the break pipe: It should enable to leave the current sequence.
+ # BREAK = "break"
+
+
+class SpecificPipe:
+ """Container for specific pipe codes and related utilities."""
+
+ @staticmethod
+ def is_continue(pipe_code: str) -> bool:
+ """Check if the pipe code is CONTINUE using match/case for exhaustive enum checking."""
+ try:
+ enum_value = SpecificPipeCodesEnum(pipe_code)
+ except ValueError:
+ return False
+
+ match enum_value:
+ case SpecificPipeCodesEnum.CONTINUE:
+ return True
+ # case SpecificPipeCodesEnum.BREAK: # Uncomment when BREAK is implemented
+ # return False
diff --git a/pipelex/core/registry_models.py b/pipelex/core/registry_models.py
index 91d15bfee..413e79663 100644
--- a/pipelex/core/registry_models.py
+++ b/pipelex/core/registry_models.py
@@ -16,7 +16,6 @@ from pipelex.core.stuffs.stuff_content import (
TextAndImagesContent,
TextContent,
)
-from pipelex.libraries.pipelines.meta.pipeline_draft import PipelexBundleBlueprint, PipelineDraft
from pipelex.pipe_controllers.batch.pipe_batch import PipeBatch
from pipelex.pipe_controllers.batch.pipe_batch_factory import PipeBatchFactory
from pipelex.pipe_controllers.condition.pipe_condition import PipeCondition
@@ -83,8 +82,6 @@ class PipelexRegistryModels(RegistryModels):
PDFContent,
TextAndImagesContent,
PageContent,
- PipelexBundleBlueprint,
- PipelineDraft,
]
EXPERIMENTAL: ClassVar[List[ModelType]] = [
diff --git a/pipelex/core/stuffs/stuff_factory.py b/pipelex/core/stuffs/stuff_factory.py
index 74555cbce..b2d202367 100644
--- a/pipelex/core/stuffs/stuff_factory.py
+++ b/pipelex/core/stuffs/stuff_factory.py
@@ -7,8 +7,7 @@ from pipelex.client.protocol import StuffContentOrData
from pipelex.core.concepts.concept import Concept
from pipelex.core.concepts.concept_blueprint import ConceptBlueprint
from pipelex.core.concepts.concept_factory import ConceptFactory
-from pipelex.core.concepts.concept_native import NATIVE_CONCEPTS_DATA, NativeConceptEnum
-from pipelex.core.domains.domain import SpecialDomain
+from pipelex.core.concepts.concept_native import NATIVE_CONCEPTS_DATA, NativeConceptEnum, NativeConceptManager
from pipelex.core.stuffs.stuff import Stuff
from pipelex.core.stuffs.stuff_content import (
ListContent,
@@ -43,7 +42,7 @@ class StuffFactory:
@classmethod
def make_from_str(cls, str_value: str, name: str) -> Stuff:
return cls.make_stuff(
- concept=ConceptFactory.make_native_concept(native_concept_data=NATIVE_CONCEPTS_DATA[NativeConceptEnum.TEXT]),
+ concept=ConceptFactory.make_native_concept(native_concept_data=NativeConceptManager.get_native_concept_data(NativeConceptEnum.TEXT)),
content=TextContent(text=str_value),
name=name,
)
@@ -174,9 +173,7 @@ class StuffFactory:
native_concept_class_names = [data.content_class_name for data in NATIVE_CONCEPTS_DATA.values()]
if concept_class_name in native_concept_class_names:
- concept = get_concept_provider().get_required_concept(
- concept_string=SpecialDomain.NATIVE.value + "." + concept_class_name.split("Content")[0]
- )
+ concept = get_concept_provider().get_native_concept(native_concept=NativeConceptEnum(concept_class_name.split("Content")[0]))
return cls.make_stuff(
concept=concept,
content=content,
@@ -226,7 +223,7 @@ class StuffFactory:
if not concept_code:
raise StuffFactoryError("Stuff content data dict is badly formed: no concept code")
content_value = stuff_content_dict["content"]
- if ConceptBlueprint.is_native_concept_string_or_concept_code(concept_string_or_concept_code=concept_code):
+ if NativeConceptManager.is_native_concept(concept_string_or_code=concept_code):
concept = ConceptFactory.make_native_concept(native_concept_data=NATIVE_CONCEPTS_DATA[NativeConceptEnum(concept_code)])
content = StuffContentFactory.make_stuff_content_from_concept_with_fallback(
concept=concept,
diff --git a/pipelex/create/build_blueprint.py b/pipelex/create/build_blueprint.py
deleted file mode 100644
index 8643fda34..000000000
--- a/pipelex/create/build_blueprint.py
+++ /dev/null
@@ -1,45 +0,0 @@
-from __future__ import annotations
-
-from typing import Optional
-
-from pipelex import log, pretty_print
-from pipelex.core.interpreter import PipelexInterpreter
-from pipelex.create.helpers import get_support_file
-from pipelex.libraries.pipelines.meta.pipeline_draft import PipelexBundleBlueprint
-from pipelex.pipeline.execute import execute_pipeline
-from pipelex.tools.misc.file_utils import save_text_to_path
-from pipelex.tools.misc.json_utils import save_as_json_to_path
-
-
-async def do_build_blueprint(
- domain: str,
- pipeline_name: str,
- requirements: str,
- output_path: Optional[str],
- validate: bool,
-) -> None:
- pipe_output = await execute_pipeline(
- pipe_code="build_blueprint",
- input_memory={
- "domain": domain,
- "pipeline_name": pipeline_name,
- "requirements": requirements,
- "draft_pipeline_rules": get_support_file(subpath="create/draft_pipelines.md"),
- "build_pipeline_rules": get_support_file(subpath="create/build_pipelines.md"),
- "create_structured_output_rules": get_support_file(subpath="create/structures.md"),
- },
- )
- pretty_print(pipe_output, title="Pipe Output")
-
- blueprint = pipe_output.main_stuff_as(content_type=PipelexBundleBlueprint)
- pipeline_draft = pipe_output.working_memory.get_stuff_as_str(name="pipeline_draft")
-
- # Save or display result
- output_path_base = output_path or "pipelex/libraries/pipelines/temp/gen_blueprint"
- draft_path = f"{output_path_base}_draft.md"
- save_text_to_path(text=pipeline_draft, path=draft_path, create_directory=True)
- rough_plx_path = f"{output_path_base}_rough.plx"
- save_text_to_path(text=PipelexInterpreter.make_plx_content(blueprint=blueprint), path=rough_plx_path)
- rough_json_path = f"{output_path_base}_rough.json"
- save_as_json_to_path(object_to_save=blueprint, path=rough_json_path)
- log.info(f"✅ Rough blueprint saved to '{output_path_base}'")
diff --git a/pipelex/create/draft_pipeline.py b/pipelex/create/draft_pipeline.py
deleted file mode 100644
index b5f5034ca..000000000
--- a/pipelex/create/draft_pipeline.py
+++ /dev/null
@@ -1,60 +0,0 @@
-from __future__ import annotations
-
-from typing import Optional
-
-import typer
-
-from pipelex import pretty_print
-from pipelex.create.helpers import get_support_file
-from pipelex.libraries.pipelines.meta.pipeline_draft import PipelineDraft
-from pipelex.pipeline.execute import execute_pipeline
-from pipelex.tools.misc.file_utils import save_text_to_path
-from pipelex.tools.misc.toml_utils import dict_to_toml, save_toml_to_path
-
-
-async def do_draft_pipeline_text(
- domain: str,
- pipeline_name: str,
- requirements: str,
- output_path: Optional[str],
-) -> None:
- pipe_output = await execute_pipeline(
- pipe_code="draft_pipeline_text",
- input_memory={
- "domain": domain,
- "pipeline_name": pipeline_name,
- "requirements": requirements,
- "rules": get_support_file(subpath="create/draft_pipelines.md"),
- },
- )
- draft_text = pipe_output.main_stuff_as_str
- pretty_print(draft_text, title="Pipeline Draft Text")
-
- # Save or display result
- output_path = output_path or "pipelex/libraries/pipelines/temp/pipeline_draft.md"
- save_text_to_path(text=draft_text, path=output_path)
- typer.echo(f"✅ Pipeline draft text saved to '{output_path}'")
-
-
-async def do_draft_pipeline(
- pipeline_name: str,
- requirements: str,
- output_path: Optional[str],
-) -> None:
- pipe_output = await execute_pipeline(
- pipe_code="draft_pipeline",
- input_memory={
- "pipeline_name": pipeline_name,
- "requirements": requirements,
- "draft_pipeline_rules": get_support_file(subpath="create/draft_pipelines.md"),
- },
- )
- draft = pipe_output.main_stuff_as(content_type=PipelineDraft)
- pretty_print(draft, title="Pipeline Draft")
- draft_dict = draft.smart_dump()
- pretty_print(dict_to_toml(data=draft_dict), title="Pipeline Draft (PLX)")
-
- # Save or display result
- output_path = output_path or "pipelex/libraries/pipelines/temp/generated_blueprint.plx"
- save_toml_to_path(data=draft_dict, path=output_path)
- typer.echo(f"✅ Blueprint saved to '{output_path}'")
diff --git a/pipelex/create/helpers.py b/pipelex/create/helpers.py
deleted file mode 100644
index 664f4df53..000000000
--- a/pipelex/create/helpers.py
+++ /dev/null
@@ -1,55 +0,0 @@
-from pathlib import Path
-from typing import Dict, List, Optional
-
-from pipelex.tools.misc.file_utils import load_text_from_path
-
-
-class PipelexSupportFiles:
- """Dictionary-like interface to access all files in .pipelex by their subpath."""
-
- def __init__(self, base_path: str = ".pipelex"):
- self.base_path = Path(base_path)
- self._file_cache: Dict[str, str] = {}
-
- def __getitem__(self, subpath: str) -> str:
- """Get file content by subpath (e.g., 'design_pipelines.md')."""
- if subpath not in self._file_cache:
- file_path = self.base_path / subpath
- if not file_path.exists():
- raise KeyError(f"File '{subpath}' not found in {self.base_path}")
- self._file_cache[subpath] = load_text_from_path(str(file_path))
- return self._file_cache[subpath]
-
- def __contains__(self, subpath: str) -> bool:
- """Check if a file exists by subpath."""
- file_path = self.base_path / subpath
- return file_path.exists() and file_path.is_file()
-
- def get(self, subpath: str, default: Optional[str] = None) -> Optional[str]:
- """Get file content by subpath with optional default."""
- try:
- return self[subpath]
- except KeyError:
- return default
-
- def keys(self) -> List[str]:
- """Get all available file subpaths."""
- if not self.base_path.exists():
- return []
-
- files: List[str] = []
- for file_path in self.base_path.rglob("*"):
- if file_path.is_file():
- subpath = str(file_path.relative_to(self.base_path))
- files.append(subpath)
- return sorted(files)
-
- def clear_cache(self) -> None:
- """Clear the internal file cache."""
- self._file_cache.clear()
-
-
-def get_support_file(subpath: str) -> str:
- """Get pipeline creation rules from design_pipelines.md."""
- support_files = PipelexSupportFiles()
- return support_files[subpath]
diff --git a/pipelex/hub.py b/pipelex/hub.py
index e2727e294..9cefd0ba8 100644
--- a/pipelex/hub.py
+++ b/pipelex/hub.py
@@ -6,9 +6,10 @@ from pipelex import log
from pipelex.cogt.content_generation.content_generator_protocol import (
ContentGeneratorProtocol,
)
-from pipelex.cogt.imgg.imgg_worker_abstract import ImggWorkerAbstract
+from pipelex.cogt.img_gen.img_gen_worker_abstract import ImgGenWorkerAbstract
from pipelex.cogt.inference.inference_manager_protocol import InferenceManagerProtocol
from pipelex.cogt.llm.llm_worker_abstract import LLMWorkerAbstract
+from pipelex.cogt.models.model_deck import ModelDeck
from pipelex.cogt.models.model_manager_abstract import ModelManagerAbstract
from pipelex.cogt.ocr.ocr_worker_abstract import OcrWorkerAbstract
from pipelex.core.concepts.concept import Concept
@@ -330,6 +331,10 @@ def get_models_manager() -> ModelManagerAbstract:
return get_pipelex_hub().get_required_models_manager()
+def get_model_deck() -> ModelDeck:
+ return get_models_manager().get_model_deck()
+
+
def get_plugin_manager() -> PluginManager:
return get_pipelex_hub().get_plugin_manager()
@@ -344,10 +349,10 @@ def get_llm_worker(
return get_inference_manager().get_llm_worker(llm_handle=llm_handle)
-def get_imgg_worker(
- imgg_handle: str,
-) -> ImggWorkerAbstract:
- return get_inference_manager().get_imgg_worker(imgg_handle=imgg_handle)
+def get_img_gen_worker(
+ img_gen_handle: str,
+) -> ImgGenWorkerAbstract:
+ return get_inference_manager().get_img_gen_worker(img_gen_handle=img_gen_handle)
def get_ocr_worker(
diff --git a/pipelex/libraries/library_manager.py b/pipelex/libraries/library_manager.py
index 724c39b1e..6a0ad2922 100644
--- a/pipelex/libraries/library_manager.py
+++ b/pipelex/libraries/library_manager.py
@@ -165,8 +165,22 @@ class LibraryManager(LibraryManagerAbstract):
return pipes
+ @override
+ def remove_from_blueprint(self, blueprint: PipelexBundleBlueprint) -> None:
+ if blueprint.pipe is not None:
+ self.pipe_library.remove_pipes_by_codes(pipe_codes=list(blueprint.pipe.keys()))
+
+ # Remove concepts (they may depend on domain)
+ if blueprint.concept is not None:
+ concept_codes_to_remove = [
+ ConceptFactory.construct_concept_string_with_domain(domain=blueprint.domain, concept_code=concept_code)
+ for concept_code in blueprint.concept.keys()
+ ]
+ self.concept_library.remove_concepts_by_codes(concept_codes=concept_codes_to_remove)
+
+ self.domain_library.remove_domain_by_code(domain_code=blueprint.domain)
+
def _load_domain_from_blueprint(self, blueprint: PipelexBundleBlueprint) -> Domain:
- """Create a Domain from blueprint."""
return DomainFactory.make_from_blueprint(
blueprint=DomainBlueprint(
code=blueprint.domain,
@@ -178,7 +192,6 @@ class LibraryManager(LibraryManagerAbstract):
)
def _load_concepts_from_blueprint(self, blueprint: PipelexBundleBlueprint) -> List[Concept]:
- """Create Concepts from blueprint."""
concepts: List[Concept] = []
if blueprint.concept is not None:
@@ -195,7 +208,6 @@ class LibraryManager(LibraryManagerAbstract):
return concepts
def _load_pipes_from_blueprint(self, blueprint: PipelexBundleBlueprint) -> List[PipeAbstract]:
- """Create Pipes from blueprint."""
pipes: List[PipeAbstract] = []
if blueprint.pipe is not None:
for pipe_name, pipe_blueprint in blueprint.pipe.items():
diff --git a/pipelex/libraries/library_manager_abstract.py b/pipelex/libraries/library_manager_abstract.py
index 21cbd33bc..37e77c21b 100644
--- a/pipelex/libraries/library_manager_abstract.py
+++ b/pipelex/libraries/library_manager_abstract.py
@@ -34,3 +34,7 @@ class LibraryManagerAbstract(ABC):
@abstractmethod
def load_from_blueprint(self, blueprint: PipelexBundleBlueprint) -> List[PipeAbstract]:
pass
+
+ @abstractmethod
+ def remove_from_blueprint(self, blueprint: PipelexBundleBlueprint) -> None:
+ pass
diff --git a/pipelex/libraries/pipelines/builder/builder.plx b/pipelex/libraries/pipelines/builder/builder.plx
new file mode 100644
index 000000000..d610ba2bf
--- /dev/null
+++ b/pipelex/libraries/pipelines/builder/builder.plx
@@ -0,0 +1,376 @@
+domain = "builder"
+definition = "Auto-generate a Pipelex bundle (concepts + pipes) from a short user brief."
+
+[concept]
+UserBrief = "A short, natural-language description of what the user wants."
+PlanDraftText = "Natural-language pipeline plan text describing sequences, inputs, outputs."
+PipelexBundleSpec = "A Pipelex bundle spec."
+PipeFailure = "Details of a single pipe failure during dry run."
+DryRunResult = "A result of a dry run of a pipelex bundle spec."
+DomainInformation = "A domain information object."
+
+# ────────────────────────────────────────────────────────────────────────────────
+# Main
+# ────────────────────────────────────────────────────────────────────────────────
+[pipe]
+[pipe.pipe_builder]
+type = "PipeSequence"
+description = "This pipe is going to be the entry point for the builder. It will take a UserBrief and return a PipelexBundleSpec."
+inputs = { brief = "UserBrief" }
+output = "PipelexBundleSpec"
+steps = [
+ { pipe = "draft_planning_text", result = "plan_draft" },
+ { pipe = "parallel_draft_to_specs" },
+ { pipe = "materialize_concept_spec_drafts", result = "concept_spec_drafts" },
+ { pipe = "materialize_pipe_signatures", result = "pipe_signatures" },
+ { pipe = "pipe_builder_domain_information", result = "domain_information" },
+ { pipe = "build_concept_spec", batch_over = "concept_spec_drafts", batch_as = "concept_spec_draft", result = "concept_specs" },
+ { pipe = "create_pipes_from_signatures", batch_over = "pipe_signatures", batch_as = "pipe_signature", result = "pipe_specs" },
+ { pipe = "compile_in_pipelex_bundle_spec", result = "pipelex_bundle_spec" }
+ { pipe = "validate_pipelex_bundle_spec", result = "pipelex_bundle_spec" }
+]
+
+[pipe.pipe_builder_domain_information]
+type = "PipeLLM"
+description = "Turn the brief into a DomainInformation object."
+inputs = { brief = "UserBrief" }
+output = "DomainInformation"
+prompt_template = """
+Based on the brief output the "domain" of this pipe, and a definition of what it would represent.
+
+Brief:
+@brief
+
+For example, if the pipe is about generating a compliance matrix out of a RFP, the domain would be "rfp_compliance_matrix"...
+It should be not more than 4 words, in snake_case.
+For the definition, i would like to see a short description of what the bundle would represent.
+"""
+
+# ────────────────────────────────────────────────────────────────────────────────
+# STAGE 1 — plan (natural language pseudo-code, but explicit about IO + sequencing)
+# ────────────────────────────────────────────────────────────────────────────────
+
+[pipe.draft_planning_text]
+type = "PipeLLM"
+description = "Turn the brief into a pseudo-code plan describing controllers, pipes, their inputs/outputs."
+inputs = { brief = "UserBrief" }
+output = "PlanDraftText"
+llm = "llm_to_engineer"
+prompt_template = """
+Return a PlanDraftText that narrates the pipeline as pseudo-steps (no code):
+- Explicitly describe where a sequence/parallel/condition/batch is used
+- For each pipe: state the pipe's description, inputs (by name), and outputs (by name),
+- Keep it coherent: children pipes referenced by parent sequences must be named consistently
+
+
+Here is a description of the pipes:
+We have pipe controllers:
+- PipeLLM: A pipe that uses an LLM to generate a text, or a structured object. It is a vision LLM that can read images.
+ CRITICAL: When extracting MULTIPLE items (articles, employees, products), use multiple_output = true with SINGULAR concepts!
+ - Create concept "Article" (not "Articles") with fields "item_name", "quantity" (not "item_names", "quantities")
+ - Then set multiple_output = true to get a list of Article objects
+- PipeSequence: A pipe that executes a sequence of pipes: It needs to reference the pipes it will execute.
+- PipeParallel: A pipe that executes a few pipes in parallel. It needs to reference the pipes it will execute. The results of each pipe will be in the working memory. The output MUST BE "Dynamic".
+- PipeCondition: A pipe that based on a specific condition, branches to a specific pipe. You have to explain what the expression of the condition is,
+ and what the different pipes are that can be executed based on the condition. It needs to reference the pipes it will execute.
+- PipeBatch: A pipe that executes a batch of pipes in parallel. It needs to reference the pipe it will execute.
+- PipeImgGen: A pipe that uses an LLM to generate an image. VERY IMPORTANT: IF YOU DECIDE TO CREATE A PIPEIMGEN, YOU ALSO HAVE TO CREATE A PIPELLM THAT WILL WRITE THE PROMPT, AND THAT NEEDS TO PRECEED THE PIPEIMGEN, based on the necessary elements.
+That means that in the MAIN pipeline, the prompt should NOT be an input. It should be a step that generates the prompt.
+- PipeOcr: A pipe that uses an OCR technology to extract text from an image.
+VERY IMPORTANT: THE INPUT OF THE PIPEOCR MUST BE NAMED "ocr_input" and it must be either an image or a pdf or a concept which refines one of them.
+Usually there is no need to rename a variable or create a new concept, just call it ocr_input, and we'll use the basic PDF concept.
+Thats mean that in the sequence, the input should be ocr_input.
+So the input of PipeOcr is {ocr_input: "PDF"} or {ocr_input: "Image"} or a concept which refines one of them.
+
+Be very detailed, process by steps.
+
+Brief:
+@brief
+
+LIMIT TO 10 DIFFERENT PIPES FOR NOW
+"""
+
+# ────────────────────────────────────────────────────────────────────────────────
+# STAGE 2 — textual specs (still TEXT, not structured objects yet)
+# ────────────────────────────────────────────────────────────────────────────────
+[pipe.parallel_draft_to_specs]
+type = "PipeParallel"
+description = "Generate ConceptSpecDraftsText and PipeSignaturesText in parallel from plan draft."
+inputs = { plan_draft = "PlanDraftText", brief = "UserBrief" }
+output = "Dynamic"
+parallels = [
+ { pipe = "draft_to_conceptspecs_text", result = "concept_spec_drafts_text" },
+ { pipe = "draft_to_pipesignatures_text", result = "pipe_signatures_text" },
+]
+
+[pipe.draft_to_conceptspecs_text]
+type = "PipeLLM"
+description = "From PlanDraftText (+ brief), extract ConceptSpecsText (codes, descriptions, structure hints) in TEXT."
+inputs = { plan_draft = "PlanDraftText", brief = "UserBrief" }
+output = "Text"
+llm = "llm_to_engineer"
+prompt_template = """
+You will receive a plan for a Pipelex pipeline.
+Each pipeline will take inputs and output. Those inputs/output are represented as concepts.
+
+Return ConceptSpecsText capturing all concepts used in the plan:
+- Use PascalCase for concept codes
+- Provide a short description per concept
+- Include structure hints as plain text (fields, types) IF IT IS needed.
+
+CRITICAL RULE: Concepts must represent SINGLE ENTITIES, never collections!
+- Create "Article" not "Articles"
+- Create "Employee" not "Employees"
+- Use SINGULAR field names: "item_name" not "item_names", "quantity" not "quantities"
+- If you need multiple items, the PipeLLM will use multiple_output=true to generate a LIST of the concept
+
+If you need structure for your concept, to isolate/extract some precise information, assign a structure:
+Here is how the structure as to be described:
+A dict with:
+- key: the field name in snake_case (ALWAYS SINGULAR)
+- value: a dict with:
+ - definition: the definition of the field, in natural language
+ - type: the type of the field (text, integer, boolean, number)
+ - required: whether the field is required
+ - default_value: the default value of the field
+
+You can have multiple fields if needed, but each field should represent a single value.
+
+Otherwise, there are native concepts that you can use:
+If the concept you want to create is JUST a text, assign "Text" to the 'refines' field, and no structure field.
+If the concept you want to create is JUST an image, assign "Image" to the 'refines' field, and no structure field.
+If the concept you want to create is JUST a PDF, assign "PDF" to the 'refines' field, and no structure field.
+If the concept you want to create is JUST a Number, assign "Number" to the 'refines' field, and no structure field.
+
+DO NOT redefine native concepts:
+- Text
+- Image
+- PDF
+- Number
+- Page
+If you need one of these, you will later on use them, but you should NOT REDEFINE THEM.
+Plan:
+@plan_draft
+
+Brief:
+@brief
+
+Remember: Create concepts for SINGLE entities. Lists are handled by PipeLLM with multiple_output=true.
+"""
+
+[pipe.draft_to_pipesignatures_text]
+type = "PipeLLM"
+description = "From PlanDraftText (+ brief), extract PipeSignaturesText in TEXT."
+inputs = { plan_draft = "PlanDraftText", brief = "UserBrief" }
+output = "Text"
+llm = "llm_to_engineer"
+prompt_template = """
+Return PipeSignaturesText listing every pipe to build:
+- For each pipe: give a unique snake_case pipe_code, type, definition, inputs (by concept code/name), output, and important_features
+- Controller pipes must reference children by their codes consistently
+- The Pipe Controllers, if they mention pipes, they should always mention existing pipes.
+- Add as much details as possible for the description.
+
+Here are the ESSENTIAL features for each pipe type that should be included in important_features (only include these key ones):
+
+**PipeLLM**: A pipe that uses an LLM to generate a text, or a structured object. It is a vision LLM that can read images.
+The inputs of the PipeLLM should be:
+The variables tagged in the prompt template (with $ or @). If there are no variables, the inputs should be empty.
+The ouput should be the concept code of the output
+- prompt_template: The prompt template with variable substitution ($ for inline, @ for blocks)
+- multiple_output: true if generating multiple number of outputs: That means it will output a LIST of the CONCEPT!
+
+CRITICAL RULE FOR PIPELLM:
+- If extracting MULTIPLE items (like multiple articles, employees, products), use multiple_output = true
+- The concept should represent ONE SINGLE item (Article, Employee, Product)
+- DO NOT create concepts with plural field names like "item_names", "quantities"
+- Instead: use multiple_output = true with singular concept fields like "item_name", "quantity"
+- Example: To extract multiple articles, create concept "Article" with fields "item_name", "quantity", then use multiple_output = true
+
+**PipeSequence**: A pipe that executes a sequence of pipes: It needs to reference the pipes it will execute.
+The inputs of the PipeSequence should be all the necessary inputs in the below steps, and the inputs that are NOT generated by intermediate steps.
+The output should be the concept code of the output of the last step.
+- steps: List of pipe codes to execute in order, with result names
+- Each step format: {"pipe": "pipe_code", "result": "result_name"}
+- Can include batch operations: {"pipe": "pipe_code", "batch_over": "list_input", "batch_as": "item_name", "result": "result_name"}
+
+**PipeParallel**: A pipe that executes a few pipes in parallel. It needs to reference the pipes it will execute.
+The inputs of the PipeParallel should be all the necessary inputs in the below steps
+The output should be the concept code of the output of the last step.
+The results of each pipe will be in the working memory.
+- parallels: List of pipes to execute in parallel
+- Each parallel format: {"pipe": "pipe_code", "result": "result_name"}
+
+**PipeCondition**: A pipe that based on a specific condition, branches to a specific pipe. You have to explain what the expression of the condition is,
+ and what the different pipes are that can be executed based on the condition. It needs to reference the pipes it will execute.
+The inputs of the PipeCondition should be all the necessary inputs in the below steps
+The output should be the concept code of the output of all the steps, except if the outputs are different, then its "Dynamic"
+- expression: Direct expression to evaluate (e.g., "task_result.status")
+- pipe_map: Dictionary mapping condition results to pipe codes (e.g., {"completed": "success_pipe", "failed": "failure_pipe"})
+- default_pipe_code: Fallback pipe when no conditions match
+
+**PipeBatch**: A pipe that executes a batch of pipes in parallel. It needs to reference the pipe it will execute.
+- branch_pipe_code: The pipe code to execute for each item
+- input_list_name: Name of the list to iterate over
+- input_item_name: Name for individual items within each execution
+
+**PipeImgGen**: A pipe that uses an LLM to generate an image.
+The inputs of the PipeImgGen should be: {prompt: ImgGenPrompt}
+The output should be the concept code that refines Image.
+- img_gen_prompt: Static prompt for image generation (if using static prompt)
+- nb_output: Number of images to generate (default 1)
+VERY IMPORTANT: IF YOU DECIDE TO CREATE A PIPEIMGEN, YOU ALSO HAVE TO CREATE A PIPELLM THAT WILL WRITE THE PROMPT, AND THAT NEEDS TO PRECEED THE PIPEIMGEN, based on the necessary elements.
+THERFORE, the OUTPUT OF THIS PIPELLM should be a VARIABLE NAMED "prompt" that will be used as input for the PipeImgGen.
+That means that in the MAIN pipeline, the prompt should NOT be an input. It should be a step that generates the prompt.
+
+**PipeOcr**: A pipe that uses an LLM to extract text from an image.
+- The INPUTS of this pipe is only "ocr_input" and it must be either an image or a pdf or a concept which refines one of them.
+- Usually there is no need to rename a variable or create a new concept, just call it ocr_input, and we'll use the basic PDF concept. Thats mean that in the sequence, the input should be ocr_input.
+So the input of PipeOcr is {ocr_input: "PDF"} or {ocr_input: "Image"} or a concept which refines one of them.
+
+**PipeFunc**: A pipe that executes a custom Python function.
+- function_name: Name of the Python function to call
+
+**PipeJinja2**: A pipe that uses Jinja2 to render a template.
+- jinja2: Raw Jinja2 template string OR
+- jinja2_name: Name reference to a template (use one or the other)
+
+Plan:
+@plan_draft
+
+Brief:
+@brief
+
+No more than 10 PipeSignatures
+"""
+
+# ────────────────────────────────────────────────────────────────────────────────
+# STAGE 3 — materialize: TEXT → real objects (ConceptSpec[], PipeSignature[])
+# ────────────────────────────────────────────────────────────────────────────────
+
+[pipe.materialize_concept_spec_drafts]
+type = "PipeLLM"
+description = "Turn ConceptSpecsText into ConceptSpec objects."
+inputs = { concept_spec_drafts_text = "Text", brief = "UserBrief" }
+output = "concept.ConceptSpecDraft"
+multiple_output = true
+llm = "llm_to_engineer"
+prompt_template = """
+Materialize ConceptSpec objects from the ConceptSpecsText.
+Do not change the information in the input. Just organize the information
+
+ConceptSpecs:
+@concept_spec_drafts_text
+
+Brief:
+@brief
+
+LIMIT TO A MAXIMUM OF 5 fields for now
+"""
+
+[pipe.materialize_pipe_signatures]
+type = "PipeLLM"
+description = "Turn PipeSignaturesText into PipeSignature objects that reference the ConceptSpec objects."
+inputs = { pipe_signatures_text = "Text", concept_spec_drafts = "concept.ConceptSpecDraft", brief = "UserBrief" }
+output = "pipe.PipeSignature"
+multiple_output = true
+llm = "llm_to_engineer"
+prompt_template = """
+Materialize PipeSignature objects from the PipeSignaturesText.
+- pipe_code MUST be snake_case
+- inputs must be a Dict[str, ConceptSpecDraft] referencing the provided ConceptSpecDraft objects. If Its the concept itself, use the concept code in PascalCase.
+- output must be a ConceptSpec from the provided set. If Its the concept itself, use the concept code in PascalCase.
+- important_features must be a Dict containing the pipe-specific features mentioned in the text
+
+VERY IMPORTANT: A pipe has inputs, and an output. The inputs are a dict of keys in snake_case, corresponding to the variables names in the working memory, and the values are the concept codes in PascalCase.
+The output is a concept code in PascalCase.
+The field "result" is corresponding to the name of the result of the pipe. It will be used in the inputs of the next pipes.
+It is important that they link each other in the right way.
+
+The output concept should be a concepts should be in PascalCase
+
+IMPORTANT:
+- THE MAIN PIPE SHOULD CONTAIN IN ITS NAME "main_pipeline"
+- IF THERE IS A PipeIMG, VERIFIES THAT THE INPUT PROMPT IS ACTUALLY GENERATED BY A PIPELLM BEFORE THE PIPEIMG.
+THIS PIPELLM SHOULD NAME THE RESULT OF ITS PIPE "prompt".
+
+PipeSignatures:
+@pipe_signatures_text
+
+ConceptSpecDrafts:
+@concept_spec_drafts
+
+Brief:
+@brief
+
+No more than 10 PipeSignatures
+"""
+
+[pipe.compile_in_pipelex_bundle_spec]
+type = "PipeFunc"
+description = "Compile the pipelex bundle spec."
+inputs = { pipe_specs = "PipeSpec", concept_specs = "ConceptSpec" }
+output = "PipelexBundleSpec"
+function_name = "compile_in_pipelex_bundle_spec"
+
+[pipe.validate_pipelex_bundle_spec]
+type = "PipeSequence"
+description = "Validate the pipelex bundle spec with iterative fixing."
+inputs = { pipelex_bundle_spec = "PipelexBundleSpec" }
+output = "PipelexBundleSpec"
+steps = [
+ { pipe = "validate_dry_run", result = "failed_pipes" },
+ { pipe = "check_validation_status", result = "validation_status" },
+ { pipe = "handle_validation_result", result = "pipelex_bundle_spec" }
+]
+
+[pipe.check_validation_status]
+type = "PipeJinja2"
+description = "Check if validation failed by examining if failed_pipes list is empty."
+inputs = { failed_pipes = "PipeFailure" }
+output = "Text"
+jinja2 = "{% if failed_pipes.content.items|length > 0 %}FAILURE{% else %}SUCCESS{% endif %}"
+
+[pipe.handle_validation_result]
+type = "PipeCondition"
+description = "Handle validation result - continue if success or fix failures once."
+inputs = { pipelex_bundle_spec = "PipelexBundleSpec", failed_pipes = "PipeFailure", validation_status = "Text" }
+output = "PipelexBundleSpec"
+expression = "validation_status.text"
+
+[pipe.handle_validation_result.pipe_map]
+SUCCESS = "continue"
+FAILURE = "fix_failing_pipes_once"
+
+[pipe.fix_failing_pipes_once]
+type = "PipeSequence"
+description = "Fix failing pipes once and return the result."
+inputs = { pipelex_bundle_spec = "PipelexBundleSpec", failed_pipes = "PipeFailure" }
+output = "PipelexBundleSpec"
+steps = [
+ { pipe = "fix_failing_pipe", batch_over = "failed_pipes", batch_as = "failed_pipe", result = "fixed_pipes" },
+ { pipe = "reconstruct_bundle_with_all_fixes", result = "pipelex_bundle_spec" },
+ { pipe = "validate_pipelex_bundle_spec", result = "pipelex_bundle_spec" }
+]
+
+[pipe.validate_dry_run]
+type = "PipeFunc"
+description = "Validate the pipelex bundle spec and return only failed pipes."
+inputs = { pipelex_bundle_spec = "PipelexBundleSpec" }
+output = "PipeFailure"
+function_name = "validate_dry_run"
+
+[pipe.continue]
+type = "PipeJinja2"
+description = "Continue with successful validation - return the bundle unchanged."
+inputs = { pipelex_bundle_spec = "PipelexBundleSpec" }
+output = "PipelexBundleSpec"
+jinja2 = "{{ pipelex_bundle_spec }}"
+
+[pipe.reconstruct_bundle_with_all_fixes]
+type = "PipeFunc"
+description = "Reconstruct the bundle spec with all the fixed pipes."
+inputs = { pipelex_bundle_spec = "PipelexBundleSpec", fixed_pipes = "Dynamic" }
+output = "PipelexBundleSpec"
+function_name = "reconstruct_bundle_with_all_fixes"
+
diff --git a/pipelex/libraries/pipelines/builder/builder.py b/pipelex/libraries/pipelines/builder/builder.py
new file mode 100644
index 000000000..89b7bc6da
--- /dev/null
+++ b/pipelex/libraries/pipelines/builder/builder.py
@@ -0,0 +1,282 @@
+from typing import Annotated, Dict, List, Optional, Union, cast
+
+from pydantic import ConfigDict, Field, ValidationError, field_validator
+
+from pipelex.core.bundles.pipelex_bundle_blueprint import PipeBlueprintUnion, PipelexBundleBlueprint
+from pipelex.core.concepts.concept_blueprint import ConceptBlueprint
+from pipelex.core.domains.domain_blueprint import DomainBlueprint
+from pipelex.core.memory.working_memory import WorkingMemory
+from pipelex.core.stuffs.stuff_content import ListContent, StructuredContent
+from pipelex.hub import get_library_manager
+from pipelex.libraries.pipelines.builder.concept.concept_spec import ConceptSpec
+from pipelex.libraries.pipelines.builder.pipe.pipe_batch_spec import PipeBatchSpec
+from pipelex.libraries.pipelines.builder.pipe.pipe_condition_spec import PipeConditionSpec
+from pipelex.libraries.pipelines.builder.pipe.pipe_func_spec import PipeFuncSpec
+from pipelex.libraries.pipelines.builder.pipe.pipe_img_spec import PipeImgGenSpec
+from pipelex.libraries.pipelines.builder.pipe.pipe_jinja2_spec import PipeJinja2Spec
+from pipelex.libraries.pipelines.builder.pipe.pipe_llm_spec import PipeLLMSpec
+from pipelex.libraries.pipelines.builder.pipe.pipe_ocr_spec import PipeOcrSpec
+from pipelex.libraries.pipelines.builder.pipe.pipe_parallel_spec import PipeParallelSpec
+from pipelex.libraries.pipelines.builder.pipe.pipe_sequence_spec import PipeSequenceSpec
+from pipelex.libraries.pipelines.builder.pipe.pipe_signature import PipeSignature
+from pipelex.pipe_works.pipe_dry import dry_run_pipes
+from pipelex.tools.typing.pydantic_utils import format_pydantic_validation_error
+from pipelex.types import StrEnum
+
+
+class DomainInformation(StructuredContent):
+ domain: str = Field(description="The domain of the pipeline library.")
+ definition: str = Field(description="The definition of the pipeline library.")
+
+
+class PipeBuilderError(Exception):
+ pass
+
+
+class PipelexBundleSpecDraft(StructuredContent):
+ """Complete spec of a pipeline library TOML file."""
+
+ domain: str = Field(description="The domain of the pipeline library.")
+ definition: str = Field(description="The definition of the pipeline library.")
+
+ concept: Dict[str, ConceptSpec] = Field(default_factory=dict, description="The concepts of the pipeline library.")
+
+ pipe: Dict[str, PipeSignature] = Field(default_factory=dict, description="The pipes of the pipeline library.")
+
+
+PipeSpecUnion = Annotated[
+ Union[
+ # Pipe operators
+ PipeFuncSpec,
+ PipeImgGenSpec,
+ PipeJinja2Spec,
+ PipeLLMSpec,
+ PipeOcrSpec,
+ # Pipe controllers
+ PipeBatchSpec,
+ PipeConditionSpec,
+ PipeParallelSpec,
+ PipeSequenceSpec,
+ ],
+ Field(discriminator="type"),
+]
+
+
+class PipelexBundleSpec(StructuredContent):
+ """Complete spec of a Pipelex bundle TOML definition.
+
+ Represents the top-level structure of a Pipelex bundle, which defines a domain
+ with its concepts, pipes, and configuration. Bundles are the primary unit of
+ organization for Pipelex workflows, loaded from TOML files.
+
+ Attributes:
+ domain: The domain identifier for this bundle in snake_case format.
+ Serves as the namespace for all concepts and pipes within.
+ definition: Natural language description of the pipeline's purpose and functionality.
+ system_prompt: Default system prompt applied to all LLM pipes in the bundle
+ unless overridden at the pipe level.
+ system_prompt_to_structure: System prompt specifically for output structuring
+ operations across the bundle.
+ prompt_template_to_structure: Template for structuring prompts used in output
+ formatting operations.
+ concept: Dictionary of concept definitions used in this domain. Keys are concept
+ codes in PascalCase format, values are ConceptBlueprint instances or
+ string references to existing concepts.
+ pipe: Dictionary of pipe definitions for data transformation. Keys are pipe
+ codes in snake_case format, values are specific pipe spec types
+ (PipeLLM, PipeImgGen, PipeSequence, etc.).
+
+ Validation Rules:
+ 1. Domain must be in valid snake_case format.
+ 2. Concept keys must be in PascalCase format.
+ 3. Pipe keys must be in snake_case format.
+ 4. Extra fields are forbidden (strict mode).
+ 5. Pipe types must match their blueprint discriminator.
+ """
+
+ model_config = ConfigDict(extra="forbid")
+
+ domain: str
+ definition: Optional[str] = None
+ system_prompt: Optional[str] = None
+ system_prompt_to_structure: Optional[str] = None
+ prompt_template_to_structure: Optional[str] = None
+
+ concept: Optional[Dict[str, Union[ConceptSpec, str]]] = Field(default_factory=dict)
+
+ pipe: Optional[Dict[str, PipeSpecUnion]] = Field(default_factory=dict)
+
+ @field_validator("domain", mode="before")
+ @classmethod
+ def validate_domain_syntax(cls, domain: str) -> str:
+ DomainBlueprint.validate_domain_code(code=domain)
+ return domain
+
+ def to_blueprint(self) -> PipelexBundleBlueprint:
+ concept: Optional[Dict[str, Union[ConceptBlueprint, str]]] = None
+
+ if self.concept:
+ concept = {}
+ for concept_code, concept_blueprint in self.concept.items():
+ if isinstance(concept_blueprint, ConceptSpec):
+ concept[concept_code] = concept_blueprint.to_blueprint()
+ else:
+ concept[concept_code] = ConceptBlueprint(definition=concept_code, structure=concept_blueprint)
+
+ pipe: Optional[Dict[str, PipeBlueprintUnion]] = None
+ if self.pipe:
+ pipe = {}
+ for pipe_code, pipe_blueprint in self.pipe.items():
+ pipe[pipe_code] = pipe_blueprint.to_blueprint()
+
+ return PipelexBundleBlueprint(
+ domain=self.domain,
+ definition=self.definition,
+ prompt_template_to_structure=self.prompt_template_to_structure,
+ system_prompt=self.system_prompt,
+ system_prompt_to_structure=self.system_prompt_to_structure,
+ pipe=pipe,
+ concept=concept,
+ )
+
+
+# TODO: Put this in a factory. Investigate why it is necessary.
+def _convert_pipe_spec(pipe_spec: PipeSpecUnion) -> PipeSpecUnion:
+ pipe_type_to_class: Dict[str, type] = {
+ "PipeFunc": PipeFuncSpec,
+ "PipeImgGen": PipeImgGenSpec,
+ "PipeJinja2": PipeJinja2Spec,
+ "PipeLLM": PipeLLMSpec,
+ "PipeOcr": PipeOcrSpec,
+ "PipeBatch": PipeBatchSpec,
+ "PipeCondition": PipeConditionSpec,
+ "PipeParallel": PipeParallelSpec,
+ "PipeSequence": PipeSequenceSpec,
+ }
+
+ pipe_class = pipe_type_to_class.get(pipe_spec.type)
+ if pipe_class is None:
+ msg = f"Unknown pipe type: {pipe_spec.type}"
+ raise PipeBuilderError(msg)
+ return cast(PipeSpecUnion, pipe_class(**pipe_spec.model_dump(serialize_as_any=True)))
+
+
+async def compile_in_pipelex_bundle_spec(working_memory: WorkingMemory) -> PipelexBundleSpec:
+ """Construct a PipelexBundleSpec from working memory containing concept and pipe blueprints.
+
+ Args:
+ working_memory: WorkingMemory containing concept_blueprints and pipe_blueprints stuffs.
+
+ Returns:
+ PipelexBundleSpec: The constructed pipeline spec.
+ """
+ # The working memory actually contains ConceptSpec objects (not ConceptSpecDraft)
+ # but they may have been deserialized incorrectly
+ concept_specs = working_memory.get_stuff_as_list(
+ name="concept_specs",
+ item_type=ConceptSpec,
+ )
+
+ pipe_specs = cast(ListContent[PipeSpecUnion], working_memory.get_stuff(name="pipe_specs").content)
+ domain_information = working_memory.get_stuff_as(name="domain_information", content_type=DomainInformation)
+
+ # Properly validate and reconstruct concept specs to ensure proper Pydantic validation
+ validated_concepts: Dict[str, Union[ConceptSpec, str]] = {}
+ for concept_spec in concept_specs.items:
+ try:
+ # Re-create the ConceptSpec to ensure proper Pydantic validation
+ # This handles any serialization/deserialization issues from working memory
+ validated_concept = ConceptSpec(**concept_spec.model_dump(serialize_as_any=True))
+ validated_concepts[validated_concept.the_concept_code] = validated_concept
+ except ValidationError as exc:
+ msg = f"Failed to validate concept spec {concept_spec.the_concept_code}: {format_pydantic_validation_error(exc)}"
+ raise PipeBuilderError(msg) from exc
+
+ return PipelexBundleSpec(
+ domain=domain_information.domain,
+ definition=domain_information.definition,
+ concept=validated_concepts,
+ pipe={pipe_spec.the_pipe_code: _convert_pipe_spec(pipe_spec) for pipe_spec in pipe_specs.items},
+ )
+
+
+class DryRunStatus(StrEnum):
+ SUCCESS = "SUCCESS"
+ FAILURE = "FAILURE"
+
+
+class ValidateDryRunError(Exception):
+ """Raised when validating the dry run of a pipelex bundle blueprint."""
+
+ pass
+
+
+class PipeFailure(StructuredContent):
+ """Details of a single pipe failure during dry run."""
+
+ pipe: PipeSpecUnion = Field(description="The failing pipe spec blueprint with pipe code")
+ error_message: str = Field(description="The error message for this pipe")
+
+
+class DryRunResult(StructuredContent):
+ """A result of a dry run of a pipelex bundle blueprint."""
+
+ status: DryRunStatus
+ failed_pipes: List[PipeFailure] = Field(default_factory=list, description="List of pipes that failed during dry run")
+
+
+async def validate_dry_run(working_memory: WorkingMemory) -> ListContent[PipeFailure]:
+ pipelex_bundle_spec = working_memory.get_stuff_as(name="pipelex_bundle_spec", content_type=PipelexBundleSpec)
+ pipelex_bundle_blueprint = pipelex_bundle_spec.to_blueprint()
+
+ library_manager = get_library_manager()
+
+ pipes = library_manager.load_from_blueprint(blueprint=pipelex_bundle_blueprint)
+ dry_run_result = await dry_run_pipes(pipes=pipes, raise_on_failure=False)
+ library_manager.remove_from_blueprint(blueprint=pipelex_bundle_blueprint)
+
+ pipe_type_to_spec_class = {
+ "PipeFunc": PipeFuncSpec,
+ "PipeImgGen": PipeImgGenSpec,
+ "PipeJinja2": PipeJinja2Spec,
+ "PipeLLM": PipeLLMSpec,
+ "PipeOcr": PipeOcrSpec,
+ "PipeBatch": PipeBatchSpec,
+ "PipeCondition": PipeConditionSpec,
+ "PipeParallel": PipeParallelSpec,
+ "PipeSequence": PipeSequenceSpec,
+ }
+
+ failed_pipes: List[PipeFailure] = []
+ for pipe_code, dry_run_output in dry_run_result.items():
+ if dry_run_output.status == DryRunStatus.FAILURE:
+ if pipelex_bundle_spec.pipe and pipe_code in pipelex_bundle_spec.pipe:
+ pipe_spec = pipelex_bundle_spec.pipe[pipe_code]
+ spec_class = pipe_type_to_spec_class.get(pipe_spec.type)
+ if not spec_class:
+ msg = f"Unknown pipe type: {pipe_spec.type}"
+ raise ValidateDryRunError(msg)
+ pipe_spec = spec_class(**pipe_spec.model_dump(serialize_as_any=True))
+ failed_pipes.append(
+ PipeFailure(
+ pipe=pipe_spec,
+ error_message=dry_run_output.error_message or "",
+ )
+ )
+
+ return ListContent[PipeFailure](items=failed_pipes)
+
+
+async def reconstruct_bundle_with_all_fixes(working_memory: WorkingMemory) -> PipelexBundleSpec:
+ pipelex_bundle_spec = working_memory.get_stuff_as(name="pipelex_bundle_spec", content_type=PipelexBundleSpec)
+ fixed_pipes_list = cast(ListContent[PipeSpecUnion], working_memory.get_stuff(name="fixed_pipes").content)
+
+ if not pipelex_bundle_spec.pipe:
+ msg = "No pipes section found in bundle spec"
+ raise PipeBuilderError(msg)
+
+ for fixed_pipe_blueprint in fixed_pipes_list.items:
+ pipe_code = fixed_pipe_blueprint.the_pipe_code
+ pipelex_bundle_spec.pipe[pipe_code] = fixed_pipe_blueprint
+
+ return pipelex_bundle_spec
diff --git a/pipelex/libraries/pipelines/builder/concept/concept.plx b/pipelex/libraries/pipelines/builder/concept/concept.plx
new file mode 100644
index 000000000..aa81136bd
--- /dev/null
+++ b/pipelex/libraries/pipelines/builder/concept/concept.plx
@@ -0,0 +1,43 @@
+domain = "concept"
+definition = "Build and process concepts for Pipelex bundles from signatures and drafts."
+
+[concept]
+ConceptStructureSpec = "A concept spec with structure but without full implementation."
+ConceptSpec = "A specification for a concept including its code, description, and a structure draft as plain text."
+ConceptSpecDraft = "A specification for a concept including its code, description, and a structure draft as plain text."
+
+[pipe.build_concept_spec]
+type = "PipeSequence"
+description = "Create a ConceptSpec from a brief, existing concepts, and concept rules."
+inputs = { concept_spec_draft = "ConceptSpecDraft"}
+output = "ConceptSpec"
+steps = [
+ { pipe = "spec_draft_to_structure", result = "concept_spec_structures" },
+ { pipe = "spec_draft_to_concept_spec", result = "concept_spec" }
+]
+
+[pipe.spec_draft_to_structure]
+type = "PipeLLM"
+description = "Convert the ConceptSpec (with its structure draft) into a proper ConceptStructureSpec."
+inputs = { concept_spec_draft = "ConceptSpecDraft" }
+output = "ConceptStructureSpec"
+multiple_output = true
+llm = "llm_to_engineer"
+prompt_template = """
+Create a ConceptStructureSpec from the ConceptSpecDraft.
+Please focus only on the structure.
+
+The field "definition" IS NOT a structure. It is a general definition of the concept.
+If the field "structure" is empty, return an empty list.
+
+ConceptSpecDraft:
+@concept_spec_draft
+"""
+
+[pipe.spec_draft_to_concept_spec]
+type = "PipeFunc"
+description = "Generate the final ConceptSpec using the spec and structure manually."
+inputs = { concept_spec_draft = "ConceptSpecDraft", concept_spec_structures = "ConceptStructureSpec"}
+output = "ConceptSpec"
+function_name = "create_concept_spec"
+
diff --git a/pipelex/libraries/pipelines/builder/concept/concept_spec.py b/pipelex/libraries/pipelines/builder/concept/concept_spec.py
new file mode 100644
index 000000000..b0d217298
--- /dev/null
+++ b/pipelex/libraries/pipelines/builder/concept/concept_spec.py
@@ -0,0 +1,290 @@
+from datetime import datetime
+from typing import Any, Dict, Optional, Union, cast
+
+from pydantic import ConfigDict, Field, field_validator, model_validator
+from typing_extensions import Self
+
+from pipelex.core.concepts.concept_blueprint import (
+ ConceptBlueprint,
+ ConceptBlueprintError,
+ ConceptStructureBlueprint,
+ ConceptStructureBlueprintError,
+ ConceptStructureBlueprintFieldType,
+)
+from pipelex.core.concepts.concept_native import NativeConceptEnum, NativeConceptManager
+from pipelex.core.concepts.exceptions import ConceptCodeError, ConceptStringError, ConceptStringOrConceptCodeError
+from pipelex.core.domains.domain import SpecialDomain
+from pipelex.core.domains.domain_blueprint import DomainBlueprint
+from pipelex.core.memory.working_memory import WorkingMemory
+from pipelex.core.stuffs.stuff_content import StructuredContent
+from pipelex.tools.misc.string_utils import is_pascal_case
+from pipelex.types import StrEnum
+
+
+class ConceptStructureSpecFieldType(StrEnum):
+ TEXT = "text"
+ INTEGER = "integer"
+ BOOLEAN = "boolean"
+ NUMBER = "number"
+ DATE = "date"
+
+
+class ConceptSpecError(Exception):
+ pass
+
+
+class ConceptStructureSpec(StructuredContent):
+ """Spec defining a field in the structure of a concept, used as a Pydantic V2 model.
+
+ This class represents the schema for a single field in a concept's structure. It supports
+ various field types including text, list, dict, integer, boolean, number, and date, as well
+ as choice-based fields (enums).
+
+ Attributes:
+ the_field_name: Field name. Must be snake_case.
+ definition: Natural language description of the field's purpose and usage.
+ type: The field's data type.
+ required: Whether the field is mandatory. Defaults to True unless explicitly set to False.
+ default_value: Default value for the field. Must match the specified type, and for choice
+ fields must be one of the valid choices. When provided, type must be specified
+ (unless choices are provided).
+
+ Validation Rules:
+ 3. Default values: When default_value is provided:
+ - For typed fields: type must be specified and default_value must match that type
+ - Type validation includes: text (str), integer (int), boolean (bool),
+ number (int/float), dict (dict)
+ """
+
+ the_field_name: str = Field(description="Field name. Must be snake_case.")
+ definition: str
+ type: Optional[ConceptStructureSpecFieldType] = Field(default=None, description="The type of the field.")
+ required: Optional[bool] = True
+ default_value: Optional[Any] = None
+
+ @model_validator(mode="after")
+ def validate_structure_blueprint(self) -> Self:
+ """Validate the structure blueprint according to type rules."""
+ # Check default_value type is the same as type
+ if self.default_value is not None and self.type is not None:
+ self._validate_default_value_type()
+ return self
+
+ def _validate_default_value_type(self) -> None:
+ """Validate that default_value matches the specified type."""
+ if self.default_value is None:
+ return
+
+ match self.type:
+ case None:
+ return
+ case ConceptStructureSpecFieldType.TEXT:
+ if not isinstance(self.default_value, str):
+ self._raise_type_mismatch_error("str", type(self.default_value).__name__)
+ case ConceptStructureSpecFieldType.INTEGER:
+ if not isinstance(self.default_value, int):
+ self._raise_type_mismatch_error("int", type(self.default_value).__name__)
+ case ConceptStructureSpecFieldType.BOOLEAN:
+ if not isinstance(self.default_value, bool):
+ self._raise_type_mismatch_error("bool", type(self.default_value).__name__)
+ case ConceptStructureSpecFieldType.NUMBER:
+ if not isinstance(self.default_value, (int, float)):
+ self._raise_type_mismatch_error("number (int or float)", type(self.default_value).__name__)
+ case ConceptStructureSpecFieldType.DATE:
+ if not isinstance(self.default_value, datetime):
+ self._raise_type_mismatch_error("date", type(self.default_value).__name__)
+
+ def _raise_type_mismatch_error(self, expected_type_name: str, actual_type_name: str) -> None:
+ """Raise a type mismatch error with consistent formatting."""
+ raise ConceptStructureBlueprintError(
+ f"default_value type mismatch: expected {expected_type_name} for type '{self.type}', but got {actual_type_name}"
+ )
+
+ def to_blueprint(self) -> ConceptStructureBlueprint:
+ """Convert this ConceptStructureBlueprint to the core ConceptStructureBlueprint."""
+ # Convert the type enum value - self.type is already a ConceptStructureBlueprintFieldType enum
+ # We need to get the corresponding value in the core enum
+ core_type = None
+ if self.type is not None:
+ # Get the string value and use it to get the core enum value
+ core_type = ConceptStructureBlueprintFieldType(self.type)
+
+ return ConceptStructureBlueprint(
+ definition=self.definition,
+ type=core_type,
+ required=self.required,
+ default_value=self.default_value,
+ )
+
+
+class ConceptSpecDraft(StructuredContent):
+ the_concept_code: str = Field(description="Concept code. Must be PascalCase.")
+ definition: str = Field(description="Description of the concept, in natural language.")
+ structure: str = Field(
+ description="A description of a dict with fieldnames as keys, and values being a dict with: definition, type, required, default_value"
+ )
+ refines: Optional[str] = Field(
+ default=None,
+ description="The native concept this concept extends (Text, Image, PDF, TextAndImages, Number, Page) "
+ "in PascalCase format. Cannot be used together with 'structure'.",
+ )
+
+
+class ConceptSpec(StructuredContent):
+ """Spec defining a concept that can be used in the Pipelex framework.
+
+ A concept represents a structured data type that can either define its own structure
+ or refine an existing native concept. Concepts are fundamental building blocks in
+ Pipelex workflows for data validation and transformation.
+
+ Attributes:
+ the_concept_code: Concept code. Must be PascalCase.
+ definition: Natural language description of what the concept represents and its purpose.
+ structure: The concept's field structure. Can be either:
+ - A string referring to another concept
+ - A dictionary where keys are field names (in snake_case) and values are
+ either strings (concept references) or ConceptStructureBlueprint instances
+ Cannot be used together with 'refines'.
+ refines: The native concept this concept extends (Text, Image, PDF, TextAndImages,
+ Number, Page) in PascalCase format. Cannot be used together with 'structure'.
+
+ Validation Rules:
+ 1. Mutual exclusivity: A concept must have either 'structure' or 'refines', but not both.
+ 2. Field names: When structure is a dict, all keys must be valid snake_case identifiers.
+ 3. Concept codes: Must be in PascalCase format (letters and numbers only, starting
+ with uppercase, no dots).
+ 4. Concept strings: Format is "domain.ConceptCode" where domain is lowercase and
+ ConceptCode is PascalCase.
+ 5. Native concepts: When refining, must be one of the valid native concepts.
+ 6. Structure values: In structure dict, values must be either valid concept strings
+ or ConceptStructureBlueprint instances.
+ """
+
+ model_config = ConfigDict(extra="forbid")
+
+ the_concept_code: str = Field(description="Concept code. Must be PascalCase.")
+ definition: str
+ structure: Optional[Union[str, Dict[str, Union[str, ConceptStructureSpec]]]] = None
+ refines: Optional[str] = None
+
+ @classmethod
+ def is_native_concept_code(cls, concept_code: str) -> bool:
+ ConceptSpec.validate_concept_code(concept_code=concept_code)
+ return concept_code in [native_concept for native_concept in [native_concept for native_concept in NativeConceptEnum]]
+
+ @classmethod
+ def validate_concept_code(cls, concept_code: str) -> None:
+ if not is_pascal_case(concept_code):
+ raise ConceptCodeError(
+ f"Concept code '{concept_code}' must be PascalCase (letters and numbers only, starting with uppercase, without `.`)"
+ )
+
+ @classmethod
+ def validate_concept_string_or_concept_code(cls, concept_string_or_code: str) -> None:
+ if concept_string_or_code.count(".") > 1:
+ raise ConceptStringOrConceptCodeError(
+ f"concept_string_or_code '{concept_string_or_code}' is invalid. "
+ "It should either contain a domain in snake_case and a concept code in PascalCase separated by one dot, "
+ "or be a concept code in PascalCase."
+ )
+
+ elif concept_string_or_code.count(".") == 1:
+ domain, concept_code = concept_string_or_code.split(".")
+ DomainBlueprint.validate_domain_code(code=domain)
+ cls.validate_concept_code(concept_code=concept_code)
+ else:
+ cls.validate_concept_code(concept_code=concept_string_or_code)
+
+ @staticmethod
+ def validate_concept_string(concept_string: str) -> None:
+ """Validate that a concept code follows PascalCase convention."""
+ if "." not in concept_string:
+ raise ConceptStringError(
+ f"Concept string '{concept_string}' is invalid. It should contain a domain in snake_case "
+ "and a concept code in PascalCase separated by one dot."
+ )
+ elif concept_string.count(".") > 1:
+ raise ConceptStringError(
+ f"Concept string '{concept_string}' is invalid. It should contain a domain in snake_case "
+ "and a concept code in PascalCase separated by one dot."
+ )
+ else:
+ domain, concept_code = concept_string.split(".", 1)
+
+ DomainBlueprint.validate_domain_code(domain)
+ if not is_pascal_case(concept_code):
+ raise ConceptCodeError(
+ f"Concept code '{concept_code}' must be PascalCase (letters and numbers only, starting with uppercase, without `.`)"
+ )
+
+ # Validate that if the concept code is among the native concepts, the domain MUST be native.
+ if concept_code in [concept.value for concept in [native_concept for native_concept in NativeConceptEnum]]:
+ if domain != SpecialDomain.NATIVE:
+ raise ConceptStringError(
+ f"Concept string '{concept_string}' is invalid. "
+ f"Concept code '{concept_code}' is a native concept, so the domain must be '{SpecialDomain.NATIVE}', "
+ f"or nothing, but not '{domain}'"
+ )
+
+ # Validate that if the domain is native, the concept code is a native concept
+ if SpecialDomain.is_native(domain=domain):
+ if concept_code not in [native_concept for native_concept in NativeConceptEnum]:
+ raise ConceptStringError(
+ f"Concept string '{concept_string}' is invalid. "
+ f"Concept code '{concept_code}' is not a native concept, so the domain must not be '{SpecialDomain.NATIVE}'."
+ )
+
+ @field_validator("refines", mode="before")
+ @classmethod
+ def validate_refines(cls, refines: Optional[str] = None) -> Optional[str]:
+ if refines is not None:
+ if not NativeConceptManager.is_native_concept(refines):
+ raise ConceptBlueprintError(f"Forbidden to refine a non-native concept: '{refines}'. Refining non-native concepts will come soon.")
+ cls.validate_concept_string_or_concept_code(concept_string_or_code=refines)
+ return refines
+
+ @model_validator(mode="before")
+ def model_validate_spec(cls, values: Dict[str, Any]) -> Dict[str, Any]:
+ if values.get("refines") and values.get("structure"):
+ raise ConceptSpecError(
+ f"Forbidden to have refines and structure at the same time: `{values.get('refines')}` "
+ f"and `{values.get('structure')}` for concept that has the definition `{values.get('definition')}`"
+ )
+ return values
+
+ def to_blueprint(self) -> ConceptBlueprint:
+ """Convert this ConceptBlueprint to the original core ConceptBlueprint."""
+ # TODO: Clarify concept structure blueprint
+ converted_structure: Optional[Union[str, Dict[str, Union[str, ConceptStructureBlueprint]]]] = None
+ if self.structure:
+ converted_structure = {}
+ if isinstance(self.structure, str):
+ converted_structure = self.structure
+ else:
+ for field_name, field_spec in cast(Dict[str, ConceptStructureSpec], self.structure).items():
+ converted_structure[field_name] = field_spec.to_blueprint()
+
+ return ConceptBlueprint(definition=self.definition, structure=converted_structure, refines=self.refines)
+
+
+async def create_concept_spec(working_memory: WorkingMemory) -> ConceptSpec:
+ concept_spec_draft = working_memory.get_stuff_as(name="concept_spec_draft", content_type=ConceptSpecDraft)
+ concept_spec_structures_stuff = working_memory.get_stuff_as_list(name="concept_spec_structures", item_type=ConceptStructureSpec)
+
+ structure_dict: Dict[str, Union[str, ConceptStructureSpec]] = {}
+ for structure_item in concept_spec_structures_stuff.items:
+ structure_spec = ConceptStructureSpec(
+ the_field_name=structure_item.the_field_name,
+ definition=structure_item.definition,
+ type=structure_item.type,
+ required=structure_item.required,
+ default_value=structure_item.default_value,
+ )
+ structure_dict[structure_item.the_field_name] = structure_spec
+
+ return ConceptSpec(
+ the_concept_code=concept_spec_draft.the_concept_code,
+ definition=concept_spec_draft.definition,
+ structure=structure_dict,
+ refines=concept_spec_draft.refines,
+ )
diff --git a/pipelex/libraries/pipelines/builder/pipe/inputs_spec.py b/pipelex/libraries/pipelines/builder/pipe/inputs_spec.py
new file mode 100644
index 000000000..4ea6bcc66
--- /dev/null
+++ b/pipelex/libraries/pipelines/builder/pipe/inputs_spec.py
@@ -0,0 +1,40 @@
+from typing import Optional
+
+from pydantic import Field, field_validator
+
+from pipelex.core.concepts.concept_blueprint import ConceptBlueprint
+from pipelex.core.pipes.pipe_input_blueprint import InputRequirementBlueprint
+from pipelex.core.pipes.pipe_run_params import PipeOutputMultiplicity
+from pipelex.core.stuffs.stuff_content import StructuredContent
+
+
+class InputRequirementSpec(StructuredContent):
+ """Spec specifying input requirements for a pipe in the Pipelex framework.
+
+ Defines the concept type and multiplicity constraints for pipe inputs, ensuring
+ proper data validation and flow control in pipeline execution.
+
+ Attributes:
+ concept: The concept code or concept string for the input. Must be in PascalCase
+ format. When using concept strings, format is "domain.ConceptCode" with
+ domain in lowercase and ConceptCode in PascalCase.
+ multiplicity: Optional constraint on input cardinality (e.g., single, multiple).
+ Defines how many instances of the concept the pipe can process.
+
+ Validation Rules:
+ 1. Concept must be a valid concept code (PascalCase) or concept string (domain.ConceptCode).
+ 2. Domain and concept code are separated by a dot when using full concept strings.
+ 3. Concept validation is performed using ConceptBlueprint.validate_concept_string_or_concept_code.
+ """
+
+ concept: str = Field(description="Concept code or concept string in PascalCase format")
+ multiplicity: Optional[PipeOutputMultiplicity] = None
+
+ @field_validator("concept", mode="before")
+ @classmethod
+ def validate_concept_string(cls, concept_string: str) -> str:
+ ConceptBlueprint.validate_concept_string_or_concept_code(concept_string_or_code=concept_string)
+ return concept_string
+
+ def to_blueprint(self) -> InputRequirementBlueprint:
+ return InputRequirementBlueprint(concept=self.concept, multiplicity=self.multiplicity)
diff --git a/pipelex/libraries/pipelines/builder/pipe/pipe.plx b/pipelex/libraries/pipelines/builder/pipe/pipe.plx
new file mode 100644
index 000000000..8c9f8da62
--- /dev/null
+++ b/pipelex/libraries/pipelines/builder/pipe/pipe.plx
@@ -0,0 +1,480 @@
+domain = "pipe"
+definition = "Build and process pipes."
+
+[concept]
+PipeSignature = "Pseudo-Pipelex step: code, type, description, inputs, output, optional children refs."
+PipeSpec = "A structured spec for a pipe (union)."
+# Pipe controllers
+PipeBatchSpec = "A structured spec for a pipe batch."
+PipeConditionSpec = "A structured spec for a pipe condition."
+PipeParallelSpec = "A structured spec for a pipe parallel."
+PipeSequenceSpec = "A structured spec for a pipe sequence."
+# Pipe operators
+PipeFuncSpec = "A structured spec for a pipe func."
+PipeImgGenSpec = "A structured spec for a pipe img gen."
+PipeJinja2Spec = "A structured spec for a pipe jinja2."
+PipeLLMSpec = "A structured spec for a pipe llm."
+PipeOcrSpec = "A structured spec for a pipe ocr."
+PipeFailure = "Details of a single pipe failure during dry run."
+
+[pipe]
+# ────────────────────────────────────────────────────────────────────────────────
+# NEW ENTRY POINT — takes PipeSignature[] + ConceptSpecs → PipeSpecs[]
+# ────────────────────────────────────────────────────────────────────────────────
+[pipe.create_pipes_from_signatures]
+type = "PipeSequence"
+description = "PipeSignature[] + ConceptSpecs → PipeSpecs[] (linked & ready)."
+inputs = { pipe_signature = "PipeSignature", concept_specs = "concept.ConceptSpec" }
+output = "Dynamic"
+steps = [
+ { pipe = "generate_pipe_spec", result = "pipe_spec" },
+ { pipe = "compile_one_signature_spec", result = "compiled_spec" },
+]
+
+[pipe.generate_pipe_spec]
+type = "PipeLLM"
+description = "Generate a PipeSpec from a PipeSignature."
+inputs = { pipe_signature = "PipeSignature", concept_specs = "concept.ConceptSpec" }
+output = "PipeSpec"
+llm = "llm_to_engineer"
+prompt_template = """
+Return a PipeSpec for this signature.
+
+Signature:
+@pipe_signature
+
+and here are the existing concepts:
+@concept_specs
+
+The inputs keys should be snake_case.
+The values should be a concept code in PascalCase.
+"""
+
+# ────────────────────────────────────────────────────────────────────────────────
+# CORE: signature → route to spec emitter
+# ────────────────────────────────────────────────────────────────────────────────
+[pipe.compile_one_signature_spec]
+type = "PipeCondition"
+description = "Route by signature.type to the correct spec emitter."
+inputs = { pipe_signature = "PipeSignature", concept_specs = "ConceptSpec", pipe_spec = "PipeSpec" }
+output = "Dynamic"
+expression = "pipe_signature.type"
+
+[pipe.compile_one_signature_spec.pipe_map]
+PipeSequence = "emit_sequence_from_signature"
+PipeParallel = "emit_parallel_from_signature"
+PipeCondition = "emit_condition_from_signature"
+PipeBatch = "emit_batch_from_signature"
+PipeLLM = "emit_llm_from_signature"
+PipeOcr = "emit_ocr_from_signature"
+PipeImgGen = "emit_imggen_from_signature"
+PipeJinja2 = "emit_jinja_from_signature"
+PipeFunc = "emit_func_from_signature"
+
+# ────────────────────────────────────────────────────────────────────────────────
+# EMITTERS — one per pipe type, all using the same minimal contract
+# (Optionally, emitters could be extended later to read from concept_index.)
+# ────────────────────────────────────────────────────────────────────────────────
+
+[pipe.emit_sequence_from_signature]
+type = "PipeLLM"
+description = "Build a PipeSequenceSpec from the signature (children referenced by code)."
+inputs = { pipe_signature = "PipeSignature", concept_specs = "concept.ConceptSpec", pipe_spec = "PipeSpec" }
+output = "PipeSequenceSpec"
+llm = "llm_to_engineer"
+prompt_template = """
+Return a PipeSequenceSpec for this signature.
+The Pipe sequence NEEDS to have at least one step.
+Orchestrate all the necessary steps to achieve the goal of the pipe.
+
+Signature:
+@pipe_signature
+
+Here is the base PipeSpec:
+@pipe_spec
+
+And here are the concepts you can use:
+@concept_specs
+"""
+
+[pipe.emit_parallel_from_signature]
+type = "PipeLLM"
+description = "Build a PipeParallelSpec from the signature."
+inputs = { pipe_signature = "PipeSignature", concept_specs = "concept.ConceptSpec", pipe_spec = "PipeSpec" }
+output = "PipeParallelSpec"
+llm = "llm_to_engineer"
+prompt_template = """
+Return a PipeParallelSpec for this signature.
+
+Signature:
+@pipe_signature
+
+Here is the base PipeSpec:
+@pipe_spec
+
+And here are the concepts you can use:
+@concept_specs
+"""
+
+[pipe.emit_condition_from_signature]
+type = "PipeLLM"
+description = "Build a PipeConditionSpec from the signature (provide expression/pipe_map consistent with children)."
+inputs = { pipe_signature = "PipeSignature", concept_specs = "concept.ConceptSpec", pipe_spec = "PipeSpec" }
+output = "PipeConditionSpec"
+llm = "llm_to_engineer"
+prompt_template = """
+Return a PipeConditionSpec for this signature.
+
+Signature:
+@pipe_signature
+
+Here is the base PipeSpec:
+@pipe_spec
+
+And here are the concepts you can use:
+@concept_specs
+"""
+
+[pipe.emit_batch_from_signature]
+type = "PipeLLM"
+description = "Build a PipeBatchSpec from the signature (choose branch_pipe_code/params)."
+inputs = { pipe_signature = "PipeSignature", concept_specs = "concept.ConceptSpec", pipe_spec = "PipeSpec" }
+output = "PipeBatchSpec"
+llm = "llm_to_engineer"
+prompt_template = """
+Return a PipeBatchSpec for this signature.
+
+Signature:
+@pipe_signature
+
+Here is the base PipeSpec:
+@pipe_spec
+
+And here are the concepts you can use:
+@concept_specs
+"""
+
+[pipe.emit_llm_from_signature]
+type = "PipeLLM"
+description = "Build a PipeLLMSpec from the signature."
+inputs = { pipe_signature = "PipeSignature", concept_specs = "concept.ConceptSpec", pipe_spec = "PipeSpec" }
+output = "PipeLLMSpec"
+llm = "llm_to_engineer"
+prompt_template = """
+Return a PipeLLMSpec for this signature.
+
+THe prompt is the field "prompt_template" in the PipeLLMSpec.
+Signature:
+@pipe_signature
+
+Here is the base PipeSpec:
+@pipe_spec
+
+And here are the concepts you can use:
+@concept_specs
+"""
+
+[pipe.emit_ocr_from_signature]
+type = "PipeLLM"
+description = "Build a PipeOcrSpec from the signature."
+inputs = { pipe_signature = "PipeSignature", concept_specs = "concept.ConceptSpec", pipe_spec = "PipeSpec" }
+output = "PipeOcrSpec"
+prompt_template = """
+Return a PipeOcrSpec for this signature.
+
+VERY IMPORTANT: THE INPUT OF THE PIPEOCR MUST BE NAMED "ocr_input" and it must be either an image or a pdf or a concept which refines one of them.
+Signature:
+@pipe_signature
+
+Here is the base PipeSpec:
+@pipe_spec
+
+And here are the concepts you can use:
+@concept_specs
+"""
+
+[pipe.emit_imggen_from_signature]
+type = "PipeLLM"
+description = "Build a PipeImgGenSpec from the signature."
+inputs = { pipe_signature = "PipeSignature", concept_specs = "concept.ConceptSpec", pipe_spec = "PipeSpec" }
+output = "PipeImgGenSpec"
+prompt_template = """
+Return a PipeImgGenSpec for this signature.
+The inputs for the image has to be only:
+input_name : prompt
+concept : A concept that refines Text. It should be text
+The output concept should be a concept in Pascal case that refines Image.
+For example:
+```
+inputs = { prompt: ImgGenPrompt }
+```
+if ImgGenPrompt is a text concept.
+
+IMPORTANT: imgg_prompt SHOULD BE NONE
+IMPORTANT: img_gen_prompt_var_name SHOULD BE NONE
+The prompt will need to be be generated by a pipe with the necessary elements.
+
+Signature:
+@pipe_signature
+
+Here is the base PipeSpec:
+@pipe_spec
+
+And here are the concepts you can use:
+@concept_specs
+"""
+
+[pipe.emit_jinja_from_signature]
+type = "PipeLLM"
+description = "Build a PipeJinja2Spec from the signature."
+inputs = { pipe_signature = "PipeSignature", concept_specs = "concept.ConceptSpec", pipe_spec = "PipeSpec" }
+output = "PipeJinja2Spec"
+prompt_template = """
+Return a PipeJinja2Spec for this signature.
+
+Signature:
+@pipe_signature
+
+Here is the base PipeSpec:
+@pipe_spec
+
+And here are the concepts you can use:
+@concept_specs
+"""
+
+[pipe.emit_func_from_signature]
+type = "PipeLLM"
+description = "Build a PipeFuncSpec from the signature."
+inputs = { pipe_signature = "PipeSignature", concept_specs = "concept.ConceptSpec", pipe_spec = "PipeSpec" }
+output = "PipeFuncSpec"
+llm = "llm_to_engineer"
+prompt_template = """
+Return a PipeFuncSpec for this signature.
+
+Signature:
+@pipe_signature
+
+Here is the base PipeSpec:
+@pipe_spec
+
+And here are the concepts you can use:
+@concept_specs
+"""
+
+# ────────────────────────────────────────────────────────────────────────────────
+# PIPE FIXERS — one per pipe type, all fixing specific pipe type issues
+# ────────────────────────────────────────────────────────────────────────────────
+
+[pipe.fix_failing_pipe]
+type = "PipeCondition"
+description = "Route to specific pipe fixer based on the failing pipe type."
+inputs = { pipelex_bundle_spec = "PipelexBundleSpec", failed_pipe = "PipeFailure" }
+output = "Dynamic"
+expression = "failed_pipe.pipe.type"
+
+[pipe.fix_failing_pipe.pipe_map]
+PipeLLM = "fix_failing_llm_pipe"
+PipeImgGen = "fix_failing_imggen_pipe"
+PipeOcr = "fix_failing_ocr_pipe"
+PipeFunc = "fix_failing_func_pipe"
+PipeJinja2 = "fix_failing_jinja2_pipe"
+PipeSequence = "fix_failing_sequence_pipe"
+PipeParallel = "fix_failing_parallel_pipe"
+PipeCondition = "fix_failing_condition_pipe"
+PipeBatch = "fix_failing_batch_pipe"
+
+[pipe.fix_failing_llm_pipe]
+type = "PipeLLM"
+description = "Fix a failing PipeLLM spec based on its specific error."
+inputs = { pipelex_bundle_spec = "PipelexBundleSpec", failed_pipe = "PipeFailure" }
+output = "PipeLLMSpec"
+llm = "llm_to_engineer"
+prompt_template = """
+Fix this failing PipeLLM spec.
+
+Failing pipe:
+@failed_pipe.pipe
+
+Error message:
+@failed_pipe.error_message
+
+Please provide only the corrected PipeLLMSpec. Common LLM pipe issues to fix:
+- Missing input variables in the pipe inputs that are referenced in prompt_template
+- Incorrect variable names in prompt templates (use $ for inline, @ for blocks)
+- Wrong concept types for inputs/outputs
+- Missing llm configuration
+- Invalid prompt template syntax
+"""
+
+[pipe.fix_failing_imggen_pipe]
+type = "PipeLLM"
+description = "Fix a failing PipeImgGen spec based on its specific error."
+inputs = { pipelex_bundle_spec = "PipelexBundleSpec", failed_pipe = "PipeFailure" }
+output = "PipeImgGenSpec"
+llm = "llm_to_engineer"
+prompt_template = """
+Fix this failing PipeImgGen spec.
+
+Failing pipe:
+@failed_pipe.pipe
+
+Error message:
+@failed_pipe.error_message
+
+Please provide only the corrected PipeImgGenSpec. Common ImgGen pipe issues to fix:
+- Missing or incorrect prompt input (should be text concept)
+- Wrong img_gen_prompt_var_name (should be None or "prompt")
+- Invalid img_gen choice
+- Missing required inputs for dynamic prompt generation
+"""
+
+[pipe.fix_failing_ocr_pipe]
+type = "PipeLLM"
+description = "Fix a failing PipeOcr spec based on its specific error."
+inputs = { pipelex_bundle_spec = "PipelexBundleSpec", failed_pipe = "PipeFailure" }
+output = "PipeOcrSpec"
+llm = "llm_to_engineer"
+prompt_template = """
+Fix this failing PipeOcr spec.
+
+VERY IMPORTANT: THE INPUT OF THE PIPEOCR MUST BE NAMED "ocr_input" and it must be either an image or a pdf or a concept which refines one of them.
+Failing pipe:
+@failed_pipe.pipe
+
+Error message:
+@failed_pipe.error_message
+
+Please provide only the corrected PipeOcrSpec. Common OCR pipe issues to fix:
+- Input must be named 'ocr_input' and be of type Image or PDF
+- Output should typically be Page (native concept)
+- Missing or incorrect input concept types
+"""
+
+[pipe.fix_failing_func_pipe]
+type = "PipeLLM"
+description = "Fix a failing PipeFunc spec based on its specific error."
+inputs = { pipelex_bundle_spec = "PipelexBundleSpec", failed_pipe = "PipeFailure" }
+output = "PipeFuncSpec"
+llm = "llm_to_engineer"
+prompt_template = """
+Fix this failing PipeFunc spec.
+
+Failing pipe:
+@failed_pipe.pipe
+
+Error message:
+@failed_pipe.error_message
+
+Please provide only the corrected PipeFuncSpec. Common Func pipe issues to fix:
+- Missing or incorrect function_name
+- Wrong input/output concept types for the function
+- Function not available in registry
+"""
+
+[pipe.fix_failing_jinja2_pipe]
+type = "PipeLLM"
+description = "Fix a failing PipeJinja2 spec based on its specific error."
+inputs = { pipelex_bundle_spec = "PipelexBundleSpec", failed_pipe = "PipeFailure" }
+output = "PipeJinja2Spec"
+llm = "llm_to_engineer"
+prompt_template = """
+Fix this failing PipeJinja2 spec.
+
+Failing pipe:
+@failed_pipe.pipe
+
+Error message:
+@failed_pipe.error_message
+
+Please provide only the corrected PipeJinja2Spec. Common Jinja2 pipe issues to fix:
+- Invalid Jinja2 template syntax
+- Missing input variables referenced in template
+- Wrong concept types for inputs/outputs
+"""
+
+[pipe.fix_failing_sequence_pipe]
+type = "PipeLLM"
+description = "Fix a failing PipeSequence spec based on its specific error."
+inputs = { pipelex_bundle_spec = "PipelexBundleSpec", failed_pipe = "PipeFailure" }
+output = "PipeSequenceSpec"
+llm = "llm_to_engineer"
+prompt_template = """
+Fix this failing PipeSequence spec.
+
+Failing pipe:
+@failed_pipe.pipe
+
+Error message:
+@failed_pipe.error_message
+
+Please provide only the corrected PipeSequenceSpec. Common Sequence pipe issues to fix:
+- Missing input variables needed by sub-pipes in steps
+- Referenced pipe codes in steps that don't exist
+- Circular dependencies in step order
+- Wrong result names in steps
+"""
+
+[pipe.fix_failing_parallel_pipe]
+type = "PipeLLM"
+description = "Fix a failing PipeParallel spec based on its specific error."
+inputs = { pipelex_bundle_spec = "PipelexBundleSpec", failed_pipe = "PipeFailure" }
+output = "PipeParallelSpec"
+llm = "llm_to_engineer"
+prompt_template = """
+Fix this failing PipeParallel spec.
+
+Failing pipe:
+@failed_pipe.pipe
+
+Error message:
+@failed_pipe.error_message
+
+Please provide only the corrected PipeParallelSpec. Common Parallel pipe issues to fix:
+- Missing input variables needed by parallel sub-pipes
+- Referenced pipe codes that don't exist
+- Incompatible output types from parallel branches
+"""
+
+[pipe.fix_failing_condition_pipe]
+type = "PipeLLM"
+description = "Fix a failing PipeCondition spec based on its specific error."
+inputs = { pipelex_bundle_spec = "PipelexBundleSpec", failed_pipe = "PipeFailure" }
+output = "PipeConditionSpec"
+llm = "llm_to_engineer"
+prompt_template = """
+Fix this failing PipeCondition spec.
+
+Failing pipe:
+@failed_pipe.pipe
+
+Error message:
+@failed_pipe.error_message
+
+Please provide only the corrected PipeConditionSpec. Common Condition pipe issues to fix:
+- Invalid expression or expression_template syntax
+- Referenced pipe codes in pipe_map that don't exist
+- Missing input variables referenced in expression
+- Incompatible output types from different condition branches
+"""
+
+[pipe.fix_failing_batch_pipe]
+type = "PipeLLM"
+description = "Fix a failing PipeBatch spec based on its specific error."
+inputs = { pipelex_bundle_spec = "PipelexBundleSpec", failed_pipe = "PipeFailure" }
+output = "PipeBatchSpec"
+llm = "llm_to_engineer"
+prompt_template = """
+Fix this failing PipeBatch spec.
+
+Failing pipe:
+@failed_pipe.pipe
+
+Error message:
+@failed_pipe.error_message
+
+Please provide only the corrected PipeBatchSpec. Common Batch pipe issues to fix:
+- Missing branch_pipe_code or referenced pipe that doesn't exist
+- Wrong input types for batch processing (should be ListContent)
+- Missing batch parameters configuration
+"""
+
diff --git a/pipelex/libraries/pipelines/builder/pipe/pipe_batch_spec.py b/pipelex/libraries/pipelines/builder/pipe/pipe_batch_spec.py
new file mode 100644
index 000000000..558ba5aec
--- /dev/null
+++ b/pipelex/libraries/pipelines/builder/pipe/pipe_batch_spec.py
@@ -0,0 +1,56 @@
+from typing import Literal, Optional
+
+from pydantic import Field
+from typing_extensions import override
+
+from pipelex.libraries.pipelines.builder.pipe.pipe_signature import PipeSpec
+from pipelex.pipe_controllers.batch.pipe_batch_blueprint import PipeBatchBlueprint
+
+
+class PipeBatchSpec(PipeSpec):
+ """Spec for batch processing pipe operations in the Pipelex framework.
+
+ PipeBatch enables parallel execution of a single pipe across multiple items
+ in a list. Each item is processed independently, making it ideal for data
+ transformation, enrichment, or analysis tasks on collections.
+
+ This controller is commonly used within PipeSequence for inline batch processing,
+ where the batch configuration is specified directly in the sequence step using
+ batch_over and batch_as parameters in SubPipeBlueprint.
+
+ Attributes:
+ the_pipe_code: Pipe code. Must be snake_case.
+ type: Fixed to "PipeBatch" for this pipe type.
+ branch_pipe_code: The pipe code to execute for each item in the input list.
+ This pipe is instantiated once per item in parallel.
+ input_list_name: Name of the list in WorkingMemory to iterate over.
+ Defaults to the PipeBatch's main input name if not specified.
+ input_item_name: Name assigned to individual items within each execution branch.
+ This is how the branch pipe accesses its specific input item.
+
+ Validation Rules:
+ 1. branch_pipe_code must reference an existing pipe in the pipeline.
+ 2. When input_list_name is specified, it must reference a list in context.
+ 3. The branch pipe should be designed to process single items.
+ """
+
+ type: Literal["PipeBatch"] = "PipeBatch"
+ category: Literal["PipeController"] = "PipeController"
+ the_pipe_code: str = Field(description="Pipe code. Must be snake_case.")
+ branch_pipe_code: str
+ input_list_name: Optional[str] = None
+ input_item_name: Optional[str] = None
+
+ @override
+ def to_blueprint(self) -> PipeBatchBlueprint:
+ base_blueprint = super().to_blueprint()
+ return PipeBatchBlueprint(
+ definition=base_blueprint.definition,
+ inputs=base_blueprint.inputs,
+ output=base_blueprint.output,
+ type=self.type,
+ category=self.category,
+ branch_pipe_code=self.branch_pipe_code,
+ input_list_name=self.input_list_name,
+ input_item_name=self.input_item_name,
+ )
diff --git a/pipelex/libraries/pipelines/builder/pipe/pipe_condition_spec.py b/pipelex/libraries/pipelines/builder/pipe/pipe_condition_spec.py
new file mode 100644
index 000000000..0127a4719
--- /dev/null
+++ b/pipelex/libraries/pipelines/builder/pipe/pipe_condition_spec.py
@@ -0,0 +1,77 @@
+from typing import Dict, Literal, Optional
+
+from pydantic import Field, RootModel
+from typing_extensions import override
+
+from pipelex.libraries.pipelines.builder.pipe.pipe_signature import PipeSpec
+from pipelex.pipe_controllers.condition.pipe_condition_blueprint import PipeConditionBlueprint, PipeConditionPipeMapBlueprint
+
+PipeConditionPipeMapRoot = Dict[str, str]
+
+
+class PipeConditionPipeMapSpec(RootModel[PipeConditionPipeMapRoot]):
+ """Spec for condition-to-pipe mapping in PipeCondition.
+
+ Maps condition values to pipe codes for conditional execution.
+
+ Attributes:
+ root: Dictionary mapping condition results (keys) to pipe codes (values).
+ Each key represents a possible condition outcome, and its value
+ is the pipe code to execute when that condition is met.
+ """
+
+ root: PipeConditionPipeMapRoot = Field(default_factory=dict)
+
+
+class PipeConditionSpec(PipeSpec):
+ """Spec for conditional pipe execution in the Pipelex framework.
+
+ PipeCondition enables branching logic in pipelines by evaluating expressions
+ and executing different pipes based on the results. Supports template-based
+ and direct expression evaluation with default fallback options.
+
+ Attributes:
+ the_pipe_code: Pipe code. Must be snake_case.
+ type: Fixed to "PipeCondition" for this pipe type.
+ expression_template: Template for building the expression to evaluate.
+ Supports variable substitution for dynamic conditions.
+ expression: Direct expression to evaluate. Typically uses the result
+ of the previous pipe. Mutually exclusive with expression_template.
+ pipe_map: Mapping of condition results to pipe codes. Each condition
+ outcome triggers execution of its associated pipe.
+ default_pipe_code: Fallback pipe to execute when no conditions in pipe_map
+ match the expression result.
+ add_alias_from_expression_to: Optional name to store the expression result
+ in the context for later reference.
+
+ Validation Rules:
+ 1. Either expression or expression_template should be provided, not both.
+ 2. pipe_map keys must be strings representing possible condition outcomes.
+ 3. All pipe codes in pipe_map and default_pipe_code must be valid pipe references.
+ """
+
+ type: Literal["PipeCondition"] = "PipeCondition"
+ category: Literal["PipeController"] = "PipeController"
+ the_pipe_code: str = Field(description="Pipe code. Must be snake_case.")
+ expression_template: Optional[str] = None
+ expression: Optional[str] = None
+ pipe_map: PipeConditionPipeMapSpec = Field(default_factory=PipeConditionPipeMapSpec)
+ default_pipe_code: Optional[str] = None
+ add_alias_from_expression_to: Optional[str] = None
+
+ @override
+ def to_blueprint(self) -> PipeConditionBlueprint:
+ base_blueprint = super().to_blueprint()
+ pipe_map_dict = PipeConditionPipeMapBlueprint(root=dict(self.pipe_map.root))
+ return PipeConditionBlueprint(
+ definition=base_blueprint.definition,
+ inputs=base_blueprint.inputs,
+ output=base_blueprint.output,
+ type=self.type,
+ category=self.category,
+ expression_template=self.expression_template,
+ expression=self.expression,
+ pipe_map=pipe_map_dict,
+ default_pipe_code=self.default_pipe_code,
+ add_alias_from_expression_to=self.add_alias_from_expression_to,
+ )
diff --git a/pipelex/libraries/pipelines/builder/pipe/pipe_func_spec.py b/pipelex/libraries/pipelines/builder/pipe/pipe_func_spec.py
new file mode 100644
index 000000000..48773f9d4
--- /dev/null
+++ b/pipelex/libraries/pipelines/builder/pipe/pipe_func_spec.py
@@ -0,0 +1,36 @@
+from typing import Literal
+
+from pydantic import Field
+from typing_extensions import override
+
+from pipelex.libraries.pipelines.builder.pipe.pipe_signature import PipeSpec
+from pipelex.pipe_operators.func.pipe_func_blueprint import PipeFuncBlueprint
+
+
+class PipeFuncSpec(PipeSpec):
+ """
+ PipeFunc enables calling custom functions in the Pipelex framework.
+
+ Attributes:
+ the_pipe_code: Pipe code. Must be snake_case.
+ type: Fixed to "PipeFunc" for this pipe type.
+ category: Fixed to "PipeOperator" for this pipe category.
+ function_name: The name of the function to call.
+ """
+
+ type: Literal["PipeFunc"] = "PipeFunc"
+ category: Literal["PipeOperator"] = "PipeOperator"
+ the_pipe_code: str = Field(description="Pipe code. Must be snake_case.")
+ function_name: str = Field(description="The name of the function to call.")
+
+ @override
+ def to_blueprint(self) -> PipeFuncBlueprint:
+ base_blueprint = super().to_blueprint()
+ return PipeFuncBlueprint(
+ definition=base_blueprint.definition,
+ inputs=base_blueprint.inputs,
+ output=base_blueprint.output,
+ type=self.type,
+ category=self.category,
+ function_name=self.function_name,
+ )
diff --git a/pipelex/libraries/pipelines/builder/pipe/pipe_img_spec.py b/pipelex/libraries/pipelines/builder/pipe/pipe_img_spec.py
new file mode 100644
index 000000000..331b3dbae
--- /dev/null
+++ b/pipelex/libraries/pipelines/builder/pipe/pipe_img_spec.py
@@ -0,0 +1,85 @@
+from typing import Literal, Optional, Union
+
+from pydantic import Field
+from typing_extensions import override
+
+from pipelex.cogt.img_gen.img_gen_job_components import AspectRatio, Background, OutputFormat
+from pipelex.cogt.img_gen.img_gen_setting import ImgGenChoice
+from pipelex.libraries.pipelines.builder.pipe.pipe_signature import PipeSpec
+from pipelex.pipe_operators.img_gen.pipe_img_gen_blueprint import PipeImgGenBlueprint
+
+
+class PipeImgGenSpec(PipeSpec):
+ """Specs for image generation pipe operations in the Pipelex framework.
+
+ PipeImgGen enables AI-powered image generation using various models like DALL-E or
+ diffusion models. Supports static and dynamic prompts with configurable generation
+ parameters.
+
+ Attributes:
+ the_pipe_code: Pipe code. Must be snake_case.
+ type: Fixed to "PipeImgGen" for this pipe type.
+ img_gen_prompt: Static text prompt for image generation. Use this or dynamic input.
+ img_gen: Image generation choice (e.g., 'gpt-image-1'). Defaults to global config.
+ aspect_ratio: Desired image aspect ratio (e.g., '16:9', '1:1').
+ quality: Generated image quality setting (e.g., 'standard', 'hd').
+ nb_steps: Number of diffusion steps for diffusion models. More steps increase detail
+ but take longer. Must be > 0.
+ guidance_scale: Prompt adherence strength. Higher values mean closer adherence to prompt.
+ Must be > 0.
+ is_moderated: Whether to apply content moderation to generated images.
+ safety_tolerance: Content moderation tolerance level. Must be between 1 and 6.
+ is_raw: Whether to return raw image data instead of processed format.
+ seed: Random seed for reproducibility. Use integer value or 'auto' for random seed.
+ nb_output: Number of images to generate. Defaults to single image. Must be >= 1.
+ img_gen_prompt_var_name: Variable name for dynamic prompt generation from inputs. Do not assign anything
+
+ Validation Rules:
+ 1. Quality and nb_steps are mutually exclusive (cannot specify both).
+ 2. nb_steps must be greater than 0 when specified.
+ 3. guidance_scale must be greater than 0 when specified.
+ 4. safety_tolerance must be between 1 and 6 inclusive.
+ 5. nb_output must be at least 1 when specified.
+
+ Raises:
+ PipeDefinitionError: When validation rules are violated or mutually exclusive
+ fields are set simultaneously.
+ """
+
+ type: Literal["PipeImgGen"] = "PipeImgGen"
+ category: Literal["PipeOperator"] = "PipeOperator"
+ the_pipe_code: str = Field(description="Pipe code. Must be snake_case.")
+ img_gen_prompt: Optional[str] = None
+ img_gen_prompt_var_name: Optional[str] = None
+
+ # New ImgGenChoice pattern (like LLM)
+ img_gen: Optional[ImgGenChoice] = None
+
+ # One-time settings (not in ImgGenSetting)
+ aspect_ratio: Optional[AspectRatio] = Field(default=None, strict=False)
+ is_raw: Optional[bool] = None
+ seed: Optional[Union[int, Literal["auto"]]] = None
+ nb_output: Optional[int] = Field(default=None, ge=1)
+ background: Optional[Background] = Field(default=None, strict=False)
+ output_format: Optional[OutputFormat] = Field(default=None, strict=False)
+
+ @override
+ def to_blueprint(self) -> PipeImgGenBlueprint:
+ """Convert this PipeImgGenBlueprint to the core PipeImgGenBlueprint."""
+ base_blueprint = super().to_blueprint()
+ return PipeImgGenBlueprint(
+ definition=base_blueprint.definition,
+ inputs=base_blueprint.inputs,
+ output=base_blueprint.output,
+ type=self.type,
+ category=self.category,
+ img_gen_prompt=self.img_gen_prompt,
+ img_gen=self.img_gen,
+ aspect_ratio=self.aspect_ratio,
+ background=self.background,
+ output_format=self.output_format,
+ is_raw=self.is_raw,
+ seed=self.seed,
+ nb_output=self.nb_output,
+ img_gen_prompt_var_name=None, # Core expects None, builder uses "prompt" as default
+ )
diff --git a/pipelex/libraries/pipelines/builder/pipe/pipe_jinja2_spec.py b/pipelex/libraries/pipelines/builder/pipe/pipe_jinja2_spec.py
new file mode 100644
index 000000000..4b6fc56dd
--- /dev/null
+++ b/pipelex/libraries/pipelines/builder/pipe/pipe_jinja2_spec.py
@@ -0,0 +1,58 @@
+from typing import Any, Dict, Literal, Optional
+
+from pydantic import Field
+from typing_extensions import override
+
+from pipelex.core.stuffs.stuff_content import StructuredContent
+from pipelex.libraries.pipelines.builder.pipe.pipe_signature import PipeSpec
+from pipelex.pipe_operators.jinja2.pipe_jinja2_blueprint import PipeJinja2Blueprint
+from pipelex.tools.templating.jinja2_template_category import Jinja2TemplateCategory
+from pipelex.tools.templating.templating_models import PromptingStyle, TagStyle, TextFormat
+
+
+class PromptingStyleSpec(StructuredContent):
+ tag_style: TagStyle = Field(strict=False)
+ text_format: TextFormat = Field(TextFormat.PLAIN, strict=False)
+
+
+class Jinja2Spec(StructuredContent):
+ jinja2_name: Optional[str] = Field(default=None, description="Name of the Jinja2 template to use")
+ jinja2: Optional[str] = Field(default=None, description="Raw Jinja2 template string")
+ prompting_style: Optional[PromptingStyleSpec] = Field(default=None, description="Style of prompting to use (typically for different LLMs)")
+ template_category: Jinja2TemplateCategory = Field(
+ default=Jinja2TemplateCategory.LLM_PROMPT,
+ description="Category of the template (could also be HTML, MARKDOWN, MERMAID, etc.), influences Jinja2 rendering environment config",
+ )
+ extra_context: Optional[Dict[str, Any]] = Field(default=None, description="Additional context variables for template rendering")
+
+
+class PipeJinja2Spec(PipeSpec, Jinja2Spec):
+ type: Literal["PipeJinja2"] = "PipeJinja2"
+ category: Literal["PipeOperator"] = "PipeOperator"
+ the_pipe_code: str = Field(description="Pipe code. Must be snake_case.")
+
+ @override
+ def to_blueprint(self) -> PipeJinja2Blueprint:
+ base_blueprint = super().to_blueprint()
+
+ if self.prompting_style:
+ prompting_style = (
+ self.prompting_style
+ if isinstance(self.prompting_style, PromptingStyle)
+ else PromptingStyle(tag_style=self.prompting_style.tag_style, text_format=self.prompting_style.text_format)
+ )
+ else:
+ prompting_style = None
+
+ return PipeJinja2Blueprint(
+ definition=base_blueprint.definition,
+ inputs=base_blueprint.inputs,
+ output=base_blueprint.output,
+ type=self.type,
+ category=self.category,
+ jinja2_name=self.jinja2_name,
+ jinja2=self.jinja2,
+ prompting_style=prompting_style,
+ template_category=self.template_category,
+ extra_context=self.extra_context,
+ )
diff --git a/pipelex/libraries/pipelines/builder/pipe/pipe_llm_spec.py b/pipelex/libraries/pipelines/builder/pipe/pipe_llm_spec.py
new file mode 100644
index 000000000..b3d8d8ce9
--- /dev/null
+++ b/pipelex/libraries/pipelines/builder/pipe/pipe_llm_spec.py
@@ -0,0 +1,160 @@
+from typing import Literal, Optional, Union
+
+from pydantic import Field, field_validator, model_validator
+from typing_extensions import Self, override
+
+from pipelex.cogt.llm.llm_setting import LLMChoice, LLMSetting
+from pipelex.core.stuffs.stuff_content import StructuredContent
+from pipelex.exceptions import PipeDefinitionError
+from pipelex.libraries.pipelines.builder.pipe.pipe_signature import PipeSpec
+from pipelex.pipe_operators.llm.pipe_llm_blueprint import PipeLLMBlueprint, StructuringMethod
+from pipelex.tools.typing.validation_utils import has_more_than_one_among_attributes_from_lists
+
+
+class LLMSettingSpec(StructuredContent):
+ llm_handle: str
+ temperature: float = Field(..., ge=0, le=1)
+ max_tokens: Optional[int] = None
+
+ @field_validator("max_tokens", mode="before")
+ @classmethod
+ def validate_max_tokens(cls, value: Union[int, Literal["auto"], None]) -> Optional[int]:
+ if value is None:
+ return None
+ elif isinstance(value, str) and value == "auto":
+ return None
+ elif isinstance(value, int): # pyright: ignore[reportUnnecessaryIsInstance]
+ return value
+
+
+LLMChoiceSpec = Union[LLMSettingSpec, str]
+
+
+class PipeLLMSpec(PipeSpec):
+ """Spec for LLM-based pipe operations in the Pipelex framework.
+
+ PipeLLM enables Large Language Model processing to generate text or structured output.
+ Supports text, structured data, and image inputs with flexible prompt configuration
+ and output structuring methods.
+
+ Attributes:
+ the_pipe_code: Pipe code. Must be snake_case.
+ type: Fixed to "PipeLLM" for this pipe type.
+ system_prompt_template: Template for system prompt with inline variables using $ syntax.
+ system_prompt_template_name: Name reference to a system prompt template.
+ Mutually exclusive with other system_prompt fields.
+ system_prompt_name: Name reference to a system prompt.
+ Mutually exclusive with other system_prompt fields.
+ system_prompt: Direct system-level prompt to guide LLM behavior. Can be inline text
+ or file reference ('file:path/to/prompt.md'). Mutually exclusive with
+ other system_prompt fields.
+ prompt_template: User prompt template with variable substitution. Use $ for inline
+ variables (e.g., $topic) and @ for entire input content (e.g., @text_to_summarize).
+ Note: Don't use @ or $ for image variables. Mutually exclusive with other
+ prompt fields.
+ template_name: Name reference to a prompt template. Mutually exclusive with other prompt fields.
+ prompt_name: Name reference to a prompt. Mutually exclusive with other prompt fields.
+ prompt: Static user prompt without variable injection. Mutually exclusive with other prompt fields.
+ llm: LLM preset(s) configuration. Can be single preset or mapping for different
+ generation modes (e.g., main, object_direct).
+ llm_to_structure: LLM preset specifically for output structuring in preliminary_text mode.
+ structuring_method: Method for structured output generation ('direct' or 'preliminary_text').
+ Defaults to global configuration.
+ prompt_template_to_structure: Prompt template for second step in preliminary_text mode.
+ system_prompt_to_structure: System prompt for structuring step in preliminary_text mode.
+ nb_output: Fixed number of outputs to generate (e.g., 3 for exactly 3 outputs).
+ Must be > 0. Mutually exclusive with multiple_output.
+ multiple_output: Enables variable-length list generation. Default is false (single output).
+ Set to true for indeterminate number of outputs. Mutually exclusive with nb_output.
+
+ Validation Rules:
+ 1. System prompt fields are mutually exclusive (only one can be set).
+ 2. User prompt fields are mutually exclusive (only one can be set).
+ 3. Output cardinality: nb_output and multiple_output are mutually exclusive.
+ 4. nb_output must be greater than 0 when specified.
+ 5. Structuring method must be 'direct' or 'preliminary_text' when specified.
+ """
+
+ type: Literal["PipeLLM"] = "PipeLLM"
+ category: Literal["PipeOperator"] = "PipeOperator"
+ the_pipe_code: str = Field(description="Pipe code. Must be snake_case.")
+ system_prompt_template: Optional[str] = None
+ system_prompt_template_name: Optional[str] = None
+ system_prompt_name: Optional[str] = None
+ system_prompt: Optional[str] = None
+
+ prompt_template: Optional[str] = None
+ template_name: Optional[str] = None
+ prompt_name: Optional[str] = None
+ prompt: Optional[str] = None
+
+ llm: Optional[LLMChoiceSpec] = None
+ llm_to_structure: Optional[LLMChoiceSpec] = None
+
+ structuring_method: Optional[StructuringMethod] = None
+ prompt_template_to_structure: Optional[str] = None
+ system_prompt_to_structure: Optional[str] = None
+
+ nb_output: Optional[int] = None
+ multiple_output: Optional[bool] = None
+
+ @field_validator("nb_output", mode="after")
+ def validate_nb_output(cls, value: Optional[int] = None) -> Optional[int]:
+ if value and value < 1:
+ raise PipeDefinitionError("PipeLLMBlueprint nb_output must be greater than 0")
+ return value
+
+ @model_validator(mode="after")
+ def validate_multiple_output(self) -> Self:
+ if excess_attributes_list := has_more_than_one_among_attributes_from_lists(
+ self,
+ attributes_lists=[
+ ["nb_output", "multiple_output"],
+ ["system_prompt", "system_prompt_name", "system_prompt_template", "system_prompt_template_name"],
+ ["prompt", "prompt_name", "prompt_template", "template_name"],
+ ],
+ ):
+ raise PipeDefinitionError(f"PipeLLMBlueprint should have no more than one of {excess_attributes_list} among them")
+ return self
+
+ @override
+ def to_blueprint(self) -> PipeLLMBlueprint:
+ base_blueprint = super().to_blueprint()
+ llm: Optional[LLMChoice] = None
+ if isinstance(self.llm, LLMSettingSpec):
+ llm = LLMSetting(llm_handle=self.llm.llm_handle, temperature=self.llm.temperature, max_tokens=self.llm.max_tokens)
+ elif isinstance(self.llm, str):
+ llm = self.llm
+
+ llm_to_structure: Optional[LLMChoice] = None
+ if isinstance(self.llm_to_structure, LLMSettingSpec):
+ llm_to_structure = LLMSetting(
+ llm_handle=self.llm_to_structure.llm_handle,
+ temperature=self.llm_to_structure.temperature,
+ max_tokens=self.llm_to_structure.max_tokens,
+ )
+ elif isinstance(self.llm_to_structure, str):
+ llm_to_structure = self.llm_to_structure
+
+ return PipeLLMBlueprint(
+ definition=base_blueprint.definition,
+ inputs=base_blueprint.inputs,
+ output=base_blueprint.output,
+ type=self.type,
+ category=self.category,
+ system_prompt_template=self.system_prompt_template,
+ system_prompt_template_name=self.system_prompt_template_name,
+ system_prompt_name=self.system_prompt_name,
+ system_prompt=self.system_prompt,
+ prompt_template=self.prompt_template,
+ template_name=self.template_name,
+ prompt_name=self.prompt_name,
+ prompt=self.prompt,
+ llm=llm,
+ llm_to_structure=llm_to_structure,
+ structuring_method=self.structuring_method,
+ prompt_template_to_structure=self.prompt_template_to_structure,
+ system_prompt_to_structure=self.system_prompt_to_structure,
+ nb_output=self.nb_output,
+ multiple_output=self.multiple_output,
+ )
diff --git a/pipelex/libraries/pipelines/builder/pipe/pipe_ocr_spec.py b/pipelex/libraries/pipelines/builder/pipe/pipe_ocr_spec.py
new file mode 100644
index 000000000..299197280
--- /dev/null
+++ b/pipelex/libraries/pipelines/builder/pipe/pipe_ocr_spec.py
@@ -0,0 +1,59 @@
+from typing import Literal, Optional
+
+from pydantic import Field
+from typing_extensions import override
+
+from pipelex.libraries.pipelines.builder.pipe.pipe_signature import PipeSpec
+from pipelex.pipe_operators.ocr.pipe_ocr_blueprint import PipeOcrBlueprint
+
+
+class PipeOcrSpec(PipeSpec):
+ """Spec for OCR (Optical Character Recognition) pipe operations in the Pipelex framework.
+
+ PipeOcr enables text extraction from images and documents using OCR technology.
+ Supports various OCR platforms and output configurations including image detection,
+ caption generation, and page rendering.
+
+ VERY IMPORTANT: THE INPUT OF THE PIPEOCR MUST BE NAMED "ocr_input" and it must be either an image or a pdf or a concept which refines one of them.
+
+ Attributes:
+ the_pipe_code: Pipe code. Must be snake_case.
+ type: Fixed to "PipeOcr" for this pipe type.
+ ocr: Needs to be "base_ocr_mistral".
+ page_images: Whether to include detected images in the OCR output. When enabled,
+ extracts and returns embedded images found in documents.
+ page_image_captions: Whether to generate captions for detected images using AI.
+ Useful for understanding image content in documents.
+ page_views: Whether to include rendered page views in the output. Provides
+ visual representation of document pages.
+ page_views_dpi: DPI (dots per inch) resolution for rendered page views.
+ Higher values provide better quality but larger file sizes.
+ Defaults to configuration setting.
+
+ Validation Rules:
+ 1. OCR model must be "base_ocr_mistral".
+ 2. Boolean flags (page_images, page_image_captions, page_views) are optional.
+ 3. page_views_dpi should be a positive integer when specified.
+ """
+
+ type: Literal["PipeOcr"] = "PipeOcr"
+ category: Literal["PipeOperator"] = "PipeOperator"
+ the_pipe_code: str = Field(description="Pipe code. Must be snake_case.")
+ ocr: str = "base_ocr_mistral"
+ page_images: Optional[bool] = None
+ page_image_captions: Optional[bool] = None
+ page_views: Optional[bool] = None
+ page_views_dpi: Optional[int] = None
+
+ @override
+ def to_blueprint(self) -> PipeOcrBlueprint:
+ base_blueprint = super().to_blueprint()
+
+ return PipeOcrBlueprint(
+ definition=base_blueprint.definition,
+ inputs=base_blueprint.inputs,
+ output=base_blueprint.output,
+ type=self.type,
+ category=self.category,
+ ocr=self.ocr,
+ )
diff --git a/pipelex/libraries/pipelines/builder/pipe/pipe_parallel_spec.py b/pipelex/libraries/pipelines/builder/pipe/pipe_parallel_spec.py
new file mode 100644
index 000000000..be95d3f2c
--- /dev/null
+++ b/pipelex/libraries/pipelines/builder/pipe/pipe_parallel_spec.py
@@ -0,0 +1,62 @@
+from typing import List, Literal, Optional
+
+from pydantic import Field, field_validator
+from typing_extensions import override
+
+from pipelex.libraries.pipelines.builder.concept.concept_spec import ConceptBlueprint
+from pipelex.libraries.pipelines.builder.pipe.pipe_signature import PipeSpec
+from pipelex.libraries.pipelines.builder.pipe.sub_pipe_spec import SubPipeSpec
+from pipelex.pipe_controllers.parallel.pipe_parallel_blueprint import PipeParallelBlueprint
+
+
+class PipeParallelSpec(PipeSpec):
+ """Spec for parallel pipe execution in the Pipelex framework.
+
+ PipeParallel enables concurrent execution of multiple pipes, improving performance
+ for independent operations. All parallel pipes receive the same input context
+ and their outputs can be combined or kept separate.
+
+ Attributes:
+ the_pipe_code: Pipe code. Must be snake_case.
+ type: Fixed to "PipeParallel" for this pipe type.
+ parallels: List of SubPipeSpec instances to execute concurrently.
+ All pipes run simultaneously with access to the same input context.
+ add_each_output: Whether to include individual pipe outputs in the combined
+ result. Default is True. When False, only combined_output is used.
+ combined_output: Optional concept string/code for the combined output structure.
+ When specified, all parallel outputs are merged into this concept.
+
+ Validation Rules:
+ 1. Parallels list must not be empty.
+ 2. Each parallel step must be a valid SubPipeSpec.
+ 3. combined_output, when specified, must be a valid concept string or code.
+ 4. Pipe codes in parallels must reference existing pipes.
+ """
+
+ type: Literal["PipeParallel"] = "PipeParallel"
+ category: Literal["PipeController"] = "PipeController"
+ the_pipe_code: str = Field(description="Pipe code. Must be snake_case.")
+ parallels: List[SubPipeSpec]
+ add_each_output: bool = True
+ combined_output: Optional[str] = None
+
+ @field_validator("combined_output", mode="before")
+ def validate_combined_output(cls, combined_output: str) -> str:
+ if combined_output:
+ ConceptBlueprint.validate_concept_string_or_concept_code(concept_string_or_code=combined_output)
+ return combined_output
+
+ @override
+ def to_blueprint(self) -> PipeParallelBlueprint:
+ base_blueprint = super().to_blueprint()
+ core_parallels = [parallel.to_blueprint() for parallel in self.parallels]
+ return PipeParallelBlueprint(
+ definition=base_blueprint.definition,
+ inputs=base_blueprint.inputs,
+ output=base_blueprint.output,
+ type=self.type,
+ category=self.category,
+ parallels=core_parallels,
+ add_each_output=self.add_each_output,
+ combined_output=self.combined_output,
+ )
diff --git a/pipelex/libraries/pipelines/builder/pipe/pipe_sequence_spec.py b/pipelex/libraries/pipelines/builder/pipe/pipe_sequence_spec.py
new file mode 100644
index 000000000..bda5a78a4
--- /dev/null
+++ b/pipelex/libraries/pipelines/builder/pipe/pipe_sequence_spec.py
@@ -0,0 +1,47 @@
+from typing import List, Literal
+
+from pydantic import Field
+from typing_extensions import override
+
+from pipelex.libraries.pipelines.builder.pipe.pipe_signature import PipeSpec
+from pipelex.libraries.pipelines.builder.pipe.sub_pipe_spec import SubPipeSpec
+from pipelex.pipe_controllers.sequence.pipe_sequence_blueprint import PipeSequenceBlueprint
+
+
+class PipeSequenceSpec(PipeSpec):
+ """Spec for sequential pipe execution in the Pipelex framework.
+
+ PipeSequence orchestrates the execution of multiple pipes in a defined order,
+ where each pipe's output can be used as input for subsequent pipes. This enables
+ building complex data processing workflows with step-by-step transformations.
+
+ Attributes:
+ the_pipe_code: Pipe code. Must be snake_case.
+ type: Fixed to "PipeSequence" for this pipe type.
+ steps: Ordered list of SubPipeBlueprint instances defining the pipes
+ to execute. Each step runs after the previous one completes,
+ with access to all prior outputs in the context.
+
+ Validation Rules:
+ 1. Steps list must not be empty.
+ 2. Each step must be a valid SubPipeBlueprint instance.
+ 3. Pipe codes referenced in steps must exist in the pipeline.
+ """
+
+ type: Literal["PipeSequence"] = "PipeSequence"
+ category: Literal["PipeController"] = "PipeController"
+ the_pipe_code: str = Field(description="Pipe code. Must be snake_case.")
+ steps: List[SubPipeSpec]
+
+ @override
+ def to_blueprint(self) -> PipeSequenceBlueprint:
+ base_blueprint = super().to_blueprint()
+ core_steps = [step.to_blueprint() for step in self.steps]
+ return PipeSequenceBlueprint(
+ definition=base_blueprint.definition,
+ inputs=base_blueprint.inputs,
+ output=base_blueprint.output,
+ type=self.type,
+ category=self.category,
+ steps=core_steps,
+ )
diff --git a/pipelex/libraries/pipelines/builder/pipe/pipe_signature.py b/pipelex/libraries/pipelines/builder/pipe/pipe_signature.py
new file mode 100644
index 000000000..0d6219e3f
--- /dev/null
+++ b/pipelex/libraries/pipelines/builder/pipe/pipe_signature.py
@@ -0,0 +1,102 @@
+from typing import Any, Dict, Optional, Union
+
+from pydantic import Field, field_validator
+
+from pipelex.core.pipes.exceptions import PipeBlueprintError
+from pipelex.core.pipes.pipe_blueprint import AllowedPipeCategories, AllowedPipeTypes, PipeBlueprint
+from pipelex.core.pipes.pipe_input_blueprint import InputRequirementBlueprint
+from pipelex.core.stuffs.stuff_content import StructuredContent
+from pipelex.libraries.pipelines.builder.concept.concept_spec import ConceptBlueprint, ConceptSpecDraft
+from pipelex.libraries.pipelines.builder.pipe.inputs_spec import InputRequirementSpec
+from pipelex.tools.misc.string_utils import is_snake_case
+
+
+class PipeSignature(StructuredContent):
+ code: str = Field(description="Pipe code. Must be snake_case.")
+ type: AllowedPipeTypes = Field(description="Pipe type.")
+ category: AllowedPipeCategories = Field(description="Pipe category.")
+ definition: str = Field(description="What the pipe does")
+ inputs: Dict[str, ConceptSpecDraft] = Field(description="Pipe inputs: key is the concept code in pascal Case.")
+ result: str = Field(description="The name of the result of the pipe. Must be snake_case. It will be used in the inputs of the next pipes.")
+ output: ConceptSpecDraft = Field(description="Concept as output")
+ important_features: Optional[Dict[str, Any]] = Field(
+ default=None,
+ description="Important features specific to this pipe type "
+ "(e.g., referenced pipe codes for controllers, specific configuration for operators)",
+ )
+
+
+class PipeSpec(StructuredContent):
+ """Spec defining a pipe component in the Pipelex framework.
+
+ Pipes are the fundamental processing units in Pipelex workflows. They transform
+ input concepts into output concepts through various operations like LLM processing,
+ image generation, OCR, or custom functions.
+
+ Attributes:
+ type: The pipe type (PipeFunc, PipeLLM, PipeImgGen, PipeOcr, PipeBatch,
+ PipeCondition, PipeParallel, PipeSequence). Uses Any type to avoid
+ type override conflicts but validated at runtime.
+ category: The pipe category (PipeOperator, PipeController). Uses Any type to avoid
+ category override conflicts but validated at runtime.
+ The pipe controllers are PipeSequence, PipeParallel, PipeCondition, PipeBatch.
+ The pipe operators are PipeFunc, PipeLLM, PipeImgGen, PipeOcr, PipeJinja2.
+ definition: Natural language description of what the pipe does.
+ inputs: Input concept specifications. should be an InputRequirementBlueprint
+ Dictionary keys are input names in snake_case, values are concept specifications in PascalCase.
+ output: Output concept code in PascalCase format.
+
+ Validation Rules:
+ 1. Pipe type: Must be one of the AllowedPipeTypes enum values.
+ 2. Output concept: Must be valid concept string or code in PascalCase.
+ 3. Input concepts: When provided, must use PascalCase for concept references.
+ 4. Pipe codes: When validating pipe codes, must be in snake_case format.
+ """
+
+ type: Any = Field(description=f"Pipe type. Must be one of: {AllowedPipeTypes}")
+ category: Any = Field(description=f"Pipe category. Must be one of: {AllowedPipeCategories}")
+ definition: Optional[str] = Field(description="Natural language description of what the pipe does.")
+ inputs: Optional[Dict[str, Union[str, InputRequirementSpec]]] = Field(
+ description=(
+ "Input concept specifications. Can be either: "
+ "InputRequirementSpec with additional constraints"
+ "Dictionary keys are input names, values are concept specifications. If Its the concept itself, use the concept code in PascalCase."
+ )
+ )
+ output: str = Field(description="Output concept code in PascalCase format!! Very important")
+
+ @field_validator("type", mode="after")
+ def validate_pipe_type(cls, value: Any) -> Any:
+ allowed_types = [_type.value for _type in AllowedPipeTypes]
+ if value not in allowed_types:
+ raise PipeBlueprintError(f"Invalid pipe type '{value}'. Must be one of: {allowed_types}")
+ return value
+
+ @field_validator("output", mode="before")
+ def validate_concept_string_or_concept_code(cls, output: str) -> str:
+ ConceptBlueprint.validate_concept_string_or_concept_code(concept_string_or_code=output)
+ return output
+
+ @classmethod
+ def validate_pipe_code_syntax(cls, pipe_code: str) -> str:
+ if not is_snake_case(pipe_code):
+ raise PipeBlueprintError(f"Invalid pipe code syntax '{pipe_code}'. Must be in snake_case.")
+ return pipe_code
+
+ def to_blueprint(self) -> PipeBlueprint:
+ converted_inputs: Optional[Dict[str, Union[str, InputRequirementBlueprint]]] = None
+ if self.inputs is not None:
+ converted_inputs = {}
+ for input_name, input_spec in self.inputs.items():
+ if isinstance(input_spec, InputRequirementSpec):
+ converted_inputs[input_name] = input_spec.to_blueprint()
+ else:
+ converted_inputs[input_name] = InputRequirementBlueprint(concept=input_spec)
+
+ return PipeBlueprint(
+ definition=self.definition,
+ inputs=converted_inputs,
+ output=self.output,
+ type=self.type,
+ category=self.category,
+ )
diff --git a/pipelex/libraries/pipelines/builder/pipe/sub_pipe_spec.py b/pipelex/libraries/pipelines/builder/pipe/sub_pipe_spec.py
new file mode 100644
index 000000000..aafc03094
--- /dev/null
+++ b/pipelex/libraries/pipelines/builder/pipe/sub_pipe_spec.py
@@ -0,0 +1,76 @@
+from typing import Optional, Union
+
+from pydantic import ConfigDict, model_validator
+from typing_extensions import Self
+
+from pipelex.core.stuffs.stuff_content import StructuredContent
+from pipelex.exceptions import PipeDefinitionError
+from pipelex.pipe_controllers.sub_pipe_blueprint import SubPipeBlueprint
+from pipelex.tools.typing.validation_utils import has_more_than_one_among_attributes_from_list
+
+
+class SubPipeSpec(StructuredContent):
+ """Spec for a single step within a pipe controller.
+
+ SubPipeSpec defines individual pipe executions within controller pipes
+ (PipeSequence, PipeParallel, PipeBatch, PipeCondition). Supports output
+ cardinality control and batch processing configuration.
+
+ Attributes:
+ the_pipe_code: The pipe code to execute. Must reference an existing pipe in the pipeline.
+ result: Optional name to assign to the pipe's output in the context.
+ If not specified, output is added directly to context.
+ nb_output: Fixed number of outputs to generate. Mutually exclusive with
+ multiple_output.
+ multiple_output: When true, allows LLM to determine the number of outputs.
+ Mutually exclusive with nb_output.
+ batch_over: Name of the list in context to iterate over for batch processing.
+ When false (default), no batching occurs. When specified as string,
+ references a list in context. Requires batch_as when set.
+ batch_as: Name to assign to the current item during batch iteration.
+ Required when batch_over is specified.
+
+ Validation Rules:
+ 1. nb_output and multiple_output are mutually exclusive.
+ 2. batch_over and batch_as must be specified together (both or neither).
+ 3. pipe must reference a valid pipe code.
+ 4. result, when specified, should follow naming conventions.
+ """
+
+ model_config = ConfigDict(extra="forbid")
+
+ the_pipe_code: str
+ result: Optional[str] = None
+ nb_output: Optional[int] = None
+ multiple_output: Optional[bool] = None
+ batch_over: Union[bool, str] = False
+ batch_as: Optional[str] = None
+
+ @model_validator(mode="after")
+ def validate_multiple_output(self) -> Self:
+ if has_more_than_one_among_attributes_from_list(self, attributes_list=["nb_output", "multiple_output"]):
+ raise PipeDefinitionError("PipeStepBlueprint should have no more than '1' of nb_output or multiple_output")
+ return self
+
+ @model_validator(mode="after")
+ def validate_batch_params(self) -> Self:
+ batch_over_is_specified = self.batch_over is not False and self.batch_over != ""
+ batch_as_is_specified = self.batch_as is not None and self.batch_as != ""
+
+ if batch_over_is_specified and not batch_as_is_specified:
+ raise PipeDefinitionError(f"In pipe '{self.the_pipe_code}': When 'batch_over' is specified, 'batch_as' must also be provided")
+
+ if batch_as_is_specified and not batch_over_is_specified:
+ raise PipeDefinitionError(f"In pipe '{self.the_pipe_code}': When 'batch_as' is specified, 'batch_over' must also be provided")
+
+ return self
+
+ def to_blueprint(self) -> SubPipeBlueprint:
+ return SubPipeBlueprint(
+ pipe=self.the_pipe_code,
+ result=self.result,
+ nb_output=self.nb_output,
+ multiple_output=self.multiple_output,
+ batch_over=self.batch_over,
+ batch_as=self.batch_as,
+ )
diff --git a/pipelex/libraries/pipelines/documents.plx b/pipelex/libraries/pipelines/documents.plx
index 0f8b290ef..374b62e2c 100644
--- a/pipelex/libraries/pipelines/documents.plx
+++ b/pipelex/libraries/pipelines/documents.plx
@@ -17,7 +17,7 @@ inputs = { ocr_input = "PDF" }
output = "Page"
page_images = true
page_views = false
-ocr_model = "mistral-ocr"
+ocr = "mistral-ocr"
[pipe.ocr_page_contents_and_views_from_pdf]
type = "PipeOcr"
@@ -26,5 +26,5 @@ inputs = { ocr_input = "PDF" }
output = "Page"
page_images = true
page_views = true
-ocr_model = "mistral-ocr"
+ocr = "mistral-ocr"
diff --git a/pipelex/libraries/pipelines/images.plx b/pipelex/libraries/pipelines/images.plx
index 55d5ea470..43338aabf 100644
--- a/pipelex/libraries/pipelines/images.plx
+++ b/pipelex/libraries/pipelines/images.plx
@@ -13,34 +13,6 @@ definition = "Photo"
refines = "Image"
[pipe]
-
-#################################################################
-# Vision: PipeLLM taking images as input
-#################################################################
-
-[pipe.describe_image]
-type = "PipeLLM"
-definition = "Describe an image"
-inputs = { image = "Image" }
-output = "VisualDescription"
-system_prompt = "You are a very good observer."
-llm = "llm_to_describe_img"
-structuring_method = "preliminary_text"
-prompt_template = """
-Describe the provided image in great detail.
-"""
-
-[pipe.describe_photo]
-type = "PipeLLM"
-definition = "Describe a photo"
-inputs = { photo = "Photo" }
-output = "VisualDescription"
-system_prompt = "You are a very good observer."
-llm = "llm_to_describe_img"
-prompt_template = """
-Describe the provided photo and how it was shot: scene, lighting, camera, etc.
-"""
-
#################################################################
# Image generation: PipeImgGen generating images as output
#################################################################
diff --git a/pipelex/libraries/pipelines/meta/meta_pipeline.plx b/pipelex/libraries/pipelines/meta/meta_pipeline.plx
deleted file mode 100644
index 791c63900..000000000
--- a/pipelex/libraries/pipelines/meta/meta_pipeline.plx
+++ /dev/null
@@ -1,68 +0,0 @@
-domain = "meta"
-definition = "Meta domain is the domain of pipelines about pipelines"
-
-[concept]
-PipelineDraft = "Rough solution to create a pipeline"
-PipelineDraftText = "Rough solution to create a pipeline in text format"
-PipeDraft = "Rough solution to create a pipe"
-ConceptBlueprint = "Details enabling to create a concept"
-PipeBlueprint = "Details enabling to create a pipe"
-
-[concept.PipelexBundleBlueprint]
-definition = "Details enabling to create a pipeline"
-structure = "PipelexBundleBlueprint"
-
-[pipe]
-[pipe.build_blueprint]
-type = "PipeSequence"
-definition = "Build a pipeline blueprint from natural language requirements"
-inputs = { draft_pipeline_rules = "Text", build_pipeline_rules = "Text", create_structured_output_rules = "Text", requirements = "Text", pipeline_name = "Text", domain = "Text" }
-output = "PipelexBundleBlueprint"
-steps = [
- { pipe = "draft_pipeline_text", result = "pipeline_draft" },
- { pipe = "structure_blueprint", result = "pipeline_blueprint" },
-]
-
-[pipe.draft_pipeline_text]
-type = "PipeLLM"
-definition = "Generate a rough pipeline draft from natural language requirements"
-inputs = { draft_pipeline_rules = "Text", create_structured_output_rules = "Text", requirements = "Text", pipeline_name = "Text", domain = "Text" }
-output = "PipelineDraftText"
-llm = "llm_to_pipelex"
-prompt_template = """
-You are a Pipelex pipeline architect. Build a pipeline draft text showing how we solve the problem.
-
-@draft_pipeline_rules
-
-Do create structured output for better concept representation.
-
-@create_structured_output_rules
----
-Name of the pipeline: $pipeline_name.
-
-Domain: $domain.
-
-Requirements: $requirements
-"""
-
-[pipe.structure_blueprint]
-type = "PipeLLM"
-definition = "Generate a pipeline blueprint from natural language requirements"
-inputs = { build_pipeline_rules = "Text", pipeline_draft = "Text", pipeline_name = "Text", domain = "Text", requirements = "Text" }
-output = "PipelexBundleBlueprint"
-llm = "llm_to_pipelex"
-prompt_template = """
-You are a Pipelex pipeline architect. Build a structured pipeline blueprint from the provided pipeline draft text.
-
-@build_pipeline_rules
-
-@pipeline_draft
-
----
-Name of the pipeline: $pipeline_name.
-
-Domain: $domain.
-
-Requirements: $requirements.
-"""
-
diff --git a/pipelex/libraries/pipelines/meta/pipeline_draft.py b/pipelex/libraries/pipelines/meta/pipeline_draft.py
deleted file mode 100644
index e2954da67..000000000
--- a/pipelex/libraries/pipelines/meta/pipeline_draft.py
+++ /dev/null
@@ -1,34 +0,0 @@
-from typing import Dict
-
-from pydantic import Field
-
-from pipelex.core.bundles.pipelex_bundle_blueprint import PipelexBundleBlueprint as PipelexBundleBlueprintBaseModel
-from pipelex.core.stuffs.stuff_content import StructuredContent
-
-
-class PipeDraft(StructuredContent):
- code: str
- type: str
- definition: str
- inputs: Dict[str, str]
- output: str
-
-
-class PipelineDraft(StructuredContent):
- """Complete blueprint of a pipeline library PLX file."""
-
- # Domain information (required)
- domain: str
- definition: str
-
- # Concepts section - concept_name -> definition (string) or blueprint (dict)
- concept: Dict[str, str] = Field(default_factory=dict)
-
- # Pipes section - pipe_name -> blueprint dict
- pipe: Dict[str, PipeDraft] = Field(default_factory=dict)
-
-
-class PipelexBundleBlueprint(PipelexBundleBlueprintBaseModel, StructuredContent):
- """Complete blueprint of a pipelex bundle PLX file."""
-
- pass
diff --git a/pipelex/libraries/templates/base_templates.toml b/pipelex/libraries/templates/base_templates.toml
index 087d362da..484fecacf 100644
--- a/pipelex/libraries/templates/base_templates.toml
+++ b/pipelex/libraries/templates/base_templates.toml
@@ -1,10 +1,11 @@
[generic_prompts]
structure_from_preliminary_text_system = "You are a data modeling expert specialized in extracting structure from text."
+# TODO: Should be a proper jinja2
structure_from_preliminary_text_user = """
Your job is to extract and structure information from a text.
Here is the text:
-{{ preliminary_text|tag("text") }}
+{preliminary_text}
Now generate the JSON in the required format.
Do not create information that is not in the text.
diff --git a/pipelex/pipe_controllers/batch/pipe_batch.py b/pipelex/pipe_controllers/batch/pipe_batch.py
index 015bc1b0f..05bf56c3f 100644
--- a/pipelex/pipe_controllers/batch/pipe_batch.py
+++ b/pipelex/pipe_controllers/batch/pipe_batch.py
@@ -1,5 +1,5 @@
import asyncio
-from typing import Any, Coroutine, List, Optional, Set, cast
+from typing import Any, Coroutine, List, Literal, Optional, Set, cast
import shortuuid
from pydantic import model_validator
@@ -8,7 +8,7 @@ from typing_extensions import Self, override
from pipelex import log
from pipelex.config import get_config
from pipelex.core.memory.working_memory import MAIN_STUFF_NAME, WorkingMemory
-from pipelex.core.pipes.pipe_input_spec import PipeInputSpec
+from pipelex.core.pipes.pipe_input import PipeInputSpec
from pipelex.core.pipes.pipe_output import PipeOutput
from pipelex.core.pipes.pipe_run_params import BatchParams, PipeRunMode, PipeRunParams
from pipelex.core.stuffs.stuff import Stuff
@@ -25,7 +25,7 @@ from pipelex.pipeline.job_metadata import JobMetadata
class PipeBatch(PipeController):
- """Runs a PipeSequence in parallel for each item in a list."""
+ type: Literal["PipeBatch"] = "PipeBatch"
branch_pipe_code: str
batch_params: Optional[BatchParams] = None
@@ -68,7 +68,7 @@ class PipeBatch(PipeController):
return required_variables
@override
- def needed_inputs(self) -> PipeInputSpec:
+ def needed_inputs(self, visited_pipes: Optional[Set[str]] = None) -> PipeInputSpec:
return self.inputs
async def _run_batch_pipe(
diff --git a/pipelex/pipe_controllers/batch/pipe_batch_blueprint.py b/pipelex/pipe_controllers/batch/pipe_batch_blueprint.py
index 2abf365cf..80707fdef 100644
--- a/pipelex/pipe_controllers/batch/pipe_batch_blueprint.py
+++ b/pipelex/pipe_controllers/batch/pipe_batch_blueprint.py
@@ -5,7 +5,7 @@ from pipelex.core.pipes.pipe_blueprint import PipeBlueprint
class PipeBatchBlueprint(PipeBlueprint):
type: Literal["PipeBatch"] = "PipeBatch"
+ category: Literal["PipeController"] = "PipeController"
branch_pipe_code: str
-
input_list_name: Optional[str] = None
input_item_name: Optional[str] = None
diff --git a/pipelex/pipe_controllers/batch/pipe_batch_factory.py b/pipelex/pipe_controllers/batch/pipe_batch_factory.py
index 91977a2c2..00bc0b8b7 100644
--- a/pipelex/pipe_controllers/batch/pipe_batch_factory.py
+++ b/pipelex/pipe_controllers/batch/pipe_batch_factory.py
@@ -4,7 +4,7 @@ from typing_extensions import override
from pipelex.core.concepts.concept_factory import ConceptFactory
from pipelex.core.pipes.pipe_factory import PipeFactoryProtocol
-from pipelex.core.pipes.pipe_input_spec_factory import PipeInputSpecFactory
+from pipelex.core.pipes.pipe_input_factory import PipeInputSpecFactory
from pipelex.core.pipes.pipe_run_params import BatchParams
from pipelex.hub import get_concept_provider
from pipelex.pipe_controllers.batch.pipe_batch import PipeBatch
@@ -23,7 +23,7 @@ class PipeBatchFactory(PipeFactoryProtocol[PipeBatchBlueprint, PipeBatch]):
) -> PipeBatch:
output_domain_and_code = ConceptFactory.make_domain_and_concept_code_from_concept_string_or_concept_code(
domain=domain,
- concept_string_or_concept_code=blueprint.output_concept_string_or_concept_code,
+ concept_string_or_code=blueprint.output,
concept_codes_from_the_same_domain=concept_codes_from_the_same_domain,
)
return PipeBatch(
diff --git a/pipelex/pipe_controllers/condition/pipe_condition.py b/pipelex/pipe_controllers/condition/pipe_condition.py
index f7d5652ad..129e08210 100644
--- a/pipelex/pipe_controllers/condition/pipe_condition.py
+++ b/pipelex/pipe_controllers/condition/pipe_condition.py
@@ -1,4 +1,4 @@
-from typing import Dict, List, Optional, Set, Union, cast
+from typing import Dict, List, Literal, Optional, Set, Union, cast
import shortuuid
from pydantic import field_validator, model_validator
@@ -7,13 +7,14 @@ from typing_extensions import Self, override
from pipelex import log
from pipelex.config import StaticValidationReaction, get_config
from pipelex.core.concepts.concept_factory import ConceptFactory
-from pipelex.core.concepts.concept_native import NATIVE_CONCEPTS_DATA, NativeConceptEnum
+from pipelex.core.concepts.concept_native import NATIVE_CONCEPTS_DATA, NativeConceptEnum, NativeConceptManager
from pipelex.core.memory.working_memory import WorkingMemory
-from pipelex.core.pipes.pipe_input_spec import PipeInputSpec
-from pipelex.core.pipes.pipe_input_spec_blueprint import InputRequirementBlueprint
-from pipelex.core.pipes.pipe_input_spec_factory import PipeInputSpecFactory
+from pipelex.core.pipes.pipe_input import PipeInputSpec
+from pipelex.core.pipes.pipe_input_blueprint import InputRequirementBlueprint
+from pipelex.core.pipes.pipe_input_factory import PipeInputSpecFactory
from pipelex.core.pipes.pipe_output import PipeOutput
from pipelex.core.pipes.pipe_run_params import PipeRunParams
+from pipelex.core.pipes.specific_pipe import SpecificPipe
from pipelex.exceptions import (
DryRunError,
PipeConditionError,
@@ -35,6 +36,7 @@ from pipelex.tools.typing.validation_utils import has_exactly_one_among_attribut
class PipeCondition(PipeController):
+ type: Literal["PipeCondition"] = "PipeCondition"
expression_template: Optional[str] = None
expression: Optional[str] = None
# TODO: rething this pipe_map.
@@ -51,20 +53,31 @@ class PipeCondition(PipeController):
Validate the output for the pipe condition.
The output of the pipe condition should match the output of all the conditional pipes, and the default pipe.
"""
+ # This pipe CONTINUE enables to leave a PipeCondition and continue the sequence.
+ # This system though has to be rethink. It might not be the best solution
for pipe_condition_pipe_map in self.pipe_map:
- pipe = get_required_pipe(pipe_code=pipe_condition_pipe_map.pipe_code)
- if self.output.concept_string != pipe.output.concept_string:
- raise PipeConditionError(
- f"The output concept code '{self.output.concept_string}' of the pipe '{self.code}' is "
- f"not matching the output concept code '{pipe.output.concept_string}' of the pipe '{pipe_condition_pipe_map.pipe_code}'"
- )
+ if not SpecificPipe.is_continue(pipe_condition_pipe_map.pipe_code):
+ pipe = get_required_pipe(pipe_code=pipe_condition_pipe_map.pipe_code)
+ if self.output.concept_string not in (
+ pipe.output.concept_string,
+ NativeConceptManager.get_native_concept_string(NativeConceptEnum.DYNAMIC),
+ ):
+ msg = (
+ f"The output concept code '{self.output.concept_string}' of the pipe '{self.code}' is "
+ f"not matching the output concept code '{pipe.output.concept_string}' of the pipe '{pipe_condition_pipe_map.pipe_code}'"
+ )
+ raise PipeConditionError(msg)
if self.default_pipe_code:
default_pipe = get_required_pipe(pipe_code=self.default_pipe_code)
- if self.output.concept_string != default_pipe.output.concept_string:
- raise PipeConditionError(
+ if self.output.concept_string not in (
+ default_pipe.output.concept_string,
+ NativeConceptManager.get_native_concept_string(NativeConceptEnum.DYNAMIC),
+ ):
+ msg = (
f"The output concept code '{self.output.concept_string}' of the pipe '{self.code}' is "
f"not matching the output concept code '{default_pipe.output.concept_string}' of the default pipe '{self.default_pipe_code}'"
)
+ raise PipeConditionError(msg)
@field_validator("pipe_map")
@classmethod
@@ -134,14 +147,17 @@ class PipeCondition(PipeController):
return self
@override
- def needed_inputs(self) -> PipeInputSpec:
- """
- Calculate the inputs needed by this PipeCondition.
+ def needed_inputs(self, visited_pipes: Optional[Set[str]] = None) -> PipeInputSpec:
+ if visited_pipes is None:
+ visited_pipes = set()
+
+ # If we've already visited this pipe, stop recursion
+ if self.code in visited_pipes:
+ return PipeInputSpecFactory.make_empty()
+
+ # Add this pipe to visited set for recursive calls
+ visited_pipes_with_current = visited_pipes | {self.code}
- The inputs are:
- 1. Inputs needed by the condition expression/expression_template
- 2. Inputs needed by ALL possible target pipes (since we don't know which will be chosen)
- """
needed_inputs = PipeInputSpecFactory.make_empty()
# 1. Add the variables from the expression/expression_template
@@ -164,9 +180,13 @@ class PipeCondition(PipeController):
# 2. Add the inputs needed by all possible target pipes
for pipe_condition_pipe_map in self.pipe_map:
- pipe = get_required_pipe(pipe_code=pipe_condition_pipe_map.pipe_code)
- for input_name, requirement in pipe.needed_inputs().items:
- needed_inputs.add_requirement(variable_name=input_name, concept=requirement.concept)
+ if not SpecificPipe.is_continue(pipe_condition_pipe_map.pipe_code):
+ pipe = get_required_pipe(pipe_code=pipe_condition_pipe_map.pipe_code)
+ # Use the centralized recursion detection
+ pipe_needed_inputs = pipe.needed_inputs(visited_pipes_with_current)
+
+ for input_name, requirement in pipe_needed_inputs.items:
+ needed_inputs.add_requirement(variable_name=input_name, concept=requirement.concept)
return needed_inputs
@@ -233,7 +253,11 @@ class PipeCondition(PipeController):
@override
def pipe_dependencies(self) -> Set[str]:
- pipe_codes = [pipe_condition_pipe_map.pipe_code for pipe_condition_pipe_map in self.pipe_map]
+ pipe_codes = [
+ pipe_condition_pipe_map.pipe_code
+ for pipe_condition_pipe_map in self.pipe_map
+ if not SpecificPipe.is_continue(pipe_condition_pipe_map.pipe_code)
+ ]
if self.default_pipe_code:
pipe_codes.append(self.default_pipe_code)
return set(pipe_codes)
@@ -269,7 +293,7 @@ class PipeCondition(PipeController):
# TODO: use jinja2 directly without going though a pipe
pipe_jinja2 = PipeJinja2Factory.make_from_blueprint(
domain=self.domain,
- pipe_code="adhoc_for_pipe_condition",
+ pipe_code="evaluation_for_pipe_condition",
blueprint=pipe_jinja2_blueprint,
)
jinja2_job_metadata = job_metadata.copy_with_update(
@@ -321,6 +345,9 @@ class PipeCondition(PipeController):
error_msg += f"\n\nPipe map: {self.pipe_map}"
raise PipeConditionError(error_msg)
+ if SpecificPipe.is_continue(chosen_pipe_code):
+ return PipeOutput(working_memory=working_memory)
+
condition_details = self._make_pipe_condition_details(
evaluated_expression=evaluated_expression,
chosen_pipe_code=chosen_pipe_code,
diff --git a/pipelex/pipe_controllers/condition/pipe_condition_blueprint.py b/pipelex/pipe_controllers/condition/pipe_condition_blueprint.py
index 30ab6852b..6898cb51c 100644
--- a/pipelex/pipe_controllers/condition/pipe_condition_blueprint.py
+++ b/pipelex/pipe_controllers/condition/pipe_condition_blueprint.py
@@ -13,6 +13,7 @@ class PipeConditionPipeMapBlueprint(RootModel[PipeConditionPipeMapRoot]):
class PipeConditionBlueprint(PipeBlueprint):
type: Literal["PipeCondition"] = "PipeCondition"
+ category: Literal["PipeController"] = "PipeController"
expression_template: Optional[str] = None
expression: Optional[str] = None
pipe_map: PipeConditionPipeMapBlueprint = Field(default_factory=PipeConditionPipeMapBlueprint)
diff --git a/pipelex/pipe_controllers/condition/pipe_condition_factory.py b/pipelex/pipe_controllers/condition/pipe_condition_factory.py
index cebdaedeb..c43d2d4ce 100644
--- a/pipelex/pipe_controllers/condition/pipe_condition_factory.py
+++ b/pipelex/pipe_controllers/condition/pipe_condition_factory.py
@@ -4,7 +4,7 @@ from typing_extensions import override
from pipelex.core.concepts.concept_factory import ConceptFactory
from pipelex.core.pipes.pipe_factory import PipeFactoryProtocol
-from pipelex.core.pipes.pipe_input_spec_factory import PipeInputSpecFactory
+from pipelex.core.pipes.pipe_input_factory import PipeInputSpecFactory
from pipelex.hub import get_concept_provider
from pipelex.pipe_controllers.condition.pipe_condition import PipeCondition
from pipelex.pipe_controllers.condition.pipe_condition_blueprint import PipeConditionBlueprint, PipeConditionPipeMapBlueprint
@@ -29,7 +29,7 @@ class PipeConditionFactory(PipeFactoryProtocol[PipeConditionBlueprint, PipeCondi
) -> PipeCondition:
output_domain_and_code = ConceptFactory.make_domain_and_concept_code_from_concept_string_or_concept_code(
domain=domain,
- concept_string_or_concept_code=blueprint.output_concept_string_or_concept_code,
+ concept_string_or_code=blueprint.output,
concept_codes_from_the_same_domain=concept_codes_from_the_same_domain,
)
return PipeCondition(
diff --git a/pipelex/pipe_controllers/parallel/pipe_parallel.py b/pipelex/pipe_controllers/parallel/pipe_parallel.py
index cd34c4fae..933152bcb 100644
--- a/pipelex/pipe_controllers/parallel/pipe_parallel.py
+++ b/pipelex/pipe_controllers/parallel/pipe_parallel.py
@@ -1,5 +1,5 @@
import asyncio
-from typing import Any, Coroutine, Dict, List, Optional, Set
+from typing import Any, Coroutine, Dict, List, Literal, Optional, Set
from pydantic import model_validator
from typing_extensions import Self, override
@@ -8,8 +8,8 @@ from pipelex import log
from pipelex.config import StaticValidationReaction, get_config
from pipelex.core.concepts.concept import Concept
from pipelex.core.memory.working_memory import WorkingMemory
-from pipelex.core.pipes.pipe_input_spec import PipeInputSpec
-from pipelex.core.pipes.pipe_input_spec_factory import PipeInputSpecFactory
+from pipelex.core.pipes.pipe_input import PipeInputSpec
+from pipelex.core.pipes.pipe_input_factory import PipeInputSpecFactory
from pipelex.core.pipes.pipe_output import PipeOutput
from pipelex.core.pipes.pipe_run_params import PipeRunMode, PipeRunParams
from pipelex.core.stuffs.stuff import Stuff
@@ -23,7 +23,7 @@ from pipelex.pipeline.job_metadata import JobMetadata
class PipeParallel(PipeController):
- """Runs a list of pipes in parallel to produce a list of results."""
+ type: Literal["PipeParallel"] = "PipeParallel"
parallel_sub_pipes: List[SubPipe]
add_each_output: bool
@@ -34,34 +34,35 @@ class PipeParallel(PipeController):
return set()
@override
- def needed_inputs(self) -> PipeInputSpec:
- """
- Calculate the inputs needed by this PipeParallel.
- This is the inputs needed by ALL parallel sub-pipes since they all run simultaneously.
- """
+ def needed_inputs(self, visited_pipes: Optional[Set[str]] = None) -> PipeInputSpec:
+ if visited_pipes is None:
+ visited_pipes = set()
+
+ # If we've already visited this pipe, stop recursion
+ if self.code in visited_pipes:
+ return PipeInputSpecFactory.make_empty()
+
+ # Add this pipe to visited set for recursive calls
+ visited_pipes_with_current = visited_pipes | {self.code}
+
needed_inputs = PipeInputSpecFactory.make_empty()
for sub_pipe in self.parallel_sub_pipes:
pipe = get_required_pipe(pipe_code=sub_pipe.pipe_code)
-
- # Get the inputs needed by this parallel pipe
- pipe_needed_inputs = pipe.needed_inputs()
-
- # Handle batching: if this sub_pipe has batch_params, exclude the batch_as input
- # since it's provided by the batching mechanism
+ # Use the centralized recursion detection
+ pipe_needed_inputs = pipe.needed_inputs(visited_pipes_with_current)
if sub_pipe.batch_params:
- batch_as_input = sub_pipe.batch_params.input_item_stuff_name
- # Create a new PipeInputSpec without the batch_as input
- filtered_needed_inputs = PipeInputSpecFactory.make_empty()
- for var_name, requirement in pipe_needed_inputs.root.items():
- if var_name != batch_as_input:
- filtered_needed_inputs.add_requirement(variable_name=var_name, concept=requirement.concept)
- pipe_needed_inputs = filtered_needed_inputs
-
- # Add all inputs from this parallel pipe
- for var_name, requirement in pipe_needed_inputs.root.items():
- needed_inputs.add_requirement(variable_name=var_name, concept=requirement.concept)
-
+ needed_inputs.add_requirement(
+ variable_name=sub_pipe.batch_params.input_list_stuff_name,
+ concept=pipe_needed_inputs.get_required_input_requirement(variable_name=sub_pipe.batch_params.input_item_stuff_name).concept,
+ multiplicity=True,
+ )
+ for input_name, requirement in pipe_needed_inputs.items:
+ if input_name != sub_pipe.batch_params.input_item_stuff_name:
+ needed_inputs.add_requirement(input_name, requirement.concept, requirement.multiplicity)
+ else:
+ for input_name, requirement in pipe_needed_inputs.items:
+ needed_inputs.add_requirement(input_name, requirement.concept, requirement.multiplicity)
return needed_inputs
@model_validator(mode="after")
@@ -313,7 +314,6 @@ class PipeParallel(PipeController):
stuff=combined_output_stuff,
name=output_name,
)
-
return PipeOutput(
working_memory=working_memory,
pipeline_run_id=job_metadata.pipeline_run_id,
diff --git a/pipelex/pipe_controllers/parallel/pipe_parallel_blueprint.py b/pipelex/pipe_controllers/parallel/pipe_parallel_blueprint.py
index 9dc11a673..b4769bb86 100644
--- a/pipelex/pipe_controllers/parallel/pipe_parallel_blueprint.py
+++ b/pipelex/pipe_controllers/parallel/pipe_parallel_blueprint.py
@@ -9,6 +9,7 @@ from pipelex.pipe_controllers.sub_pipe_blueprint import SubPipeBlueprint
class PipeParallelBlueprint(PipeBlueprint):
type: Literal["PipeParallel"] = "PipeParallel"
+ category: Literal["PipeController"] = "PipeController"
parallels: List[SubPipeBlueprint]
add_each_output: bool = True
combined_output: Optional[str] = None
@@ -16,5 +17,5 @@ class PipeParallelBlueprint(PipeBlueprint):
@field_validator("combined_output", mode="before")
def validate_combined_output(cls, combined_output: str) -> str:
if combined_output:
- ConceptBlueprint.validate_concept_string_or_concept_code(concept_string_or_concept_code=combined_output)
+ ConceptBlueprint.validate_concept_string_or_concept_code(concept_string_or_code=combined_output)
return combined_output
diff --git a/pipelex/pipe_controllers/parallel/pipe_parallel_factory.py b/pipelex/pipe_controllers/parallel/pipe_parallel_factory.py
index a2d9b5224..bbcdc3c7b 100644
--- a/pipelex/pipe_controllers/parallel/pipe_parallel_factory.py
+++ b/pipelex/pipe_controllers/parallel/pipe_parallel_factory.py
@@ -4,7 +4,7 @@ from typing_extensions import override
from pipelex.core.concepts.concept_factory import ConceptFactory
from pipelex.core.pipes.pipe_factory import PipeFactoryProtocol
-from pipelex.core.pipes.pipe_input_spec_factory import PipeInputSpecFactory
+from pipelex.core.pipes.pipe_input_factory import PipeInputSpecFactory
from pipelex.exceptions import PipeDefinitionError
from pipelex.hub import get_concept_provider
from pipelex.pipe_controllers.parallel.pipe_parallel import PipeParallel
@@ -35,7 +35,7 @@ class PipeParallelFactory(PipeFactoryProtocol[PipeParallelBlueprint, PipeParalle
if blueprint.combined_output:
combined_output_domain_and_code = ConceptFactory.make_domain_and_concept_code_from_concept_string_or_concept_code(
domain=domain,
- concept_string_or_concept_code=blueprint.output_concept_string_or_concept_code,
+ concept_string_or_code=blueprint.output,
concept_codes_from_the_same_domain=concept_codes_from_the_same_domain,
)
combined_output = get_concept_provider().get_required_concept(
@@ -48,7 +48,7 @@ class PipeParallelFactory(PipeFactoryProtocol[PipeParallelBlueprint, PipeParalle
output_domain_and_code = ConceptFactory.make_domain_and_concept_code_from_concept_string_or_concept_code(
domain=domain,
- concept_string_or_concept_code=blueprint.output_concept_string_or_concept_code,
+ concept_string_or_code=blueprint.output,
concept_codes_from_the_same_domain=concept_codes_from_the_same_domain,
)
return PipeParallel(
diff --git a/pipelex/pipe_controllers/pipe_controller.py b/pipelex/pipe_controllers/pipe_controller.py
index 5dcb428f0..5cb192290 100644
--- a/pipelex/pipe_controllers/pipe_controller.py
+++ b/pipelex/pipe_controllers/pipe_controller.py
@@ -1,5 +1,5 @@
from abc import abstractmethod
-from typing import Optional
+from typing import Literal, Optional
from typing_extensions import override
@@ -12,6 +12,8 @@ from pipelex.pipeline.job_metadata import JobMetadata
class PipeController(PipeAbstract):
+ category: Literal["PipeController"] = "PipeController"
+
@override
async def run_pipe(
self,
@@ -19,6 +21,7 @@ class PipeController(PipeAbstract):
working_memory: WorkingMemory,
pipe_run_params: PipeRunParams,
output_name: Optional[str] = None,
+ print_intermediate_outputs: Optional[bool] = False,
) -> PipeOutput:
pipe_run_params.push_pipe_to_stack(pipe_code=self.code)
self.monitor_pipe_stack(pipe_run_params=pipe_run_params)
@@ -32,9 +35,10 @@ class PipeController(PipeAbstract):
case PipeRunMode.LIVE:
indent_level = len(pipe_run_params.pipe_stack) - 1
indent = " " * indent_level
- label = f"{indent}{self.class_name}: {self.code}".ljust(80)
- output = self.output.code
- log.info(f"{label} → {output}")
+ label = (
+ f"{indent}{'[yellow]↳[/yellow]' if indent_level > 0 else ''} Running [blue]{self.class_name}[/blue] → [green]{self.code}[/green]"
+ )
+ log.info(f"{label} → [red]{self.output.code}[/red]")
pipe_output = await self._run_controller_pipe(
job_metadata=job_metadata,
working_memory=working_memory,
@@ -42,12 +46,11 @@ class PipeController(PipeAbstract):
output_name=output_name,
)
case PipeRunMode.DRY:
- name = f"Dry {self.class_name}"
+ name = f"Dry running [blue]{self.class_name}[/blue]"
indent_level = len(pipe_run_params.pipe_stack) - 1
indent = " " * indent_level
- label = f"{indent}{name}: {self.code}".ljust(80)
- output = self.output.code
- log.info(f"{label} → {output}")
+ label = f"{indent}{'[yellow]↳[/yellow]' if indent_level > 0 else ''} {name}: [green]{self.code}[/green]"
+ log.info(f"{label} → [red]{self.output.code}[/red]")
pipe_output = await self._dry_run_controller_pipe(
job_metadata=job_metadata,
working_memory=working_memory,
diff --git a/pipelex/pipe_controllers/sequence/pipe_sequence.py b/pipelex/pipe_controllers/sequence/pipe_sequence.py
index 2de58e9ef..38e008ca1 100644
--- a/pipelex/pipe_controllers/sequence/pipe_sequence.py
+++ b/pipelex/pipe_controllers/sequence/pipe_sequence.py
@@ -1,4 +1,4 @@
-from typing import List, Optional, Set
+from typing import List, Literal, Optional, Set
from pydantic import model_validator
from typing_extensions import Self, override
@@ -6,8 +6,8 @@ from typing_extensions import Self, override
from pipelex import log
from pipelex.config import StaticValidationReaction, get_config
from pipelex.core.memory.working_memory import WorkingMemory
-from pipelex.core.pipes.pipe_input_spec import PipeInputSpec
-from pipelex.core.pipes.pipe_input_spec_factory import PipeInputSpecFactory
+from pipelex.core.pipes.pipe_input import PipeInputSpec
+from pipelex.core.pipes.pipe_input_factory import PipeInputSpecFactory
from pipelex.core.pipes.pipe_output import PipeOutput
from pipelex.core.pipes.pipe_run_params import PipeRunMode, PipeRunParams
from pipelex.exceptions import (
@@ -16,6 +16,7 @@ from pipelex.exceptions import (
StaticValidationErrorType,
)
from pipelex.hub import get_required_pipe
+from pipelex.pipe_controllers.parallel.pipe_parallel import PipeParallel
from pipelex.pipe_controllers.pipe_controller import PipeController
from pipelex.pipe_controllers.sequence.exceptions import PipeSequenceError
from pipelex.pipe_controllers.sub_pipe import SubPipe
@@ -23,50 +24,57 @@ from pipelex.pipeline.job_metadata import JobMetadata
class PipeSequence(PipeController):
+ type: Literal["PipeSequence"] = "PipeSequence"
sequential_sub_pipes: List[SubPipe]
@override
- def needed_inputs(self) -> PipeInputSpec:
- """
- Calculate the inputs needed by this sequence.
- This is the inputs needed by all pipes in the sequence MINUS the outputs generated by previous steps.
- """
+ def needed_inputs(self, visited_pipes: Optional[Set[str]] = None) -> PipeInputSpec:
+ if visited_pipes is None:
+ visited_pipes = set()
+
+ # If we've already visited this pipe, stop recursion
+ if self.code in visited_pipes:
+ return PipeInputSpecFactory.make_empty()
+
+ # Add this pipe to visited set for recursive calls
+ visited_pipes_with_current = visited_pipes | {self.code}
needed_inputs = PipeInputSpecFactory.make_empty()
generated_outputs: Set[str] = set()
for sequential_sub_pipe in self.sequential_sub_pipes:
- sub_pipe_needed_inputs = get_required_pipe(pipe_code=sequential_sub_pipe.pipe_code).needed_inputs()
+ sub_pipe = get_required_pipe(pipe_code=sequential_sub_pipe.pipe_code)
+ # Use the centralized recursion detection
+ sub_pipe_needed_inputs = sub_pipe.needed_inputs(visited_pipes_with_current)
+
+ if isinstance(sub_pipe, PipeParallel):
+ if sub_pipe.add_each_output:
+ for sub_parallel_pipe in sub_pipe.parallel_sub_pipes:
+ if sub_pipe.add_each_output and sub_parallel_pipe.output_name:
+ generated_outputs.add(sub_parallel_pipe.output_name)
+ elif sub_parallel_pipe.output_name:
+ generated_outputs.add(sub_parallel_pipe.output_name)
- # Handle batching: if this sub_pipe has batch_params, exclude the batch_as input
- # since it's provided by the batching mechanism.
- # Instead we must add the batch_over variable nale.
if sequential_sub_pipe.batch_params:
- batch_as_input = sequential_sub_pipe.batch_params.input_item_stuff_name
- batch_item_concept_code = sub_pipe_needed_inputs.get_required_input_requirement(variable_name=batch_as_input)
- # Create a new PipeInputSpec without the batch_as input
- # TODO: this should be done with a simple remove_requirement() method on the PipeInputSpec
- filtered_needed_inputs = PipeInputSpecFactory.make_empty()
- for var_name, requirement in sub_pipe_needed_inputs.items:
- if var_name != batch_as_input:
- filtered_needed_inputs.add_requirement(variable_name=var_name, concept=requirement.concept)
- sub_pipe_needed_inputs = filtered_needed_inputs
- batch_over = sequential_sub_pipe.batch_params.input_list_stuff_name
- sub_pipe_needed_inputs.add_requirement(
- variable_name=batch_over,
- concept=batch_item_concept_code.concept,
- multiplicity=True,
- )
-
- # Add inputs that haven't been generated by previous steps
- for var_name, requirement in sub_pipe_needed_inputs.items:
- if var_name not in generated_outputs:
- needed_inputs.add_requirement(var_name, requirement.concept, requirement.multiplicity)
+ if sequential_sub_pipe.batch_params.input_list_stuff_name not in generated_outputs:
+ needed_inputs.add_requirement(
+ variable_name=sequential_sub_pipe.batch_params.input_list_stuff_name,
+ concept=sub_pipe_needed_inputs.get_required_input_requirement(
+ variable_name=sequential_sub_pipe.batch_params.input_item_stuff_name
+ ).concept,
+ multiplicity=True,
+ )
+ for input_name, requirement in sub_pipe_needed_inputs.items:
+ if input_name != sequential_sub_pipe.batch_params.input_item_stuff_name and input_name not in generated_outputs:
+ needed_inputs.add_requirement(input_name, requirement.concept, requirement.multiplicity)
+ else:
+ for input_name, requirement in sub_pipe_needed_inputs.items:
+ if input_name not in generated_outputs:
+ needed_inputs.add_requirement(input_name, requirement.concept, requirement.multiplicity)
# Add this step's output to generated outputs
if sequential_sub_pipe.output_name:
generated_outputs.add(sequential_sub_pipe.output_name)
-
return needed_inputs
@override
@@ -175,8 +183,7 @@ class PipeSequence(PipeController):
evolving_memory = working_memory
for sub_pipe_index, sub_pipe in enumerate(self.sequential_sub_pipes):
- sub_pipe_run_params: PipeRunParams
- # only the last step should apply the final_stuff_code
+ # Only the last step should apply the final_stuff_code
if sub_pipe_index == len(self.sequential_sub_pipes) - 1:
sub_pipe_run_params = pipe_run_params.model_copy()
else:
diff --git a/pipelex/pipe_controllers/sequence/pipe_sequence_blueprint.py b/pipelex/pipe_controllers/sequence/pipe_sequence_blueprint.py
index 2cee016ac..f20fe2582 100644
--- a/pipelex/pipe_controllers/sequence/pipe_sequence_blueprint.py
+++ b/pipelex/pipe_controllers/sequence/pipe_sequence_blueprint.py
@@ -6,4 +6,5 @@ from pipelex.pipe_controllers.sub_pipe_blueprint import SubPipeBlueprint
class PipeSequenceBlueprint(PipeBlueprint):
type: Literal["PipeSequence"] = "PipeSequence"
+ category: Literal["PipeController"] = "PipeController"
steps: List[SubPipeBlueprint]
diff --git a/pipelex/pipe_controllers/sequence/pipe_sequence_factory.py b/pipelex/pipe_controllers/sequence/pipe_sequence_factory.py
index 4f79ca82d..c801765a4 100644
--- a/pipelex/pipe_controllers/sequence/pipe_sequence_factory.py
+++ b/pipelex/pipe_controllers/sequence/pipe_sequence_factory.py
@@ -4,7 +4,7 @@ from typing_extensions import override
from pipelex.core.concepts.concept_factory import ConceptFactory
from pipelex.core.pipes.pipe_factory import PipeFactoryProtocol
-from pipelex.core.pipes.pipe_input_spec_factory import PipeInputSpecFactory
+from pipelex.core.pipes.pipe_input_factory import PipeInputSpecFactory
from pipelex.hub import get_concept_provider
from pipelex.pipe_controllers.sequence.pipe_sequence import PipeSequence
from pipelex.pipe_controllers.sequence.pipe_sequence_blueprint import PipeSequenceBlueprint
@@ -23,7 +23,7 @@ class PipeSequenceFactory(PipeFactoryProtocol[PipeSequenceBlueprint, PipeSequenc
) -> PipeSequence:
output_domain_and_code = ConceptFactory.make_domain_and_concept_code_from_concept_string_or_concept_code(
domain=domain,
- concept_string_or_concept_code=blueprint.output_concept_string_or_concept_code,
+ concept_string_or_code=blueprint.output,
concept_codes_from_the_same_domain=concept_codes_from_the_same_domain,
)
return PipeSequence(
diff --git a/pipelex/pipe_controllers/sub_pipe.py b/pipelex/pipe_controllers/sub_pipe.py
index 2feeb93b6..a2da43fe6 100644
--- a/pipelex/pipe_controllers/sub_pipe.py
+++ b/pipelex/pipe_controllers/sub_pipe.py
@@ -4,9 +4,10 @@ from pydantic import BaseModel
from pipelex import log
from pipelex.core.memory.working_memory import WorkingMemory
-from pipelex.core.pipes.pipe_input_spec_blueprint import InputRequirementBlueprint
+from pipelex.core.pipes.pipe_input_blueprint import InputRequirementBlueprint
from pipelex.core.pipes.pipe_output import PipeOutput
from pipelex.core.pipes.pipe_run_params import BatchParams, PipeOutputMultiplicity, PipeRunMode, PipeRunParams
+from pipelex.core.stuffs.stuff_content import ListContent
from pipelex.exceptions import PipeInputError, WorkingMemoryStuffNotFoundError
from pipelex.hub import get_pipe_router, get_pipeline_tracker, get_required_pipe
from pipelex.pipe_controllers.batch.pipe_batch_blueprint import PipeBatchBlueprint
@@ -41,23 +42,19 @@ class SubPipe(BaseModel):
# Case 1: Batch processing
if batch_params := self.batch_params:
try:
- input_list_stuff = working_memory.get_stuff(name=batch_params.input_list_stuff_name)
+ working_memory.get_typed_object_or_attribute(name=batch_params.input_list_stuff_name, wanted_type=ListContent)
except WorkingMemoryStuffNotFoundError as exc:
raise PipeInputError(
f"Input list stuff named '{batch_params.input_list_stuff_name}' required by sub_pipe '{self.pipe_code}' "
f"of pipe '{calling_pipe_code}' not found in working memory: {exc}"
) from exc
- sub_pipe.inputs.add_requirement(variable_name=batch_params.input_list_stuff_name, concept=input_list_stuff.concept)
-
- # Create blueprint for PipeBatch
pipe_batch_blueprint = PipeBatchBlueprint(
definition=f"Batch processing for {self.pipe_code}",
branch_pipe_code=self.pipe_code,
output=sub_pipe.output.code,
input_list_name=batch_params.input_list_stuff_name,
input_item_name=batch_params.input_item_stuff_name,
- # inputs should be of type: Dict[str, InputRequirementBlueprint]
inputs={
batch_params.input_item_stuff_name: InputRequirementBlueprint(
concept=sub_pipe.inputs.root[batch_params.input_item_stuff_name].concept.concept_string
diff --git a/pipelex/pipe_operators/func/pipe_func.py b/pipelex/pipe_operators/func/pipe_func.py
index be895d503..0ee4e2afc 100644
--- a/pipelex/pipe_operators/func/pipe_func.py
+++ b/pipelex/pipe_operators/func/pipe_func.py
@@ -1,4 +1,4 @@
-from typing import List, Optional, Set, cast, get_type_hints
+from typing import List, Literal, Optional, Set, cast, get_type_hints
from typing_extensions import override
@@ -6,7 +6,7 @@ from pipelex import log
from pipelex.core.concepts.concept_factory import ConceptFactory
from pipelex.core.memory.working_memory import WorkingMemory
from pipelex.core.memory.working_memory_factory import WorkingMemoryFactory
-from pipelex.core.pipes.pipe_input_spec import PipeInputSpec, TypedNamedInputRequirement
+from pipelex.core.pipes.pipe_input import PipeInputSpec, TypedNamedInputRequirement
from pipelex.core.pipes.pipe_output import PipeOutput
from pipelex.core.pipes.pipe_run_params import PipeRunParams
from pipelex.core.stuffs.stuff_content import ListContent, StuffContent, TextContent
@@ -22,6 +22,7 @@ class PipeFuncOutput(PipeOutput):
class PipeFunc(PipeOperator):
+ type: Literal["PipeFunc"] = "PipeFunc"
function_name: str
@override
@@ -29,7 +30,7 @@ class PipeFunc(PipeOperator):
return set()
@override
- def needed_inputs(self) -> PipeInputSpec:
+ def needed_inputs(self, visited_pipes: Optional[Set[str]] = None) -> PipeInputSpec:
return self.inputs
@override
@@ -50,7 +51,7 @@ class PipeFunc(PipeOperator):
if not callable(function):
raise ValueError(f"Function '{self.function_name}' is not callable")
- func_output_object = function(working_memory=working_memory)
+ func_output_object = await function(working_memory=working_memory)
the_content: StuffContent
if isinstance(func_output_object, StuffContent):
the_content = func_output_object
@@ -108,7 +109,7 @@ class PipeFunc(PipeOperator):
raise DryRunError(f"Function '{self.function_name}' has no return type annotation")
else:
if not issubclass(return_type, StuffContent):
- raise ValueError(f"Function '{self.function_name}' return type {return_type} is not a subclass of StuffContent")
+ raise DryRunError(f"Function '{self.function_name}' return type {return_type} is not a subclass of StuffContent")
requirement = TypedNamedInputRequirement(
variable_name="mock_output",
diff --git a/pipelex/pipe_operators/func/pipe_func_blueprint.py b/pipelex/pipe_operators/func/pipe_func_blueprint.py
index ded1ad92e..5d9af5be2 100644
--- a/pipelex/pipe_operators/func/pipe_func_blueprint.py
+++ b/pipelex/pipe_operators/func/pipe_func_blueprint.py
@@ -1,8 +1,11 @@
from typing import Literal
+from pydantic import Field
+
from pipelex.core.pipes.pipe_blueprint import PipeBlueprint
class PipeFuncBlueprint(PipeBlueprint):
type: Literal["PipeFunc"] = "PipeFunc"
- function_name: str
+ category: Literal["PipeOperator"] = "PipeOperator"
+ function_name: str = Field(description="The name of the function to call.")
diff --git a/pipelex/pipe_operators/func/pipe_func_factory.py b/pipelex/pipe_operators/func/pipe_func_factory.py
index e5745616b..2ddcc294b 100644
--- a/pipelex/pipe_operators/func/pipe_func_factory.py
+++ b/pipelex/pipe_operators/func/pipe_func_factory.py
@@ -4,7 +4,7 @@ from typing_extensions import override
from pipelex.core.concepts.concept_factory import ConceptFactory
from pipelex.core.pipes.pipe_factory import PipeFactoryProtocol
-from pipelex.core.pipes.pipe_input_spec_factory import PipeInputSpecFactory
+from pipelex.core.pipes.pipe_input_factory import PipeInputSpecFactory
from pipelex.hub import get_concept_provider
from pipelex.pipe_operators.func.pipe_func import PipeFunc
from pipelex.pipe_operators.func.pipe_func_blueprint import PipeFuncBlueprint
@@ -22,7 +22,7 @@ class PipeFuncFactory(PipeFactoryProtocol[PipeFuncBlueprint, PipeFunc]):
) -> PipeFunc:
output_domain_and_code = ConceptFactory.make_domain_and_concept_code_from_concept_string_or_concept_code(
domain=domain,
- concept_string_or_concept_code=blueprint.output_concept_string_or_concept_code,
+ concept_string_or_code=blueprint.output,
concept_codes_from_the_same_domain=concept_codes_from_the_same_domain,
)
return PipeFunc(
diff --git a/pipelex/pipe_operators/img_gen/pipe_img_gen.py b/pipelex/pipe_operators/img_gen/pipe_img_gen.py
index 290b4569c..237a33ed9 100644
--- a/pipelex/pipe_operators/img_gen/pipe_img_gen.py
+++ b/pipelex/pipe_operators/img_gen/pipe_img_gen.py
@@ -6,15 +6,16 @@ from typing_extensions import Self, override
from pipelex import log
from pipelex.cogt.content_generation.content_generator_dry import ContentGeneratorDry
from pipelex.cogt.content_generation.content_generator_protocol import ContentGeneratorProtocol
-from pipelex.cogt.imgg.imgg_handle import ImggHandle
-from pipelex.cogt.imgg.imgg_job_components import AspectRatio, Background, ImggJobParams, Quality
-from pipelex.cogt.imgg.imgg_prompt import ImggPrompt
+from pipelex.cogt.img_gen.img_gen_job_components import AspectRatio, Background, ImgGenJobParams, OutputFormat, Quality
+from pipelex.cogt.img_gen.img_gen_prompt import ImgGenPrompt
+from pipelex.cogt.img_gen.img_gen_setting import ImgGenChoice, ImgGenSetting
+from pipelex.cogt.models.model_deck_check import check_img_gen_choice_with_deck
from pipelex.config import StaticValidationReaction, get_config
from pipelex.core.concepts.concept_factory import ConceptFactory
from pipelex.core.concepts.concept_native import NATIVE_CONCEPTS_DATA, NativeConceptEnum
from pipelex.core.memory.working_memory import WorkingMemory
-from pipelex.core.pipes.pipe_input_spec import PipeInputSpec
-from pipelex.core.pipes.pipe_input_spec_factory import PipeInputSpecFactory
+from pipelex.core.pipes.pipe_input import PipeInputSpec
+from pipelex.core.pipes.pipe_input_factory import PipeInputSpecFactory
from pipelex.core.pipes.pipe_output import PipeOutput
from pipelex.core.pipes.pipe_run_params import PipeOutputMultiplicity, PipeRunMode, PipeRunParams, output_multiplicity_to_apply
from pipelex.core.pipes.pipe_run_params_factory import PipeRunParamsFactory
@@ -29,7 +30,7 @@ from pipelex.exceptions import (
UnexpectedPipeDefinitionError,
WorkingMemoryStuffNotFoundError,
)
-from pipelex.hub import get_concept_provider, get_content_generator
+from pipelex.hub import get_concept_provider, get_content_generator, get_model_deck
from pipelex.pipe_operators.pipe_operator import PipeOperator
from pipelex.pipeline.job_metadata import JobMetadata
@@ -53,22 +54,29 @@ DEFAULT_PROMPT_VAR_NAME = "prompt"
class PipeImgGen(PipeOperator):
- imgg_prompt: Optional[str] = None
- # TODO: wrap this up in imgg llm_presets like for llm
- imgg_handle: Optional[ImggHandle] = None
- aspect_ratio: Optional[AspectRatio] = Field(default=None, strict=False)
+ type: Literal["PipeImgGen"] = "PipeImgGen"
+ img_gen_prompt: Optional[str] = None
+ img_gen_prompt_var_name: Optional[str] = None
+
+ # New ImgGenChoice pattern (like LLM)
+ img_gen: Optional[ImgGenChoice] = None
+
+ # Legacy individual settings (for backwards compatibility)
+ img_gen_handle: Optional[str] = None
+ quality: Optional[Quality] = Field(default=None, strict=False)
nb_steps: Optional[int] = Field(default=None, gt=0)
guidance_scale: Optional[float] = Field(default=None, gt=0)
is_moderated: Optional[bool] = None
- background: Optional[Background] = None
- quality: Optional[Quality] = Field(default=None, strict=False)
safety_tolerance: Optional[int] = Field(default=None, ge=1, le=6)
+
+ # One-time settings (not in ImgGenSetting)
+ aspect_ratio: Optional[AspectRatio] = Field(default=None, strict=False)
is_raw: Optional[bool] = None
seed: Optional[Union[int, Literal["auto"]]] = None
+ background: Optional[Background] = Field(default=None, strict=False)
+ output_format: Optional[OutputFormat] = Field(default=None, strict=False)
output_multiplicity: PipeOutputMultiplicity
- img_gen_prompt_var_name: Optional[str] = None
-
@field_validator("img_gen_prompt_var_name")
@classmethod
def validate_input_var_name_not_provided_as_attribute(cls, value: Optional[str]) -> Optional[str]:
@@ -81,6 +89,12 @@ class PipeImgGen(PipeOperator):
self._validate_inputs()
return self
+ @override
+ def validate_with_libraries(self):
+ self._validate_inputs()
+ if self.img_gen:
+ check_img_gen_choice_with_deck(img_gen_choice=self.img_gen)
+
@override
def validate_output(self):
if not get_concept_provider().is_compatible(
@@ -94,11 +108,11 @@ class PipeImgGen(PipeOperator):
)
@override
- def needed_inputs(self) -> PipeInputSpec:
+ def needed_inputs(self, visited_pipes: Optional[Set[str]] = None) -> PipeInputSpec:
needed_inputs = PipeInputSpecFactory.make_empty()
- if self.imgg_prompt:
+ if self.img_gen_prompt:
needed_inputs.add_requirement(
- variable_name="imgg_prompt",
+ variable_name="img_gen_prompt",
concept=ConceptFactory.make_native_concept(
native_concept_data=NATIVE_CONCEPTS_DATA[NativeConceptEnum.TEXT],
),
@@ -120,10 +134,10 @@ class PipeImgGen(PipeOperator):
static_validation_config = get_config().pipelex.static_validation_config
default_reaction = static_validation_config.default_reaction
reactions = static_validation_config.reactions
- # check that we have either an imgg_prompt passed as attribute or as a single text input
- if self.imgg_prompt:
+ # check that we have either an img_gen_prompt passed as attribute or as a single text input
+ if self.img_gen_prompt:
if self.inputs.items:
- raise PipeDefinitionError("img_gen_prompt_var_name must be None if imgg_prompt is provided")
+ raise PipeDefinitionError("img_gen_prompt_var_name must be None if img_gen_prompt is provided")
else:
# we're good with the prompt provided as attribute
return
@@ -203,41 +217,63 @@ class PipeImgGen(PipeOperator):
applied_output_multiplicity = multiplicity_resolution.resolved_multiplicity
log.debug("Getting image generation prompt from context")
- if self.imgg_prompt:
- imgg_prompt_text = self.imgg_prompt
+ if self.img_gen_prompt:
+ img_gen_prompt_text = self.img_gen_prompt
elif stuff_name := self.img_gen_prompt_var_name:
try:
- imgg_prompt_text = working_memory.get_stuff_as_str(stuff_name)
+ img_gen_prompt_text = working_memory.get_stuff_as_str(stuff_name)
except WorkingMemoryStuffNotFoundError as exc:
raise PipeInputError(f"Could not find a valid user image named '{stuff_name}' in the working_memory: {exc}") from exc
else:
raise UnexpectedPipeDefinitionError("You must provide an image gen prompt either as attribute of the pipe or as a single text input")
- imgg_config = get_config().cogt.imgg_config
- imgg_param_defaults = imgg_config.imgg_param_defaults
+ img_gen_config = get_config().cogt.img_gen_config
+ img_gen_param_defaults = img_gen_config.img_gen_param_defaults
+ model_deck = get_model_deck()
+
+ # Get ImgGenSetting either from img_gen choice or legacy settings
+ img_gen_setting: ImgGenSetting
+ if self.img_gen is not None:
+ # New pattern: use img_gen choice (preset or inline setting)
+ img_gen_setting = model_deck.get_img_gen_setting(self.img_gen)
+ elif self.img_gen_handle is not None:
+ # Legacy pattern: create ImgGenSetting from individual settings
+ img_gen_setting = ImgGenSetting(
+ img_gen_handle=self.img_gen_handle,
+ quality=self.quality,
+ nb_steps=self.nb_steps,
+ guidance_scale=self.guidance_scale or img_gen_param_defaults.guidance_scale,
+ is_moderated=self.is_moderated if self.is_moderated is not None else img_gen_param_defaults.is_moderated,
+ safety_tolerance=self.safety_tolerance or img_gen_param_defaults.safety_tolerance,
+ )
+ else:
+ # Use default from model deck
+ img_gen_setting = model_deck.get_img_gen_setting(model_deck.img_gen_choice_default)
- seed_setting = self.seed or imgg_param_defaults.seed
+ # Process one-time settings
+ seed_setting = self.seed or img_gen_param_defaults.seed
seed: Optional[int]
if isinstance(seed_setting, str) and seed_setting == "auto":
seed = None
else:
seed = seed_setting
- # TODO: refacto this as a model update
- imgg_job_params = ImggJobParams(
- aspect_ratio=self.aspect_ratio or imgg_param_defaults.aspect_ratio,
- background=self.background or imgg_param_defaults.background,
- quality=self.quality or imgg_param_defaults.quality,
- nb_steps=self.nb_steps or imgg_param_defaults.nb_steps,
- guidance_scale=self.guidance_scale or imgg_param_defaults.guidance_scale,
- is_moderated=self.is_moderated or imgg_param_defaults.is_moderated,
- safety_tolerance=self.safety_tolerance or imgg_param_defaults.safety_tolerance,
- is_raw=self.is_raw or imgg_param_defaults.is_raw,
- output_format=imgg_param_defaults.output_format,
+ # Build ImggJobParams from ImgGenSetting + one-time settings
+ img_gen_job_params = ImgGenJobParams(
+ aspect_ratio=self.aspect_ratio or img_gen_param_defaults.aspect_ratio,
+ background=self.background or img_gen_param_defaults.background,
+ quality=img_gen_setting.quality,
+ nb_steps=img_gen_setting.nb_steps,
+ guidance_scale=img_gen_setting.guidance_scale,
+ is_moderated=img_gen_setting.is_moderated,
+ safety_tolerance=img_gen_setting.safety_tolerance,
+ is_raw=self.is_raw if self.is_raw is not None else img_gen_param_defaults.is_raw,
+ output_format=self.output_format or img_gen_param_defaults.output_format,
seed=seed,
)
- imgg_handle = self.imgg_handle or imgg_config.default_imgg_handle
- log.debug(f"Using imgg handle: {imgg_handle}")
+ # Get the image generation handle
+ img_gen_handle = img_gen_setting.img_gen_handle
+ log.debug(f"Using img_gen handle: {img_gen_handle}")
the_content: StuffContent
image_urls: List[str] = []
@@ -259,20 +295,20 @@ class PipeImgGen(PipeOperator):
if nb_images > 1:
generated_image_list = await content_generator.make_image_list(
job_metadata=job_metadata,
- imgg_handle=imgg_handle,
- imgg_prompt=ImggPrompt(
- positive_text=imgg_prompt_text,
+ img_gen_handle=img_gen_handle,
+ img_gen_prompt=ImgGenPrompt(
+ positive_text=img_gen_prompt_text,
),
nb_images=nb_images,
- imgg_job_params=imgg_job_params,
- imgg_job_config=imgg_config.imgg_job_config,
+ img_gen_job_params=img_gen_job_params,
+ img_gen_job_config=img_gen_config.img_gen_job_config,
)
image_content_items: List[StuffContent] = []
for generated_image in generated_image_list:
image_content_items.append(
ImageContent(
url=generated_image.url,
- source_prompt=imgg_prompt_text,
+ source_prompt=img_gen_prompt_text,
)
)
image_urls.append(generated_image.url)
@@ -283,12 +319,12 @@ class PipeImgGen(PipeOperator):
else:
generated_image = await content_generator.make_single_image(
job_metadata=job_metadata,
- imgg_handle=imgg_handle,
- imgg_prompt=ImggPrompt(
- positive_text=imgg_prompt_text,
+ img_gen_handle=img_gen_handle,
+ img_gen_prompt=ImgGenPrompt(
+ positive_text=img_gen_prompt_text,
),
- imgg_job_params=imgg_job_params,
- imgg_job_config=imgg_config.imgg_job_config,
+ img_gen_job_params=img_gen_job_params,
+ img_gen_job_config=img_gen_config.img_gen_job_config,
)
generated_image_url = generated_image.url
@@ -296,7 +332,7 @@ class PipeImgGen(PipeOperator):
the_content = ImageContent(
url=generated_image_url,
- source_prompt=imgg_prompt_text,
+ source_prompt=img_gen_prompt_text,
)
log.verbose(the_content, title="Single image content")
diff --git a/pipelex/pipe_operators/img_gen/pipe_img_gen_blueprint.py b/pipelex/pipe_operators/img_gen/pipe_img_gen_blueprint.py
index a08ad0f00..27e39ba0d 100644
--- a/pipelex/pipe_operators/img_gen/pipe_img_gen_blueprint.py
+++ b/pipelex/pipe_operators/img_gen/pipe_img_gen_blueprint.py
@@ -1,37 +1,25 @@
from typing import Literal, Optional, Union
-from pydantic import Field, model_validator
-from typing_extensions import Self
+from pydantic import Field
-from pipelex.cogt.imgg.imgg_handle import ImggHandle
-from pipelex.cogt.imgg.imgg_job_components import AspectRatio, Quality
+from pipelex.cogt.img_gen.img_gen_job_components import AspectRatio, Background, OutputFormat
+from pipelex.cogt.img_gen.img_gen_setting import ImgGenChoice
from pipelex.core.pipes.pipe_blueprint import PipeBlueprint
-from pipelex.exceptions import PipeDefinitionError
-from pipelex.tools.typing.validation_utils import has_more_than_one_among_attributes_from_lists
class PipeImgGenBlueprint(PipeBlueprint):
type: Literal["PipeImgGen"] = "PipeImgGen"
+ category: Literal["PipeOperator"] = "PipeOperator"
img_gen_prompt: Optional[str] = None
- imgg_handle: Optional[ImggHandle] = None
+ img_gen_prompt_var_name: Optional[str] = None
+
+ # New ImgGenChoice pattern (like LLM)
+ img_gen: Optional[ImgGenChoice] = None
+
+ # One-time settings (not in ImgGenSetting)
aspect_ratio: Optional[AspectRatio] = Field(default=None, strict=False)
- quality: Optional[Quality] = Field(default=None, strict=False)
- nb_steps: Optional[int] = Field(default=None, gt=0)
- guidance_scale: Optional[float] = Field(default=None, gt=0)
- is_moderated: Optional[bool] = None
- safety_tolerance: Optional[int] = Field(default=None, ge=1, le=6)
is_raw: Optional[bool] = None
seed: Optional[Union[int, Literal["auto"]]] = None
nb_output: Optional[int] = Field(default=None, ge=1)
- img_gen_prompt_var_name: Optional[str] = None
-
- @model_validator(mode="after")
- def validate_imgg_prompt_and_imgg_prompt_stuff_name(self) -> Self:
- if excess_attributes_list := has_more_than_one_among_attributes_from_lists(
- self,
- [
- ["quality", "nb_steps"],
- ],
- ):
- raise PipeDefinitionError(f"PipeImgGenBlueprint should have no more than one of {excess_attributes_list} among them")
- return self
+ background: Optional[Background] = Field(default=None, strict=False)
+ output_format: Optional[OutputFormat] = Field(default=None, strict=False)
diff --git a/pipelex/pipe_operators/img_gen/pipe_img_gen_factory.py b/pipelex/pipe_operators/img_gen/pipe_img_gen_factory.py
index 14cb5e58d..1e0a3163d 100644
--- a/pipelex/pipe_operators/img_gen/pipe_img_gen_factory.py
+++ b/pipelex/pipe_operators/img_gen/pipe_img_gen_factory.py
@@ -4,7 +4,7 @@ from typing_extensions import override
from pipelex.core.concepts.concept_factory import ConceptFactory
from pipelex.core.pipes.pipe_factory import PipeFactoryProtocol
-from pipelex.core.pipes.pipe_input_spec_factory import PipeInputSpecFactory
+from pipelex.core.pipes.pipe_input_factory import PipeInputSpecFactory
from pipelex.hub import get_concept_provider
from pipelex.pipe_operators.img_gen.pipe_img_gen import PipeImgGen
from pipelex.pipe_operators.img_gen.pipe_img_gen_blueprint import PipeImgGenBlueprint
@@ -22,7 +22,7 @@ class PipeImgGenFactory(PipeFactoryProtocol[PipeImgGenBlueprint, PipeImgGen]):
) -> PipeImgGen:
output_domain_and_code = ConceptFactory.make_domain_and_concept_code_from_concept_string_or_concept_code(
domain=domain,
- concept_string_or_concept_code=blueprint.output_concept_string_or_concept_code,
+ concept_string_or_code=blueprint.output,
concept_codes_from_the_same_domain=concept_codes_from_the_same_domain,
)
return PipeImgGen(
@@ -38,14 +38,12 @@ class PipeImgGenFactory(PipeFactoryProtocol[PipeImgGenBlueprint, PipeImgGen]):
)
),
output_multiplicity=blueprint.nb_output or 1,
- imgg_prompt=blueprint.img_gen_prompt,
- imgg_handle=blueprint.imgg_handle,
+ img_gen_prompt=blueprint.img_gen_prompt,
+ img_gen_prompt_var_name=blueprint.img_gen_prompt_var_name,
+ img_gen=blueprint.img_gen,
aspect_ratio=blueprint.aspect_ratio,
- nb_steps=blueprint.nb_steps,
- guidance_scale=blueprint.guidance_scale,
- is_moderated=blueprint.is_moderated,
- safety_tolerance=blueprint.safety_tolerance,
is_raw=blueprint.is_raw,
seed=blueprint.seed,
- img_gen_prompt_var_name=blueprint.img_gen_prompt_var_name,
+ background=blueprint.background,
+ output_format=blueprint.output_format,
)
diff --git a/pipelex/pipe_operators/jinja2/pipe_jinja2.py b/pipelex/pipe_operators/jinja2/pipe_jinja2.py
index c882f3ee8..c12ff796a 100644
--- a/pipelex/pipe_operators/jinja2/pipe_jinja2.py
+++ b/pipelex/pipe_operators/jinja2/pipe_jinja2.py
@@ -1,4 +1,4 @@
-from typing import Any, ClassVar, Dict, Optional, Set
+from typing import Any, ClassVar, Dict, Literal, Optional, Set
from jinja2 import TemplateSyntaxError
from pydantic import ConfigDict, model_validator
@@ -12,8 +12,8 @@ from pipelex.core.concepts.concept import Concept
from pipelex.core.concepts.concept_factory import ConceptFactory
from pipelex.core.concepts.concept_native import NATIVE_CONCEPTS_DATA, NativeConceptEnum
from pipelex.core.memory.working_memory import WorkingMemory
-from pipelex.core.pipes.pipe_input_spec import PipeInputSpec
-from pipelex.core.pipes.pipe_input_spec_factory import PipeInputSpecFactory
+from pipelex.core.pipes.pipe_input import PipeInputSpec
+from pipelex.core.pipes.pipe_input_factory import PipeInputSpecFactory
from pipelex.core.pipes.pipe_output import PipeOutput
from pipelex.core.pipes.pipe_run_params import PipeRunMode, PipeRunParams
from pipelex.core.pipes.pipe_run_params_factory import PipeRunParamsFactory
@@ -38,6 +38,7 @@ class PipeJinja2Output(PipeOutput):
class PipeJinja2(PipeOperator):
+ type: Literal["PipeJinja2"] = "PipeJinja2"
model_config = ConfigDict(extra="forbid", strict=False)
adhoc_pipe_code: ClassVar[str] = "jinja2_render"
@@ -86,7 +87,7 @@ class PipeJinja2(PipeOperator):
log.debug(f"Validated jinja2 template '{self.jinja2_name}':\n{the_template}")
@override
- def needed_inputs(self) -> PipeInputSpec:
+ def needed_inputs(self, visited_pipes: Optional[Set[str]] = None) -> PipeInputSpec:
needed_inputs = PipeInputSpecFactory.make_empty()
for input_name, requirement in self.inputs.root.items():
needed_inputs.add_requirement(variable_name=input_name, concept=requirement.concept)
diff --git a/pipelex/pipe_operators/jinja2/pipe_jinja2_blueprint.py b/pipelex/pipe_operators/jinja2/pipe_jinja2_blueprint.py
index 4cdf09ee0..0fefced7c 100644
--- a/pipelex/pipe_operators/jinja2/pipe_jinja2_blueprint.py
+++ b/pipelex/pipe_operators/jinja2/pipe_jinja2_blueprint.py
@@ -6,3 +6,4 @@ from pipelex.tools.templating.jinja2_blueprint import Jinja2Blueprint
class PipeJinja2Blueprint(PipeBlueprint, Jinja2Blueprint):
type: Literal["PipeJinja2"] = "PipeJinja2"
+ category: Literal["PipeOperator"] = "PipeOperator"
diff --git a/pipelex/pipe_operators/jinja2/pipe_jinja2_factory.py b/pipelex/pipe_operators/jinja2/pipe_jinja2_factory.py
index abfe83e9e..3fda17ba7 100644
--- a/pipelex/pipe_operators/jinja2/pipe_jinja2_factory.py
+++ b/pipelex/pipe_operators/jinja2/pipe_jinja2_factory.py
@@ -4,8 +4,8 @@ from typing_extensions import override
from pipelex.core.concepts.concept_factory import ConceptFactory
from pipelex.core.pipes.pipe_factory import PipeFactoryProtocol
-from pipelex.core.pipes.pipe_input_spec import PipeInputSpec
-from pipelex.core.pipes.pipe_input_spec_factory import PipeInputSpecFactory
+from pipelex.core.pipes.pipe_input import PipeInputSpec
+from pipelex.core.pipes.pipe_input_factory import PipeInputSpecFactory
from pipelex.exceptions import PipeDefinitionError
from pipelex.hub import get_concept_provider
from pipelex.pipe_operators.jinja2.pipe_jinja2 import PipeJinja2
@@ -37,7 +37,7 @@ class PipeJinja2Factory(PipeFactoryProtocol[PipeJinja2Blueprint, PipeJinja2]):
output_domain_and_code = ConceptFactory.make_domain_and_concept_code_from_concept_string_or_concept_code(
domain=domain,
- concept_string_or_concept_code=blueprint.output_concept_string_or_concept_code,
+ concept_string_or_code=blueprint.output,
concept_codes_from_the_same_domain=concept_codes_from_the_same_domain,
)
return PipeJinja2(
diff --git a/pipelex/pipe_operators/llm/pipe_llm.py b/pipelex/pipe_operators/llm/pipe_llm.py
index efa908d35..d481fb107 100644
--- a/pipelex/pipe_operators/llm/pipe_llm.py
+++ b/pipelex/pipe_operators/llm/pipe_llm.py
@@ -1,4 +1,4 @@
-from typing import List, Optional, Set, Type, cast
+from typing import List, Literal, Optional, Set, Type, cast
from pydantic import model_validator
from typing_extensions import Self, override
@@ -10,15 +10,15 @@ from pipelex.cogt.llm.llm_prompt import LLMPrompt
from pipelex.cogt.llm.llm_prompt_factory_abstract import LLMPromptFactoryAbstract
from pipelex.cogt.llm.llm_prompt_spec import LLMPromptSpec
from pipelex.cogt.llm.llm_prompt_template import LLMPromptTemplate
-from pipelex.cogt.llm.llm_setting import LLMSetting, LLMSettingChoices, LLMSettingOrPresetId
-from pipelex.cogt.models.model_deck_check import check_llm_setting_with_deck
+from pipelex.cogt.llm.llm_setting import LLMChoice, LLMSetting, LLMSettingChoices
+from pipelex.cogt.models.model_deck_check import check_llm_choice_with_deck
from pipelex.config import StaticValidationReaction, get_config
from pipelex.core.concepts.concept_factory import ConceptFactory
from pipelex.core.concepts.concept_native import NativeConceptEnum
from pipelex.core.domains.domain import Domain, SpecialDomain
from pipelex.core.memory.working_memory import WorkingMemory
-from pipelex.core.pipes.pipe_input_spec import PipeInputSpec
-from pipelex.core.pipes.pipe_input_spec_factory import PipeInputSpecFactory
+from pipelex.core.pipes.pipe_input import PipeInputSpec
+from pipelex.core.pipes.pipe_input_factory import PipeInputSpecFactory
from pipelex.core.pipes.pipe_output import PipeOutput
from pipelex.core.pipes.pipe_run_params import (
PipeOutputMultiplicity,
@@ -39,7 +39,7 @@ from pipelex.hub import (
get_class_registry,
get_concept_provider,
get_content_generator,
- get_models_manager,
+ get_model_deck,
get_optional_pipe,
get_required_concept,
get_required_domain,
@@ -49,7 +49,7 @@ from pipelex.hub import (
from pipelex.pipe_operators.llm.pipe_llm_blueprint import StructuringMethod
from pipelex.pipe_operators.pipe_operator import PipeOperator
from pipelex.pipeline.job_metadata import JobMetadata
-from pipelex.tools.typing.type_inspector import get_type_structure
+from pipelex.tools.typing.structure_printer import StructurePrinter
class PipeLLMOutput(PipeOutput):
@@ -57,6 +57,7 @@ class PipeLLMOutput(PipeOutput):
class PipeLLM(PipeOperator):
+ type: Literal["PipeLLM"] = "PipeLLM"
llm_prompt_spec: LLMPromptSpec
llm_choices: Optional[LLMSettingChoices] = None
structuring_method: Optional[StructuringMethod] = None
@@ -73,10 +74,10 @@ class PipeLLM(PipeOperator):
@model_validator(mode="after")
def validate_output_concept_consistency(self) -> Self:
if self.structuring_method is not None:
- if self.output.structure_class_name == NativeConceptEnum.TEXT.value:
+ if self.output.structure_class_name == NativeConceptEnum.TEXT:
raise PipeDefinitionError(
f"Output concept '{self.output.code}' is considered a Text concept, "
- f"so it cannot be structured. Maybe you forgot to add '{NativeConceptEnum.TEXT.value}' to the class registry?"
+ f"so it cannot be structured. Maybe you forgot to add '{NativeConceptEnum.TEXT}' to the class registry?"
)
return self
@@ -89,8 +90,8 @@ class PipeLLM(PipeOperator):
if self.system_prompt_to_structure:
get_template(template_name=self.system_prompt_to_structure)
if self.llm_choices:
- for llm_setting in self.llm_choices.list_used_presets():
- check_llm_setting_with_deck(llm_setting_or_preset_id=llm_setting)
+ for llm_choice in self.llm_choices.list_choices():
+ check_llm_choice_with_deck(llm_choice=llm_choice)
@override
def validate_output(self):
@@ -104,7 +105,7 @@ class PipeLLM(PipeOperator):
)
@override
- def needed_inputs(self) -> PipeInputSpec:
+ def needed_inputs(self, visited_pipes: Optional[Set[str]] = None) -> PipeInputSpec:
"""Needed inputs are the inputs needed to run the pipe, specified in the inputs attribute of the pipe"""
# The images are not tagged in the prompt_template.
# Therefore if an image is provided in the inputs, it becomes a needed input.
@@ -236,20 +237,18 @@ class PipeLLM(PipeOperator):
# interpret / unwrap the arguments
log.debug(f"PipeLLM pipe_code = {self.code}")
output_concept = self.output
- if self.output.code == SpecialDomain.NATIVE.value + "." + NativeConceptEnum.DYNAMIC.value:
+ if self.output.code == SpecialDomain.NATIVE + "." + NativeConceptEnum.DYNAMIC:
# TODO: This DYNAMIC_OUTPUT_CONCEPT should not be a field in the params attribute of PipeRunParams.
# It should be an attribute of PipeRunParams.
output_concept_code = pipe_run_params.dynamic_output_concept_code or pipe_run_params.params.get(PipeRunParamKey.DYNAMIC_OUTPUT_CONCEPT)
if not output_concept_code:
- output_concept_code = SpecialDomain.NATIVE.value + "." + NativeConceptEnum.TEXT.value
+ output_concept_code = SpecialDomain.NATIVE + "." + NativeConceptEnum.TEXT
else:
output_concept = get_concept_provider().get_required_concept(
concept_string=ConceptFactory.construct_concept_string_with_domain(domain=self.domain, concept_code=output_concept_code)
)
- # self.llm_prompt_spec.output = output_concept
-
multiplicity_resolution = output_multiplicity_to_apply(
base_multiplicity=self.output_multiplicity,
override_multiplicity=pipe_run_params.output_multiplicity,
@@ -259,31 +258,31 @@ class PipeLLM(PipeOperator):
fixed_nb_output = multiplicity_resolution.specific_output_count
# Collect what LLM settings we have for this particular PipeLLM
- llm_for_text_choice: Optional[LLMSettingOrPresetId] = None
- llm_for_object_choice: Optional[LLMSettingOrPresetId] = None
+ llm_for_text_choice: Optional[LLMChoice] = None
+ llm_for_object_choice: Optional[LLMChoice] = None
if self.llm_choices:
llm_for_text_choice = self.llm_choices.for_text
llm_for_object_choice = self.llm_choices.for_object
- llm_deck = get_models_manager().get_model_deck()
+ model_deck = get_model_deck()
# Choice of main LLM for text first from this PipeLLM setting (self.llm_choices)
# or from the llm_choice_overrides or fallback on the llm_choice_defaults
- llm_setting_or_preset_id_for_text: LLMSettingOrPresetId = (
- llm_for_text_choice or llm_deck.llm_choice_overrides.for_text or llm_deck.llm_choice_defaults.for_text
+ llm_setting_or_preset_id_for_text: LLMChoice = (
+ llm_for_text_choice or model_deck.llm_choice_overrides.for_text or model_deck.llm_choice_defaults.for_text
)
- llm_setting_main: LLMSetting = llm_deck.get_llm_setting(llm_setting_or_preset_id=llm_setting_or_preset_id_for_text)
+ llm_setting_main: LLMSetting = model_deck.get_llm_setting(llm_choice=llm_setting_or_preset_id_for_text)
# Choice of main LLM for object from this PipeLLM setting (self.llm_choices)
# OR FROM THE llm_for_text_choice (if any)
# then fallback on the llm_choice_overrides or llm_choice_defaults
- llm_setting_or_preset_id_for_object: LLMSettingOrPresetId = (
- llm_for_object_choice or llm_for_text_choice or llm_deck.llm_choice_overrides.for_object or llm_deck.llm_choice_defaults.for_object
+ llm_setting_or_preset_id_for_object: LLMChoice = (
+ llm_for_object_choice or llm_for_text_choice or model_deck.llm_choice_overrides.for_object or model_deck.llm_choice_defaults.for_object
)
- llm_setting_for_object: LLMSetting = llm_deck.get_llm_setting(llm_setting_or_preset_id=llm_setting_or_preset_id_for_object)
+ llm_setting_for_object: LLMSetting = model_deck.get_llm_setting(llm_choice=llm_setting_or_preset_id_for_object)
if (not self.llm_prompt_spec.prompting_style) and (
- inference_model := llm_deck.get_optional_inference_model(model_handle=llm_setting_main.llm_handle)
+ inference_model := model_deck.get_optional_inference_model(model_handle=llm_setting_main.llm_handle)
):
# Note: the case where we don't get an inference model corresponds to the use of an external LLM Plugin
# TODO: improve this by making it possible to get the inference model for external LLM Plugins
@@ -295,7 +294,9 @@ class PipeLLM(PipeOperator):
is_with_preliminary_text = (
self.structuring_method == StructuringMethod.PRELIMINARY_TEXT
) or get_config().pipelex.structure_config.is_default_text_then_structure
-
+ log.verbose(
+ f"is_with_preliminary_text: {is_with_preliminary_text} for pipe {self.code} because the structuring_method is {self.structuring_method}"
+ )
# Append output structure prompt if needed
output_structure_prompt: Optional[str] = PipeLLM.get_output_structure_prompt(
concept_string=pipe_run_params.dynamic_output_concept_code or output_concept.concept_string,
@@ -317,7 +318,7 @@ class PipeLLM(PipeOperator):
)
the_content: StuffContent
- if output_concept.structure_class_name == NativeConceptEnum.TEXT.value and not is_multiple_output:
+ if output_concept.structure_class_name == NativeConceptEnum.TEXT and not is_multiple_output:
log.debug(f"PipeLLM generating a single text output: {self.class_name}_gen_text")
generated_text: str = await content_generator.make_llm_text(
job_metadata=job_metadata,
@@ -344,6 +345,7 @@ class PipeLLM(PipeOperator):
case StructuringMethod.DIRECT:
llm_prompt_2_factory = None
case StructuringMethod.PRELIMINARY_TEXT:
+ log.verbose(f"Creating llm_prompt_2_factory for pipe {self.code} with structuring_method {structuring_method}")
pipe = get_required_pipe(pipe_code=self.code)
# TODO: run_pipe() could get the domain at the same time as the pip_code
domain = get_required_domain(domain=pipe.domain)
@@ -441,6 +443,7 @@ class PipeLLM(PipeOperator):
# We're generating a list of objects using preliminary text
method_desc = "text_then_object"
log.dev(f"{task_desc} by {method_desc}")
+ log.verbose(f"llm_prompt_2_factory: {llm_prompt_2_factory}")
generated_objects = await content_generator.make_text_then_object_list(
job_metadata=job_metadata,
@@ -472,6 +475,7 @@ class PipeLLM(PipeOperator):
# We're generating a single object using preliminary text
method_desc = "text_then_object"
log.verbose(f"{task_desc} by {method_desc}")
+ log.verbose(f"llm_prompt_2_factory: {llm_prompt_2_factory}")
generated_object = await content_generator.make_text_then_object(
job_metadata=job_metadata,
object_class=content_class,
@@ -520,29 +524,33 @@ class PipeLLM(PipeOperator):
if not output_class:
return ""
- class_structure = get_type_structure(output_class, base_class=StuffContent)
+ class_structure = StructurePrinter().get_type_structure(tp=output_class, base_class=StuffContent)
if not class_structure:
return ""
-
class_structure_str = "\n".join(class_structure)
# TODO: use proper prompt templating for this
if is_with_preliminary_text:
- output_structure_prompt = (
- f"\n\n---\nRequested output format: The requested output will be used to define the following class: {class_name}\n"
- f"{class_structure_str}\n"
- "You do NOT need to output a formatted JSON object, another LLM will take care of that. "
- "If you cannot find a value that is Optional, output None for that field. "
- "However, you MUST clearly output the values for each of these fields in your response.\n---\n"
- "DO NOT create information. If the information is not present, output None."
- )
+ output_structure_prompt = f"""
+
+---
+The instance we want to generate will be for the following class:
+{class_structure_str}
+
+Don't bother with JSON formatting, we'll do that as a second step.
+For now, just output markdown with the details of the instance.
+DO NOT create information.
+If some information is not present for an attribute, output the default value or None according to the attribute definition.
+"""
else:
- output_structure_prompt = (
- f"\n\n---\nRequested output format: The output must conform to the following BaseModel: {class_name}\n"
- f"{class_structure_str}\n"
- "If you cannot find a value that is Optional, output None for that field. "
- "However, you MUST clearly output the values for each of these fields in your response.\n---\n"
- "DO NOT create information. If the information is not present, output None."
- )
+ output_structure_prompt = f"""
+
+---
+The instance we want to generate will be for the following class:
+{class_structure_str}
+
+DO NOT create information.
+If some information is not present for an attribute, output the default value or None according to the attribute definition.
+"""
return output_structure_prompt
diff --git a/pipelex/pipe_operators/llm/pipe_llm_blueprint.py b/pipelex/pipe_operators/llm/pipe_llm_blueprint.py
index 497adaa6b..84605a5b3 100644
--- a/pipelex/pipe_operators/llm/pipe_llm_blueprint.py
+++ b/pipelex/pipe_operators/llm/pipe_llm_blueprint.py
@@ -3,7 +3,7 @@ from typing import Literal, Optional
from pydantic import field_validator, model_validator
from typing_extensions import Self
-from pipelex.cogt.llm.llm_setting import LLMSettingOrPresetId
+from pipelex.cogt.llm.llm_setting import LLMChoice
from pipelex.core.pipes.pipe_blueprint import PipeBlueprint
from pipelex.exceptions import PipeDefinitionError
from pipelex.tools.typing.validation_utils import has_more_than_one_among_attributes_from_lists
@@ -17,6 +17,7 @@ class StructuringMethod(StrEnum):
class PipeLLMBlueprint(PipeBlueprint):
type: Literal["PipeLLM"] = "PipeLLM"
+ category: Literal["PipeOperator"] = "PipeOperator"
system_prompt_template: Optional[str] = None
system_prompt_template_name: Optional[str] = None
system_prompt_name: Optional[str] = None
@@ -27,8 +28,8 @@ class PipeLLMBlueprint(PipeBlueprint):
prompt_name: Optional[str] = None
prompt: Optional[str] = None
- llm: Optional[LLMSettingOrPresetId] = None
- llm_to_structure: Optional[LLMSettingOrPresetId] = None
+ llm: Optional[LLMChoice] = None
+ llm_to_structure: Optional[LLMChoice] = None
structuring_method: Optional[StructuringMethod] = None
prompt_template_to_structure: Optional[str] = None
diff --git a/pipelex/pipe_operators/llm/pipe_llm_factory.py b/pipelex/pipe_operators/llm/pipe_llm_factory.py
index c3658de28..a4e3f1011 100644
--- a/pipelex/pipe_operators/llm/pipe_llm_factory.py
+++ b/pipelex/pipe_operators/llm/pipe_llm_factory.py
@@ -6,8 +6,8 @@ from pipelex.cogt.llm.llm_prompt_spec import LLMPromptSpec
from pipelex.cogt.llm.llm_setting import LLMSettingChoices
from pipelex.core.concepts.concept_factory import ConceptFactory
from pipelex.core.pipes.pipe_factory import PipeFactoryProtocol
-from pipelex.core.pipes.pipe_input_spec_blueprint import InputRequirementBlueprint
-from pipelex.core.pipes.pipe_input_spec_factory import PipeInputSpecFactory
+from pipelex.core.pipes.pipe_input_blueprint import InputRequirementBlueprint
+from pipelex.core.pipes.pipe_input_factory import PipeInputSpecFactory
from pipelex.core.pipes.pipe_run_params import make_output_multiplicity
from pipelex.exceptions import PipeDefinitionError
from pipelex.hub import get_concept_provider, get_optional_domain
@@ -80,7 +80,7 @@ class PipeLLMFactory(PipeFactoryProtocol[PipeLLMBlueprint, PipeLLM]):
concept_string = requirement.concept
domain_and_code = ConceptFactory.make_domain_and_concept_code_from_concept_string_or_concept_code(
domain=domain,
- concept_string_or_concept_code=concept_string,
+ concept_string_or_code=concept_string,
concept_codes_from_the_same_domain=concept_codes_from_the_same_domain,
)
concept = get_concept_provider().get_required_concept(
@@ -116,7 +116,7 @@ class PipeLLMFactory(PipeFactoryProtocol[PipeLLMBlueprint, PipeLLM]):
output_domain_and_code = ConceptFactory.make_domain_and_concept_code_from_concept_string_or_concept_code(
domain=domain,
- concept_string_or_concept_code=blueprint.output_concept_string_or_concept_code,
+ concept_string_or_code=blueprint.output,
concept_codes_from_the_same_domain=concept_codes_from_the_same_domain,
)
output_concept_domain, output_concept_code = output_domain_and_code.domain, output_domain_and_code.concept_code
diff --git a/pipelex/pipe_operators/ocr/pipe_ocr.py b/pipelex/pipe_operators/ocr/pipe_ocr.py
index 5961c30b5..3df996622 100644
--- a/pipelex/pipe_operators/ocr/pipe_ocr.py
+++ b/pipelex/pipe_operators/ocr/pipe_ocr.py
@@ -1,4 +1,4 @@
-from typing import List, Optional, Set
+from typing import List, Literal, Optional, Set
from pydantic import model_validator
from typing_extensions import Self, override
@@ -6,14 +6,16 @@ from typing_extensions import Self, override
from pipelex import log
from pipelex.cogt.content_generation.content_generator_dry import ContentGeneratorDry
from pipelex.cogt.content_generation.content_generator_protocol import ContentGeneratorProtocol
+from pipelex.cogt.models.model_deck_check import check_ocr_choice_with_deck
from pipelex.cogt.ocr.ocr_input import OcrInput
from pipelex.cogt.ocr.ocr_job_components import OcrJobConfig, OcrJobParams
+from pipelex.cogt.ocr.ocr_setting import OcrChoice, OcrSetting
from pipelex.config import StaticValidationReaction, get_config
from pipelex.core.concepts.concept_native import NativeConceptEnum
from pipelex.core.memory.working_memory import WorkingMemory
-from pipelex.core.pipes.pipe_input_spec import PipeInputSpec
-from pipelex.core.pipes.pipe_input_spec_blueprint import InputRequirementBlueprint
-from pipelex.core.pipes.pipe_input_spec_factory import PipeInputSpecFactory
+from pipelex.core.pipes.pipe_input import PipeInputSpec
+from pipelex.core.pipes.pipe_input_blueprint import InputRequirementBlueprint
+from pipelex.core.pipes.pipe_input_factory import PipeInputSpecFactory
from pipelex.core.pipes.pipe_output import PipeOutput
from pipelex.core.pipes.pipe_run_params import PipeRunMode, PipeRunParams
from pipelex.core.stuffs.stuff_content import ImageContent, ListContent, PageContent, TextAndImagesContent, TextContent
@@ -26,6 +28,7 @@ from pipelex.exceptions import (
from pipelex.hub import (
get_concept_provider,
get_content_generator,
+ get_model_deck,
)
from pipelex.pipe_operators.pipe_operator import PipeOperator
from pipelex.pipeline.job_metadata import JobMetadata
@@ -40,7 +43,8 @@ PIPE_OCR_INPUT_NAME = "ocr_input"
class PipeOcr(PipeOperator):
- ocr_handle: str
+ type: Literal["PipeOcr"] = "PipeOcr"
+ ocr_choice: Optional[OcrChoice]
should_caption_images: bool
should_include_images: bool
should_include_page_views: bool
@@ -54,6 +58,12 @@ class PipeOcr(PipeOperator):
self._validate_inputs()
return self
+ @override
+ def validate_with_libraries(self):
+ self._validate_inputs()
+ if self.ocr_choice:
+ check_ocr_choice_with_deck(ocr_choice=self.ocr_choice)
+
@override
def required_variables(self) -> Set[str]:
return {PIPE_OCR_INPUT_NAME}
@@ -137,7 +147,7 @@ class PipeOcr(PipeOperator):
raise missing_input_var_error
@override
- def needed_inputs(self) -> PipeInputSpec:
+ def needed_inputs(self, visited_pipes: Optional[Set[str]] = None) -> PipeInputSpec:
return PipeInputSpecFactory.make_from_blueprint(
domain=self.domain,
blueprint={PIPE_OCR_INPUT_NAME: InputRequirementBlueprint(concept=self.inputs.root[PIPE_OCR_INPUT_NAME].concept.concept_string)},
@@ -165,11 +175,16 @@ class PipeOcr(PipeOperator):
else:
raise PipeDefinitionError("PipeOcr should have a non-None image_stuff_name or pdf_stuff_name")
+ ocr_choice: OcrChoice = self.ocr_choice or get_model_deck().ocr_choice_default
+ ocr_setting: OcrSetting = get_model_deck().get_ocr_setting(ocr_choice=ocr_choice)
+
ocr_job_params = OcrJobParams(
should_include_images=self.should_include_images,
should_caption_images=self.should_caption_images,
should_include_page_views=self.should_include_page_views,
page_views_dpi=self.page_views_dpi,
+ max_nb_images=ocr_setting.max_nb_images,
+ image_min_size=ocr_setting.image_min_size,
)
ocr_input = OcrInput(
image_uri=image_uri,
@@ -177,7 +192,7 @@ class PipeOcr(PipeOperator):
)
ocr_output = await content_generator.make_ocr_extract_pages(
ocr_input=ocr_input,
- ocr_handle=self.ocr_handle,
+ ocr_handle=ocr_setting.ocr_handle,
job_metadata=job_metadata,
ocr_job_params=ocr_job_params,
ocr_job_config=OcrJobConfig(),
diff --git a/pipelex/pipe_operators/ocr/pipe_ocr_blueprint.py b/pipelex/pipe_operators/ocr/pipe_ocr_blueprint.py
index 410a9c969..610eb5e96 100644
--- a/pipelex/pipe_operators/ocr/pipe_ocr_blueprint.py
+++ b/pipelex/pipe_operators/ocr/pipe_ocr_blueprint.py
@@ -1,11 +1,13 @@
from typing import Literal, Optional
+from pipelex.cogt.ocr.ocr_setting import OcrChoice
from pipelex.core.pipes.pipe_blueprint import PipeBlueprint
class PipeOcrBlueprint(PipeBlueprint):
type: Literal["PipeOcr"] = "PipeOcr"
- ocr_model: str
+ category: Literal["PipeOperator"] = "PipeOperator"
+ ocr: Optional[OcrChoice] = None
page_images: Optional[bool] = None
page_image_captions: Optional[bool] = None
page_views: Optional[bool] = None
diff --git a/pipelex/pipe_operators/ocr/pipe_ocr_factory.py b/pipelex/pipe_operators/ocr/pipe_ocr_factory.py
index 3c93930b1..9767858db 100644
--- a/pipelex/pipe_operators/ocr/pipe_ocr_factory.py
+++ b/pipelex/pipe_operators/ocr/pipe_ocr_factory.py
@@ -5,7 +5,7 @@ from typing_extensions import override
from pipelex.config import get_config
from pipelex.core.concepts.concept_factory import ConceptFactory
from pipelex.core.pipes.pipe_factory import PipeFactoryProtocol
-from pipelex.core.pipes.pipe_input_spec_factory import PipeInputSpecFactory
+from pipelex.core.pipes.pipe_input_factory import PipeInputSpecFactory
from pipelex.hub import get_concept_provider
from pipelex.pipe_operators.ocr.pipe_ocr import PipeOcr
from pipelex.pipe_operators.ocr.pipe_ocr_blueprint import PipeOcrBlueprint
@@ -23,7 +23,7 @@ class PipeOcrFactory(PipeFactoryProtocol[PipeOcrBlueprint, PipeOcr]):
) -> PipeOcr:
output_domain_and_code = ConceptFactory.make_domain_and_concept_code_from_concept_string_or_concept_code(
domain=domain,
- concept_string_or_concept_code=blueprint.output_concept_string_or_concept_code,
+ concept_string_or_code=blueprint.output,
concept_codes_from_the_same_domain=concept_codes_from_the_same_domain,
)
@@ -39,7 +39,7 @@ class PipeOcrFactory(PipeFactoryProtocol[PipeOcrBlueprint, PipeOcr]):
inputs=PipeInputSpecFactory.make_from_blueprint(
domain=domain, blueprint=blueprint.inputs or {}, concept_codes_from_the_same_domain=concept_codes_from_the_same_domain
),
- ocr_handle=blueprint.ocr_model,
+ ocr_choice=blueprint.ocr,
should_include_images=blueprint.page_images or False,
should_caption_images=blueprint.page_image_captions or False,
should_include_page_views=blueprint.page_views or False,
diff --git a/pipelex/pipe_operators/pipe_operator.py b/pipelex/pipe_operators/pipe_operator.py
index 09bc7145e..c8f3010b0 100644
--- a/pipelex/pipe_operators/pipe_operator.py
+++ b/pipelex/pipe_operators/pipe_operator.py
@@ -1,19 +1,22 @@
from abc import abstractmethod
-from typing import Optional
+from typing import Literal, Optional
from typing_extensions import override
-from pipelex import log
+from pipelex import log, pretty_print, pretty_print_md
from pipelex.core.memory.working_memory import WorkingMemory
from pipelex.core.pipes.pipe_abstract import PipeAbstract
from pipelex.core.pipes.pipe_output import PipeOutput
from pipelex.core.pipes.pipe_run_params import PipeRunMode, PipeRunParams
+from pipelex.core.stuffs.stuff_content import TextContent
from pipelex.hub import get_activity_manager
from pipelex.pipeline.activity.activity_models import ActivityReport
from pipelex.pipeline.job_metadata import JobMetadata
class PipeOperator(PipeAbstract):
+ category: Literal["PipeOperator"] = "PipeOperator"
+
@override
async def run_pipe(
self,
@@ -21,6 +24,7 @@ class PipeOperator(PipeAbstract):
working_memory: WorkingMemory,
pipe_run_params: PipeRunParams,
output_name: Optional[str] = None,
+ print_intermediate_outputs: Optional[bool] = False,
) -> PipeOutput:
pipe_run_params.push_pipe_to_stack(pipe_code=self.code)
self.monitor_pipe_stack(pipe_run_params=pipe_run_params)
@@ -33,25 +37,27 @@ class PipeOperator(PipeAbstract):
match pipe_run_params.run_mode:
case PipeRunMode.LIVE:
if self.class_name not in ["PipeJinja2", "PipeLLMPrompt"]:
- name = self.class_name
+ name = f"Running [cyan]{self.class_name}[/cyan]"
indent_level = len(pipe_run_params.pipe_stack) - 1
indent = " " * indent_level
- label = f"{indent}{name}: {self.code}".ljust(80)
- output = self.output.code
- log.info(f"{label} → {output}")
+ label = f"{indent}{'[yellow]↳[/yellow]' if indent_level > 0 else ''} {name} → [green]{self.code}[/green]"
+ log.info(f"{label} → [red]{self.output.code}[/red]")
pipe_output = await self._run_operator_pipe(
job_metadata=job_metadata,
working_memory=working_memory,
pipe_run_params=pipe_run_params,
output_name=output_name,
)
+ if isinstance(pipe_output.main_stuff.content, TextContent):
+ pretty_print_md(pipe_output.main_stuff_as_str, title=f"PipeOutput of pipe {self.code}")
+ else:
+ pretty_print(pipe_output.main_stuff, title=f"PipeOutput of pipe {self.code}")
case PipeRunMode.DRY:
- name = f"Dry {self.class_name}"
+ name = f"Dry run [cyan]{self.class_name}[/cyan]"
indent_level = len(pipe_run_params.pipe_stack) - 1
indent = " " * indent_level
- label = f"{indent}{name}: {self.code}".ljust(80)
- output = self.output.code
- log.info(f"{label} → {output}")
+ label = f"{indent}{'[yellow]↳[/yellow]' if indent_level > 0 else ''} {name}: [green]{self.code}[/green]"
+ log.info(f"{label} → [red]{self.output.code}[/red]")
pipe_output = await self._dry_run_operator_pipe(
job_metadata=job_metadata,
working_memory=working_memory,
diff --git a/pipelex/pipe_works/pipe_dry.py b/pipelex/pipe_works/pipe_dry.py
index 738446cfc..eb7695b9f 100644
--- a/pipelex/pipe_works/pipe_dry.py
+++ b/pipelex/pipe_works/pipe_dry.py
@@ -2,61 +2,79 @@ import asyncio
import functools
import time
from concurrent.futures import ThreadPoolExecutor
-from typing import Dict, List, Tuple
+from typing import Dict, List, Optional
+
+from pydantic import BaseModel
from pipelex import log
from pipelex.config import get_config
from pipelex.core.memory.working_memory_factory import WorkingMemoryFactory
from pipelex.core.pipes.pipe_abstract import PipeAbstract
-from pipelex.core.pipes.pipe_input_spec import PipeInputSpec, TypedNamedInputRequirement
+from pipelex.core.pipes.pipe_input import PipeInputSpec, TypedNamedInputRequirement
from pipelex.core.pipes.pipe_run_params import PipeRunMode
from pipelex.core.pipes.pipe_run_params_factory import PipeRunParamsFactory
from pipelex.core.stuffs.stuff_content import StuffContent, TextContent
-from pipelex.hub import get_class_registry, get_pipe_provider
+from pipelex.hub import get_class_registry
from pipelex.pipeline.job_metadata import JobMetadata
+from pipelex.types import StrEnum
-async def dry_run_all_pipes():
- all_pipes = get_pipe_provider().get_pipes()
- await dry_run_pipes(pipes=all_pipes)
+class DryRunError(Exception):
+ """Raised when a dry run fails due to missing inputs or other validation issues."""
+ pass
-async def dry_run_single_pipe(pipe_code: str):
- """
- Dry run a single pipe by its code.
- Args:
- pipe_code: The code of the pipe to dry run
+class DryRunStatus(StrEnum):
+ SUCCESS = "SUCCESS"
+ FAILURE = "FAILURE"
+ WARNING = "WARNING"
- Returns:
- Status string: "SUCCESS" or error message
- """
- # Get the pipe using the hub function
- pipe = get_pipe_provider().get_required_pipe(pipe_code=pipe_code)
- # Run the single pipe
- await dry_run_pipes(pipes=[pipe])
+class DryRunOutput(BaseModel):
+ pipe_code: str
+ status: DryRunStatus
+ error_message: Optional[str] = None
+ warning_message: Optional[str] = None
-async def dry_run_pipe_codes(pipe_codes: List[str]) -> Dict[str, str]:
+async def dry_run_pipe(pipe: PipeAbstract, raise_on_failure: bool = False) -> DryRunOutput:
"""
- Dry run a list of pipe codes.
+ Dry run a single pipe directly without parallelization.
"""
- pipe_provider = get_pipe_provider()
- pipes = [pipe_provider.get_required_pipe(pipe_code=pipe_code) for pipe_code in pipe_codes]
- return await dry_run_pipes(pipes=pipes)
-
-
-async def dry_run_pipe(pipe: PipeAbstract):
- """
- Dry run a pipe.
+ allowed_to_fail_pipes = get_config().pipelex.dry_run_config.allowed_to_fail_pipes
+ needed_inputs_for_factory = _convert_to_working_memory_format(needed_inputs_spec=pipe.needed_inputs())
+
+ working_memory = WorkingMemoryFactory.make_for_dry_run(needed_inputs=needed_inputs_for_factory)
+ try:
+ pipe.validate_with_libraries()
+ await pipe.run_pipe(
+ job_metadata=JobMetadata(job_name=f"dry_run_{pipe.code}"),
+ working_memory=working_memory,
+ pipe_run_params=PipeRunParamsFactory.make_run_params(pipe_run_mode=PipeRunMode.DRY),
+ )
+ except Exception as exc:
+ if pipe.code in allowed_to_fail_pipes:
+ warning_message = f"Allowed to fail dry run for pipe '{pipe.code}': {exc}"
+ log.warning(warning_message)
+ return DryRunOutput(pipe_code=pipe.code, status=DryRunStatus.WARNING, warning_message=warning_message)
+ elif raise_on_failure:
+ raise exc
+ else:
+ error_message = f"Dry run failed for pipe '{pipe.code}': {exc}"
+ log.error(error_message)
+ return DryRunOutput(pipe_code=pipe.code, status=DryRunStatus.FAILURE, error_message=error_message)
+ log.info(f"Pipe '{pipe.code}' dry run completed successfully")
+ return DryRunOutput(pipe_code=pipe.code, status=DryRunStatus.SUCCESS)
+
+
+async def dry_run_pipes(pipes: List[PipeAbstract], run_in_parallel: bool = True, raise_on_failure: bool = True) -> Dict[str, DryRunOutput]:
"""
- return await dry_run_pipes(pipes=[pipe])
-
+ Dry run pipes with optional parallelization.
-async def dry_run_pipes(pipes: List[PipeAbstract]) -> Dict[str, str]:
- """
- Dry run all pipes in the library using ThreadPoolExecutor for true parallelism.
+ Args:
+ pipes: List of pipes to dry run
+ run_in_parallel: If True, run pipes in parallel using ThreadPoolExecutor. If False, run sequentially.
For each pipe, this method:
1. Gets the pipe's needed inputs
@@ -68,84 +86,42 @@ async def dry_run_pipes(pipes: List[PipeAbstract]) -> Dict[str, str]:
"""
start_time = time.time()
- results: Dict[str, str] = {}
- pipe_provider = get_pipe_provider()
- pipe_provider.validate_with_libraries()
- # Get the list of pipes that are allowed to fail from config
+ results: Dict[str, DryRunOutput] = {}
allowed_to_fail_pipes = get_config().pipelex.dry_run_config.allowed_to_fail_pipes
- log.info(f"Starting dry run for {len(pipes)} pipes...")
-
- # Define a function that will run in a thread
- def run_pipe_in_thread(pipe: PipeAbstract) -> Tuple[str, str]:
- """Execute pipe.run_pipe in a thread and return its status."""
- # try:
- # This function runs in a separate thread
- try:
- needed_inputs = pipe.needed_inputs()
- log.debug(f"Needed inputs for {pipe.code}: {needed_inputs}")
- needed_inputs_for_factory = _convert_to_working_memory_format(needed_inputs_spec=needed_inputs)
-
- log.debug(f"Needed inputs for {pipe.code} converted to working memory format: {needed_inputs_for_factory}")
- working_memory = WorkingMemoryFactory.make_for_dry_run(needed_inputs=needed_inputs_for_factory)
+ if run_in_parallel:
- # Create a new event loop for this thread
+ def run_pipe_in_thread(pipe: PipeAbstract) -> DryRunOutput:
+ """Parallel execution using ThreadPoolExecutor"""
loop = asyncio.new_event_loop()
asyncio.set_event_loop(loop)
+ return loop.run_until_complete(dry_run_pipe(pipe, raise_on_failure=raise_on_failure))
- try:
- # Run the pipe in this thread's event loop
- loop.run_until_complete(
- pipe.run_pipe(
- job_metadata=JobMetadata(job_name=f"dry_run_{pipe.code}"),
- working_memory=working_memory,
- pipe_run_params=PipeRunParamsFactory.make_run_params(pipe_run_mode=PipeRunMode.DRY),
- )
- )
- result = (pipe.code, "SUCCESS")
- log.debug(f"✓ Pipe {pipe.code} dry run completed successfully")
- return result
- finally:
- loop.close()
-
- except Exception as exc:
- error_msg = f"FAILED: {str(exc)}"
-
- # Check if this pipe is allowed to fail
- if pipe.code in allowed_to_fail_pipes:
- log.debug(f"✗ Pipe '{pipe.code}' dry run failed: {exc} (this is normal, allowed by config)")
- else:
- log.error(f"✗ Pipe '{pipe.code}' dry run failed: {exc}")
+ with ThreadPoolExecutor() as executor:
+ futures = [asyncio.get_running_loop().run_in_executor(executor, functools.partial(run_pipe_in_thread, pipe)) for pipe in pipes]
+ for future in asyncio.as_completed(futures):
+ output = await future
+ results[output.pipe_code] = output
+ else:
+ for pipe in pipes:
+ results[pipe.code] = await dry_run_pipe(pipe, raise_on_failure=raise_on_failure)
- return (pipe.code, error_msg)
+ successful_pipes = [pipe_code for pipe_code, status in results.items() if status.status == DryRunStatus.SUCCESS]
+ failed_pipes = [pipe_code for pipe_code, status in results.items() if status.status != DryRunStatus.SUCCESS]
- # Get the event loop for the main thread
- loop = asyncio.get_running_loop()
-
- # Execute pipes in thread pool
- with ThreadPoolExecutor() as executor:
- # Schedule all pipe executions to the thread pool
- futures = [loop.run_in_executor(executor, functools.partial(run_pipe_in_thread, pipe)) for pipe in pipes]
-
- # Wait for all executions to complete
- for future in asyncio.as_completed(futures):
- pipe_code, status = await future
- results[pipe_code] = status
-
- successful_pipes = [pipe_code for pipe_code, status in results.items() if status == "SUCCESS"]
- failed_pipes = [pipe_code for pipe_code, status in results.items() if status != "SUCCESS"]
-
- # Filter out pipes that are allowed to fail
unexpected_failures = {pipe_code: results[pipe_code] for pipe_code in failed_pipes if pipe_code not in allowed_to_fail_pipes}
- log.info(f"Dry run completed: '{len(successful_pipes)}' successful, '{len(failed_pipes)}' failed, in '{time.time() - start_time:.2f}' seconds")
-
+ log.info(
+ f"Dry run completed: '{len(successful_pipes)}' successful, '{len(failed_pipes)}' failed, "
+ f"'{len(allowed_to_fail_pipes)}' allowed to fail, in '{time.time() - start_time:.2f}' seconds"
+ )
if unexpected_failures:
unexpected_failures_details = "\n".join([f"'{pipe_code}': {results[pipe_code]}" for pipe_code in unexpected_failures])
- raise Exception(f"Dry run failed with '{len(unexpected_failures)}' unexpected pipe failures:\n{unexpected_failures_details}")
-
- if failed_pipes and not unexpected_failures:
- log.info("All failures were expected (allowed by config)")
+ if raise_on_failure:
+ raise DryRunError(f"Dry run failed with '{len(unexpected_failures)}' unexpected pipe failures:\n{unexpected_failures_details}")
+ else:
+ log.error(f"Dry run failed with '{len(unexpected_failures)}' unexpected pipe failures:\n{unexpected_failures_details}")
+ return results
return results
diff --git a/pipelex/pipelex.py b/pipelex/pipelex.py
index 2c511b3f5..000d5574f 100644
--- a/pipelex/pipelex.py
+++ b/pipelex/pipelex.py
@@ -9,7 +9,6 @@ from kajson.class_registry_abstract import ClassRegistryAbstract
from kajson.kajson_manager import KajsonManager
from kajson.singleton import MetaSingleton
from pydantic import ValidationError
-from rich import print
from typing_extensions import Self
from pipelex import log
@@ -251,13 +250,11 @@ class Pipelex(metaclass=MetaSingleton):
self.class_registry.teardown()
func_registry.teardown()
- project_name = get_config().project_name
log.debug(f"{PACKAGE_NAME} version {PACKAGE_VERSION} teardown done for {get_config().project_name} (except config & logs)")
self.pipelex_hub.reset_config()
# Clear the singleton instance from metaclass
if self.__class__ in MetaSingleton.instances:
del MetaSingleton.instances[self.__class__]
- print(f"{PACKAGE_NAME} version {PACKAGE_VERSION} config reset done for {project_name}")
# TODO: add kwargs to make() so that subclasses can employ specific parameters
@classmethod
@@ -311,7 +308,6 @@ class Pipelex(metaclass=MetaSingleton):
pipelex_instance = cls(config_dir_path=config_dir_path)
pipelex_instance.setup()
pipelex_instance.setup_libraries()
- log.info(f"Pipelex {PACKAGE_VERSION} initialized.")
return pipelex_instance
@classmethod
diff --git a/pipelex/pipelex.toml b/pipelex/pipelex.toml
index 6505f8e56..1a07639cb 100644
--- a/pipelex/pipelex.toml
+++ b/pipelex/pipelex.toml
@@ -59,7 +59,7 @@ pipelex = "INFO"
is_show_time = false
is_show_level = true
is_link_path_enabled = true
-is_markup_enabled = false
+is_markup_enabled = true
highlighter_name = "json"
is_rich_tracebacks = true
is_tracebacks_word_wrap = true
@@ -81,7 +81,7 @@ inference_config_path = ".pipelex/inference"
[cogt.inference_manager_config]
is_auto_setup_preset_llm = true
-is_auto_setup_preset_imgg = true
+is_auto_setup_preset_img_gen = true
is_auto_setup_preset_ocr = true
[cogt.llm_config]
@@ -91,23 +91,18 @@ default_max_images = 100
is_openai_structured_output_enabled = false
[cogt.llm_config.llm_job_config]
-max_retries = 1
+max_retries = 3
is_streaming_enabled = false
####################################################################################################
# Image generation config
####################################################################################################
-[cogt.imgg_config]
-imgg_handles = ["fal-ai/flux-pro", "fal-ai/fast-lightning-sdxl"]
-default_imgg_handle = "fal-ai/flux-pro/v1.1-ultra" # use "fal-ai/fast-lightning-sdxl" for SDXL Lightning cheap and fast
-
-[cogt.imgg_config.imgg_job_config]
+[cogt.img_gen_config.img_gen_job_config]
is_sync_mode = false
-[cogt.imgg_config.imgg_param_defaults]
-# quality and nb_steps cannot be set at the same time
-# nb_steps = 1 # 28 is a good default for Flux, possible values are [1,2,4,8] for SDXL Lightning
+[cogt.img_gen_config.img_gen_param_defaults]
+nb_steps = 1 # 28 is a good default for Flux, possible values are [1,2,4,8] for SDXL Lightning
quality = "low"
aspect_ratio = "square" # "square", "landscape_4_3", "landscape_16_9", "landscape_21_9", "portrait_3_4", "portrait_9_16", "portrait_9_21"
guidance_scale = 3.5
@@ -118,7 +113,7 @@ is_raw = false
output_format = "jpg"
seed = "auto"
-[cogt.imgg_config.fal_config]
+[cogt.img_gen_config.fal_config]
flux_map_quality_to_steps = { "low" = 14, "medium" = 28, "high" = 56 }
sdxl_lightning_map_quality_to_steps = { "low" = 2, "medium" = 4, "high" = 8 }
@@ -208,9 +203,11 @@ text_gen_truncate_length = 256
nb_list_items = 3
nb_ocr_pages = 4
allowed_to_fail_pipes = [
- "infinite_loop_1",
- "build_blueprint",
- "structure_blueprint",
+ "infinite_loop_1", # Loop but only for testing purposes
+ "fix_failing_pipes_once", # Loop but its normal
+ "validate_pipelex_bundle_spec", # Loop but its normal
+ "handle_validation_result", # Complex case of a sub pipe needed a ListContent because its batching on, but the dry_run cannot know.
+ "pipe_builder", # because of the 'fix_failing_pipes_once', 'validate_pipelex_bundle_spec', 'handle_validation_result'
]
image_urls = [
"https://storage.googleapis.com/public_test_files_7fa6_4277_9ab/fashion/fashion_photo_1.jpg",
diff --git a/pipelex/pipeline/activity/activity_handler.py b/pipelex/pipeline/activity/activity_handler.py
index 2d9a38396..9b896d5ae 100644
--- a/pipelex/pipeline/activity/activity_handler.py
+++ b/pipelex/pipeline/activity/activity_handler.py
@@ -25,9 +25,9 @@ class ActivityHandlerForResultFiles:
self.result_dir_path = result_dir_path
self.images_dir_path = os.path.join(result_dir_path, "images")
ensure_path(self.images_dir_path)
- imgg_config = get_config().cogt.imgg_config
- imgg_param_defaults = imgg_config.imgg_param_defaults
- self.image_output_format = imgg_param_defaults.output_format
+ img_gen_config = get_config().cogt.img_gen_config
+ img_gen_param_defaults = img_gen_config.img_gen_param_defaults
+ self.image_output_format = img_gen_param_defaults.output_format
self.already_handled_stuff: Set[str] = set()
def _generate_stuff_id(self, stuff: Stuff) -> str:
diff --git a/pipelex/pipeline/execute.py b/pipelex/pipeline/execute.py
index d34b19253..84926a787 100644
--- a/pipelex/pipeline/execute.py
+++ b/pipelex/pipeline/execute.py
@@ -97,9 +97,6 @@ async def execute_pipeline(
pipe_run_mode=pipe_run_mode,
)
- if working_memory:
- working_memory.pretty_print_summary()
-
pipe_job = PipeJobFactory.make_pipe_job(
pipe=pipe,
pipe_run_params=pipe_run_params,
diff --git a/pipelex/pipeline/job_metadata.py b/pipelex/pipeline/job_metadata.py
index 6ce2810dc..f3f94b4dc 100644
--- a/pipelex/pipeline/job_metadata.py
+++ b/pipelex/pipeline/job_metadata.py
@@ -10,7 +10,7 @@ from pipelex.types import StrEnum
class JobCategory(StrEnum):
MOCK_JOB = "mock_job"
LLM_JOB = "llm_job"
- IMGG_JOB = "imgg_job"
+ IMG_GEN_JOB = "img_gen_job"
PROMPTING_JOB = "prompting_job"
JINJA2_JOB = "jinja2_job"
OCR_JOB = "ocr_job"
@@ -19,7 +19,7 @@ class JobCategory(StrEnum):
class UnitJobId(StrEnum):
LLM_GEN_TEXT = "llm_gen_text"
LLM_GEN_OBJECT = "llm_gen_object"
- IMGG_TEXT_TO_IMAGE = "imgg_text_to_image"
+ IMG_GEN_TEXT_TO_IMAGE = "img_gen_text_to_image"
OCR_EXTRACT_PAGES = "ocr_extract_pages"
diff --git a/pipelex/pipeline/track/pipeline_tracker.py b/pipelex/pipeline/track/pipeline_tracker.py
index eda8c2d9f..af4cb3a81 100644
--- a/pipelex/pipeline/track/pipeline_tracker.py
+++ b/pipelex/pipeline/track/pipeline_tracker.py
@@ -324,7 +324,7 @@ class PipelineTracker(PipelineTrackerProtocol):
def _print_mermaid_flowchart_code_and_url(self, title: Optional[str] = None, subtitle: Optional[str] = None):
if not self.nx_graph.nodes:
- log.info("No nodes in the pipeline tracker")
+ log.debug("No nodes in the pipeline tracker")
return
if self.start_node is None:
raise JobHistoryError("Start node is not set")
@@ -338,7 +338,7 @@ class PipelineTracker(PipelineTrackerProtocol):
def _print_mermaid_flowchart_url(self, title: Optional[str] = None, subtitle: Optional[str] = None) -> Optional[str]:
if not self.nx_graph.nodes:
- log.info("No nodes in the pipeline tracker")
+ log.debug("No nodes in the pipeline tracker")
return None
if self.start_node is None:
raise JobHistoryError("Start node is not set")
diff --git a/pipelex/plugins/anthropic/anthropic_factory.py b/pipelex/plugins/anthropic/anthropic_factory.py
index bad120729..f7df8abc9 100644
--- a/pipelex/plugins/anthropic/anthropic_factory.py
+++ b/pipelex/plugins/anthropic/anthropic_factory.py
@@ -15,10 +15,10 @@ from pipelex import log
from pipelex.cogt.exceptions import CogtError
from pipelex.cogt.image.prompt_image import (
PromptImage,
- PromptImageBytes,
+ PromptImageBase64,
PromptImagePath,
- PromptImageTypedBytes,
- PromptImageTypedBytesOrUrl,
+ PromptImageTypedBase64,
+ PromptImageTypedUrlOrBase64,
PromptImageUrl,
)
from pipelex.cogt.image.prompt_image_factory import PromptImageFactory
@@ -87,7 +87,7 @@ class AnthropicFactory:
# images_block_params: List[ImageBlockParam] = []
for prepped_image in prepped_user_images:
image_block_param: ImageBlockParam
- if isinstance(prepped_image, PromptImageTypedBytes):
+ if isinstance(prepped_image, PromptImageTypedBase64):
mime = prepped_image.file_type.mime
image_block_param = {
"type": "image",
@@ -121,7 +121,7 @@ class AnthropicFactory:
@staticmethod
def openai_typed_user_message(
user_content_txt: str,
- prepped_user_images: Optional[List[PromptImageTypedBytesOrUrl]] = None,
+ prepped_user_images: Optional[List[PromptImageTypedUrlOrBase64]] = None,
) -> ChatCompletionMessageParam:
text_block_param: TextBlockParam = {"type": "text", "text": user_content_txt}
message: MessageParam
@@ -130,7 +130,7 @@ class AnthropicFactory:
images_block_params: List[ImageBlockParam] = []
for prepped_image in prepped_user_images:
image_block_param_in_loop: ImageBlockParam
- if isinstance(prepped_image, PromptImageTypedBytes):
+ if isinstance(prepped_image, PromptImageTypedBase64):
mime = prepped_image.file_type.mime
image_block_param_in_loop = {
"type": "image",
@@ -171,17 +171,17 @@ class AnthropicFactory:
async def _prep_image_for_anthropic(
cls,
prompt_image: PromptImage,
- ) -> PromptImageTypedBytesOrUrl:
- typed_bytes_or_url: PromptImageTypedBytesOrUrl
- if isinstance(prompt_image, PromptImageBytes):
- typed_bytes_or_url = prompt_image.make_prompt_image_typed_bytes()
+ ) -> PromptImageTypedUrlOrBase64:
+ typed_bytes_or_url: PromptImageTypedUrlOrBase64
+ if isinstance(prompt_image, PromptImageBase64):
+ typed_bytes_or_url = prompt_image.make_prompt_image_typed_base64()
elif isinstance(prompt_image, PromptImageUrl):
- image_bytes = await PromptImageFactory().make_promptimagebytes_from_url_async(prompt_image)
+ image_bytes = await PromptImageFactory.make_promptimagebase64_from_url_async(prompt_image)
file_type = detect_file_type_from_base64(image_bytes.base_64)
- typed_bytes_or_url = PromptImageTypedBytes(base_64=image_bytes.base_64, file_type=file_type)
+ typed_bytes_or_url = PromptImageTypedBase64(base_64=image_bytes.base_64, file_type=file_type)
elif isinstance(prompt_image, PromptImagePath):
b64 = await load_binary_as_base64_async(prompt_image.file_path)
- typed_bytes_or_url = PromptImageTypedBytes(base_64=b64, file_type=prompt_image.get_file_type())
+ typed_bytes_or_url = PromptImageTypedBase64(base_64=b64, file_type=prompt_image.get_file_type())
else:
raise AnthropicFactoryError(f"Unsupported PromptImage type: '{type(prompt_image).__name__}'")
return typed_bytes_or_url
@@ -200,7 +200,7 @@ class AnthropicFactory:
if system_content := llm_prompt.system_text:
messages.append(ChatCompletionSystemMessageParam(role="system", content=system_content))
- prepped_user_images: Optional[List[PromptImageTypedBytesOrUrl]]
+ prepped_user_images: Optional[List[PromptImageTypedUrlOrBase64]]
if llm_prompt.user_images:
tasks_to_prep_images = [cls._prep_image_for_anthropic(prompt_image) for prompt_image in llm_prompt.user_images]
prepped_user_images = await asyncio.gather(*tasks_to_prep_images)
diff --git a/pipelex/plugins/anthropic/anthropic_llm_worker.py b/pipelex/plugins/anthropic/anthropic_llm_worker.py
index 5db6beccf..1f7c1037a 100644
--- a/pipelex/plugins/anthropic/anthropic_llm_worker.py
+++ b/pipelex/plugins/anthropic/anthropic_llm_worker.py
@@ -21,6 +21,24 @@ class AnthropicExtraField(StrEnum):
CLAUDE_4_TOKENS_LIMIT = "claude_4_tokens_limit"
+class AnthropicLLMWorkerError(Exception):
+ """Base exception for Anthropic LLM Worker errors."""
+
+ pass
+
+
+class AnthropicBadRequestError(AnthropicLLMWorkerError):
+ """Raised when Anthropic API returns a BadRequestError."""
+
+ pass
+
+
+class AnthropicInstructorError(AnthropicLLMWorkerError):
+ """Raised when Instructor encounters an error with Anthropic."""
+
+ pass
+
+
class AnthropicLLMWorker(LLMWorkerInternalAbstract):
def __init__(
self,
diff --git a/pipelex/plugins/bedrock/bedrock_factory.py b/pipelex/plugins/bedrock/bedrock_factory.py
index 3bd56c354..f12bb584c 100644
--- a/pipelex/plugins/bedrock/bedrock_factory.py
+++ b/pipelex/plugins/bedrock/bedrock_factory.py
@@ -1,10 +1,9 @@
from pipelex import log
-from pipelex.cogt.exceptions import CogtError, LLMCapabilityError, PromptImageFormatError
-from pipelex.cogt.image.prompt_image import PromptImageBytes
+from pipelex.cogt.exceptions import CogtError, LLMCapabilityError
from pipelex.cogt.llm.llm_job import LLMJob
from pipelex.cogt.model_backends.backend import InferenceBackend
from pipelex.plugins.bedrock.bedrock_client_protocol import BedrockClientProtocol
-from pipelex.plugins.bedrock.bedrock_message import BedrockContentItem, BedrockImage, BedrockMessage, BedrockSource, ImageFormat
+from pipelex.plugins.bedrock.bedrock_message import BedrockContentItem, BedrockMessage
from pipelex.plugins.plugin_sdk_registry import Plugin
from pipelex.types import StrEnum
@@ -68,17 +67,7 @@ class BedrockFactory:
message = BedrockMessage(role="user", content=[])
if user_text := llm_job.llm_prompt.user_text:
message.content.append(BedrockContentItem(text=user_text))
- if user_images := llm_job.llm_prompt.user_images:
+ if llm_job.llm_prompt.user_images:
raise LLMCapabilityError("BedrockFactory does not support images. Skipping images.")
- for user_image in user_images:
- if isinstance(user_image, PromptImageBytes):
- image_bytes = user_image.image_bytes
- image = BedrockImage(
- format=ImageFormat.JPEG,
- source=BedrockSource(bytes=image_bytes),
- )
- message.content.append(BedrockContentItem(image=image))
- else:
- raise PromptImageFormatError("Only PromptImageBytes is supported for BedrockFactory.")
return message
diff --git a/pipelex/plugins/fal/fal_config.py b/pipelex/plugins/fal/fal_config.py
index 93e4cb9e9..ea3d9e252 100644
--- a/pipelex/plugins/fal/fal_config.py
+++ b/pipelex/plugins/fal/fal_config.py
@@ -2,7 +2,7 @@ from typing import Dict
from pydantic import field_validator
-from pipelex.cogt.imgg.imgg_job_components import Quality
+from pipelex.cogt.img_gen.img_gen_job_components import Quality
from pipelex.tools.config.config_model import ConfigModel
from pipelex.tools.exceptions import ConfigValidationError
diff --git a/pipelex/plugins/fal/fal_factory.py b/pipelex/plugins/fal/fal_factory.py
index 4d4780466..f94705e92 100644
--- a/pipelex/plugins/fal/fal_factory.py
+++ b/pipelex/plugins/fal/fal_factory.py
@@ -1,29 +1,23 @@
from typing import Any, Dict, List, Optional
from pipelex import log
-from pipelex.cogt.exceptions import ImggGeneratedTypeError, ImggParameterError
+from pipelex.cogt.exceptions import ImgGenGeneratedTypeError, ImgGenParameterError
from pipelex.cogt.image.generated_image import GeneratedImage
-from pipelex.cogt.imgg.imgg_engine import ImggEngine
-from pipelex.cogt.imgg.imgg_handle import ImggHandle
-from pipelex.cogt.imgg.imgg_job import ImggJob
-from pipelex.cogt.imgg.imgg_job_components import AspectRatio, OutputFormat, Quality
+from pipelex.cogt.img_gen.img_gen_job import ImgGenJob
+from pipelex.cogt.img_gen.img_gen_job_components import AspectRatio, OutputFormat, Quality
from pipelex.config import get_config
class FalFactory:
- @staticmethod
- def make_fal_application(imgg_engine: ImggEngine) -> str:
- return f"{imgg_engine.imgg_platform}/{imgg_engine.imgg_model_name}"
-
@classmethod
def make_nb_steps_from_quality_for_flux_pro(cls, quality: Quality) -> int:
- flux_map_quality_to_steps = get_config().cogt.imgg_config.fal_config.flux_map_quality_to_steps
- return flux_map_quality_to_steps[quality.value]
+ flux_map_quality_to_steps = get_config().cogt.img_gen_config.fal_config.flux_map_quality_to_steps
+ return flux_map_quality_to_steps[quality]
@classmethod
def make_nb_steps_from_quality_for_sdxl_lightning(cls, quality: Quality) -> int:
- sdxl_lightning_map_quality_to_steps = get_config().cogt.imgg_config.fal_config.sdxl_lightning_map_quality_to_steps
- return sdxl_lightning_map_quality_to_steps[quality.value]
+ sdxl_lightning_map_quality_to_steps = get_config().cogt.img_gen_config.fal_config.sdxl_lightning_map_quality_to_steps
+ return sdxl_lightning_map_quality_to_steps[quality]
@classmethod
def make_image_size_for_flux_1(cls, aspect_ratio: AspectRatio) -> str:
@@ -43,7 +37,7 @@ class FalFactory:
case AspectRatio.PORTRAIT_9_21:
return "portrait_21_9"
case AspectRatio.LANDSCAPE_3_2 | AspectRatio.PORTRAIT_2_3:
- raise ImggParameterError(f"Aspect ratio '{aspect_ratio}' is not supported by Flux-1 image generation model")
+ raise ImgGenParameterError(f"Aspect ratio '{aspect_ratio}' is not supported by Flux-1 image generation model")
@classmethod
def make_aspect_ratio_for_flux_1_1_ultra(cls, aspect_ratio: AspectRatio) -> str:
@@ -63,7 +57,7 @@ class FalFactory:
case AspectRatio.PORTRAIT_9_21:
return "9:21"
case AspectRatio.LANDSCAPE_3_2 | AspectRatio.PORTRAIT_2_3:
- raise ImggParameterError(f"Aspect ratio '{aspect_ratio}' is not supported by Flux-1.1 Ultra image generation model")
+ raise ImgGenParameterError(f"Aspect ratio '{aspect_ratio}' is not supported by Flux-1.1 Ultra image generation model")
@classmethod
def make_output_format_for_flux(cls, output_format: OutputFormat) -> str:
@@ -73,26 +67,28 @@ class FalFactory:
case OutputFormat.JPG:
return "jpeg"
case OutputFormat.WEBP:
- raise ImggParameterError("Output format WebP is not supported for Flux")
+ raise ImgGenParameterError("Output format WebP is not supported for Flux")
@classmethod
def make_fal_arguments(
cls,
fal_application: str,
- imgg_job: ImggJob,
+ img_gen_job: ImgGenJob,
nb_images: int,
) -> Dict[str, Any]:
- params = imgg_job.job_params
+ params = img_gen_job.job_params
args_dict: Dict[str, Any]
num_inference_steps: Optional[int]
match fal_application:
- case ImggHandle.FLUX_1_PRO_LEGACY | ImggHandle.FLUX_1_1_PRO:
+ case "fal-ai/flux-pro" | "fal-ai/flux-pro/v1.1":
num_inference_steps = params.nb_steps
- if not num_inference_steps and (quality := params.quality):
- num_inference_steps = cls.make_nb_steps_from_quality_for_flux_pro(quality=quality)
+ if not num_inference_steps:
+ if not params.quality:
+ raise ImgGenParameterError(f"Either nb_steps or quality must be set for image generation with '{fal_application}'")
+ num_inference_steps = cls.make_nb_steps_from_quality_for_flux_pro(quality=params.quality)
args_dict = {
- "prompt": imgg_job.imgg_prompt.positive_text,
+ "prompt": img_gen_job.img_gen_prompt.positive_text,
"image_size": cls.make_image_size_for_flux_1(params.aspect_ratio),
"num_inference_steps": num_inference_steps,
"guidance_scale": params.guidance_scale,
@@ -101,11 +97,11 @@ class FalFactory:
"safety_tolerance": params.safety_tolerance,
"seed": params.seed,
"output_format": cls.make_output_format_for_flux(params.output_format),
- "sync_mode": imgg_job.job_config.is_sync_mode,
+ "sync_mode": img_gen_job.job_config.is_sync_mode,
}
- case ImggHandle.FLUX_1_1_ULTRA:
+ case "fal-ai/flux-pro/v1.1-ultra":
args_dict = {
- "prompt": imgg_job.imgg_prompt.positive_text,
+ "prompt": img_gen_job.img_gen_prompt.positive_text,
"aspect_ratio": cls.make_aspect_ratio_for_flux_1_1_ultra(params.aspect_ratio),
"num_images": nb_images,
"enable_safety_checker": params.is_moderated,
@@ -113,9 +109,9 @@ class FalFactory:
"raw": params.is_raw,
"seed": params.seed,
"output_format": cls.make_output_format_for_flux(params.output_format),
- "sync_mode": imgg_job.job_config.is_sync_mode,
+ "sync_mode": img_gen_job.job_config.is_sync_mode,
}
- case ImggHandle.SDXL_LIGHTNING:
+ case "fal-ai/fast-lightning-sdxl":
num_inference_steps = params.nb_steps
if not num_inference_steps and (quality := params.quality):
num_inference_steps = cls.make_nb_steps_from_quality_for_sdxl_lightning(quality=quality)
@@ -124,16 +120,16 @@ class FalFactory:
log.warning(f"Number of inference steps {num_inference_steps}' for SDXL Lightning must be one of {acceptable_steps}")
num_inference_steps = 8
args_dict = {
- "prompt": imgg_job.imgg_prompt.positive_text,
+ "prompt": img_gen_job.img_gen_prompt.positive_text,
"image_size": cls.make_image_size_for_flux_1(params.aspect_ratio),
"num_inference_steps": num_inference_steps,
"num_images": nb_images,
"seed": params.seed,
"output_format": cls.make_output_format_for_flux(params.output_format),
- "sync_mode": imgg_job.job_config.is_sync_mode,
+ "sync_mode": img_gen_job.job_config.is_sync_mode,
}
case _:
- raise ImggParameterError(f"Invalid fal application: '{fal_application}'")
+ raise ImgGenParameterError(f"Invalid fal application: '{fal_application}'")
return args_dict
@@ -143,15 +139,15 @@ class FalFactory:
fal_image_dict = images[0]
image_url = fal_image_dict["url"]
if not isinstance(image_url, str):
- raise ImggGeneratedTypeError("Image url is not a string")
+ raise ImgGenGeneratedTypeError("Image url is not a string")
# TODO: if the url is actual image data, send it to cloud storage?
width = fal_image_dict["width"]
if not isinstance(width, int):
- raise ImggGeneratedTypeError("Image width is not an integer")
+ raise ImgGenGeneratedTypeError("Image width is not an integer")
height = fal_image_dict["height"]
if not isinstance(height, int):
- raise ImggGeneratedTypeError("Image height is not an integer")
+ raise ImgGenGeneratedTypeError("Image height is not an integer")
generated_image = GeneratedImage(
url=image_url,
@@ -168,14 +164,14 @@ class FalFactory:
for fal_image_dict in fal_image_dicts:
image_url = fal_image_dict["url"]
if not isinstance(image_url, str):
- raise ImggGeneratedTypeError("Image url is not a string")
+ raise ImgGenGeneratedTypeError("Image url is not a string")
width = fal_image_dict["width"]
if not isinstance(width, int):
- raise ImggGeneratedTypeError("Image width is not an integer")
+ raise ImgGenGeneratedTypeError("Image width is not an integer")
height = fal_image_dict["height"]
if not isinstance(height, int):
- raise ImggGeneratedTypeError("Image height is not an integer")
+ raise ImgGenGeneratedTypeError("Image height is not an integer")
generated_image = GeneratedImage(
url=image_url,
diff --git a/pipelex/plugins/fal/fal_imgg_worker.py b/pipelex/plugins/fal/fal_img_gen_worker.py
similarity index 78%
rename from pipelex/plugins/fal/fal_imgg_worker.py
rename to pipelex/plugins/fal/fal_img_gen_worker.py
index f3884a5ee..3e9df2102 100644
--- a/pipelex/plugins/fal/fal_imgg_worker.py
+++ b/pipelex/plugins/fal/fal_img_gen_worker.py
@@ -6,21 +6,21 @@ from typing_extensions import override
from pipelex import log
from pipelex.cogt.exceptions import SdkTypeError
from pipelex.cogt.image.generated_image import GeneratedImage
-from pipelex.cogt.imgg.imgg_engine import ImggEngine
-from pipelex.cogt.imgg.imgg_job import ImggJob
-from pipelex.cogt.imgg.imgg_worker_abstract import ImggWorkerAbstract
+from pipelex.cogt.img_gen.img_gen_job import ImgGenJob
+from pipelex.cogt.img_gen.img_gen_worker_abstract import ImgGenWorkerAbstract
+from pipelex.cogt.model_backends.model_spec import InferenceModelSpec
from pipelex.plugins.fal.fal_factory import FalFactory
from pipelex.reporting.reporting_protocol import ReportingProtocol
-class FalImggWorker(ImggWorkerAbstract):
+class FalImgGenWorker(ImgGenWorkerAbstract):
def __init__(
self,
sdk_instance: Any,
- imgg_engine: ImggEngine,
+ inference_model: InferenceModelSpec,
reporting_delegate: Optional[ReportingProtocol] = None,
):
- super().__init__(imgg_engine=imgg_engine, reporting_delegate=reporting_delegate)
+ super().__init__(inference_model=inference_model, reporting_delegate=reporting_delegate)
if not isinstance(sdk_instance, AsyncClient):
raise SdkTypeError(f"Provided Imgg sdk_instance is not of type fal_client.AsyncClient: it's a '{type(sdk_instance)}'")
@@ -30,17 +30,17 @@ class FalImggWorker(ImggWorkerAbstract):
@override
async def _gen_image(
self,
- imgg_job: ImggJob,
+ img_gen_job: ImgGenJob,
) -> GeneratedImage:
- application = FalFactory.make_fal_application(imgg_engine=self.imgg_engine)
+ fal_application = self.inference_model.model_id
arguments = FalFactory.make_fal_arguments(
- fal_application=application,
- imgg_job=imgg_job,
+ fal_application=fal_application,
+ img_gen_job=img_gen_job,
nb_images=1,
)
- log.verbose(arguments, title=f"Fal arguments, application={application}")
+ log.verbose(arguments, title=f"Fal arguments, application={fal_application}")
handler = await self.fal_async_client.submit(
- application=application,
+ application=fal_application,
arguments=arguments,
)
@@ -62,13 +62,13 @@ class FalImggWorker(ImggWorkerAbstract):
@override
async def _gen_image_list(
self,
- imgg_job: ImggJob,
+ img_gen_job: ImgGenJob,
nb_images: int,
) -> List[GeneratedImage]:
- application = FalFactory.make_fal_application(imgg_engine=self.imgg_engine)
+ application = self.inference_model.model_id
arguments = FalFactory.make_fal_arguments(
fal_application=application,
- imgg_job=imgg_job,
+ img_gen_job=img_gen_job,
nb_images=nb_images,
)
handler = await self.fal_async_client.submit(
diff --git a/pipelex/plugins/google/__init__.py b/pipelex/plugins/google/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/pipelex/plugins/google/google_factory.py b/pipelex/plugins/google/google_factory.py
new file mode 100644
index 000000000..06a77b7fe
--- /dev/null
+++ b/pipelex/plugins/google/google_factory.py
@@ -0,0 +1,93 @@
+import asyncio
+from typing import List, Optional
+
+from google import genai
+from google.genai import types as genai_types
+
+from pipelex.cogt.exceptions import CogtError
+from pipelex.cogt.image.prompt_image import (
+ PromptImage,
+ PromptImageBase64,
+ PromptImagePath,
+ PromptImageUrl,
+)
+from pipelex.cogt.image.prompt_image_factory import PromptImageFactory
+from pipelex.cogt.llm.llm_prompt import LLMPrompt
+from pipelex.cogt.model_backends.backend import InferenceBackend
+from pipelex.cogt.usage.token_category import NbTokensByCategoryDict, TokenCategory
+from pipelex.tools.misc.base_64_utils import load_binary_async
+
+
+class GoogleFactoryError(CogtError):
+ pass
+
+
+class GoogleFactory:
+ @staticmethod
+ def make_google_client(backend: InferenceBackend) -> genai.Client:
+ """Create a Google Gemini API client."""
+ return genai.Client(api_key=backend.api_key)
+
+ @classmethod
+ async def prepare_image_part(cls, prompt_image: PromptImage) -> genai_types.Part:
+ """Convert a PromptImage to Google genai Part format."""
+ image_bytes: bytes
+ mime_type: str
+
+ if isinstance(prompt_image, PromptImageBase64):
+ image_bytes = prompt_image.get_decoded_bytes()
+ mime_type = prompt_image.get_mime_type()
+ return genai_types.Part.from_bytes(data=image_bytes, mime_type=mime_type)
+ elif isinstance(prompt_image, PromptImagePath):
+ image_bytes = await load_binary_async(prompt_image.file_path)
+ mime_type = prompt_image.get_mime_type()
+ return genai_types.Part.from_bytes(data=image_bytes, mime_type=mime_type)
+ elif isinstance(prompt_image, PromptImageUrl):
+ prompt_image_binary = await PromptImageFactory.make_promptimagebinary_from_url_async(prompt_image)
+ image_bytes = prompt_image_binary.binary
+ mime_type = prompt_image_binary.get_mime_type()
+ return genai_types.Part.from_bytes(data=image_bytes, mime_type=mime_type)
+ else:
+ raise GoogleFactoryError(f"Unsupported PromptImage type: '{type(prompt_image).__name__}'")
+
+ @classmethod
+ async def prepare_user_contents(cls, llm_prompt: LLMPrompt) -> genai_types.ContentListUnion:
+ """Prepare contents for Google genai API."""
+
+ # Build list of parts for multimodal content
+ parts: List[genai_types.Part] = []
+
+ # Add text content if present
+ if llm_prompt.user_text:
+ parts.append(genai_types.Part.from_text(text=llm_prompt.user_text))
+
+ # Add image parts if present
+ if llm_prompt.user_images:
+ # Prepare all images in parallel
+ image_tasks = [cls.prepare_image_part(image) for image in llm_prompt.user_images]
+ image_parts = await asyncio.gather(*image_tasks)
+ parts.extend(image_parts)
+
+ return genai_types.Content(parts=parts, role="user")
+
+ @classmethod
+ def extract_token_usage(cls, usage_metadata: Optional[genai_types.GenerateContentResponseUsageMetadata]) -> NbTokensByCategoryDict:
+ """Extract token usage from Google's usage metadata."""
+ if not usage_metadata:
+ return {}
+
+ nb_tokens_by_category: NbTokensByCategoryDict = {}
+
+ # Add input tokens
+ if usage_metadata.prompt_token_count:
+ nb_tokens_by_category[TokenCategory.INPUT] = usage_metadata.prompt_token_count
+
+ # Add output tokens
+ if usage_metadata.candidates_token_count:
+ nb_tokens_by_category[TokenCategory.OUTPUT] = usage_metadata.candidates_token_count
+
+ # Add cached tokens if available
+ if usage_metadata.cached_content_token_count:
+ nb_tokens_by_category[TokenCategory.INPUT_CACHED] = usage_metadata.cached_content_token_count
+
+ return nb_tokens_by_category
diff --git a/pipelex/plugins/google/google_llm_worker.py b/pipelex/plugins/google/google_llm_worker.py
new file mode 100644
index 000000000..9b03e9d6e
--- /dev/null
+++ b/pipelex/plugins/google/google_llm_worker.py
@@ -0,0 +1,138 @@
+from typing import Optional, Type, cast
+
+import instructor
+from google import genai
+from google.genai import types
+from openai.types.chat import ChatCompletionMessageParam
+from typing_extensions import override
+
+from pipelex import log
+from pipelex.cogt.exceptions import LLMCompletionError
+from pipelex.cogt.llm.llm_job import LLMJob
+from pipelex.cogt.llm.llm_worker_internal_abstract import LLMWorkerInternalAbstract
+from pipelex.cogt.llm.structured_output import StructureMethod
+from pipelex.cogt.model_backends.model_spec import InferenceModelSpec
+from pipelex.cogt.usage.token_category import NbTokensByCategoryDict, TokenCategory
+from pipelex.plugins.google.google_factory import GoogleFactory
+from pipelex.reporting.reporting_protocol import ReportingProtocol
+from pipelex.tools.typing.pydantic_utils import BaseModelTypeVar
+
+
+class GoogleLLMWorkerError(Exception):
+ """Base exception for Google LLM Worker errors."""
+
+ pass
+
+
+class GoogleLLMWorker(LLMWorkerInternalAbstract):
+ def __init__(
+ self,
+ sdk_instance: genai.Client,
+ inference_model: InferenceModelSpec,
+ structure_method: Optional[StructureMethod] = None,
+ reporting_delegate: Optional[ReportingProtocol] = None,
+ ):
+ super().__init__(
+ inference_model=inference_model,
+ structure_method=structure_method,
+ reporting_delegate=reporting_delegate,
+ )
+ genai_client: genai.Client = sdk_instance
+ self.genai_async_client = genai_client.aio
+ if structure_method:
+ instructor_mode = structure_method.as_instructor_mode()
+ log.debug(f"Google structure mode: {structure_method} --> {instructor_mode}")
+ self.instructor_for_objects = instructor.from_genai(client=sdk_instance, mode=instructor_mode, use_async=True)
+ else:
+ self.instructor_for_objects = instructor.from_genai(client=sdk_instance, use_async=True)
+
+ @override
+ async def _gen_text(
+ self,
+ llm_job: LLMJob,
+ ) -> str:
+ """Generate text using Google Gemini API."""
+ # Prepare contents (text and images)
+ contents = await GoogleFactory.prepare_user_contents(llm_job.llm_prompt)
+
+ # Build generation config
+ generation_config = types.GenerateContentConfig(
+ temperature=llm_job.job_params.temperature,
+ max_output_tokens=llm_job.job_params.max_tokens,
+ candidate_count=1, # Generate one candidate
+ )
+
+ # Add system instruction if present (as part of config)
+ if llm_job.llm_prompt.system_text:
+ generation_config.system_instruction = llm_job.llm_prompt.system_text
+
+ # Generate content using async client
+ response = await self.genai_async_client.models.generate_content(
+ model=self.inference_model.model_id,
+ contents=contents,
+ config=generation_config,
+ )
+
+ # Extract text from response
+ if not response.candidates:
+ raise LLMCompletionError(f"No candidates returned from model: {self.inference_model.desc}")
+
+ candidate = response.candidates[0]
+ if not candidate.content or not candidate.content.parts:
+ raise LLMCompletionError(f"No content parts in response from model: {self.inference_model.desc}")
+
+ # Extract text from the first part
+ text_content = candidate.content.parts[0].text
+ if not text_content:
+ raise LLMCompletionError(f"No text content in response from model: {self.inference_model.desc}")
+
+ # Track token usage if available
+ if llm_job.job_report.llm_tokens_usage and response.usage_metadata:
+ llm_job.job_report.llm_tokens_usage.nb_tokens_by_category = GoogleFactory.extract_token_usage(response.usage_metadata)
+
+ return text_content
+
+ @override
+ async def _gen_object(
+ self,
+ llm_job: LLMJob,
+ schema: Type[BaseModelTypeVar],
+ ) -> BaseModelTypeVar:
+ """Generate structured output using Google Gemini API with instructor."""
+ # Prepare contents (text and images)
+ contents = await GoogleFactory.prepare_user_contents(llm_job.llm_prompt)
+
+ # Build generation config
+ generation_config = types.GenerateContentConfig(
+ system_instruction=llm_job.llm_prompt.system_text,
+ temperature=llm_job.job_params.temperature,
+ max_output_tokens=llm_job.job_params.max_tokens,
+ candidate_count=1,
+ )
+
+ result_object, completion = await self.instructor_for_objects.chat.completions.create_with_completion(
+ messages=[cast(ChatCompletionMessageParam, contents)],
+ response_model=schema,
+ max_retries=llm_job.job_config.max_retries,
+ model=self.inference_model.model_id,
+ generation_config=generation_config,
+ )
+ if not isinstance(result_object, schema):
+ raise GoogleLLMWorkerError(f"Google Gemini API returned an object that is not of type {schema}: {result_object}")
+
+ # Track token usage if available from completion
+ if llm_job.job_report.llm_tokens_usage:
+ # Instructor may provide usage information in the completion object
+ if hasattr(completion, "usage_metadata"):
+ llm_job.job_report.llm_tokens_usage.nb_tokens_by_category = GoogleFactory.extract_token_usage(completion.usage_metadata)
+ elif hasattr(completion, "usage"):
+ # Fallback to standard usage format
+ usage = completion.usage
+ nb_tokens: NbTokensByCategoryDict = {}
+ if hasattr(usage, "prompt_tokens"):
+ nb_tokens[TokenCategory.INPUT] = usage.prompt_tokens
+ if hasattr(usage, "completion_tokens"):
+ nb_tokens[TokenCategory.OUTPUT] = usage.completion_tokens
+ llm_job.job_report.llm_tokens_usage.nb_tokens_by_category = nb_tokens
+
+ return result_object
diff --git a/pipelex/plugins/mistral/mistral_factory.py b/pipelex/plugins/mistral/mistral_factory.py
index bed02e2b4..100934d72 100644
--- a/pipelex/plugins/mistral/mistral_factory.py
+++ b/pipelex/plugins/mistral/mistral_factory.py
@@ -20,13 +20,14 @@ from openai.types.chat import (
)
from pipelex.cogt.exceptions import PromptImageFormatError
-from pipelex.cogt.image.prompt_image import PromptImage, PromptImageBytes, PromptImagePath, PromptImageUrl
+from pipelex.cogt.image.prompt_image import PromptImage, PromptImageBase64, PromptImagePath, PromptImageUrl
from pipelex.cogt.llm.llm_job import LLMJob
from pipelex.cogt.model_backends.backend import InferenceBackend
from pipelex.cogt.ocr.ocr_output import ExtractedImageFromPage, OcrOutput, Page
from pipelex.cogt.usage.token_category import NbTokensByCategoryDict, TokenCategory
from pipelex.plugins.openai.openai_factory import OpenAIFactory
-from pipelex.tools.misc.base_64_utils import encode_to_base64, load_binary_as_base64
+from pipelex.tools.misc.base_64_utils import load_binary_as_base64
+from pipelex.tools.misc.filetype_utils import detect_file_type_from_base64, detect_file_type_from_path
class MistralFactory:
@@ -71,12 +72,12 @@ class MistralFactory:
return ImageURLChunk(image_url=prompt_image.url)
elif isinstance(prompt_image, PromptImagePath):
image_bytes = load_binary_as_base64(prompt_image.file_path).decode("utf-8")
- # TODO: use actual image type
- return ImageURLChunk(image_url=f"data:image/png;base64,{image_bytes}")
- elif isinstance(prompt_image, PromptImageBytes):
- image_bytes = encode_to_base64(prompt_image.base_64).decode("utf-8")
- # TODO: use actual image type
- return ImageURLChunk(image_url=f"data:image/png;base64,{image_bytes}")
+ file_type = detect_file_type_from_path(prompt_image.file_path)
+ return ImageURLChunk(image_url=f"data:{file_type.mime};base64,{image_bytes}")
+ elif isinstance(prompt_image, PromptImageBase64):
+ image_bytes = prompt_image.base_64.decode("utf-8")
+ file_type = detect_file_type_from_base64(prompt_image.base_64)
+ return ImageURLChunk(image_url=f"data:{file_type.mime};base64,{image_bytes}")
else:
raise PromptImageFormatError(f"prompt_image of type {type(prompt_image)} is not supported")
diff --git a/pipelex/plugins/mistral/mistral_ocr_worker.py b/pipelex/plugins/mistral/mistral_ocr_worker.py
index 07f8389c8..c63004b01 100644
--- a/pipelex/plugins/mistral/mistral_ocr_worker.py
+++ b/pipelex/plugins/mistral/mistral_ocr_worker.py
@@ -44,13 +44,13 @@ class MistralOcrWorker(OcrWorkerAbstract):
) -> OcrOutput:
# TODO: report usage
if image_uri := ocr_job.ocr_input.image_uri:
- ocr_output = await self.make_ocr_output_from_image(
+ ocr_output = await self._make_ocr_output_from_image(
image_uri=image_uri,
should_caption_image=ocr_job.job_params.should_caption_images,
)
elif pdf_uri := ocr_job.ocr_input.pdf_uri:
- ocr_output = await self.make_ocr_output_from_pdf(
+ ocr_output = await self._make_ocr_output_from_pdf(
pdf_uri=pdf_uri,
should_include_images=ocr_job.job_params.should_include_images,
should_caption_images=ocr_job.job_params.should_caption_images,
@@ -60,7 +60,7 @@ class MistralOcrWorker(OcrWorkerAbstract):
raise OcrInputError("No image nor PDF URI provided in OcrJob")
return ocr_output
- async def make_ocr_output_from_image(
+ async def _make_ocr_output_from_image(
self,
image_uri: str,
should_caption_image: bool = False,
@@ -78,7 +78,7 @@ class MistralOcrWorker(OcrWorkerAbstract):
image_path=image_path,
)
- async def make_ocr_output_from_pdf(
+ async def _make_ocr_output_from_pdf(
self,
pdf_uri: str,
should_include_images: bool,
diff --git a/pipelex/plugins/openai/openai_factory.py b/pipelex/plugins/openai/openai_factory.py
index e46a6b9d9..9bf8427a2 100644
--- a/pipelex/plugins/openai/openai_factory.py
+++ b/pipelex/plugins/openai/openai_factory.py
@@ -14,10 +14,9 @@ from openai.types.completion_usage import CompletionUsage
from pipelex import log
from pipelex.cogt.exceptions import CogtError, LLMPromptParameterError
-from pipelex.cogt.image.prompt_image import PromptImage, PromptImageBytes, PromptImagePath, PromptImageUrl
+from pipelex.cogt.image.prompt_image import PromptImage, PromptImageBase64, PromptImagePath, PromptImageUrl
from pipelex.cogt.llm.llm_job import LLMJob
from pipelex.cogt.model_backends.backend import InferenceBackend
-from pipelex.cogt.model_backends.model_spec import InferenceModelSpec
from pipelex.cogt.usage.token_category import NbTokensByCategoryDict, TokenCategory
from pipelex.plugins.plugin_sdk_registry import Plugin
from pipelex.tools.misc.base_64_utils import load_binary_as_base64
@@ -74,7 +73,6 @@ class OpenAIFactory:
def make_simple_messages(
cls,
llm_job: LLMJob,
- inference_model: InferenceModelSpec,
) -> List[ChatCompletionMessageParam]:
"""
Makes a list of messages with a system message (if provided) and followed by a user message.
@@ -82,7 +80,7 @@ class OpenAIFactory:
llm_prompt = llm_job.llm_prompt
messages: List[ChatCompletionMessageParam] = []
user_contents: List[ChatCompletionContentPartParam] = []
- if inference_model.is_system_prompt_supported and (system_content := llm_prompt.system_text):
+ if system_content := llm_prompt.system_text:
messages.append(ChatCompletionSystemMessageParam(role="system", content=system_content))
# TODO: confirm that we can prompt without user_contents, for instance if we have only images,
# otherwise consider using a default user_content
@@ -103,13 +101,13 @@ class OpenAIFactory:
if isinstance(prompt_image, PromptImageUrl):
url = prompt_image.url
openai_image_url = ImageURL(url=url, detail="high")
- elif isinstance(prompt_image, PromptImageBytes):
+ elif isinstance(prompt_image, PromptImageBase64):
# TODO: manage image type
url_with_bytes: str = f"data:image/jpeg;base64,{prompt_image.base_64.decode('utf-8')}"
openai_image_url = ImageURL(url=url_with_bytes, detail="high")
elif isinstance(prompt_image, PromptImagePath):
image_bytes = load_binary_as_base64(path=prompt_image.file_path)
- return cls.make_openai_image_url(PromptImageBytes(base_64=image_bytes))
+ return cls.make_openai_image_url(PromptImageBase64(base_64=image_bytes))
else:
raise LLMPromptParameterError(f"prompt_image of type {type(prompt_image)} is not supported")
return openai_image_url
diff --git a/pipelex/plugins/openai/openai_imgg_factory.py b/pipelex/plugins/openai/openai_img_gen_factory.py
similarity index 87%
rename from pipelex/plugins/openai/openai_imgg_factory.py
rename to pipelex/plugins/openai/openai_img_gen_factory.py
index 0a1c2cc1d..4d99ca48d 100644
--- a/pipelex/plugins/openai/openai_imgg_factory.py
+++ b/pipelex/plugins/openai/openai_img_gen_factory.py
@@ -1,7 +1,7 @@
from typing import Literal
-from pipelex.cogt.exceptions import ImggParameterError
-from pipelex.cogt.imgg.imgg_job_components import AspectRatio, Background, OutputFormat, Quality
+from pipelex.cogt.exceptions import ImgGenParameterError
+from pipelex.cogt.img_gen.img_gen_job_components import AspectRatio, Background, OutputFormat, Quality
GptImage1SizeType = Literal["1024x1024", "1536x1024", "1024x1536"]
GptImage1OutputFormatType = Literal["png", "jpeg", "webp"]
@@ -10,7 +10,7 @@ GptImage1QualityType = Literal["low", "medium", "high"]
GptImage1BackgroundType = Literal["transparent", "opaque", "auto"]
-class OpenAIImggFactory:
+class OpenAIImgGenFactory:
@classmethod
def image_size_for_gpt_image_1(cls, aspect_ratio: AspectRatio) -> GptImage1SizeType:
match aspect_ratio:
@@ -28,7 +28,7 @@ class OpenAIImggFactory:
| AspectRatio.PORTRAIT_9_16
| AspectRatio.PORTRAIT_9_21
):
- raise ImggParameterError(f"Aspect ratio '{aspect_ratio}' is not supported by GPT Image 1 model")
+ raise ImgGenParameterError(f"Aspect ratio '{aspect_ratio}' is not supported by GPT Image 1 model")
@classmethod
def output_format_for_gpt_image_1(cls, output_format: OutputFormat) -> GptImage1OutputFormatType:
diff --git a/pipelex/plugins/openai/openai_imgg_worker.py b/pipelex/plugins/openai/openai_img_gen_worker.py
similarity index 54%
rename from pipelex/plugins/openai/openai_imgg_worker.py
rename to pipelex/plugins/openai/openai_img_gen_worker.py
index 83553eba8..4b8720153 100644
--- a/pipelex/plugins/openai/openai_imgg_worker.py
+++ b/pipelex/plugins/openai/openai_img_gen_worker.py
@@ -5,28 +5,28 @@ import shortuuid
from typing_extensions import override
from pipelex import log
-from pipelex.cogt.exceptions import ImggGenerationError, SdkTypeError
+from pipelex.cogt.exceptions import ImgGenGenerationError, SdkTypeError
from pipelex.cogt.image.generated_image import GeneratedImage
-from pipelex.cogt.imgg.imgg_engine import ImggEngine
-from pipelex.cogt.imgg.imgg_job import ImggJob
-from pipelex.cogt.imgg.imgg_job_components import Quality
-from pipelex.cogt.imgg.imgg_worker_abstract import ImggWorkerAbstract
-from pipelex.plugins.openai.openai_imgg_factory import OpenAIImggFactory
+from pipelex.cogt.img_gen.img_gen_job import ImgGenJob
+from pipelex.cogt.img_gen.img_gen_job_components import Quality
+from pipelex.cogt.img_gen.img_gen_worker_abstract import ImgGenWorkerAbstract
+from pipelex.cogt.model_backends.model_spec import InferenceModelSpec
+from pipelex.plugins.openai.openai_img_gen_factory import OpenAIImgGenFactory
from pipelex.reporting.reporting_protocol import ReportingProtocol
from pipelex.tools.misc.base_64_utils import save_base64_to_binary_file
from pipelex.tools.misc.file_utils import ensure_path
-TEMP_OUTPUTS_DIR = "temp/imgg_by_gpt_image"
+TEMP_OUTPUTS_DIR = "temp/img_gen_by_gpt_image"
-class OpenAIImggWorker(ImggWorkerAbstract):
+class OpenAIImgGenWorker(ImgGenWorkerAbstract):
def __init__(
self,
sdk_instance: Any,
- imgg_engine: ImggEngine,
+ inference_model: InferenceModelSpec,
reporting_delegate: Optional[ReportingProtocol] = None,
):
- super().__init__(imgg_engine=imgg_engine, reporting_delegate=reporting_delegate)
+ super().__init__(inference_model=inference_model, reporting_delegate=reporting_delegate)
if not isinstance(sdk_instance, openai.AsyncOpenAI):
raise SdkTypeError(f"Provided Imgg sdk_instance is not of type openai.AsyncOpenAI: it's a '{type(sdk_instance)}'")
@@ -36,27 +36,27 @@ class OpenAIImggWorker(ImggWorkerAbstract):
@override
async def _gen_image(
self,
- imgg_job: ImggJob,
+ img_gen_job: ImgGenJob,
) -> GeneratedImage:
- one_image_list = await self.gen_image_list(imgg_job=imgg_job, nb_images=1)
+ one_image_list = await self.gen_image_list(img_gen_job=img_gen_job, nb_images=1)
generated_image = one_image_list[0]
return generated_image
@override
async def _gen_image_list(
self,
- imgg_job: ImggJob,
+ img_gen_job: ImgGenJob,
nb_images: int,
) -> List[GeneratedImage]:
- image_size = OpenAIImggFactory.image_size_for_gpt_image_1(aspect_ratio=imgg_job.job_params.aspect_ratio)
- output_format = OpenAIImggFactory.output_format_for_gpt_image_1(output_format=imgg_job.job_params.output_format)
- moderation = OpenAIImggFactory.moderation_for_gpt_image_1(is_moderated=imgg_job.job_params.is_moderated)
- background = OpenAIImggFactory.background_for_gpt_image_1(background=imgg_job.job_params.background)
- quality = OpenAIImggFactory.quality_for_gpt_image_1(quality=imgg_job.job_params.quality or Quality.LOW)
- output_compression = OpenAIImggFactory.output_compression_for_gpt_image_1()
+ image_size = OpenAIImgGenFactory.image_size_for_gpt_image_1(aspect_ratio=img_gen_job.job_params.aspect_ratio)
+ output_format = OpenAIImgGenFactory.output_format_for_gpt_image_1(output_format=img_gen_job.job_params.output_format)
+ moderation = OpenAIImgGenFactory.moderation_for_gpt_image_1(is_moderated=img_gen_job.job_params.is_moderated)
+ background = OpenAIImgGenFactory.background_for_gpt_image_1(background=img_gen_job.job_params.background)
+ quality = OpenAIImgGenFactory.quality_for_gpt_image_1(quality=img_gen_job.job_params.quality or Quality.LOW)
+ output_compression = OpenAIImgGenFactory.output_compression_for_gpt_image_1()
result = await self.openai_client.images.generate(
- prompt=imgg_job.imgg_prompt.positive_text,
- model=self.imgg_engine.imgg_model_name,
+ prompt=img_gen_job.img_gen_prompt.positive_text,
+ model=self.inference_model.model_id,
moderation=moderation,
background=background,
quality=quality,
@@ -66,14 +66,14 @@ class OpenAIImggWorker(ImggWorkerAbstract):
n=nb_images,
)
if not result.data:
- raise ImggGenerationError("No result from OpenAI")
+ raise ImgGenGenerationError("No result from OpenAI")
generated_image_list: List[GeneratedImage] = []
image_id = shortuuid.uuid()[:4]
for image_index, image_data in enumerate(result.data):
image_base64 = image_data.b64_json
if not image_base64:
- raise ImggGenerationError("No base64 image data received from OpenAI")
+ raise ImgGenGenerationError("No base64 image data received from OpenAI")
folder_path = TEMP_OUTPUTS_DIR
ensure_path(folder_path)
diff --git a/pipelex/plugins/openai/openai_llm_worker.py b/pipelex/plugins/openai/openai_llm_worker.py
index b5c021c71..6775da12f 100644
--- a/pipelex/plugins/openai/openai_llm_worker.py
+++ b/pipelex/plugins/openai/openai_llm_worker.py
@@ -2,6 +2,7 @@ from typing import Any, Optional, Type
import instructor
import openai
+from instructor.exceptions import InstructorRetryException
from openai import NOT_GIVEN, APIConnectionError, BadRequestError, NotFoundError
from openai.types.chat import ChatCompletionMessage
from typing_extensions import override
@@ -53,10 +54,7 @@ class OpenAILLMWorker(LLMWorkerInternalAbstract):
self,
llm_job: LLMJob,
) -> str:
- messages = OpenAIFactory.make_simple_messages(
- llm_job=llm_job,
- inference_model=self.inference_model,
- )
+ messages = OpenAIFactory.make_simple_messages(llm_job=llm_job)
try:
temperature = llm_job.job_params.temperature
@@ -97,10 +95,7 @@ class OpenAILLMWorker(LLMWorkerInternalAbstract):
llm_job: LLMJob,
schema: Type[BaseModelTypeVar],
) -> BaseModelTypeVar:
- messages = OpenAIFactory.make_simple_messages(
- llm_job=llm_job,
- inference_model=self.inference_model,
- )
+ messages = OpenAIFactory.make_simple_messages(llm_job=llm_job)
try:
temperature = llm_job.job_params.temperature
if ModelConstraints.TEMPERATURE_MUST_BE_MULTIPLIED_BY_2 in self.inference_model.constraints:
@@ -108,15 +103,20 @@ class OpenAILLMWorker(LLMWorkerInternalAbstract):
if ModelConstraints.TEMPERATURE_MUST_BE_1 in self.inference_model.constraints and temperature != 1:
log.warning(f"OpenAI model {self.inference_model.desc} used with a temperature of {temperature}, but it must be 1 for this model")
temperature = 1
- result_object, completion = await self.instructor_for_objects.chat.completions.create_with_completion(
- model=self.inference_model.model_id,
- temperature=temperature,
- max_tokens=llm_job.job_params.max_tokens or NOT_GIVEN,
- seed=llm_job.job_params.seed,
- messages=messages,
- response_model=schema,
- max_retries=llm_job.job_config.max_retries,
- )
+ try:
+ result_object, completion = await self.instructor_for_objects.chat.completions.create_with_completion(
+ model=self.inference_model.model_id,
+ temperature=temperature,
+ max_tokens=llm_job.job_params.max_tokens or NOT_GIVEN,
+ seed=llm_job.job_params.seed,
+ messages=messages,
+ response_model=schema,
+ max_retries=llm_job.job_config.max_retries,
+ )
+ except InstructorRetryException as exc:
+ raise LLMCompletionError(
+ f"OpenAI instructor failed with model: {self.inference_model.desc} trying to generate schema: {schema} with error: {exc}"
+ ) from exc
except NotFoundError as exc:
raise LLMCompletionError(f"OpenAI model or deployment '{self.inference_model.model_id}' not found: {exc}") from exc
except BadRequestError as bad_request_error:
diff --git a/pipelex/plugins/openai/vertexai_factory.py b/pipelex/plugins/openai/vertexai_factory.py
index 083951f85..e8b5a71e0 100644
--- a/pipelex/plugins/openai/vertexai_factory.py
+++ b/pipelex/plugins/openai/vertexai_factory.py
@@ -2,9 +2,9 @@ from typing import Any, Dict, Tuple
from pipelex.cogt.exceptions import CogtError, MissingDependencyError
from pipelex.tools.config.config_model import ConfigModel
-from pipelex.tools.environment import ENV_DUMMY_PLACEHOLDER_VALUE
from pipelex.tools.exceptions import CredentialsError
from pipelex.tools.misc.json_utils import load_json_dict_from_path
+from pipelex.tools.misc.placeholder import value_is_placeholder
from pipelex.types import StrEnum
@@ -40,7 +40,7 @@ class VertexAIFactory(ConfigModel):
endpoint = cls._make_endpoint(gcp_project_id=gcp_project_id, gcp_location=gcp_location)
gcp_credentials_file_path = extra_config.get(VertexAIExtraField.GCP_CREDENTIALS_FILE_PATH)
- if not gcp_credentials_file_path or gcp_credentials_file_path == ENV_DUMMY_PLACEHOLDER_VALUE:
+ if not gcp_credentials_file_path or value_is_placeholder(gcp_credentials_file_path):
raise VertexAIConfigError("GCP credentials file path is not properly set for VertexAI.")
api_key = cls._make_api_key(gcp_credentials_file_path=gcp_credentials_file_path)
diff --git a/pipelex/plugins/plugin_sdk_registry.py b/pipelex/plugins/plugin_sdk_registry.py
index aca4bae8b..3fd7f8168 100644
--- a/pipelex/plugins/plugin_sdk_registry.py
+++ b/pipelex/plugins/plugin_sdk_registry.py
@@ -2,7 +2,6 @@ from typing import Any, Dict, Optional
from pydantic import BaseModel, Field, RootModel
-from pipelex.cogt.imgg.imgg_platform import ImggPlatform
from pipelex.cogt.model_backends.model_spec import InferenceModelSpec
@@ -14,14 +13,6 @@ class Plugin(BaseModel):
def sdk_handle(self) -> str:
return f"{self.sdk}@{self.backend}"
- @classmethod
- def make_for_imgg_engine(cls, imgg_platform: ImggPlatform) -> "Plugin":
- match imgg_platform:
- case ImggPlatform.FAL_AI:
- return Plugin(sdk="fal", backend="fal")
- case ImggPlatform.OPENAI:
- return Plugin(sdk="openai", backend="openai")
-
@classmethod
def make_for_inference_model(cls, inference_model: InferenceModelSpec) -> "Plugin":
return Plugin(
diff --git a/pipelex/test_extras/shared_pytest_plugins.py b/pipelex/test_extras/shared_pytest_plugins.py
index 7ff461bbf..6e5dd3880 100644
--- a/pipelex/test_extras/shared_pytest_plugins.py
+++ b/pipelex/test_extras/shared_pytest_plugins.py
@@ -1,15 +1,45 @@
+import os
+from typing import List
+
import pytest
from pytest import FixtureRequest, Parser
from rich import print
from pipelex.core.pipes.pipe_run_params import PipeRunMode
-from pipelex.tools.environment import ENV_DUMMY_PLACEHOLDER_VALUE, is_env_set, set_env
+from pipelex.tools.environment import is_env_var_set, set_env
+from pipelex.tools.misc.placeholder import make_placeholder_value, value_is_placeholder
from pipelex.tools.runtime_manager import RunMode, runtime_manager
+# List of environment variables that may need placeholders in CI
+ENV_VAR_KEYS_WHICH_MAY_NEED_PLACEHOLDERS_IN_CI = [
+ "PIPELEX_API_TOKEN",
+ "PIPELEX_API_BASE_URL",
+ "PIPELEX_INFERENCE_API_KEY",
+ "OPENAI_API_KEY",
+ "AWS_ACCESS_KEY_ID",
+ "AWS_SECRET_ACCESS_KEY",
+ "AWS_REGION",
+ "AZURE_API_BASE",
+ "AZURE_API_KEY",
+ "AZURE_API_VERSION",
+ "GCP_PROJECT_ID",
+ "GCP_LOCATION",
+ "GCP_CREDENTIALS_FILE_PATH",
+ "ANTHROPIC_API_KEY",
+ "MISTRAL_API_KEY",
+ "PERPLEXITY_API_KEY",
+ "PERPLEXITY_API_ENDPOINT",
+ "XAI_API_KEY",
+ "XAI_API_ENDPOINT",
+ "FAL_API_KEY",
+ "BLACKBOX_API_KEY",
+ "GOOGLE_API_KEY",
+]
+
@pytest.fixture(scope="session", autouse=True)
def set_run_mode():
- if is_env_set("GITHUB_ACTIONS") or is_env_set("CI"):
+ if is_env_var_set(key="GITHUB_ACTIONS") or is_env_var_set(key="CI"):
runtime_manager.set_run_mode(run_mode=RunMode.CI_TEST)
else:
runtime_manager.set_run_mode(run_mode=RunMode.UNIT_TEST)
@@ -31,43 +61,21 @@ def pipe_run_mode(request: FixtureRequest) -> PipeRunMode:
return PipeRunMode(mode_str)
-def _setup_env_var_placeholders():
+def _setup_env_var_placeholders(env_var_keys: List[str]) -> None:
"""Set placeholder environment variables when running in CI to prevent import failures.
These placeholders allow the code to import successfully, while actual inference tests
remain skipped via pytest markers.
- """
-
- # Define placeholder values for all inference-related env vars
- env_var_placeholders = {
- "PIPELEX_API_TOKEN": ENV_DUMMY_PLACEHOLDER_VALUE,
- "PIPELEX_API_BASE_URL": ENV_DUMMY_PLACEHOLDER_VALUE,
- "PIPELEX_INFERENCE_API_KEY": ENV_DUMMY_PLACEHOLDER_VALUE,
- "OPENAI_API_KEY": ENV_DUMMY_PLACEHOLDER_VALUE,
- "AWS_ACCESS_KEY_ID": ENV_DUMMY_PLACEHOLDER_VALUE,
- "AWS_SECRET_ACCESS_KEY": ENV_DUMMY_PLACEHOLDER_VALUE,
- "AWS_REGION": ENV_DUMMY_PLACEHOLDER_VALUE,
- "AZURE_API_BASE": ENV_DUMMY_PLACEHOLDER_VALUE,
- "AZURE_API_KEY": ENV_DUMMY_PLACEHOLDER_VALUE,
- "AZURE_API_VERSION": ENV_DUMMY_PLACEHOLDER_VALUE,
- "GCP_PROJECT_ID": ENV_DUMMY_PLACEHOLDER_VALUE,
- "GCP_LOCATION": ENV_DUMMY_PLACEHOLDER_VALUE,
- "GCP_CREDENTIALS_FILE_PATH": ENV_DUMMY_PLACEHOLDER_VALUE,
- "ANTHROPIC_API_KEY": ENV_DUMMY_PLACEHOLDER_VALUE,
- "MISTRAL_API_KEY": ENV_DUMMY_PLACEHOLDER_VALUE,
- "PERPLEXITY_API_KEY": ENV_DUMMY_PLACEHOLDER_VALUE,
- "PERPLEXITY_API_ENDPOINT": ENV_DUMMY_PLACEHOLDER_VALUE,
- "XAI_API_KEY": ENV_DUMMY_PLACEHOLDER_VALUE,
- "XAI_API_ENDPOINT": ENV_DUMMY_PLACEHOLDER_VALUE,
- "FAL_API_KEY": ENV_DUMMY_PLACEHOLDER_VALUE,
- "BLACKBOX_API_KEY": ENV_DUMMY_PLACEHOLDER_VALUE,
- }
+ Args:
+ env_var_keys: List of environment variable keys that need placeholders
+ """
# Set placeholders for env vars who's presence is required for the code to run properly
# even if their value is not used in the test
substitutions_counter = 0
- for key, placeholder_value in env_var_placeholders.items():
- if not is_env_set(key):
+ for key in env_var_keys:
+ if not is_env_var_set(key=key):
+ placeholder_value = make_placeholder_value(key)
set_env(key, placeholder_value)
substitutions_counter += 1
@@ -75,10 +83,35 @@ def _setup_env_var_placeholders():
print(f"[yellow]Set {substitutions_counter} placeholder environment variables[/yellow]")
+def _cleanup_placeholder_env_vars(env_var_keys: List[str]) -> None:
+ """Remove placeholder environment variables that were set during CI testing.
+
+ This function identifies and removes any environment variables that contain
+ placeholder values, cleaning up the environment after tests complete.
+
+ Args:
+ env_var_keys: List of environment variable keys to check for placeholders
+ """
+ removed_counter = 0
+
+ # Check each specified environment variable for placeholder values
+ for key in env_var_keys:
+ value = os.environ.get(key)
+ if value is not None and value_is_placeholder(value):
+ del os.environ[key]
+ removed_counter += 1
+
+ if removed_counter > 0:
+ print(f"[yellow]Cleaned up {removed_counter} placeholder environment variables[/yellow]")
+
+
@pytest.fixture(scope="session", autouse=True)
def setup_ci_environment():
"""Set up CI environment variables and configuration before any tests run."""
- # Check if we're running in CI (GitHub Actions or generic CI environment)
- if runtime_manager.is_unit_testing:
- _setup_env_var_placeholders()
+ env_var_keys = ENV_VAR_KEYS_WHICH_MAY_NEED_PLACEHOLDERS_IN_CI
+ if runtime_manager.is_ci_testing:
+ _setup_env_var_placeholders(env_var_keys=env_var_keys)
yield
+ # Cleanup placeholder environment variables after tests complete
+ if runtime_manager.is_ci_testing:
+ _cleanup_placeholder_env_vars(env_var_keys=env_var_keys)
diff --git a/pipelex/tools/config/manager.py b/pipelex/tools/config/manager.py
index 0e57889ff..53f1079a1 100644
--- a/pipelex/tools/config/manager.py
+++ b/pipelex/tools/config/manager.py
@@ -10,7 +10,7 @@ from pipelex.tools.config.config_root import (
CONFIG_BASE_OVERRIDES_BEFORE_ENV,
)
from pipelex.tools.misc.json_utils import deep_update
-from pipelex.tools.misc.toml_utils import failable_load_toml_from_path
+from pipelex.tools.misc.toml_utils import failable_load_toml_from_path, load_toml_from_path
from pipelex.tools.runtime_manager import runtime_manager
CONFIG_DIR_NAME = ".pipelex"
@@ -63,9 +63,7 @@ class ConfigManager:
Dict[str, Any]: The configuration dictionary from pipelex.toml
"""
config_path = self.pipelex_root_config_path
- config = failable_load_toml_from_path(config_path)
- if not config:
- raise ConfigException(f"Pipelex root config could not be found at {self.pipelex_specific_config_file_path}.")
+ config = load_toml_from_path(config_path)
return config
def get_local_config(self) -> Dict[str, Any]:
diff --git a/pipelex/tools/environment.py b/pipelex/tools/environment.py
index 9b31d8bf3..97f14fc59 100644
--- a/pipelex/tools/environment.py
+++ b/pipelex/tools/environment.py
@@ -1,11 +1,10 @@
import os
-from typing import Iterable, Optional
+from typing import List, Optional
from dotenv import load_dotenv
from pipelex.tools.exceptions import ToolException
-
-ENV_DUMMY_PLACEHOLDER_VALUE = "env-dummy-placeholder"
+from pipelex.tools.misc.placeholder import value_is_placeholder
load_dotenv(dotenv_path=".env", override=True)
@@ -26,46 +25,25 @@ def get_optional_env(key: str) -> Optional[str]:
return value
-def is_env_set(key: str | Iterable[str]) -> bool:
- if isinstance(key, str):
- return os.getenv(key) is not None
- for each_key in key:
- if os.getenv(each_key) is None:
+def is_env_var_set(key: str) -> bool:
+ return os.getenv(key) is not None
+
+
+def all_env_vars_are_set(keys: List[str]) -> bool:
+ for each_key in keys:
+ if not is_env_var_set(each_key):
return False
return True
-def any_is_placeholder_env(key: str | Iterable[str]) -> bool:
- if isinstance(key, str):
- return os.getenv(key) == ENV_DUMMY_PLACEHOLDER_VALUE
- for each_key in key:
- if os.getenv(each_key) == ENV_DUMMY_PLACEHOLDER_VALUE:
+def any_env_var_is_placeholder(keys: List[str]) -> bool:
+ for each_key in keys:
+ env_value = os.getenv(each_key)
+ if value_is_placeholder(env_value):
return True
- return True
+ return False
def set_env(key: str, value: str) -> None:
os.environ[key] = value
return None
-
-
-def get_rooted_path(root: str, path: Optional[str] = None) -> str:
- if path is None:
- path = ""
- if path.startswith(root):
- return path
- elif os.path.isabs(path):
- return path
- else:
- joined = os.path.join(root, path)
- # remove edning "/" if any
- if joined.endswith("/"):
- joined = joined[:-1]
- return joined
-
-
-def get_env_rooted_path(root_env: str, path: Optional[str] = None) -> str:
- root = os.getenv(root_env)
- if root is None:
- root = ""
- return get_rooted_path(root, path)
diff --git a/pipelex/tools/func_registry.py b/pipelex/tools/func_registry.py
index 645c31c0c..223e19cb9 100644
--- a/pipelex/tools/func_registry.py
+++ b/pipelex/tools/func_registry.py
@@ -1,10 +1,10 @@
+import inspect
import logging
-from typing import Any, Callable, Dict, List, Optional, TypeVar
+from typing import Any, Callable, Dict, List, Optional, TypeVar, get_type_hints
from pydantic import Field, PrivateAttr, RootModel
from pipelex.tools.exceptions import ToolException
-from pipelex.tools.log.log_levels import LOGGING_LEVEL_VERBOSE
FUNC_REGISTRY_LOGGER_CHANNEL_NAME = "func_registry"
@@ -37,11 +37,16 @@ class FuncRegistry(RootModel[FuncRegistryDict]):
name: Optional[str] = None,
should_warn_if_already_registered: bool = True,
) -> None:
- """Registers a function in the registry with a name."""
+ """Registers a function in the registry with a name if it meets eligibility criteria."""
+ if not self.is_eligible_function(func):
+ return
+
key = name or func.__name__
if key in self.root:
if should_warn_if_already_registered:
self.log(f"Function '{key}' already exists in registry")
+ else:
+ raise FuncRegistryError(f"Function '{key}' already exists in registry")
else:
self.log(f"Registered new single function '{key}' in registry")
self.root[key] = func
@@ -61,36 +66,14 @@ class FuncRegistry(RootModel[FuncRegistryDict]):
del self.root[name]
def register_functions_dict(self, functions: Dict[str, Callable[..., Any]]) -> None:
- """Registers multiple functions in the registry with names."""
- self.root.update(functions)
- nb_functions = len(functions)
- if nb_functions > 1:
- self.log(f"Registered {nb_functions} functions in registry")
- functions_list_str = "\n".join([f"{key}: {value.__name__}" for key, value in functions.items()])
- logging.log(level=LOGGING_LEVEL_VERBOSE, msg=functions_list_str)
- else:
- self.log(f"Registered single function '{list(functions.values())[0].__name__}' in registry")
+ """Registers multiple functions in the registry with names if they meet eligibility criteria."""
+ for name, func in functions.items():
+ self.register_function(func=func, name=name, should_warn_if_already_registered=False)
def register_functions(self, functions: List[Callable[..., Any]]) -> None:
- """Registers multiple functions in the registry with names."""
- if not functions:
- self.log("register_functions called with empty list of functions to register")
- return
-
+ """Registers multiple functions in the registry with names if they meet eligibility criteria."""
for func in functions:
- key = func.__name__
- if key in self.root:
- self.log(f"Function '{key}' already exists in registry, skipping")
- continue
- self.root[key] = func
-
- nb_functions = len(functions)
- if nb_functions > 1:
- self.log(f"Registered {nb_functions} functions in registry")
- functions_list_str = "\n".join([f"{func.__name__}: {func}" for func in functions])
- logging.log(level=LOGGING_LEVEL_VERBOSE, msg=functions_list_str)
- else:
- self.log(f"Registered single function '{functions[0].__name__}' in registry")
+ self.register_function(func=func, should_warn_if_already_registered=False)
def get_function(self, name: str) -> Optional[Callable[..., Any]]:
"""Retrieves a function from the registry by its name. Returns None if not found."""
@@ -99,7 +82,10 @@ class FuncRegistry(RootModel[FuncRegistryDict]):
def get_required_function(self, name: str) -> Callable[..., Any]:
"""Retrieves a function from the registry by its name. Raises an error if not found."""
if name not in self.root:
- raise FuncRegistryError(f"Function '{name}' not found in registry")
+ raise FuncRegistryError(
+ f"Function '{name}' not found in registry: \
+ See how to register a function here: https://docs.pipelex.com/pages/build-reliable-ai-workflows-with-pipelex/pipe-operators/PipeFunc"
+ )
return self.root[name]
def get_required_function_with_signature(self, name: str, expected_signature: Callable[..., T]) -> Callable[..., T]:
@@ -121,5 +107,64 @@ class FuncRegistry(RootModel[FuncRegistryDict]):
"""Checks if a function is in the registry by its name."""
return name in self.root
+ def is_eligible_function(self, func: Callable[..., Any]) -> bool:
+ """
+ Checks if a function matches the criteria for PipeFunc registration:
+ - Exactly 1 parameter named "working_memory" with type WorkingMemory
+ - Return type that is a subclass of StuffContent
+ """
+ try:
+ # Import here to avoid circular imports
+ from pipelex.core.memory.working_memory import WorkingMemory
+ from pipelex.core.stuffs.stuff_content import StuffContent
+
+ # Get function signature
+ sig = inspect.signature(func)
+ params = list(sig.parameters.values())
+
+ # Check parameter count and name
+ if len(params) != 1:
+ return False
+
+ param = params[0]
+ if param.name != "working_memory":
+ return False
+
+ # Get type hints
+ type_hints = get_type_hints(func)
+
+ # Check parameter type
+ if "working_memory" not in type_hints:
+ return False
+
+ param_type = type_hints["working_memory"]
+ if param_type != WorkingMemory:
+ return False
+
+ # Check return type
+ if "return" not in type_hints:
+ return False
+
+ return_type = type_hints["return"]
+
+ # Check if return type is a subclass of StuffContent
+ try:
+ if inspect.isclass(return_type) and issubclass(return_type, StuffContent):
+ return True
+ # Handle generic types like ListContent[SomeType]
+ if hasattr(return_type, "__origin__"):
+ origin = getattr(return_type, "__origin__")
+ if inspect.isclass(origin) and issubclass(origin, StuffContent):
+ return True
+ except TypeError:
+ # Handle cases where issubclass fails on generic types
+ pass
+
+ return False
+
+ except Exception:
+ # If we can't analyze the function, skip it
+ return False
+
func_registry = FuncRegistry()
diff --git a/pipelex/tools/func_registry_utils.py b/pipelex/tools/func_registry_utils.py
index 1b8c6b98d..c16bd06f9 100644
--- a/pipelex/tools/func_registry_utils.py
+++ b/pipelex/tools/func_registry_utils.py
@@ -1,9 +1,7 @@
import inspect
from pathlib import Path
-from typing import Any, Callable, List, get_type_hints
+from typing import Any, Callable, List
-from pipelex.core.memory.working_memory import WorkingMemory
-from pipelex.core.stuffs.stuff_content import StuffContent
from pipelex.tools.func_registry import func_registry
from pipelex.tools.typing.module_inspector import import_module_from_file
@@ -16,7 +14,9 @@ class FuncRegistryUtils:
is_recursive: bool = True,
) -> None:
"""
- Registers all functions in Python files within a folder that have:
+ Discovers and attempts to register all functions in Python files within a folder.
+ Only functions that meet the eligibility criteria will be registered:
+ - Must be an async function
- Exactly 1 parameter named "working_memory" with type WorkingMemory
- Return type that is a subclass of StuffContent
@@ -57,9 +57,7 @@ class FuncRegistryUtils:
@classmethod
def _find_functions_in_module(cls, module: Any) -> List[Callable[..., Any]]:
"""
- Finds all functions in a module that match the criteria:
- - Exactly 1 parameter named "working_memory" with type WorkingMemory
- - Return type that is a subclass of StuffContent
+ Finds all functions in a module (eligibility will be checked during registration).
"""
functions: List[Callable[..., Any]] = []
module_name = module.__name__
@@ -70,67 +68,11 @@ class FuncRegistryUtils:
if obj.__module__ != module_name:
continue
- if cls._is_eligible_function(obj):
- functions.append(obj)
+ # Add all functions - eligibility will be checked by func_registry.register_function
+ functions.append(obj)
return functions
- @classmethod
- def _is_eligible_function(cls, func: Callable[..., Any]) -> bool:
- """
- Checks if a function matches the criteria:
- - Exactly 1 parameter named "working_memory" with type WorkingMemory
- - Return type that is a subclass of StuffContent
- """
- try:
- # Get function signature
- sig = inspect.signature(func)
- params = list(sig.parameters.values())
-
- # Check parameter count and name
- if len(params) != 1:
- return False
-
- param = params[0]
- if param.name != "working_memory":
- return False
-
- # Get type hints
- type_hints = get_type_hints(func)
-
- # Check parameter type
- if "working_memory" not in type_hints:
- return False
-
- param_type = type_hints["working_memory"]
- if param_type != WorkingMemory:
- return False
-
- # Check return type
- if "return" not in type_hints:
- return False
-
- return_type = type_hints["return"]
-
- # Check if return type is a subclass of StuffContent
- try:
- if inspect.isclass(return_type) and issubclass(return_type, StuffContent):
- return True
- # Handle generic types like ListContent[SomeType]
- if hasattr(return_type, "__origin__"):
- origin = getattr(return_type, "__origin__")
- if inspect.isclass(origin) and issubclass(origin, StuffContent):
- return True
- except TypeError:
- # Handle cases where issubclass fails on generic types
- pass
-
- return False
-
- except Exception:
- # If we can't analyze the function, skip it
- return False
-
@classmethod
def _find_files_in_dir(cls, dir_path: str, pattern: str, is_recursive: bool) -> List[Path]:
"""
diff --git a/pipelex/tools/misc/base_64_utils.py b/pipelex/tools/misc/base_64_utils.py
index 409092a0a..a95d5ebaa 100644
--- a/pipelex/tools/misc/base_64_utils.py
+++ b/pipelex/tools/misc/base_64_utils.py
@@ -17,6 +17,12 @@ async def load_binary_as_base64_async(path: str) -> bytes:
return base64.b64encode(data_bytes)
+async def load_binary_async(path: str) -> bytes:
+ async with aiofiles.open(path, "rb") as fp: # type: ignore[reportUnknownMemberType]
+ data_bytes = await fp.read()
+ return data_bytes
+
+
def encode_to_base64(data_bytes: bytes) -> bytes:
b64 = base64.b64encode(data_bytes)
return b64
diff --git a/pipelex/tools/misc/placeholder.py b/pipelex/tools/misc/placeholder.py
new file mode 100644
index 000000000..a8275ebfe
--- /dev/null
+++ b/pipelex/tools/misc/placeholder.py
@@ -0,0 +1,27 @@
+from typing import Optional
+
+PLACEHOLDER_PREFIX = "placeholder"
+
+
+def make_placeholder_value(key: str) -> str:
+ """Create a placeholder value for a given key.
+
+ Args:
+ key: The key/variable name to create a placeholder for
+
+ Returns:
+ A placeholder value in the format "placeholder-for-{key}"
+ """
+ return f"{PLACEHOLDER_PREFIX}-for-{key}"
+
+
+def value_is_placeholder(value: Optional[str]) -> bool:
+ """Check if a value is a placeholder based on the prefix.
+
+ Args:
+ value: The value to check
+
+ Returns:
+ True if the value starts with the placeholder prefix, False otherwise
+ """
+ return value is not None and value.startswith(PLACEHOLDER_PREFIX)
diff --git a/pipelex/tools/misc/pretty.py b/pipelex/tools/misc/pretty.py
index d3f48f16c..49d030275 100644
--- a/pipelex/tools/misc/pretty.py
+++ b/pipelex/tools/misc/pretty.py
@@ -2,6 +2,7 @@ import shutil
from typing import Any, ClassVar, List, Optional, Union
from rich import print as rich_print
+from rich.markdown import Markdown
from rich.panel import Panel
from rich.pretty import Pretty
from rich.style import StyleType
@@ -25,6 +26,16 @@ def pretty_print(
PrettyPrinter.pretty_print(content=content, title=title, subtitle=subtitle, border_style=border_style)
+def pretty_print_md(
+ content: str,
+ title: Optional[TextType] = None,
+ subtitle: Optional[TextType] = None,
+ border_style: Optional[StyleType] = None,
+):
+ md_content = Markdown(content)
+ PrettyPrinter.pretty_print(content=md_content, title=title, subtitle=subtitle, border_style=border_style)
+
+
class PrettyPrintMode(StrEnum):
RICH = "rich"
POOR = "poor"
@@ -69,6 +80,8 @@ class PrettyPrinter:
if subtitle:
rich_print(f"\n[dim]{subtitle}[/]")
return
+ elif isinstance(content, Markdown):
+ print("\n")
else:
content = Pretty(content)
panel = Panel(
diff --git a/pipelex/tools/misc/toml_utils.py b/pipelex/tools/misc/toml_utils.py
index 3b83621cc..c14e5c286 100644
--- a/pipelex/tools/misc/toml_utils.py
+++ b/pipelex/tools/misc/toml_utils.py
@@ -232,7 +232,7 @@ def _create_ordered_inline_table(data: Mapping[str, Any]) -> Any:
inline_table_obj = inline_table()
# Define the preferred order for concept structure fields
- field_order = ["type", "definition", "required", "choices", "item_type", "key_type", "value_type"]
+ field_order = ["type", "definition", "required"]
# Add fields in preferred order first
for field in field_order:
@@ -289,6 +289,7 @@ def dict_to_toml(data: Mapping[str, Any]) -> str:
# Handle the structure field specially
for field_key, field_value in item_value.items():
+ # THIS IS CUSTOM CODE FOR THE CONCEPT STRUCUTRE FIELD. It should be generic
if field_key == "structure" and isinstance(field_value, Mapping):
# Structure should be its own table [section.item.structure]
structure_table = table()
diff --git a/pipelex/tools/runtime_manager.py b/pipelex/tools/runtime_manager.py
index 8faf4e727..2d0874e29 100644
--- a/pipelex/tools/runtime_manager.py
+++ b/pipelex/tools/runtime_manager.py
@@ -93,7 +93,7 @@ class RuntimeManager(BaseModel):
return True
@property
- def is_gha_testing(self) -> bool:
+ def is_ci_testing(self) -> bool:
match self.run_mode:
case RunMode.NORMAL:
return False
diff --git a/pipelex/tools/templating/jinja2_parsing.py b/pipelex/tools/templating/jinja2_parsing.py
index 0a3ee950c..901b426fc 100644
--- a/pipelex/tools/templating/jinja2_parsing.py
+++ b/pipelex/tools/templating/jinja2_parsing.py
@@ -1,10 +1,19 @@
+from jinja2.exceptions import TemplateSyntaxError
+
from pipelex.tools.templating.jinja2_environment import make_jinja2_env_without_loader
from pipelex.tools.templating.jinja2_template_category import Jinja2TemplateCategory
+class Jinja2ParsingError(Exception):
+ pass
+
+
def check_jinja2_parsing(
jinja2_template_source: str,
template_category: Jinja2TemplateCategory = Jinja2TemplateCategory.LLM_PROMPT,
):
jinja2_env = make_jinja2_env_without_loader(template_category=template_category)
- jinja2_env.parse(jinja2_template_source)
+ try:
+ jinja2_env.parse(jinja2_template_source)
+ except TemplateSyntaxError as exc:
+ raise Jinja2ParsingError(f"Could not parse Jinja2 template because of: {exc}. The template is:\n{jinja2_template_source}") from exc
diff --git a/pipelex/tools/typing/structure_printer.py b/pipelex/tools/typing/structure_printer.py
new file mode 100644
index 000000000..982064336
--- /dev/null
+++ b/pipelex/tools/typing/structure_printer.py
@@ -0,0 +1,325 @@
+from enum import Enum
+from typing import Any, Dict, List, Optional, Set, Type, Union, get_args, get_origin, get_type_hints
+
+from pydantic import BaseModel
+
+from pipelex.types import StrEnum
+
+
+class StructurePrinter:
+ def pretty_type(self, tp: object) -> str:
+ """Pretty print a type, with special handling for containers, literals and enums."""
+ origin = getattr(tp, "__origin__", None)
+ args = getattr(tp, "__args__", None)
+ if origin is None:
+ if isinstance(tp, type):
+ return tp.__name__
+ return str(tp)
+
+ if origin is Union and args:
+ non_none = [a for a in args if a is not type(None)]
+ if len(non_none) == 1 and len(args) == 2:
+ return f"Optional[{self.pretty_type(non_none[0])}]"
+ return f"Union[{', '.join(self.pretty_type(a) for a in args)}]"
+
+ if str(origin).endswith("Literal") and args: # Handle both typing.Literal and typing_extensions.Literal
+ # For enum values, just get their values
+ values: List[str] = []
+ for arg in args:
+ if isinstance(arg, Enum) or isinstance(arg, StrEnum):
+ values.append(f'"{arg.value}"')
+ else:
+ values.append(repr(arg))
+ # Return multi-line format for Literal fields
+ if len(values) > 1:
+ return "Literal[\n " + ",\n ".join(values) + ",\n ]"
+ return f"Literal[{', '.join(values)}]"
+
+ if (origin is list or origin is List) and args:
+ return f"List[{self.pretty_type(args[0])}]"
+ if (origin is dict or origin is Dict) and args:
+ return f"Dict[{self.pretty_type(args[0])}, {self.pretty_type(args[1])}]"
+ return str(tp)
+
+ def get_type_structure(
+ self,
+ tp: Type[Any],
+ seen_types: Optional[Set[str]] = None,
+ collected_types: Optional[Dict[str, Type[Any]]] = None,
+ collected_enums: Optional[Dict[str, Type[Enum]]] = None,
+ base_class: Type[Any] = BaseModel,
+ ) -> List[str]:
+ """
+ Get the structure of a type, listing referenced subclasses of base_class and enums.
+
+ Args:
+ tp: The type to analyze
+ seen_types: Set of already seen type names to avoid cycles
+ collected_types: Dictionary of collected types to analyze
+ collected_enums: Dictionary of collected enums
+ base_class: The base class to check for inheritance (defaults to BaseModel)
+ """
+ if seen_types is None:
+ seen_types = set()
+ if collected_types is None:
+ collected_types = {}
+ if collected_enums is None:
+ collected_enums = {}
+
+ def format_type(tp: Any) -> str:
+ """Format a type annotation nicely"""
+ origin = get_origin(tp)
+ if origin is None:
+ if isinstance(tp, type):
+ return tp.__name__
+ return str(tp)
+
+ args = get_args(tp)
+ if origin is Union:
+ non_none = [a for a in args if a is not type(None)]
+ if len(non_none) == 1:
+ return f"Optional[{format_type(non_none[0])}]"
+ return f"Union[{', '.join(format_type(a) for a in non_none)}]"
+
+ if str(origin).endswith("Literal") and args: # Handle both typing.Literal and typing_extensions.Literal
+ # For enum values, just get their values
+ values: List[str] = []
+ enum_type = None
+ for arg in args:
+ if isinstance(arg, Enum) or isinstance(arg, StrEnum):
+ values.append(arg.value)
+ if enum_type is None:
+ enum_type = type(arg)
+ else:
+ values.append(str(arg))
+ # Add enum type to collected_enums if found
+ if enum_type is not None:
+ collected_enums[enum_type.__name__] = enum_type
+ # Return multi-line format for Literal fields
+ if len(values) > 1:
+ lines: List[str] = []
+ for value in values:
+ lines.append(f'"{value}"')
+ return "Literal[\n " + ",\n ".join(lines) + ",\n ]"
+ return f"Literal[{', '.join(values)}]"
+
+ if origin in (list, List):
+ return f"List[{format_type(args[0])}]"
+ if origin in (dict, Dict):
+ return f"Dict[{format_type(args[0])}, {format_type(args[1])}]"
+ return str(tp)
+
+ def collect_types(tp: Type[Any]) -> None:
+ """Recursively collect types and enums"""
+ origin = get_origin(tp)
+ args = get_args(tp)
+
+ if origin:
+ if origin is Union:
+ non_none = [a for a in args if a is not type(None)]
+ for arg in non_none:
+ if isinstance(arg, type):
+ collect_types(arg)
+ elif hasattr(arg, "__origin__"): # Handle nested generics
+ collect_types(arg)
+ elif origin in (list, List):
+ if isinstance(args[0], type):
+ collect_types(args[0])
+ elif hasattr(args[0], "__origin__"): # Handle nested generics
+ collect_types(args[0])
+ elif origin in (dict, Dict):
+ for arg in args:
+ if isinstance(arg, type):
+ collect_types(arg)
+ elif hasattr(arg, "__origin__"): # Handle nested generics
+ collect_types(arg)
+ return
+
+ # Collect enums
+ if issubclass(tp, Enum) and tp not in collected_enums.values():
+ collected_enums[tp.__name__] = tp
+ return
+
+ # Collect model classes
+ if issubclass(tp, base_class) and tp.__name__ not in seen_types:
+ seen_types.add(tp.__name__)
+ collected_types[tp.__name__] = tp
+
+ # Only collect immediate parent class if it's a custom class
+ for base in tp.__bases__:
+ if (
+ issubclass(base, BaseModel)
+ and base is not BaseModel
+ and base.__module__ != "pydantic.main"
+ and base.__module__ != "abc"
+ and not base.__module__.startswith("pipelex.core")
+ ):
+ collect_types(base)
+
+ try:
+ type_hints = get_type_hints(tp)
+ model_fields = getattr(tp, "model_fields", {})
+
+ if model_fields:
+ for fname, _ in model_fields.items():
+ ftype = type_hints[fname]
+ collect_types(ftype)
+ elif hasattr(tp, "__annotations__"):
+ for fname, ftype in type_hints.items():
+ collect_types(ftype)
+ except (TypeError, AttributeError):
+ # Handle cases where type hints cannot be retrieved
+ pass
+
+ # Start collection
+ collect_types(tp)
+
+ # Generate output
+ output: List[str] = []
+
+ # First output the main class and its dependencies
+ for class_name, class_type in collected_types.items():
+ if output:
+ output.append("")
+
+ # Get class docstring
+ doc = class_type.__doc__ and class_type.__doc__.strip()
+ base_class_name = class_type.__bases__[0].__name__
+
+ # Get generic parameters if any
+ type_args = get_args(class_type)
+ if type_args:
+ base_class_name = f"{base_class_name}[{', '.join(arg.__name__ for arg in type_args)}]"
+
+ # Class definition with docstring
+ output.append(f"class {class_name}({base_class_name}):")
+ if doc:
+ # Split docstring into lines and format each line
+ doc_lines = [line.rstrip() for line in doc.split("\n")]
+
+ # Remove empty lines from start and end
+ while doc_lines and not doc_lines[0].strip():
+ doc_lines.pop(0)
+ while doc_lines and not doc_lines[-1].strip():
+ doc_lines.pop()
+
+ if len(doc_lines) == 1:
+ # Single line docstring
+ output.append(f' """{doc_lines[0]}"""')
+ else:
+ # Multi-line docstring
+ output.append(f' """{doc_lines[0]}')
+
+ # Add empty line after first line if there's content
+ if doc_lines[1].strip():
+ output.append("")
+
+ # Add remaining lines with proper indentation
+ for line in doc_lines[1:]:
+ if line.strip():
+ output.append(f" {line.strip()}")
+ else:
+ output.append("")
+
+ # Close the docstring
+ output.append(' """')
+
+ # Handle empty classes or classes that only inherit fields
+ try:
+ type_hints = get_type_hints(class_type)
+ model_fields = getattr(class_type, "model_fields", {})
+
+ # Get and sort fields
+ if model_fields:
+ fields = model_fields.items()
+ else:
+ fields = [(type_k, type_hints[type_k]) for type_k in sorted(type_hints.keys())]
+
+ # Check if all fields are inherited
+ parent_fields: Set[str] = set()
+ for base in class_type.__bases__:
+ try:
+ parent_fields.update(get_type_hints(base).keys())
+ except (TypeError, AttributeError):
+ continue
+
+ current_fields = set(dict(fields).keys())
+ non_inherited_fields = current_fields - parent_fields
+
+ # Output fields
+ for fname, ftype in fields:
+ if fname in non_inherited_fields or (fname == "items" and "List" in base_class_name):
+ # Initialize is_optional to False by default
+ is_optional = False
+
+ if isinstance(ftype, type) and issubclass(ftype, BaseModel):
+ ftype_str = ftype.__name__
+ else:
+ field_type = type_hints[fname]
+ ftype_str = format_type(field_type)
+ # Check if field is Optional
+ field_origin = get_origin(field_type)
+ field_args = get_args(field_type)
+ is_optional = field_origin is Union and type(None) in field_args
+
+ # Handle default values
+ field_default = None
+ field_description = None
+ if model_fields:
+ field_info = model_fields.get(fname)
+ if field_info:
+ if hasattr(field_info, "default") and field_info.default is not None:
+ # Skip PydanticUndefined default values
+ if str(field_info.default) != "PydanticUndefined":
+ field_default = field_info.default
+ if hasattr(field_info, "description"):
+ field_description = field_info.description
+
+ # Build the field line
+ field_line = f" {fname}: {ftype_str}"
+ if is_optional:
+ field_line += " = None"
+ elif field_default is not None:
+ if isinstance(field_default, bool):
+ field_line += f" = {str(field_default)}"
+ elif not isinstance(field_default, (BaseModel, list, dict)):
+ field_line += f" = {repr(field_default)}"
+
+ # Add description as a comment if available
+ # First check if there's a direct field description from model_fields
+ if field_description:
+ field_line += f" # {field_description}"
+ # Then check if the field type itself has model_fields and a description
+ # This handles nested content types that have field descriptions
+ elif hasattr(ftype, "model_fields") and fname in ftype.model_fields and hasattr(ftype.model_fields[fname], "description"): # type: ignore
+ field_line += f" # {ftype.model_fields[fname].description}" # type: ignore
+
+ # Split multi-line field lines
+ if "\n" in field_line:
+ lines = field_line.split("\n")
+ output.extend(lines)
+ else:
+ output.append(field_line)
+ continue
+
+ # If no fields were output, show inheritance comment
+ if len(output) == (2 if doc else 1):
+ output.append(" # No additional fields")
+ except (TypeError, AttributeError):
+ # If we can't get type hints, show inheritance comment
+ output.append(" # No additional fields")
+
+ # Then output all enum classes
+ for enum_name, enum_type in collected_enums.items():
+ if output:
+ output.append("")
+ output.append(f"class {enum_name}({enum_type.__bases__[0].__name__}):")
+ # Add enum docstring if available, but skip Python's default "An enumeration." docstring
+ # This ensures we only include meaningful custom docstrings
+ if enum_type.__doc__ and enum_type.__doc__.strip() != "An enumeration.":
+ doc = enum_type.__doc__.strip()
+ output.append(f' """{doc}"""')
+ for member in enum_type:
+ output.append(f' {member.name} = "{member.value}"')
+
+ return output
diff --git a/pipelex/tools/typing/type_inspector.py b/pipelex/tools/typing/type_inspector.py
deleted file mode 100644
index 6df8f2471..000000000
--- a/pipelex/tools/typing/type_inspector.py
+++ /dev/null
@@ -1,327 +0,0 @@
-from enum import Enum
-from typing import Any, Dict, List, Optional, Set, Type, Union, get_type_hints
-
-from pydantic import BaseModel
-from typing_extensions import get_args, get_origin
-
-from pipelex.types import StrEnum
-
-
-def pretty_type(tp: object) -> str:
- """Pretty print a type, with special handling for containers, literals and enums."""
- origin = getattr(tp, "__origin__", None)
- args = getattr(tp, "__args__", None)
- if origin is None:
- if isinstance(tp, type):
- return tp.__name__
- return str(tp)
-
- if origin is Union and args:
- non_none = [a for a in args if a is not type(None)]
- if len(non_none) == 1 and len(args) == 2:
- return f"Optional[{pretty_type(non_none[0])}]"
- return f"Union[{', '.join(pretty_type(a) for a in args)}]"
-
- if str(origin).endswith("Literal") and args: # Handle both typing.Literal and typing_extensions.Literal
- # For enum values, just get their values
- values: List[str] = []
- for arg in args:
- if isinstance(arg, Enum) or isinstance(arg, StrEnum):
- values.append(f'"{arg.value}"')
- else:
- values.append(repr(arg))
- # Return multi-line format for Literal fields
- if len(values) > 1:
- return "Literal[\n " + ",\n ".join(values) + ",\n ]"
- return f"Literal[{', '.join(values)}]"
-
- if (origin is list or origin is List) and args:
- return f"List[{pretty_type(args[0])}]"
- if (origin is dict or origin is Dict) and args:
- return f"Dict[{pretty_type(args[0])}, {pretty_type(args[1])}]"
- return str(tp)
-
-
-def get_type_structure(
- tp: Type[Any],
- seen_types: Optional[Set[str]] = None,
- collected_types: Optional[Dict[str, Type[Any]]] = None,
- collected_enums: Optional[Dict[str, Type[Enum]]] = None,
- base_class: Type[Any] = BaseModel,
-) -> List[str]:
- """
- Get the structure of a type, listing referenced subclasses of base_class and enums.
-
- Args:
- tp: The type to analyze
- seen_types: Set of already seen type names to avoid cycles
- collected_types: Dictionary of collected types to analyze
- collected_enums: Dictionary of collected enums
- base_class: The base class to check for inheritance (defaults to BaseModel)
- """
- if seen_types is None:
- seen_types = set()
- if collected_types is None:
- collected_types = {}
- if collected_enums is None:
- collected_enums = {}
-
- def format_type(tp: Any) -> str:
- """Format a type annotation nicely"""
- origin = get_origin(tp)
- if origin is None:
- if isinstance(tp, type):
- return tp.__name__
- return str(tp)
-
- args = get_args(tp)
- if origin is Union:
- non_none = [a for a in args if a is not type(None)]
- if len(non_none) == 1:
- return f"Optional[{format_type(non_none[0])}]"
- return f"Union[{', '.join(format_type(a) for a in non_none)}]"
-
- if str(origin).endswith("Literal") and args: # Handle both typing.Literal and typing_extensions.Literal
- # For enum values, just get their values
- values: List[str] = []
- enum_type = None
- for arg in args:
- if isinstance(arg, Enum) or isinstance(arg, StrEnum):
- values.append(arg.value)
- if enum_type is None:
- enum_type = type(arg)
- else:
- values.append(str(arg))
- # Add enum type to collected_enums if found
- if enum_type is not None:
- collected_enums[enum_type.__name__] = enum_type
- # Return multi-line format for Literal fields
- if len(values) > 1:
- lines: List[str] = []
- for value in values:
- lines.append(f'"{value}"')
- return "Literal[\n " + ",\n ".join(lines) + ",\n ]"
- return f"Literal[{', '.join(values)}]"
-
- if origin in (list, List):
- return f"List[{format_type(args[0])}]"
- if origin in (dict, Dict):
- return f"Dict[{format_type(args[0])}, {format_type(args[1])}]"
- return str(tp)
-
- def collect_types(tp: Type[Any]) -> None:
- """Recursively collect types and enums"""
- origin = get_origin(tp)
- args = get_args(tp)
-
- if origin:
- if origin is Union:
- non_none = [a for a in args if a is not type(None)]
- for arg in non_none:
- if isinstance(arg, type):
- collect_types(arg)
- elif hasattr(arg, "__origin__"): # Handle nested generics
- collect_types(arg)
- elif origin in (list, List):
- if isinstance(args[0], type):
- collect_types(args[0])
- elif hasattr(args[0], "__origin__"): # Handle nested generics
- collect_types(args[0])
- elif origin in (dict, Dict):
- for arg in args:
- if isinstance(arg, type):
- collect_types(arg)
- elif hasattr(arg, "__origin__"): # Handle nested generics
- collect_types(arg)
- return
-
- # Collect enums
- if issubclass(tp, Enum) and tp not in collected_enums.values():
- collected_enums[tp.__name__] = tp
- return
-
- # Collect model classes
- if issubclass(tp, base_class) and tp.__name__ not in seen_types:
- seen_types.add(tp.__name__)
- collected_types[tp.__name__] = tp
-
- # Only collect immediate parent class if it's a custom class
- for base in tp.__bases__:
- if (
- issubclass(base, BaseModel)
- and base is not BaseModel
- and base.__module__ != "pydantic.main"
- and base.__module__ != "abc"
- and not base.__module__.startswith("pipelex.core")
- ):
- collect_types(base)
-
- try:
- type_hints = get_type_hints(tp)
- model_fields = getattr(tp, "model_fields", {})
-
- if model_fields:
- for fname, _ in model_fields.items():
- ftype = type_hints[fname]
- collect_types(ftype)
- elif hasattr(tp, "__annotations__"):
- for fname, ftype in type_hints.items():
- collect_types(ftype)
- except (TypeError, AttributeError):
- # Handle cases where type hints cannot be retrieved
- pass
-
- # Start collection
- collect_types(tp)
-
- # Generate output
- output: List[str] = []
-
- # First output the main class and its dependencies
- for class_name, class_type in collected_types.items():
- if output:
- output.append("")
-
- # Get class docstring
- doc = class_type.__doc__ and class_type.__doc__.strip()
- base_class_name = class_type.__bases__[0].__name__
-
- # Get generic parameters if any
- type_args = get_args(class_type)
- if type_args:
- base_class_name = f"{base_class_name}[{', '.join(arg.__name__ for arg in type_args)}]"
-
- # Class definition with docstring
- output.append(f"class {class_name}({base_class_name}):")
- if doc:
- # Split docstring into lines and format each line
- doc_lines = [line.rstrip() for line in doc.split("\n")]
-
- # Remove empty lines from start and end
- while doc_lines and not doc_lines[0].strip():
- doc_lines.pop(0)
- while doc_lines and not doc_lines[-1].strip():
- doc_lines.pop()
-
- if len(doc_lines) == 1:
- # Single line docstring
- output.append(f' """{doc_lines[0]}"""')
- else:
- # Multi-line docstring
- output.append(f' """{doc_lines[0]}')
-
- # Add empty line after first line if there's content
- if doc_lines[1].strip():
- output.append("")
-
- # Add remaining lines with proper indentation
- for line in doc_lines[1:]:
- if line.strip():
- output.append(f" {line.strip()}")
- else:
- output.append("")
-
- # Close the docstring
- output.append(' """')
-
- # Handle empty classes or classes that only inherit fields
- try:
- type_hints = get_type_hints(class_type)
- model_fields = getattr(class_type, "model_fields", {})
-
- # Get and sort fields
- if model_fields:
- fields = model_fields.items()
- else:
- fields = [(type_k, type_hints[type_k]) for type_k in sorted(type_hints.keys())]
-
- # Check if all fields are inherited
- parent_fields: Set[str] = set()
- for base in class_type.__bases__:
- try:
- parent_fields.update(get_type_hints(base).keys())
- except (TypeError, AttributeError):
- continue
-
- current_fields = set(dict(fields).keys())
- non_inherited_fields = current_fields - parent_fields
-
- # Output fields
- for fname, ftype in fields:
- if fname in non_inherited_fields or (fname == "items" and "List" in base_class_name):
- # Initialize is_optional to False by default
- is_optional = False
-
- if isinstance(ftype, type) and issubclass(ftype, BaseModel):
- ftype_str = ftype.__name__
- else:
- field_type = type_hints[fname]
- ftype_str = format_type(field_type)
- # Check if field is Optional
- field_origin = get_origin(field_type)
- field_args = get_args(field_type)
- is_optional = field_origin is Union and type(None) in field_args
-
- # Handle default values
- field_default = None
- field_description = None
- if model_fields:
- field_info = model_fields.get(fname)
- if field_info:
- if hasattr(field_info, "default") and field_info.default is not None:
- # Skip PydanticUndefined default values
- if str(field_info.default) != "PydanticUndefined":
- field_default = field_info.default
- if hasattr(field_info, "description"):
- field_description = field_info.description
-
- # Build the field line
- field_line = f" {fname}: {ftype_str}"
- if is_optional:
- field_line += " = None"
- elif field_default is not None:
- if isinstance(field_default, bool):
- field_line += f" = {str(field_default)}"
- elif not isinstance(field_default, (BaseModel, list, dict)):
- field_line += f" = {repr(field_default)}"
-
- # Add description as a comment if available
- # First check if there's a direct field description from model_fields
- if field_description:
- field_line += f" # {field_description}"
- # Then check if the field type itself has model_fields and a description
- # This handles nested content types that have field descriptions
- elif hasattr(ftype, "model_fields") and fname in ftype.model_fields and hasattr(ftype.model_fields[fname], "description"): # type: ignore
- field_line += f" # {ftype.model_fields[fname].description}" # type: ignore
-
- # Split multi-line field lines
- if "\n" in field_line:
- lines = field_line.split("\n")
- output.extend(lines)
- else:
- output.append(field_line)
- continue
-
- # If no fields were output, show inheritance comment
- if len(output) == (2 if doc else 1):
- output.append(f" # Inherits from {base_class_name}")
- output.append(" # No additional fields")
- except (TypeError, AttributeError):
- # If we can't get type hints, show inheritance comment
- output.append(f" # Inherits from {base_class_name}")
- output.append(" # No additional fields")
-
- # Then output all enum classes
- for enum_name, enum_type in collected_enums.items():
- if output:
- output.append("")
- output.append(f"class {enum_name}({enum_type.__bases__[0].__name__}):")
- # Add enum docstring if available, but skip Python's default "An enumeration." docstring
- # This ensures we only include meaningful custom docstrings
- if enum_type.__doc__ and enum_type.__doc__.strip() != "An enumeration.":
- doc = enum_type.__doc__.strip()
- output.append(f' """{doc}"""')
- for member in enum_type:
- output.append(f' {member.name} = "{member.value}"')
-
- return output
diff --git a/pyproject.toml b/pyproject.toml
index 39cd1c99c..cff1cee2b 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -22,7 +22,7 @@ dependencies = [
"backports.strenum>=1.3.0 ; python_version < '3.11'",
"filetype>=1.2.0",
"httpx>=0.27.0",
- "instructor>=1.8.3,<1.10.0", # 1.10.0 causes an error with mypy
+ "instructor>=1.8.3,<1.11.0", # 1.11.x causes typing errors with mypy
"jinja2>=3.1.4",
"json2html>=1.3.0",
"kajson==0.3.0",
@@ -57,6 +57,7 @@ anthropic = ["anthropic>=0.49.0"]
bedrock = ["boto3>=1.34.131", "aioboto3>=13.4.0"]
fal = ["fal-client>=0.4.1"]
google = ["google-auth-oauthlib>=1.2.1"]
+google-genai = ["google-genai", "instructor[google-genai]"]
mistralai = ["mistralai==1.5.2"]
docs = [
"mkdocs==1.6.1",
@@ -207,13 +208,13 @@ strictSetInference = true
typeCheckingMode = "strict"
[tool.pytest.ini_options]
-addopts = "--import-mode=importlib -ra -m 'not (inference or llm or imgg or ocr or needs_output or pipelex_api)'"
+addopts = "--import-mode=importlib -ra -m 'not (inference or llm or img_gen or ocr or needs_output or pipelex_api)'"
asyncio_default_fixture_loop_scope = "session"
markers = [
"needs_output: tests that need output to be displayed",
"inference: slow and costly due to inference calls",
"llm: slow and costly due to llm inference calls",
- "imgg: slow and costly due to imgg inference calls",
+ "img_gen: slow and costly due to image generation inference calls",
"ocr: slow and costly due to ocr inference calls",
"gha_disabled: tests that should not run in GitHub Actions",
"codex_disabled: tests that should not run in Codex",
diff --git a/tests/conftest.py b/tests/conftest.py
index 15ed2ca5c..e280c2d64 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -7,7 +7,7 @@ import pipelex.pipelex
from pipelex import log
from pipelex.config import get_config
from pipelex.core.concepts.concept_provider_abstract import ConceptProviderAbstract
-from pipelex.hub import get_concept_provider
+from pipelex.hub import get_concept_provider, get_report_delegate
from tests.cases.registry import Fruit
pytest_plugins = [
@@ -20,7 +20,7 @@ TEST_OUTPUTS_DIR = "temp/test_outputs"
@pytest.fixture(scope="module", autouse=True)
def reset_pipelex_config_fixture():
# Code to run before each test
- print("[magenta]pipelex setup[/magenta]")
+ Console().print("[magenta]pipelex setup[/magenta]")
try:
pipelex_instance = pipelex.pipelex.Pipelex.make(relative_config_folder_path="../pipelex/libraries")
config = get_config()
@@ -32,7 +32,8 @@ def reset_pipelex_config_fixture():
pytest.exit(f"Critical Pipelex setup error: {exc}")
yield
# Code to run after each test
- print("[magenta]pipelex teardown[/magenta]")
+ get_report_delegate().generate_report()
+ Console().print("[magenta]pipelex teardown[/magenta]")
pipelex_instance.teardown()
diff --git a/tests/e2e/pipelex/pipes/pipe_controller/pipe_condition/test_pipe_condition_1.py b/tests/e2e/pipelex/pipes/pipe_controller/pipe_condition/test_pipe_condition_1.py
index fb5e2fded..8e7d56cbc 100644
--- a/tests/e2e/pipelex/pipes/pipe_controller/pipe_condition/test_pipe_condition_1.py
+++ b/tests/e2e/pipelex/pipes/pipe_controller/pipe_condition/test_pipe_condition_1.py
@@ -8,7 +8,7 @@ from pipelex.core.pipes.pipe_output import PipeOutput
from pipelex.core.pipes.pipe_run_params import PipeRunMode
from pipelex.core.pipes.pipe_run_params_factory import PipeRunParamsFactory
from pipelex.core.stuffs.stuff_factory import StuffFactory
-from pipelex.hub import get_pipe_router, get_report_delegate
+from pipelex.hub import get_pipe_router
from pipelex.pipeline.job_metadata import JobMetadata
from tests.test_pipelines.pipe_controllers.pipe_condition.pipe_condition import CategoryInput
@@ -55,7 +55,6 @@ class TestPipeConditionExpressionTemplate:
# Log output and generate report
pretty_print(pipe_output, title=f"PipeCondition routing test - {category}")
- get_report_delegate().generate_report()
# Basic assertions
assert pipe_output is not None
diff --git a/tests/e2e/pipelex/pipes/pipe_controller/pipe_condition/test_pipe_condition_2.py b/tests/e2e/pipelex/pipes/pipe_controller/pipe_condition/test_pipe_condition_2.py
index cbc10f3ba..3cf459fdf 100644
--- a/tests/e2e/pipelex/pipes/pipe_controller/pipe_condition/test_pipe_condition_2.py
+++ b/tests/e2e/pipelex/pipes/pipe_controller/pipe_condition/test_pipe_condition_2.py
@@ -8,7 +8,7 @@ from pipelex.core.pipes.pipe_output import PipeOutput
from pipelex.core.pipes.pipe_run_params import PipeRunMode
from pipelex.core.pipes.pipe_run_params_factory import PipeRunParamsFactory
from pipelex.core.stuffs.stuff_factory import StuffFactory
-from pipelex.hub import get_pipe_router, get_report_delegate
+from pipelex.hub import get_pipe_router
from pipelex.pipeline.job_metadata import JobMetadata
from tests.test_pipelines.pipe_controllers.pipe_condition.pipe_condition import CategoryInput
@@ -55,7 +55,6 @@ class TestPipeConditionExpression:
# Log output and generate report
pretty_print(pipe_output, title=f"PipeCondition routing test - {category}")
- get_report_delegate().generate_report()
# Basic assertions
assert pipe_output is not None
diff --git a/tests/e2e/pipelex/pipes/pipe_controller/pipe_condition/test_pipe_condition_simple.py b/tests/e2e/pipelex/pipes/pipe_controller/pipe_condition/test_pipe_condition_simple.py
index 448d41e4a..9c6fca945 100644
--- a/tests/e2e/pipelex/pipes/pipe_controller/pipe_condition/test_pipe_condition_simple.py
+++ b/tests/e2e/pipelex/pipes/pipe_controller/pipe_condition/test_pipe_condition_simple.py
@@ -7,8 +7,8 @@ from pipelex.core.concepts.concept_factory import ConceptFactory
from pipelex.core.concepts.concept_native import NativeConceptEnum
from pipelex.core.domains.domain import SpecialDomain
from pipelex.core.memory.working_memory_factory import WorkingMemoryFactory
-from pipelex.core.pipes.pipe_input_spec import TypedNamedInputRequirement
-from pipelex.core.pipes.pipe_input_spec_blueprint import InputRequirementBlueprint
+from pipelex.core.pipes.pipe_input import TypedNamedInputRequirement
+from pipelex.core.pipes.pipe_input_blueprint import InputRequirementBlueprint
from pipelex.core.pipes.pipe_run_params import PipeRunMode
from pipelex.core.pipes.pipe_run_params_factory import PipeRunParamsFactory
from pipelex.exceptions import DryRunError
@@ -28,7 +28,7 @@ class TestPipeConditionSimple:
pipe_condition_blueprint = PipeConditionBlueprint(
definition="Test condition that should fail",
inputs={"user_category": InputRequirementBlueprint(concept="test_pipe_condition.CategoryInput")},
- output=f"{SpecialDomain.NATIVE.value}.{NativeConceptEnum.TEXT.value}",
+ output=f"{SpecialDomain.NATIVE}.{NativeConceptEnum.TEXT}",
expression_template="{{ user_category.category }}",
pipe_map=PipeConditionPipeMapBlueprint(root={"small": "process_small", "medium": "process_medium", "large": "process_large"}),
default_pipe_code="process_small",
@@ -62,7 +62,7 @@ class TestPipeConditionSimple:
pipe_condition_blueprint = PipeConditionBlueprint(
definition="Test condition that should succeed",
inputs={"user_status": InputRequirementBlueprint(concept="test_pipe_condition.CategoryInput")},
- output=f"{SpecialDomain.NATIVE.value}.{NativeConceptEnum.TEXT.value}",
+ output=f"{SpecialDomain.NATIVE}.{NativeConceptEnum.TEXT}",
expression_template="{{ user_status.category }}",
pipe_map=PipeConditionPipeMapBlueprint(root={"active": "process_small", "inactive": "process_medium", "pending": "process_large"}),
default_pipe_code="process_small",
diff --git a/tests/e2e/pipelex/pipes/pipe_controller/pipe_parallel/test_pipe_parallel_1.py b/tests/e2e/pipelex/pipes/pipe_controller/pipe_parallel/test_pipe_parallel_1.py
index de9934a0b..560a63d53 100644
--- a/tests/e2e/pipelex/pipes/pipe_controller/pipe_parallel/test_pipe_parallel_1.py
+++ b/tests/e2e/pipelex/pipes/pipe_controller/pipe_parallel/test_pipe_parallel_1.py
@@ -10,7 +10,7 @@ from pipelex.core.pipes.pipe_output import PipeOutput
from pipelex.core.pipes.pipe_run_params import PipeRunMode
from pipelex.core.pipes.pipe_run_params_factory import PipeRunParamsFactory
from pipelex.core.stuffs.stuff_factory import StuffFactory
-from pipelex.hub import get_pipe_router, get_report_delegate
+from pipelex.hub import get_pipe_router
from pipelex.pipeline.job_metadata import JobMetadata
from tests.test_pipelines.pipe_controllers.pipe_parallel.pipe_parallel import ContentAnalysis, DocumentInput, LengthAnalysis
@@ -60,7 +60,6 @@ class TestPipeParallelDocumentAnalysis:
# Log output and generate report
pretty_print(pipe_output, title="PipeParallel document analysis test")
- get_report_delegate().generate_report()
# Basic assertions
assert pipe_output is not None
diff --git a/tests/e2e/pipelex/pipes/pipe_controller/pipe_sequence/test_pipe_sequence_batching.py b/tests/e2e/pipelex/pipes/pipe_controller/pipe_sequence/test_pipe_sequence_batching.py
index a51762fcc..6b5ea1fd1 100644
--- a/tests/e2e/pipelex/pipes/pipe_controller/pipe_sequence/test_pipe_sequence_batching.py
+++ b/tests/e2e/pipelex/pipes/pipe_controller/pipe_sequence/test_pipe_sequence_batching.py
@@ -7,7 +7,7 @@ import pytest
from pipelex import log
from pipelex.core.concepts.concept_factory import ConceptFactory
from pipelex.core.memory.working_memory_factory import WorkingMemoryFactory
-from pipelex.core.pipes.pipe_input_spec import TypedNamedInputRequirement
+from pipelex.core.pipes.pipe_input import TypedNamedInputRequirement
from pipelex.core.pipes.pipe_run_params import PipeRunMode
from pipelex.core.pipes.pipe_run_params_factory import PipeRunParamsFactory
from pipelex.core.stuffs.stuff_factory import StuffFactory
diff --git a/tests/e2e/pipelex/pipes/pipe_controller/pipe_sequence/test_pipe_sequence_with_input_memory.py b/tests/e2e/pipelex/pipes/pipe_controller/pipe_sequence/test_pipe_sequence_with_input_memory.py
index f48fec848..7dc3a3df8 100644
--- a/tests/e2e/pipelex/pipes/pipe_controller/pipe_sequence/test_pipe_sequence_with_input_memory.py
+++ b/tests/e2e/pipelex/pipes/pipe_controller/pipe_sequence/test_pipe_sequence_with_input_memory.py
@@ -3,7 +3,6 @@
import pytest
from pipelex import pretty_print
-from pipelex.hub import get_report_delegate
from pipelex.pipeline.execute import execute_pipeline
from tests.test_pipelines.test_tweet import OptimizedTweet
@@ -120,7 +119,6 @@ class TestPipeSequenceWithInputMemory:
# Log output and generate report
pretty_print(pipe_output, title="Pipe output for optimize_tweet_sequence")
- get_report_delegate().generate_report()
# Verify the optimized tweet is different from the draft
assert optimized_tweet.text != SAMPLE_DRAFT_TWEET
diff --git a/tests/integration/pipelex/client/test_client.py b/tests/integration/pipelex/client/test_client.py
index 214af7f7f..62777912c 100644
--- a/tests/integration/pipelex/client/test_client.py
+++ b/tests/integration/pipelex/client/test_client.py
@@ -141,7 +141,7 @@ class TestPipelexApiClient:
assert isinstance(item["justification"], str)
# Verify text structure
- assert working_memory["text"]["concept_code"] == f"{SpecialDomain.NATIVE.value}.{NativeConceptEnum.TEXT.value}"
+ assert working_memory["text"]["concept_code"] == f"{SpecialDomain.NATIVE}.{NativeConceptEnum.TEXT}"
assert "content" in working_memory["text"]
assert isinstance(working_memory["text"]["content"], str)
assert "The Dawn of Ultra-Rapid Transit" in working_memory["text"]["content"]
diff --git a/tests/integration/pipelex/cogt/test_content_generator.py b/tests/integration/pipelex/cogt/test_content_generator.py
index da3a313bf..658fb5235 100644
--- a/tests/integration/pipelex/cogt/test_content_generator.py
+++ b/tests/integration/pipelex/cogt/test_content_generator.py
@@ -6,14 +6,13 @@ from pytest import FixtureRequest
from pipelex import pretty_print
from pipelex.cogt.exceptions import LLMHandleNotFoundError
from pipelex.cogt.image.generated_image import GeneratedImage
-from pipelex.cogt.imgg.imgg_handle import ImggHandle
-from pipelex.cogt.imgg.imgg_prompt import ImggPrompt
+from pipelex.cogt.img_gen.img_gen_prompt import ImgGenPrompt
from pipelex.cogt.llm.llm_prompt import LLMPrompt
from pipelex.cogt.llm.llm_setting import LLMSetting
from pipelex.cogt.ocr.ocr_input import OcrInput
from pipelex.cogt.ocr.ocr_job_components import OcrJobConfig, OcrJobParams
from pipelex.cogt.ocr.ocr_output import OcrOutput
-from pipelex.hub import get_content_generator, get_models_manager
+from pipelex.hub import get_content_generator, get_model_deck
from pipelex.pipeline.job_metadata import JobMetadata
from tests.cases import ImageTestCases, PDFTestCases
from tests.integration.pipelex.cogt.test_data import Employee
@@ -60,7 +59,7 @@ class TestContentGenerator:
@pytest.mark.llm
@pytest.mark.inference
async def test_make_llm_text_only(self, request: FixtureRequest):
- llm_setting_main = get_models_manager().get_model_deck().get_llm_setting(llm_setting_or_preset_id="llm_for_testing_gen_text")
+ llm_setting_main = get_model_deck().get_llm_setting(llm_choice="llm_for_testing_gen_text")
text: str = await get_content_generator().make_llm_text(
job_metadata=JobMetadata(job_name=request.node.originalname), # type: ignore
@@ -74,7 +73,7 @@ class TestContentGenerator:
@pytest.mark.llm
@pytest.mark.inference
async def test_make_object_direct(self, request: FixtureRequest):
- llm_setting_for_object = get_models_manager().get_model_deck().get_llm_setting(llm_setting_or_preset_id="llm_for_testing_gen_object")
+ llm_setting_for_object = get_model_deck().get_llm_setting(llm_choice="llm_for_testing_gen_object")
person_direct: Employee = await get_content_generator().make_object_direct(
job_metadata=JobMetadata(job_name=request.node.originalname), # type: ignore
@@ -89,7 +88,7 @@ class TestContentGenerator:
@pytest.mark.llm
@pytest.mark.inference
async def test_make_object_list_direct(self, request: FixtureRequest):
- llm_setting_for_object = get_models_manager().get_model_deck().get_llm_setting(llm_setting_or_preset_id="llm_for_testing_gen_object")
+ llm_setting_for_object = get_model_deck().get_llm_setting(llm_choice="llm_for_testing_gen_object")
person_list_direct: List[Employee] = await get_content_generator().make_object_list_direct(
job_metadata=JobMetadata(job_name=request.node.originalname), # type: ignore
@@ -102,17 +101,19 @@ class TestContentGenerator:
assert isinstance(person_list_direct, list)
assert all(isinstance(person, Employee) for person in person_list_direct)
- @pytest.mark.imgg
+ @pytest.mark.img_gen
@pytest.mark.inference
async def test_make_image(self, request: FixtureRequest):
+ img_gen_handle = "fast-lightning-sdxl"
+ positive_text = "A dog with sunglasses coding on a laptop"
image: GeneratedImage = await get_content_generator().make_single_image(
job_metadata=JobMetadata(job_name=request.node.originalname), # type: ignore
- imgg_handle=ImggHandle.SDXL_LIGHTNING,
- imgg_prompt=ImggPrompt(
- positive_text="A dog with sunglasses coding on a laptop",
+ img_gen_handle=img_gen_handle,
+ img_gen_prompt=ImgGenPrompt(
+ positive_text=positive_text,
),
)
- pretty_print(image, title="make_image")
+ pretty_print(image, title=f"Image generated by '{img_gen_handle}' for '{positive_text}'")
assert isinstance(image, GeneratedImage)
async def test_make_jinja2_text(self, request: FixtureRequest):
@@ -166,4 +167,4 @@ class TestContentGenerator:
)
error = excinfo.value
pretty_print(f"Caught expected error: {error}")
- assert str(error).startswith("LLM Engine blueprint for llm_handle 'bad_handle_to_test_failure' not found")
+ assert str(error).startswith("Model handle 'bad_handle_to_test_failure' not found")
diff --git a/tests/integration/pipelex/cogt/test_data.py b/tests/integration/pipelex/cogt/test_data.py
index fa109cf77..279051250 100644
--- a/tests/integration/pipelex/cogt/test_data.py
+++ b/tests/integration/pipelex/cogt/test_data.py
@@ -63,24 +63,17 @@ class LLMVisionTestCases:
("AI Lympics PNG", PATH_IMG_PNG_1),
("AI Lympics JPEG", PATH_IMG_JPEG_1),
("Gantt Chart", PATH_IMG_GANTT_1),
+ ("Eiffel Tower", PATH_IMG_JPEG_3),
]
IMAGE_PATH_PAIRS: ClassVar[List[Tuple[str, Tuple[str, str]]]] = [ # topic, image_pair
("AI Lympics PNG", (PATH_IMG_PNG_1, PATH_IMG_PNG_2)),
]
- IMAGES_MIXED_SOURCES: ClassVar[List[Tuple[str, str]]] = [ # topic, image_uri
+ IMAGE_URLS: ClassVar[List[Tuple[str, str]]] = [ # topic, image_uri
(
"Alan Turing",
URL_WIKIPEDIA_ALAN_TURING,
),
- (
- "AI Lympics",
- PATH_IMG_PNG_1,
- ),
- (
- "Eiffel Tower",
- PATH_IMG_JPEG_3,
- ),
(
"Gantt chart",
PipeTestCases.URL_IMG_GANTT_1,
diff --git a/tests/integration/pipelex/cogt/test_image_gen.py b/tests/integration/pipelex/cogt/test_image_gen.py
deleted file mode 100644
index c7f408cf1..000000000
--- a/tests/integration/pipelex/cogt/test_image_gen.py
+++ /dev/null
@@ -1,22 +0,0 @@
-import pytest
-
-from pipelex import pretty_print
-from pipelex.cogt.imgg.imgg_job_factory import ImggJobFactory
-from pipelex.hub import get_imgg_worker
-from tests.integration.pipelex.test_data import IMGGTestCases
-
-
[email protected](loop_scope="class")
-class TestImageGeneration:
- @pytest.mark.parametrize("topic, imgg_prompt_text", IMGGTestCases.IMAGE_DESC)
- async def test_imgg_using_handle(self, imgg_handle: str, topic: str, imgg_prompt_text: str):
- imgg_worker_async = get_imgg_worker(imgg_handle=imgg_handle)
- imgg_job = ImggJobFactory.make_imgg_job_from_prompt_contents(
- positive_text=imgg_prompt_text,
- )
- generated_image = await imgg_worker_async.gen_image(
- imgg_job=imgg_job,
- )
- pretty_print(generated_image, title=f"Generated Image, topic={topic}, imgg_handle={imgg_handle}")
diff --git a/tests/integration/pipelex/cogt/test_img_gen.py b/tests/integration/pipelex/cogt/test_img_gen.py
new file mode 100644
index 000000000..8d2182a80
--- /dev/null
+++ b/tests/integration/pipelex/cogt/test_img_gen.py
@@ -0,0 +1,22 @@
+import pytest
+
+from pipelex import pretty_print
+from pipelex.cogt.img_gen.img_gen_job_factory import ImgGenJobFactory
+from pipelex.hub import get_img_gen_worker
+from tests.integration.pipelex.test_data import ImageGenTestCases
+
+
[email protected](loop_scope="class")
+class TestImageGeneration:
+ @pytest.mark.parametrize("topic, img_gen_prompt_text", ImageGenTestCases.IMAGE_DESC)
+ async def test_img_gen_using_handle(self, img_gen_handle: str, topic: str, img_gen_prompt_text: str):
+ img_gen_worker_async = get_img_gen_worker(img_gen_handle=img_gen_handle)
+ img_gen_job = ImgGenJobFactory.make_img_gen_job_from_prompt_contents(
+ positive_text=img_gen_prompt_text,
+ )
+ generated_image = await img_gen_worker_async.gen_image(
+ img_gen_job=img_gen_job,
+ )
+ pretty_print(generated_image, title=f"Image generated by '{img_gen_handle}' for topic '{topic}'")
diff --git a/tests/integration/pipelex/cogt/test_llm_gen_object.py b/tests/integration/pipelex/cogt/test_llm_gen_object.py
index d25125b5b..3a609494b 100644
--- a/tests/integration/pipelex/cogt/test_llm_gen_object.py
+++ b/tests/integration/pipelex/cogt/test_llm_gen_object.py
@@ -7,12 +7,12 @@ from pydantic import BaseModel
from pipelex import pretty_print
from pipelex.cogt.llm.llm_job_components import LLMJobParams
from pipelex.cogt.llm.llm_job_factory import LLMJobFactory
-from pipelex.hub import get_llm_worker, get_models_manager
+from pipelex.hub import get_llm_worker, get_model_deck
from tests.integration.pipelex.cogt.test_data import LLMTestCases
def get_async_worker_and_job(llm_preset_id: str, user_text: str):
- llm_setting = get_models_manager().get_model_deck().get_llm_setting(llm_setting_or_preset_id=llm_preset_id)
+ llm_setting = get_model_deck().get_llm_setting(llm_choice=llm_preset_id)
pretty_print(llm_setting, title=llm_preset_id)
pretty_print(user_text)
llm_worker = get_llm_worker(llm_handle=llm_setting.llm_handle)
diff --git a/tests/integration/pipelex/cogt/test_llm_gen_text.py b/tests/integration/pipelex/cogt/test_llm_gen_text.py
index 78dbf5dc4..b340b04b8 100644
--- a/tests/integration/pipelex/cogt/test_llm_gen_text.py
+++ b/tests/integration/pipelex/cogt/test_llm_gen_text.py
@@ -10,12 +10,12 @@ from pipelex.cogt.llm.llm_job_factory import LLMJobFactory
from pipelex.cogt.llm.llm_worker_abstract import LLMWorkerAbstract
from pipelex.cogt.llm.llm_worker_internal_abstract import LLMWorkerInternalAbstract
from pipelex.cogt.model_backends.model_constraints import ModelConstraints
-from pipelex.hub import get_llm_worker, get_models_manager, get_report_delegate
+from pipelex.hub import get_llm_worker, get_model_deck
from tests.integration.pipelex.cogt.test_data import LLMTestCases
def get_worker_and_job(llm_preset_id: str, user_text: str) -> Tuple[LLMWorkerAbstract, LLMJob]:
- llm_setting = get_models_manager().get_model_deck().get_llm_setting(llm_setting_or_preset_id=llm_preset_id)
+ llm_setting = get_model_deck().get_llm_setting(llm_choice=llm_preset_id)
pretty_print(llm_setting, title=llm_preset_id)
pretty_print(user_text)
llm_worker = get_llm_worker(llm_handle=llm_setting.llm_handle)
@@ -33,7 +33,7 @@ def get_worker_and_job(llm_preset_id: str, user_text: str) -> Tuple[LLMWorkerAbs
class TestLLMGenText:
@pytest.mark.parametrize("topic, prompt_text", LLMTestCases.SINGLE_TEXT)
async def test_gen_text_using_handle(self, llm_job_params: LLMJobParams, llm_handle: str, topic: str, prompt_text: str):
- pretty_print(prompt_text, title=topic)
+ pretty_print(prompt_text, title=f"Generating text about '{topic}' using '{llm_handle}'")
llm_worker = get_llm_worker(llm_handle=llm_handle)
llm_job = LLMJobFactory.make_llm_job_from_prompt_contents(
user_text=prompt_text,
@@ -42,7 +42,6 @@ class TestLLMGenText:
generated_text = await llm_worker.gen_text(llm_job=llm_job)
assert generated_text
pretty_print(generated_text)
- get_report_delegate().generate_report()
@pytest.mark.parametrize("topic, prompt_text", LLMTestCases.SINGLE_TEXT)
async def test_gen_text_using_llm_preset(self, llm_preset_id: str, topic: str, prompt_text: str):
diff --git a/tests/integration/pipelex/cogt/test_llm_inference.py b/tests/integration/pipelex/cogt/test_llm_inference.py
index 77cf58ff3..ae96d2c91 100644
--- a/tests/integration/pipelex/cogt/test_llm_inference.py
+++ b/tests/integration/pipelex/cogt/test_llm_inference.py
@@ -14,7 +14,7 @@ from tests.integration.pipelex.cogt.test_data import LLMTestConstants, LLMVision
@pytest.mark.asyncio(loop_scope="class")
class TestLLMInference:
async def test_simple_gen_text_from_text(self, llm_job_params: LLMJobParams, llm_handle: str):
- log.info(f"Testing llm_handle '{llm_handle}'")
+ log.info(f"test_simple_gen_text_from_text: Testing llm_handle '{llm_handle}'")
llm_worker = get_inference_manager().get_llm_worker(llm_handle=llm_handle)
log.info(f"Using llm_worker: {llm_worker.desc}")
llm_job = LLMJobFactory.make_llm_job_from_prompt_contents(
@@ -27,7 +27,7 @@ class TestLLMInference:
pretty_print(generated_text)
async def test_simple_gen_object_from_text(self, llm_job_params: LLMJobParams, llm_handle: str):
- log.info(f"Testing llm_handle '{llm_handle}'")
+ log.info(f"test_simple_gen_object_from_text: Testing llm_handle '{llm_handle}'")
llm_worker = get_inference_manager().get_llm_worker(llm_handle=llm_handle)
log.info(f"Using llm_worker: {llm_worker.desc}")
llm_job = LLMJobFactory.make_llm_job_from_prompt_contents(
@@ -43,6 +43,7 @@ class TestLLMInference:
@pytest.mark.parametrize("image_path", [LLMVisionTestCases.PATH_IMG_PNG_1])
async def test_gen_text_from_image(self, llm_job_params: LLMJobParams, llm_handle: str, image_path: str):
+ log.info(f"test_gen_text_from_image: Testing llm_handle '{llm_handle}'")
prompt_image = PromptImagePath(file_path=image_path)
llm_worker = get_inference_manager().get_llm_worker(llm_handle=llm_handle)
llm_job = LLMJobFactory.make_llm_job_from_prompt_contents(
diff --git a/tests/integration/pipelex/cogt/test_llm_report.py b/tests/integration/pipelex/cogt/test_llm_report.py
index da319b870..96807b22c 100644
--- a/tests/integration/pipelex/cogt/test_llm_report.py
+++ b/tests/integration/pipelex/cogt/test_llm_report.py
@@ -5,7 +5,7 @@ import pytest
from pipelex import pretty_print
from pipelex.cogt.llm.llm_job_factory import LLMJobFactory
-from pipelex.hub import get_llm_worker, get_models_manager, get_report_delegate
+from pipelex.hub import get_llm_worker, get_model_deck, get_report_delegate
from tests.integration.pipelex.cogt.test_data import LLMTestCases
@@ -24,10 +24,9 @@ class TestLLMReport:
generated_text = await llm_worker.gen_text(llm_job=llm_job)
assert generated_text
pretty_print(generated_text)
- get_report_delegate().generate_report()
def _get_async_worker_and_job(self, llm_preset_id: str, prompt_text: str):
- llm_setting = get_models_manager().get_model_deck().get_llm_setting(llm_setting_or_preset_id=llm_preset_id)
+ llm_setting = get_model_deck().get_llm_setting(llm_choice=llm_preset_id)
pretty_print(llm_setting, title=llm_preset_id)
pretty_print(prompt_text)
llm_worker = get_llm_worker(llm_handle=llm_setting.llm_handle)
@@ -58,5 +57,3 @@ class TestLLMReport:
tasks.append(task)
generated_texts = await asyncio.gather(*tasks)
pretty_print(generated_texts)
-
- get_report_delegate().generate_report()
diff --git a/tests/integration/pipelex/cogt/test_llm_vision.py b/tests/integration/pipelex/cogt/test_llm_vision.py
index 145adcd6b..da74826aa 100644
--- a/tests/integration/pipelex/cogt/test_llm_vision.py
+++ b/tests/integration/pipelex/cogt/test_llm_vision.py
@@ -4,7 +4,7 @@ import pytest
from pipelex import log, pretty_print
from pipelex.cogt.exceptions import LLMCapabilityError, PromptImageFormatError
-from pipelex.cogt.image.prompt_image import PromptImageBytes, PromptImagePath
+from pipelex.cogt.image.prompt_image import PromptImageBase64, PromptImagePath
from pipelex.cogt.image.prompt_image_factory import PromptImageFactory
from pipelex.cogt.llm.llm_job import LLMJobParams
from pipelex.cogt.llm.llm_job_factory import LLMJobFactory
@@ -17,7 +17,7 @@ from tests.integration.pipelex.cogt.test_data import LLMVisionTestCases
@pytest.mark.inference
@pytest.mark.asyncio(loop_scope="class")
class TestLLMVision:
- @pytest.mark.parametrize("topic, image_uri", LLMVisionTestCases.IMAGES_MIXED_SOURCES)
+ @pytest.mark.parametrize("topic, image_uri", LLMVisionTestCases.IMAGE_URLS)
async def test_gen_text_from_vision_by_url(self, llm_handle_for_vision: str, topic: str, image_uri: str):
prompt_image = PromptImageFactory.make_prompt_image_from_uri(uri=image_uri)
llm_worker = get_llm_worker(llm_handle=llm_handle_for_vision)
@@ -39,7 +39,7 @@ class TestLLMVision:
@pytest.mark.parametrize("topic, image_path", LLMVisionTestCases.IMAGE_PATHS)
async def test_gen_text_from_vision_by_bytes(self, llm_handle_for_vision: str, topic: str, image_path: str):
image_bytes = load_binary_as_base64(path=image_path)
- prompt_image = PromptImageBytes(base_64=image_bytes)
+ prompt_image = PromptImageBase64(base_64=image_bytes)
llm_worker = get_llm_worker(llm_handle=llm_handle_for_vision)
llm_job = LLMJobFactory.make_llm_job_from_prompt_contents(
user_text=LLMVisionTestCases.VISION_USER_TEXT_2,
diff --git a/tests/integration/pipelex/cogt/test_ocr.py b/tests/integration/pipelex/cogt/test_ocr.py
index 22464ff60..3818558ff 100644
--- a/tests/integration/pipelex/cogt/test_ocr.py
+++ b/tests/integration/pipelex/cogt/test_ocr.py
@@ -63,6 +63,8 @@ class TestOcr:
should_caption_images=False,
should_include_page_views=False,
page_views_dpi=72,
+ max_nb_images=None,
+ image_min_size=None,
)
ocr_job = OcrJobFactory.make_ocr_job(
ocr_input=OcrInput(pdf_uri=file_path),
diff --git a/tests/integration/pipelex/cogt/test_setup_inference_workers.py b/tests/integration/pipelex/cogt/test_setup_inference_workers.py
index 54cb856b3..697dc1731 100644
--- a/tests/integration/pipelex/cogt/test_setup_inference_workers.py
+++ b/tests/integration/pipelex/cogt/test_setup_inference_workers.py
@@ -1,20 +1,12 @@
-import pytest
-
-from pipelex import log
from pipelex.cogt.llm.llm_worker_internal_abstract import LLMWorkerInternalAbstract
from pipelex.cogt.model_backends.model_type import ModelType
-from pipelex.config import get_config
-from pipelex.hub import get_inference_manager, get_models_manager
+from pipelex.hub import get_inference_manager, get_model_deck
class TestSetupInferenceWorkers:
def test_setup_inference_workers(self):
inference_manager = get_inference_manager()
- inference_models = get_models_manager().get_model_deck().inference_models
- log.verbose(f"{len(inference_models)} LLM engine_cards found")
- for model_handle, inference_model in inference_models.items():
+ for model_handle, inference_model in get_model_deck().inference_models.items():
match inference_model.model_type:
case ModelType.LLM:
llm_worker = inference_manager.get_llm_worker(llm_handle=model_handle)
@@ -22,13 +14,5 @@ class TestSetupInferenceWorkers:
assert inference_model == llm_worker.inference_model
case ModelType.TEXT_EXTRACTOR:
_ = inference_manager.get_ocr_worker(model_handle=model_handle)
-
- def test_setup_imgg_workers(self):
- inference_manager = get_inference_manager()
- imgg_handles = get_config().cogt.imgg_config.imgg_handles
- log.verbose(f"{len(imgg_handles)} Imgg handles found")
- for imgg_handle in imgg_handles:
- imgg_worker = inference_manager.get_imgg_worker(imgg_handle=imgg_handle)
- assert imgg_worker is not None
- assert imgg_worker.imgg_engine is not None
- log.debug("Done setting up Imgg Workers (async)")
+ case ModelType.IMG_GEN:
+ _ = inference_manager.get_img_gen_worker(img_gen_handle=model_handle)
diff --git a/tests/integration/pipelex/conftest.py b/tests/integration/pipelex/conftest.py
index 0bda0d816..f284d92d7 100644
--- a/tests/integration/pipelex/conftest.py
+++ b/tests/integration/pipelex/conftest.py
@@ -2,7 +2,6 @@ from typing import Optional
import pytest
-from pipelex.cogt.imgg.imgg_handle import ImggHandle
from pipelex.cogt.llm.llm_job_components import LLMJobParams
from pipelex.plugins.plugin_sdk_registry import Plugin
@@ -19,7 +18,6 @@ def llm_preset_id(request: pytest.FixtureRequest) -> str:
return request.param
-# TODO: make it efficient to also test multiple platforms like openai/azure and mistral/anthropic/bedrock
@pytest.fixture(
params=[
# "o1",
@@ -30,24 +28,16 @@ def llm_preset_id(request: pytest.FixtureRequest) -> str:
# "o3-mini",
# "gpt-5-mini", # TODO: fix this
# "gpt-5-nano", # TODO: fix this
- # "best-gpt",
# "claude-3-haiku",
# "claude-3-5-sonnet",
# "claude-3-7-sonnet",
- # "best-claude",
# "mistral-large",
# "ministral-3b",
# "ministral-8b",
- "mistral-medium",
- "mistral-medium-2508",
+ # "mistral-medium",
+ # "mistral-medium-2508",
# "pixtral-12b",
# "pixtral-large",
- # "gemini-1-5-pro",
- # "gemini-1-5-flash",
- # "gemini-2-flash",
- # "gemini-2-pro",
- # "gemini-2-5-flash",
- # "gemini-2-5-pro",
# "bedrock-mistral-large",
# "bedrock-claude-3-7-sonnet",
# "bedrock-meta-llama-3-3-70b-instruct",
@@ -57,7 +47,6 @@ def llm_preset_id(request: pytest.FixtureRequest) -> str:
# "pipelex/claude-3.7-sonnet",
# "pipelex/gemini-2.0-flash-vertex",
# "pipelex/gemini-2.0-flash",
- # "best-claude",
# "llm_to_engineer",
# "gpt-5-nano",
# "gpt-4o-mini",
@@ -68,11 +57,12 @@ def llm_preset_id(request: pytest.FixtureRequest) -> str:
# "claude-3.5-sonnet",
# "claude-3.5-sonnet-v2"
# "claude-3.7-sonnet",
- # "gemini-2.5-flash-lite",
- # "gemini-2.5-flash",
- # "gemini-2.5-pro",
# "grok-3",
# "grok-3-mini",
+ # "base-claude",
+ "gemini-2.5-flash-lite",
+ # "gemini-2.5-flash",
+ # "gemini-2.5-pro",
]
)
def llm_handle(request: pytest.FixtureRequest) -> str:
@@ -99,8 +89,11 @@ def llm_handle(request: pytest.FixtureRequest) -> str:
# "gemini-2.5-pro",
# "gemini-2.5-flash",
# "mistral-small3.1",
- "mistral-medium",
- "mistral-medium-2508",
+ # "mistral-medium",
+ # "mistral-medium-2508",
+ "gemini-2.5-flash-lite",
+ # "gemini-2.5-flash",
+ # "gemini-2.5-pro",
# "qwen3:8b",
]
)
@@ -206,15 +199,15 @@ def llm_job_params(request: pytest.FixtureRequest) -> LLMJobParams:
@pytest.fixture(
params=[
- # ImggHandle.FLUX_1_PRO_LEGACY,
- # ImggHandle.FLUX_1_1_PRO,
- # ImggHandle.FLUX_1_1_ULTRA,
- ImggHandle.SDXL_LIGHTNING,
- # ImggHandle.OPENAI_GPT_IMAGE_1,
+ # "flux-pro",
+ "flux-pro/v1.1",
+ # "flux-pro/v1.1-ultra",
+ "fast-lightning-sdxl",
+ "gpt-image-1",
]
)
-def imgg_handle(request: pytest.FixtureRequest) -> ImggHandle:
- assert isinstance(request.param, ImggHandle)
+def img_gen_handle(request: pytest.FixtureRequest) -> str:
+ assert isinstance(request.param, str)
return request.param
@@ -237,3 +230,23 @@ def ocr_handle(request: pytest.FixtureRequest) -> str:
def ocr_handle_from_image(request: pytest.FixtureRequest) -> str:
assert isinstance(request.param, str)
return request.param
+
+
+ params=[
+ "base_ocr_mistral",
+ ]
+)
+def ocr_choice_for_pdf(request: pytest.FixtureRequest) -> str:
+ assert isinstance(request.param, str)
+ return request.param
+
+
+ params=[
+ "base_ocr_mistral",
+ ]
+)
+def ocr_choice_for_image(request: pytest.FixtureRequest) -> str:
+ assert isinstance(request.param, str)
+ return request.param
diff --git a/tests/integration/pipelex/core/memory/test_working_memory_factory.py b/tests/integration/pipelex/core/memory/test_working_memory_factory.py
index e01cb59cf..95ccdff34 100644
--- a/tests/integration/pipelex/core/memory/test_working_memory_factory.py
+++ b/tests/integration/pipelex/core/memory/test_working_memory_factory.py
@@ -9,7 +9,7 @@ class TestWorkingMemoryFactory:
"""Test deserialization of compact memory with text content."""
compact_memory: CompactMemory = {
"text_item": {
- "concept_code": NativeConceptEnum.TEXT.value,
+ "concept_code": NativeConceptEnum.TEXT,
"content": "Hello, world!",
}
}
@@ -20,7 +20,7 @@ class TestWorkingMemoryFactory:
assert "text_item" in working_memory.root
stuff = working_memory.root["text_item"]
- assert stuff.concept.code == NativeConceptEnum.TEXT.value
+ assert stuff.concept.code == NativeConceptEnum.TEXT
assert isinstance(stuff.content, TextContent)
assert stuff.content.text == "Hello, world!"
@@ -28,7 +28,7 @@ class TestWorkingMemoryFactory:
"""Test deserialization of compact memory with complex nested structured content."""
compact_memory: CompactMemory = {
"complex_page": {
- "concept_code": NativeConceptEnum.PAGE.value,
+ "concept_code": NativeConceptEnum.PAGE,
"content": {
"text_and_images": {
"text": {
@@ -60,7 +60,7 @@ class TestWorkingMemoryFactory:
assert "complex_page" in working_memory.root
stuff = working_memory.root["complex_page"]
- assert stuff.concept.code == NativeConceptEnum.PAGE.value
+ assert stuff.concept.code == NativeConceptEnum.PAGE
assert isinstance(stuff.content, PageContent)
# Verify text_and_images structure
@@ -118,11 +118,11 @@ class TestWorkingMemoryFactory:
"""Test deserialization of compact memory with multiple items."""
compact_memory: CompactMemory = {
"text1": {
- "concept_code": NativeConceptEnum.TEXT.value,
+ "concept_code": NativeConceptEnum.TEXT,
"content": "First text",
},
"text2": {
- "concept_code": NativeConceptEnum.TEXT.value,
+ "concept_code": NativeConceptEnum.TEXT,
"content": "Second text",
},
}
diff --git a/tests/integration/pipelex/pipes/controller/pipe_batch/test_pipe_batch_simple.py b/tests/integration/pipelex/pipes/controller/pipe_batch/test_pipe_batch_simple.py
index af4eda584..bd5c25b78 100644
--- a/tests/integration/pipelex/pipes/controller/pipe_batch/test_pipe_batch_simple.py
+++ b/tests/integration/pipelex/pipes/controller/pipe_batch/test_pipe_batch_simple.py
@@ -9,7 +9,7 @@ from pipelex import pretty_print
from pipelex.core.concepts.concept_factory import ConceptBlueprint, ConceptFactory
from pipelex.core.concepts.concept_native import NATIVE_CONCEPTS_DATA, NativeConceptEnum
from pipelex.core.memory.working_memory_factory import WorkingMemoryFactory
-from pipelex.core.pipes.pipe_input_spec_blueprint import InputRequirementBlueprint
+from pipelex.core.pipes.pipe_input_blueprint import InputRequirementBlueprint
from pipelex.core.pipes.pipe_run_params import PipeRunMode
from pipelex.core.pipes.pipe_run_params_factory import PipeRunParamsFactory
from pipelex.core.stuffs.stuff_content import ListContent, StuffContent, TextContent
diff --git a/tests/integration/pipelex/pipes/controller/pipe_condition/test_pipe_condition_simple.py b/tests/integration/pipelex/pipes/controller/pipe_condition/test_pipe_condition_simple.py
index 32088a1c7..56b95777a 100644
--- a/tests/integration/pipelex/pipes/controller/pipe_condition/test_pipe_condition_simple.py
+++ b/tests/integration/pipelex/pipes/controller/pipe_condition/test_pipe_condition_simple.py
@@ -10,7 +10,7 @@ from pipelex.core.concepts.concept_factory import ConceptFactory
from pipelex.core.concepts.concept_native import NATIVE_CONCEPTS_DATA, NativeConceptEnum
from pipelex.core.domains.domain import SpecialDomain
from pipelex.core.memory.working_memory_factory import WorkingMemoryFactory
-from pipelex.core.pipes.pipe_input_spec_blueprint import InputRequirementBlueprint
+from pipelex.core.pipes.pipe_input_blueprint import InputRequirementBlueprint
from pipelex.core.pipes.pipe_output import PipeOutput
from pipelex.core.pipes.pipe_run_params import PipeRunMode
from pipelex.core.pipes.pipe_run_params_factory import PipeRunParamsFactory
@@ -34,8 +34,8 @@ class TestPipeConditionSimple:
"""Test PipeCondition with long text that should trigger capitalize_long_text pipe."""
pipe_condition_blueprint = PipeConditionBlueprint(
definition="Text length condition for testing",
- inputs={"input_text": InputRequirementBlueprint(concept=f"{SpecialDomain.NATIVE.value}.{NativeConceptEnum.TEXT.value}")},
- output=f"{SpecialDomain.NATIVE.value}.{NativeConceptEnum.TEXT.value}",
+ inputs={"input_text": InputRequirementBlueprint(concept=f"{SpecialDomain.NATIVE}.{NativeConceptEnum.TEXT}")},
+ output=f"{SpecialDomain.NATIVE}.{NativeConceptEnum.TEXT}",
expression_template="{% if input_text.text|length > 5 %}long{% else %}short{% endif %}",
pipe_map=PipeConditionPipeMapBlueprint(root={"long": "capitalize_long_text", "short": "add_prefix_short_text"}),
)
@@ -99,18 +99,15 @@ class TestPipeConditionSimple:
assert isinstance(final_result_in_memory.content, TextContent)
if pipe_run_mode != PipeRunMode.DRY:
assert final_result_in_memory.content.text == "LONG: HELLO WORLD"
- assert (
- f"{final_result_in_memory.concept.domain}.{final_result_in_memory.concept.code}"
- == f"{SpecialDomain.NATIVE.value}.{NativeConceptEnum.TEXT.value}"
- )
+ assert f"{final_result_in_memory.concept.domain}.{final_result_in_memory.concept.code}" == f"{SpecialDomain.NATIVE}.{NativeConceptEnum.TEXT}"
async def test_condition_short_text_processing(self, request: FixtureRequest, pipe_run_mode: PipeRunMode):
"""Test PipeCondition with short text that should trigger add_prefix_short_text pipe."""
# Create PipeCondition instance - pipes are loaded from PLX files
pipe_condition_blueprint = PipeConditionBlueprint(
definition="Text length condition for short text testing",
- inputs={"input_text": InputRequirementBlueprint(concept=f"{SpecialDomain.NATIVE.value}.{NativeConceptEnum.TEXT.value}")},
- output=f"{SpecialDomain.NATIVE.value}.{NativeConceptEnum.TEXT.value}",
+ inputs={"input_text": InputRequirementBlueprint(concept=f"{SpecialDomain.NATIVE}.{NativeConceptEnum.TEXT}")},
+ output=f"{SpecialDomain.NATIVE}.{NativeConceptEnum.TEXT}",
expression_template="{% if input_text.text|length > 5 %}long{% else %}short{% endif %}",
pipe_map=PipeConditionPipeMapBlueprint(root={"long": "capitalize_long_text", "short": "add_prefix_short_text"}),
)
@@ -171,10 +168,7 @@ class TestPipeConditionSimple:
assert isinstance(final_result_in_memory.content, TextContent)
if pipe_run_mode != PipeRunMode.DRY:
assert final_result_in_memory.content.text == "SHORT: hi"
- assert (
- f"{final_result_in_memory.concept.domain}.{final_result_in_memory.concept.code}"
- == f"{SpecialDomain.NATIVE.value}.{NativeConceptEnum.TEXT.value}"
- )
+ assert f"{final_result_in_memory.concept.domain}.{final_result_in_memory.concept.code}" == f"{SpecialDomain.NATIVE}.{NativeConceptEnum.TEXT}"
async def test_condition_dry_run_success(self, request: FixtureRequest):
"""Test PipeCondition dry run with valid inputs using real pipe - should succeed."""
diff --git a/tests/integration/pipelex/pipes/controller/pipe_condition/test_pipe_condition_validation.py b/tests/integration/pipelex/pipes/controller/pipe_condition/test_pipe_condition_validation.py
index f15bcfa78..cd6e2d004 100644
--- a/tests/integration/pipelex/pipes/controller/pipe_condition/test_pipe_condition_validation.py
+++ b/tests/integration/pipelex/pipes/controller/pipe_condition/test_pipe_condition_validation.py
@@ -1,5 +1,5 @@
from pipelex.core.concepts.concept_factory import ConceptBlueprint, ConceptFactory
-from pipelex.core.pipes.pipe_input_spec_blueprint import InputRequirementBlueprint
+from pipelex.core.pipes.pipe_input_blueprint import InputRequirementBlueprint
from pipelex.hub import get_concept_provider
from pipelex.pipe_controllers.condition.pipe_condition_blueprint import PipeConditionBlueprint, PipeConditionPipeMapBlueprint
from pipelex.pipe_controllers.condition.pipe_condition_factory import PipeConditionFactory
diff --git a/tests/integration/pipelex/pipes/controller/pipe_parallel/test_pipe_parallel_simple.py b/tests/integration/pipelex/pipes/controller/pipe_parallel/test_pipe_parallel_simple.py
index 953330765..4529ab07b 100644
--- a/tests/integration/pipelex/pipes/controller/pipe_parallel/test_pipe_parallel_simple.py
+++ b/tests/integration/pipelex/pipes/controller/pipe_parallel/test_pipe_parallel_simple.py
@@ -1,5 +1,3 @@
-"""Simple integration test for PipeParallel controller."""
-
from typing import cast
import pytest
@@ -9,7 +7,7 @@ from pipelex.core.concepts.concept_factory import ConceptFactory
from pipelex.core.concepts.concept_native import NATIVE_CONCEPTS_DATA, NativeConceptEnum
from pipelex.core.domains.domain import SpecialDomain
from pipelex.core.memory.working_memory_factory import WorkingMemoryFactory
-from pipelex.core.pipes.pipe_input_spec_blueprint import InputRequirementBlueprint
+from pipelex.core.pipes.pipe_input_blueprint import InputRequirementBlueprint
from pipelex.core.pipes.pipe_run_params import PipeRunMode
from pipelex.core.pipes.pipe_run_params_factory import PipeRunParamsFactory
from pipelex.core.stuffs.stuff_content import TextContent
@@ -31,8 +29,8 @@ class TestPipeParallelSimple:
# Create PipeParallel instance - pipes are loaded from PLX files
pipe_parallel_blueprint = PipeParallelBlueprint(
definition="Parallel text analysis pipeline",
- inputs={"input_text": InputRequirementBlueprint(concept=f"{SpecialDomain.NATIVE.value}.{NativeConceptEnum.TEXT.value}")},
- output=f"{SpecialDomain.NATIVE.value}.{NativeConceptEnum.TEXT.value}",
+ inputs={"input_text": InputRequirementBlueprint(concept=f"{SpecialDomain.NATIVE}.{NativeConceptEnum.TEXT}")},
+ output=f"{SpecialDomain.NATIVE}.{NativeConceptEnum.TEXT}",
parallels=[
SubPipeBlueprint(pipe="analyze_sentiment", result="sentiment_result"),
SubPipeBlueprint(pipe="count_words", result="word_count_result"),
@@ -108,7 +106,7 @@ class TestPipeParallelSimple:
# Should return one of: positive, negative, neutral
if pipe_run_mode != PipeRunMode.DRY:
assert sentiment_result.content.text.lower() in ["positive", "negative", "neutral"]
- assert f"{sentiment_result.concept.domain}.{sentiment_result.concept.code}" == f"{SpecialDomain.NATIVE.value}.{NativeConceptEnum.TEXT.value}"
+ assert f"{sentiment_result.concept.domain}.{sentiment_result.concept.code}" == f"{SpecialDomain.NATIVE}.{NativeConceptEnum.TEXT}"
# Verify word count result
word_count_result = final_working_memory.get_stuff("word_count_result")
@@ -147,8 +145,8 @@ class TestPipeParallelSimple:
# Create PipeParallel instance
pipe_parallel_blueprint = PipeParallelBlueprint(
definition="Parallel text analysis pipeline for short text",
- inputs={"input_text": InputRequirementBlueprint(concept=f"{SpecialDomain.NATIVE.value}.{NativeConceptEnum.TEXT.value}")},
- output=f"{SpecialDomain.NATIVE.value}.{NativeConceptEnum.TEXT.value}",
+ inputs={"input_text": InputRequirementBlueprint(concept=f"{SpecialDomain.NATIVE}.{NativeConceptEnum.TEXT}")},
+ output=f"{SpecialDomain.NATIVE}.{NativeConceptEnum.TEXT}",
parallels=[
SubPipeBlueprint(pipe="analyze_sentiment", result="sentiment_result"),
SubPipeBlueprint(pipe="count_words", result="word_count_result"),
@@ -163,7 +161,6 @@ class TestPipeParallelSimple:
pipe_code="parallel_text_analyzer",
blueprint=pipe_parallel_blueprint,
)
-
# Create test data - shorter text
input_text_stuff = StuffFactory.make_stuff(
concept=ConceptFactory.make_native_concept(native_concept_data=NATIVE_CONCEPTS_DATA[NativeConceptEnum.TEXT]),
diff --git a/tests/integration/pipelex/pipes/controller/pipe_parallel/test_pipe_parallel_validation.py b/tests/integration/pipelex/pipes/controller/pipe_parallel/test_pipe_parallel_validation.py
index 816d2a7b0..ed89f5364 100644
--- a/tests/integration/pipelex/pipes/controller/pipe_parallel/test_pipe_parallel_validation.py
+++ b/tests/integration/pipelex/pipes/controller/pipe_parallel/test_pipe_parallel_validation.py
@@ -1,7 +1,7 @@
from pipelex.core.concepts.concept_blueprint import ConceptBlueprint
from pipelex.core.concepts.concept_factory import ConceptFactory
-from pipelex.core.pipes.pipe_input_spec import PipeInputSpec
-from pipelex.core.pipes.pipe_input_spec_blueprint import InputRequirementBlueprint
+from pipelex.core.pipes.pipe_input import PipeInputSpec
+from pipelex.core.pipes.pipe_input_blueprint import InputRequirementBlueprint
from pipelex.hub import get_concept_provider, get_pipe_provider
from pipelex.pipe_controllers.parallel.pipe_parallel_blueprint import PipeParallelBlueprint
from pipelex.pipe_controllers.parallel.pipe_parallel_factory import PipeParallelFactory
diff --git a/tests/integration/pipelex/pipes/controller/pipe_sequence/test_pipe_sequence_dry_run.py b/tests/integration/pipelex/pipes/controller/pipe_sequence/test_pipe_sequence_dry_run.py
index ff9d74352..d11958273 100644
--- a/tests/integration/pipelex/pipes/controller/pipe_sequence/test_pipe_sequence_dry_run.py
+++ b/tests/integration/pipelex/pipes/controller/pipe_sequence/test_pipe_sequence_dry_run.py
@@ -13,7 +13,7 @@ from pipelex.core.pipes.pipe_run_params_factory import PipeRunParamsFactory
from pipelex.core.stuffs.stuff import Stuff
from pipelex.core.stuffs.stuff_content import ListContent
from pipelex.core.stuffs.stuff_factory import StuffFactory
-from pipelex.hub import get_pipe_router, get_report_delegate
+from pipelex.hub import get_pipe_router
from pipelex.pipeline.job_metadata import JobMetadata
from pipelex.tools.misc.json_utils import load_json_list_from_path
from tests.test_pipelines.discord_newsletter import ChannelSummary, DiscordChannelUpdate
@@ -63,7 +63,6 @@ class TestPipeSequenceDryRun:
# Log output for debugging
pretty_print(pipe_output, title="Discord Newsletter Dry Run Output")
- get_report_delegate().generate_report()
# Basic assertions
assert pipe_output is not None
diff --git a/tests/integration/pipelex/pipes/controller/pipe_sequence/test_pipe_sequence_simple.py b/tests/integration/pipelex/pipes/controller/pipe_sequence/test_pipe_sequence_simple.py
index d75b00207..f062380f9 100644
--- a/tests/integration/pipelex/pipes/controller/pipe_sequence/test_pipe_sequence_simple.py
+++ b/tests/integration/pipelex/pipes/controller/pipe_sequence/test_pipe_sequence_simple.py
@@ -10,7 +10,7 @@ from pipelex.core.concepts.concept_blueprint import ConceptBlueprint
from pipelex.core.concepts.concept_factory import ConceptFactory
from pipelex.core.concepts.concept_native import NATIVE_CONCEPTS_DATA, NativeConceptEnum
from pipelex.core.memory.working_memory_factory import WorkingMemoryFactory
-from pipelex.core.pipes.pipe_input_spec_blueprint import InputRequirementBlueprint
+from pipelex.core.pipes.pipe_input_blueprint import InputRequirementBlueprint
from pipelex.core.pipes.pipe_run_params import PipeRunMode
from pipelex.core.pipes.pipe_run_params_factory import PipeRunParamsFactory
from pipelex.core.stuffs.stuff_content import TextContent
diff --git a/tests/integration/pipelex/pipes/controller/pipe_sequence/test_pipe_sequence_validation.py b/tests/integration/pipelex/pipes/controller/pipe_sequence/test_pipe_sequence_validation.py
index 45ddf4a72..7f1ed04e5 100644
--- a/tests/integration/pipelex/pipes/controller/pipe_sequence/test_pipe_sequence_validation.py
+++ b/tests/integration/pipelex/pipes/controller/pipe_sequence/test_pipe_sequence_validation.py
@@ -1,5 +1,5 @@
from pipelex.core.concepts.concept_factory import ConceptBlueprint, ConceptFactory
-from pipelex.core.pipes.pipe_input_spec_blueprint import InputRequirementBlueprint
+from pipelex.core.pipes.pipe_input_blueprint import InputRequirementBlueprint
from pipelex.hub import get_concept_provider
from pipelex.pipe_controllers.sequence.pipe_sequence_blueprint import PipeSequenceBlueprint
from pipelex.pipe_controllers.sequence.pipe_sequence_factory import PipeSequenceFactory
diff --git a/tests/integration/pipelex/pipes/operator/pipe_func/test_pipe_func.py b/tests/integration/pipelex/pipes/operator/pipe_func/test_pipe_func.py
index 3c82511f6..7bd3bd9d0 100644
--- a/tests/integration/pipelex/pipes/operator/pipe_func/test_pipe_func.py
+++ b/tests/integration/pipelex/pipes/operator/pipe_func/test_pipe_func.py
@@ -8,7 +8,7 @@ from pipelex.core.pipes.pipe_run_params import PipeRunMode
from pipelex.core.pipes.pipe_run_params_factory import PipeRunParamsFactory
from pipelex.core.stuffs.stuff_content import TextContent
from pipelex.core.stuffs.stuff_factory import StuffFactory
-from pipelex.hub import get_pipe_router, get_report_delegate
+from pipelex.hub import get_pipe_router
from pipelex.pipe_operators.func.pipe_func import PipeFuncOutput
from pipelex.pipe_operators.func.pipe_func_blueprint import PipeFuncBlueprint
from pipelex.pipe_operators.func.pipe_func_factory import PipeFuncFactory
@@ -57,7 +57,7 @@ if __name__ == "__main__":
pipe_func_blueprint = PipeFuncBlueprint(
definition="Function pipe for wrapping lines",
function_name="wrap_lines",
- output=NativeConceptEnum.TEXT.value,
+ output=NativeConceptEnum.TEXT,
)
pipe_job = PipeJobFactory.make_pipe_job(
@@ -79,7 +79,6 @@ if __name__ == "__main__":
log.verbose(pipe_func_output, title="pipe_func_output")
wrapped_text = pipe_func_output.main_stuff_as_text
pretty_print(wrapped_text, title="wrapped_text")
- get_report_delegate().generate_report()
# Verify the output contains wrapped lines
assert pipe_func_output is not None
diff --git a/tests/integration/pipelex/pipes/operator/pipe_imgg/test_pipe_imgg.py b/tests/integration/pipelex/pipes/operator/pipe_imgg/test_pipe_img_gen.py
similarity index 67%
rename from tests/integration/pipelex/pipes/operator/pipe_imgg/test_pipe_imgg.py
rename to tests/integration/pipelex/pipes/operator/pipe_imgg/test_pipe_img_gen.py
index 901eb9fc3..fac606553 100644
--- a/tests/integration/pipelex/pipes/operator/pipe_imgg/test_pipe_imgg.py
+++ b/tests/integration/pipelex/pipes/operator/pipe_imgg/test_pipe_img_gen.py
@@ -1,7 +1,6 @@
import pytest
from pipelex import pretty_print
-from pipelex.cogt.imgg.imgg_handle import ImggHandle
from pipelex.core.concepts.concept_native import NativeConceptEnum
from pipelex.core.pipes.pipe_run_params import PipeRunMode
from pipelex.core.pipes.pipe_run_params_factory import PipeRunParamsFactory
@@ -10,27 +9,26 @@ from pipelex.pipe_operators.img_gen.pipe_img_gen import PipeImgGenOutput
from pipelex.pipe_operators.img_gen.pipe_img_gen_blueprint import PipeImgGenBlueprint
from pipelex.pipe_operators.img_gen.pipe_img_gen_factory import PipeImgGenFactory
from pipelex.pipe_works.pipe_job_factory import PipeJobFactory
-from tests.integration.pipelex.test_data import IMGGTestCases
+from tests.integration.pipelex.test_data import ImageGenTestCases
@pytest.mark.dry_runnable
@pytest.mark.inference
@pytest.mark.asyncio(loop_scope="class")
-class TestPipeImgg:
- @pytest.mark.parametrize("topic, image_desc", IMGGTestCases.IMAGE_DESC)
+class TestPipeImgGen:
+ @pytest.mark.parametrize("topic, image_desc", ImageGenTestCases.IMAGE_DESC)
async def test_pipe_img_gen(
self,
pipe_run_mode: PipeRunMode,
- imgg_handle: ImggHandle,
+ img_gen_handle: str,
topic: str,
image_desc: str,
):
- pipe_imgg_blueprint = PipeImgGenBlueprint(
+ pipe_img_gen_blueprint = PipeImgGenBlueprint(
definition="Image generation test",
img_gen_prompt=image_desc,
- imgg_handle=imgg_handle,
- output=NativeConceptEnum.IMAGE.value,
+ output=NativeConceptEnum.IMAGE,
nb_output=1,
)
@@ -38,12 +36,12 @@ class TestPipeImgg:
pipe=PipeImgGenFactory.make_from_blueprint(
domain="generic",
pipe_code="adhoc_for_test_pipe_img_gen",
- blueprint=pipe_imgg_blueprint,
+ blueprint=pipe_img_gen_blueprint,
),
pipe_run_params=PipeRunParamsFactory.make_run_params(pipe_run_mode=pipe_run_mode),
)
- pipe_imgg_output: PipeImgGenOutput = await get_pipe_router().run_pipe_job(
+ pipe_img_gen_output: PipeImgGenOutput = await get_pipe_router().run_pipe_job(
pipe_job=pipe_job,
)
- image_urls = pipe_imgg_output.image_urls[0]
- pretty_print(image_urls, title=topic)
+ image_urls = pipe_img_gen_output.image_urls[0]
+ pretty_print(image_urls, title=f"Image generated by '{img_gen_handle}' for topic '{topic}'")
diff --git a/tests/integration/pipelex/pipes/operator/pipe_llm/test_pipe_llm.py b/tests/integration/pipelex/pipes/operator/pipe_llm/test_pipe_llm.py
index c94b51ecf..12a8c2640 100644
--- a/tests/integration/pipelex/pipes/operator/pipe_llm/test_pipe_llm.py
+++ b/tests/integration/pipelex/pipes/operator/pipe_llm/test_pipe_llm.py
@@ -4,18 +4,17 @@ import pytest
from pipelex import log, pretty_print
from pipelex.core.concepts.concept_native import NativeConceptEnum
-from pipelex.core.domains.domain import SpecialDomain
from pipelex.core.memory.working_memory_factory import WorkingMemoryFactory
-from pipelex.core.pipes.pipe_input_spec_blueprint import InputRequirementBlueprint
+from pipelex.core.pipes.pipe_input_blueprint import InputRequirementBlueprint
from pipelex.core.pipes.pipe_run_params import PipeRunMode
from pipelex.core.pipes.pipe_run_params_factory import PipeRunParamsFactory
from pipelex.core.stuffs.stuff import Stuff
-from pipelex.hub import get_pipe_router, get_report_delegate
+from pipelex.hub import get_class_registry, get_pipe_provider, get_pipe_router
from pipelex.pipe_operators.llm.pipe_llm import PipeLLMOutput
-from pipelex.pipe_operators.llm.pipe_llm_blueprint import PipeLLMBlueprint
+from pipelex.pipe_operators.llm.pipe_llm_blueprint import PipeLLMBlueprint, StructuringMethod
from pipelex.pipe_operators.llm.pipe_llm_factory import PipeLLMFactory
from pipelex.pipe_works.pipe_job_factory import PipeJobFactory
-from tests.integration.pipelex.test_data import PipeTestCases
+from tests.integration.pipelex.test_data import BasicStructuredDataTestCases, ComplexStructuredDataTestCases, PipeTestCases
@pytest.mark.dry_runnable
@@ -23,23 +22,26 @@ from tests.integration.pipelex.test_data import PipeTestCases
@pytest.mark.inference
@pytest.mark.asyncio(loop_scope="class")
class TestPipeLLM:
- async def test_pipe_llm(
+ async def test_pipe_llm_simple(
self,
pipe_run_mode: PipeRunMode,
):
pipe_llm_blueprint = PipeLLMBlueprint(
definition="LLM test for basic text generation",
- output=NativeConceptEnum.TEXT.value,
+ output=NativeConceptEnum.TEXT,
system_prompt=PipeTestCases.SYSTEM_PROMPT,
prompt=PipeTestCases.USER_PROMPT,
)
+ pipe = PipeLLMFactory.make_from_blueprint(
+ domain="documents",
+ pipe_code="adhoc_for_test_pipe_llm",
+ blueprint=pipe_llm_blueprint,
+ )
+ pipe_provider = get_pipe_provider()
+ pipe_provider.add_new_pipe(pipe)
pipe_job = PipeJobFactory.make_pipe_job(
- pipe=PipeLLMFactory.make_from_blueprint(
- domain="generic",
- pipe_code="adhoc_for_test_pipe_llm",
- blueprint=pipe_llm_blueprint,
- ),
+ pipe=pipe,
pipe_run_params=PipeRunParamsFactory.make_run_params(pipe_run_mode=pipe_run_mode),
)
pipe_llm_output: PipeLLMOutput = await get_pipe_router().run_pipe_job(
@@ -49,7 +51,97 @@ class TestPipeLLM:
log.verbose(pipe_llm_output, title="stuff")
llm_generated_text = pipe_llm_output.main_stuff_as_text
pretty_print(llm_generated_text, title="llm_generated_text")
- get_report_delegate().generate_report()
+
+ @pytest.mark.parametrize(
+ "topic,data,concept",
+ BasicStructuredDataTestCases.STRUCTURE_TEST_CASES + ComplexStructuredDataTestCases.STRUCTURE_TEST_CASES,
+ )
+ @pytest.mark.parametrize(
+ "structuring_method",
+ [
+ StructuringMethod.DIRECT,
+ StructuringMethod.PRELIMINARY_TEXT,
+ ],
+ )
+ @pytest.mark.parametrize(
+ "llm,llm_to_structure",
+ [
+ # ("gpt-4o-mini", "gpt-4o-mini"),
+ # ("gemini-2.5-flash-lite", "gemini-2.5-flash-lite"),
+ ("gemini-2.5-flash-lite", "gpt-4o-mini"),
+ ],
+ )
+ async def test_pipe_llm_structured(
+ self,
+ topic: str,
+ data: str,
+ concept: str,
+ structuring_method: StructuringMethod,
+ pipe_run_mode: PipeRunMode,
+ llm: str,
+ llm_to_structure: str,
+ ):
+ # TODO: Add assertion on generated objects vs expected results
+ pretty_print(data, title="data")
+ working_memory = WorkingMemoryFactory.make_from_text(text=data, name="data")
+
+ # Create pipe blueprint
+ pipe_llm_blueprint = PipeLLMBlueprint(
+ definition=f"Extract {concept} from text",
+ inputs={"data": "Text"},
+ output=f"test_structured_generations.{concept}",
+ prompt_template=BasicStructuredDataTestCases.EXTRACTION_PROMPT,
+ llm=llm,
+ llm_to_structure=llm_to_structure,
+ structuring_method=structuring_method,
+ )
+
+ pipe_code = f"extract_{topic}_{concept}_{structuring_method}"
+ pipe_code = pipe_code.lower().replace(" ", "_")
+ pipe = PipeLLMFactory.make_from_blueprint(
+ domain="test_structured_generations",
+ pipe_code=pipe_code,
+ blueprint=pipe_llm_blueprint,
+ )
+ pipe_provider = get_pipe_provider()
+ pipe_provider.add_new_pipe(pipe)
+
+ pipe_job = PipeJobFactory.make_pipe_job(
+ pipe=pipe,
+ working_memory=working_memory,
+ pipe_run_params=PipeRunParamsFactory.make_run_params(pipe_run_mode=pipe_run_mode),
+ )
+ pipe_llm_output: PipeLLMOutput = await get_pipe_router().run_pipe_job(
+ pipe_job=pipe_job,
+ )
+
+ # Log test information
+ log.verbose(f"Testing {topic} with {structuring_method} method", title="Test Case")
+
+ # Get the main output
+ main_stuff = pipe_llm_output.main_stuff
+ assert main_stuff is not None, f"No output generated for {topic}"
+
+ # Pretty print the structured output
+ pretty_print(main_stuff.content, title=f"{topic} - {concept} - {structuring_method}")
+
+ # If using PRELIMINARY_TEXT, also print the preliminary text
+ # if structuring_method == StructuringMethod.PRELIMINARY_TEXT:
+ # # Get preliminary text from working memory
+ # preliminary_stuff = pipe_llm_output.working_memory.get_optional_stuff("preliminary_text")
+ # if preliminary_stuff:
+ # content = preliminary_stuff.content
+ # if hasattr(content, "text"):
+ # pretty_print(content.text, title="Preliminary Text") # type: ignore
+ # else:
+ # pretty_print(str(content), title="Preliminary Text")
+
+ # Verify the output type matches expected structure
+ structure_class = get_class_registry().get_class(concept)
+ assert structure_class is not None, f"Structure class {concept} not found in registry"
+ assert isinstance(main_stuff.content, structure_class), (
+ f"Output content is not of expected type {concept}. Got {type(main_stuff.content).__name__} instead."
+ )
@pytest.mark.llm
@pytest.mark.inference
@@ -69,7 +161,7 @@ class TestPipeLLM:
pipe_llm_blueprint = PipeLLMBlueprint(
definition="LLM test for image processing with attributes",
inputs={stuff_name: InputRequirementBlueprint(concept=stuff.concept.concept_string)},
- output=f"{SpecialDomain.NATIVE.value}.{NativeConceptEnum.TEXT.value}",
+ output=NativeConceptEnum.TEXT,
system_prompt=PipeTestCases.SYSTEM_PROMPT,
prompt=PipeTestCases.MULTI_IMG_DESC_PROMPT,
)
@@ -89,6 +181,4 @@ class TestPipeLLM:
)
log.verbose(pipe_llm_output, title="stuff")
- llm_generated_text = pipe_llm_output.main_stuff_as_text
- pretty_print(llm_generated_text, title="llm_generated_text")
- get_report_delegate().generate_report()
+ _ = pipe_llm_output.main_stuff_as_text
diff --git a/tests/integration/pipelex/pipes/operator/pipe_ocr/test_pipe_ocr.py b/tests/integration/pipelex/pipes/operator/pipe_ocr/test_pipe_ocr.py
index 424af565f..3c996bb5e 100644
--- a/tests/integration/pipelex/pipes/operator/pipe_ocr/test_pipe_ocr.py
+++ b/tests/integration/pipelex/pipes/operator/pipe_ocr/test_pipe_ocr.py
@@ -7,7 +7,7 @@ from pipelex.core.concepts.concept_blueprint import ConceptBlueprint
from pipelex.core.concepts.concept_factory import ConceptFactory
from pipelex.core.concepts.concept_native import NativeConceptEnum
from pipelex.core.memory.working_memory_factory import WorkingMemoryFactory
-from pipelex.core.pipes.pipe_input_spec_blueprint import InputRequirementBlueprint
+from pipelex.core.pipes.pipe_input_blueprint import InputRequirementBlueprint
from pipelex.core.pipes.pipe_run_params import PipeRunMode
from pipelex.core.pipes.pipe_run_params_factory import PipeRunParamsFactory
from pipelex.core.stuffs.stuff_content import PageContent
@@ -42,20 +42,20 @@ class TestPipeOCR:
@pytest.mark.parametrize("image_url", PipeOcrTestCases.PIPE_OCR_IMAGE_TEST_CASES)
async def test_pipe_ocr_image(
self,
- ocr_handle_from_image: str,
+ ocr_choice_for_image: str,
pipe_run_mode: PipeRunMode,
image_url: str,
setup: Any,
):
pipe_ocr_blueprint = PipeOcrBlueprint(
definition="OCR test for image processing",
- inputs={"page_scan": InputRequirementBlueprint(concept=NativeConceptEnum.IMAGE.value)},
- output=NativeConceptEnum.TEXT_AND_IMAGES.value,
+ inputs={"page_scan": InputRequirementBlueprint(concept=NativeConceptEnum.IMAGE)},
+ output=NativeConceptEnum.TEXT_AND_IMAGES,
page_images=True,
page_image_captions=False,
page_views=True,
page_views_dpi=72,
- ocr_model=ocr_handle_from_image,
+ ocr=ocr_choice_for_image,
)
pipe_job = PipeJobFactory.make_pipe_job(
@@ -79,15 +79,15 @@ class TestPipeOCR:
@pytest.mark.parametrize("pdf_url", PipeOcrTestCases.PIPE_OCR_PDF_TEST_CASES)
async def test_pipe_ocr_pdf(
self,
- ocr_handle: str,
+ ocr_choice_for_pdf: str,
pipe_run_mode: PipeRunMode,
pdf_url: str,
):
pipe_ocr_blueprint = PipeOcrBlueprint(
definition="OCR test for PDF processing",
- inputs={PIPE_OCR_INPUT_NAME: InputRequirementBlueprint(concept=NativeConceptEnum.PDF.value)},
- output=NativeConceptEnum.TEXT_AND_IMAGES.value,
- ocr_model=ocr_handle,
+ inputs={PIPE_OCR_INPUT_NAME: InputRequirementBlueprint(concept=NativeConceptEnum.PDF)},
+ output=NativeConceptEnum.TEXT_AND_IMAGES,
+ ocr=ocr_choice_for_pdf,
page_images=True,
page_image_captions=False,
page_views=True,
diff --git a/tests/integration/pipelex/pipes/test_image_inputs.py b/tests/integration/pipelex/pipes/test_image_inputs.py
index 58f8c8e70..371b59116 100644
--- a/tests/integration/pipelex/pipes/test_image_inputs.py
+++ b/tests/integration/pipelex/pipes/test_image_inputs.py
@@ -1,7 +1,6 @@
import pytest
from pytest import FixtureRequest
-from pipelex import pretty_print
from pipelex.core.concepts.concept_factory import ConceptFactory
from pipelex.core.concepts.concept_native import NATIVE_CONCEPTS_DATA, NativeConceptEnum
from pipelex.core.memory.working_memory_factory import WorkingMemoryFactory
@@ -10,7 +9,7 @@ from pipelex.core.pipes.pipe_run_params import PipeRunMode
from pipelex.core.pipes.pipe_run_params_factory import PipeRunParamsFactory
from pipelex.core.stuffs.stuff_content import ImageContent, PageContent, TextAndImagesContent, TextContent
from pipelex.core.stuffs.stuff_factory import StuffFactory
-from pipelex.hub import get_pipe_router, get_report_delegate
+from pipelex.hub import get_pipe_router
from pipelex.pipeline.job_metadata import JobMetadata
from tests.cases import ImageTestCases
from tests.test_pipelines.misc_tests.test_structures import Article
@@ -31,22 +30,21 @@ class TestImageInputs:
"""Test that an image is indeed given to the LLM, and that it can extract extact whats on the image."""
working_memory = WorkingMemoryFactory.make_from_image(name="image", image_url=ImageTestCases.IMAGE_FILE_PATH_PNG)
- # Run the pipe
pipe_output: PipeOutput = await get_pipe_router().run_pipe_code(
pipe_code="extract_article_from_image",
pipe_run_params=PipeRunParamsFactory.make_run_params(pipe_run_mode=pipe_run_mode),
working_memory=working_memory,
job_metadata=JobMetadata(job_name=request.node.originalname), # type: ignore
)
-
- # Log output and generate report
- pretty_print(pipe_output, title="Pipe output")
- get_report_delegate().generate_report()
-
- # Verify output
if pipe_run_mode != PipeRunMode.DRY:
article = pipe_output.main_stuff_as(content_type=Article)
- assert article.title == "2037 AI-Lympics"
+ assert (
+ article.title == "2037 AI-Lympics PARIS"
+ or article.title == "2037 AI-Lympics Paris"
+ or article.title == "2037 AI-Lympics"
+ or article.title == "2037 AI-LYMPICS PARIS"
+ or article.title == "2037 AI-LYMPICS"
+ )
assert pipe_output is not None
assert pipe_output.working_memory is not None
assert pipe_output.main_stuff is not None
@@ -58,7 +56,7 @@ class TestImageInputs:
"""
# Create the page content
image_content = ImageContent(url=ImageTestCases.IMAGE_FILE_PATH_PNG)
- text_and_images = TextAndImagesContent(text=TextContent(text="This is a test page"), images=[])
+ text_and_images = TextAndImagesContent(text=TextContent(text="This is the description of the page blablabla"), images=[])
page_content = PageContent(text_and_images=text_and_images, page_view=image_content)
# Create stuff from page content
@@ -79,16 +77,16 @@ class TestImageInputs:
job_metadata=JobMetadata(job_name=request.node.originalname), # type: ignore
)
- # Log output and generate report
- pretty_print(pipe_output, title="Pipe output")
- get_report_delegate().generate_report()
-
- # Verify output
if pipe_run_mode != PipeRunMode.DRY:
article = pipe_output.main_stuff_as(content_type=Article)
- pretty_print(article, title="Article")
- assert article.title == "2037 AI-Lympics"
- assert article.description == "This is a test page"
+ assert (
+ article.title == "2037 AI-Lympics Paris"
+ or article.title == "2037 AI-Lympics PARIS"
+ or article.title == "2037 AI-Lympics"
+ or article.title == "2037 AI-LYMPICS PARIS"
+ or article.title == "2037 AI-LYMPICS"
+ )
+ assert article.description == "This is the description of the page blablabla"
assert pipe_output is not None
assert pipe_output.working_memory is not None
assert pipe_output.main_stuff is not None
diff --git a/tests/integration/pipelex/pipes/test_pipe_jinja2.py b/tests/integration/pipelex/pipes/test_pipe_jinja2.py
index 0eabb2d50..ea9c3b334 100644
--- a/tests/integration/pipelex/pipes/test_pipe_jinja2.py
+++ b/tests/integration/pipelex/pipes/test_pipe_jinja2.py
@@ -26,7 +26,7 @@ class TestPipeJinja2:
pipe_jinja2_blueprint = PipeJinja2Blueprint(
definition="Jinja2 test for any context",
jinja2=jinja2,
- output=NativeConceptEnum.TEXT.value,
+ output=NativeConceptEnum.TEXT,
extra_context={"place_holder": "[some text from test_pipe_jinja2_for_any]"},
)
@@ -54,7 +54,7 @@ class TestPipeJinja2:
definition="Jinja2 test for stuff context",
jinja2=jinja2,
prompting_style=PromptingStyle(tag_style=TagStyle.TICKS, text_format=TextFormat.MARKDOWN),
- output=NativeConceptEnum.TEXT.value,
+ output=NativeConceptEnum.TEXT,
)
pipe_job = PipeJobFactory.make_pipe_job(
diff --git a/tests/integration/pipelex/pipes/test_pipe_running_variants.py b/tests/integration/pipelex/pipes/test_pipe_running_variants.py
index 4d973209e..27c74b753 100644
--- a/tests/integration/pipelex/pipes/test_pipe_running_variants.py
+++ b/tests/integration/pipelex/pipes/test_pipe_running_variants.py
@@ -11,7 +11,7 @@ from pipelex.core.pipes.pipe_output import PipeOutput
from pipelex.core.pipes.pipe_run_params import PipeOutputMultiplicity, PipeRunMode
from pipelex.core.pipes.pipe_run_params_factory import PipeRunParamsFactory
from pipelex.core.stuffs.stuff import Stuff
-from pipelex.hub import get_library_manager, get_pipe_router, get_report_delegate
+from pipelex.hub import get_library_manager, get_pipe_router
from pipelex.pipeline.activity.activity_handler import ActivityHandlerForResultFiles
from pipelex.pipeline.job_metadata import JobMetadata
from tests.integration.pipelex.test_data import PipeTestCases
@@ -42,7 +42,6 @@ class TestPipeRunningVariants:
working_memory=working_memory,
job_metadata=JobMetadata(job_name=request.node.originalname), # type: ignore
)
- get_report_delegate().generate_report()
# Save stuff context
result_dir_path, _ = pipe_result_handler
@@ -65,7 +64,6 @@ class TestPipeRunningVariants:
working_memory=WorkingMemoryFactory.make_empty(),
job_metadata=JobMetadata(job_name=request.node.originalname), # type: ignore
)
- get_report_delegate().generate_report()
# Save stuff context
result_dir_path, _ = pipe_result_handler
@@ -97,7 +95,6 @@ class TestPipeRunningVariants:
working_memory=WorkingMemoryFactory.make_empty(),
job_metadata=JobMetadata(job_name=request.node.originalname), # type: ignore
)
- get_report_delegate().generate_report()
# Save stuff context
result_dir_path, _ = pipe_result_handler
@@ -135,4 +132,3 @@ class TestPipeRunningVariants:
)
pretty_print(exc.value, title="exception")
assert expected_error_message in str(exc.value)
- get_report_delegate().generate_report()
diff --git a/tests/integration/pipelex/plugins/test_anthropic.py b/tests/integration/pipelex/plugins/test_anthropic.py
index d66c51094..03b1eb509 100644
--- a/tests/integration/pipelex/plugins/test_anthropic.py
+++ b/tests/integration/pipelex/plugins/test_anthropic.py
@@ -8,7 +8,7 @@ from pipelex.hub import get_models_manager
from pipelex.plugins.anthropic.anthropic_exceptions import AnthropicSDKUnsupportedError
from pipelex.plugins.anthropic.anthropic_llms import anthropic_list_anthropic_models
from pipelex.plugins.plugin_sdk_registry import Plugin
-from pipelex.tools.environment import any_is_placeholder_env, is_env_set
+from pipelex.tools.environment import all_env_vars_are_set, any_env_var_is_placeholder
REQUIRED_ENV_VARS = ["ANTHROPIC_API_KEY"]
@@ -26,9 +26,9 @@ class TestAnthropic:
pytestconfig: pytest.Config,
plugin_for_anthropic: Plugin,
):
- if not is_env_set(REQUIRED_ENV_VARS):
+ if not all_env_vars_are_set(keys=REQUIRED_ENV_VARS):
pytest.skip(f"Some key(s) missing amongst {REQUIRED_ENV_VARS}")
- if any_is_placeholder_env(REQUIRED_ENV_VARS):
+ if any_env_var_is_placeholder(REQUIRED_ENV_VARS):
pytest.skip(f"Some key(s) among {REQUIRED_ENV_VARS} are a placeholder, can't be used to test listing models")
try:
backend = get_models_manager().get_required_inference_backend("anthropic")
diff --git a/tests/integration/pipelex/plugins/test_bedrock.py b/tests/integration/pipelex/plugins/test_bedrock.py
index 2d14f7d9a..57c9d4bdf 100644
--- a/tests/integration/pipelex/plugins/test_bedrock.py
+++ b/tests/integration/pipelex/plugins/test_bedrock.py
@@ -7,7 +7,7 @@ from rich import box
from rich.console import Console
from rich.table import Table
-from pipelex.tools.environment import any_is_placeholder_env, is_env_set
+from pipelex.tools.environment import all_env_vars_are_set, any_env_var_is_placeholder
warnings.filterwarnings(
"ignore",
@@ -31,9 +31,9 @@ class TestBedrock:
bedrock_provider: str,
bedrock_region_name: str,
):
- if not is_env_set(REQUIRED_ENV_VARS):
+ if not all_env_vars_are_set(keys=REQUIRED_ENV_VARS):
pytest.skip(f"Some key(s) missing amongst {REQUIRED_ENV_VARS}")
- if any_is_placeholder_env(REQUIRED_ENV_VARS):
+ if any_env_var_is_placeholder(REQUIRED_ENV_VARS):
pytest.skip(f"Some key(s) among {REQUIRED_ENV_VARS} are a placeholder, can't be used to test listing models")
client = boto3.client("bedrock", region_name=bedrock_region_name) # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType]
response: Dict[str, Any] = client.list_foundation_models(byProvider=bedrock_provider) # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType]
@@ -67,9 +67,9 @@ class TestBedrock:
pytestconfig: pytest.Config,
bedrock_region_name: str,
):
- if not is_env_set(REQUIRED_ENV_VARS):
+ if not all_env_vars_are_set(keys=REQUIRED_ENV_VARS):
pytest.skip(f"Some key(s) missing amongst {REQUIRED_ENV_VARS}")
- if any_is_placeholder_env(REQUIRED_ENV_VARS):
+ if any_env_var_is_placeholder(REQUIRED_ENV_VARS):
pytest.skip(f"Some key(s) among {REQUIRED_ENV_VARS} are a placeholder, can't be used to test listing models")
client = boto3.client("bedrock", region_name=bedrock_region_name) # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType]
response: Dict[str, Any] = client.list_inference_profiles() # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType]
diff --git a/tests/integration/pipelex/plugins/test_external_plugin.py b/tests/integration/pipelex/plugins/test_external_plugin.py
index 5d4e9befb..5dd431229 100644
--- a/tests/integration/pipelex/plugins/test_external_plugin.py
+++ b/tests/integration/pipelex/plugins/test_external_plugin.py
@@ -90,7 +90,7 @@ class TestExternalPlugin:
pipe_llm_blueprint = PipeLLMBlueprint(
definition="LLM test with external plugin",
- output=NativeConceptEnum.TEXT.value,
+ output=NativeConceptEnum.TEXT,
system_prompt=PipeTestCases.SYSTEM_PROMPT,
prompt=PipeTestCases.USER_PROMPT,
llm=LLMSetting(
diff --git a/tests/integration/pipelex/plugins/test_mistral.py b/tests/integration/pipelex/plugins/test_mistral.py
index 829cd9b1d..acfe00847 100644
--- a/tests/integration/pipelex/plugins/test_mistral.py
+++ b/tests/integration/pipelex/plugins/test_mistral.py
@@ -5,7 +5,7 @@ from rich.table import Table
from pipelex import pretty_print
from pipelex.plugins.mistral.mistral_llms import list_mistral_models
-from pipelex.tools.environment import any_is_placeholder_env, is_env_set
+from pipelex.tools.environment import all_env_vars_are_set, any_env_var_is_placeholder
REQUIRED_ENV_VARS = ["MISTRAL_API_KEY"]
@@ -17,9 +17,9 @@ class TestMistral:
# pytest -k test_mistral_list_models -s -vv
# make t VERBOSE=2 TEST=test_mistral_list_models
def test_mistral_list_models(self, pytestconfig: pytest.Config):
- if not is_env_set(REQUIRED_ENV_VARS):
+ if not all_env_vars_are_set(keys=REQUIRED_ENV_VARS):
pytest.skip(f"Some key(s) missing amongst {REQUIRED_ENV_VARS}")
- if any_is_placeholder_env(REQUIRED_ENV_VARS):
+ if any_env_var_is_placeholder(REQUIRED_ENV_VARS):
pytest.skip(f"Some key(s) among {REQUIRED_ENV_VARS} are a placeholder, can't be used to test listing models")
mistral_models_list = list_mistral_models()
if pytestconfig.get_verbosity() >= 2:
@@ -49,9 +49,9 @@ class TestMistral:
# pytest -k test_mistral_list_model_ids -s -vv
# make t VERBOSE=2 TEST=test_mistral_list_model_ids
def test_mistral_list_model_ids(self, pytestconfig: pytest.Config):
- if not is_env_set(REQUIRED_ENV_VARS):
+ if not all_env_vars_are_set(keys=REQUIRED_ENV_VARS):
pytest.skip(f"Some key(s) missing amongst {REQUIRED_ENV_VARS}")
- if any_is_placeholder_env(REQUIRED_ENV_VARS):
+ if any_env_var_is_placeholder(REQUIRED_ENV_VARS):
pytest.skip(f"Some key(s) among {REQUIRED_ENV_VARS} are a placeholder, can't be used to test listing models")
mistral_models_list = list_mistral_models()
mistral_model_ids = [{"id": model.id, "aliases": model.aliases} for model in mistral_models_list]
diff --git a/tests/integration/pipelex/plugins/test_openai.py b/tests/integration/pipelex/plugins/test_openai.py
index f76593d02..58d12d54b 100644
--- a/tests/integration/pipelex/plugins/test_openai.py
+++ b/tests/integration/pipelex/plugins/test_openai.py
@@ -4,7 +4,7 @@ from pipelex import pretty_print
from pipelex.hub import get_models_manager
from pipelex.plugins.openai.openai_llms import openai_list_available_models
from pipelex.plugins.plugin_sdk_registry import Plugin
-from pipelex.tools.environment import any_is_placeholder_env, is_env_set
+from pipelex.tools.environment import all_env_vars_are_set, any_env_var_is_placeholder
# make t VERBOSE=2 TEST=TestOpenAI
@@ -25,9 +25,9 @@ class TestOpenAI:
required_env_vars = ["AZURE_API_KEY", "AZURE_API_BASE", "AZURE_API_VERSION"]
case _:
raise ValueError(f"Plugin {plugin_for_openai} is not supported in this test")
- if not is_env_set(required_env_vars):
+ if not all_env_vars_are_set(keys=required_env_vars):
pytest.skip(f"Some key(s) missing amongst {required_env_vars}")
- if any_is_placeholder_env(required_env_vars):
+ if any_env_var_is_placeholder(required_env_vars):
pytest.skip(f"Some key(s) among {required_env_vars} are a placeholder, can't be used to test listing models")
backend = get_models_manager().get_required_inference_backend(plugin_for_openai.backend)
openai_models_list = await openai_list_available_models(
diff --git a/tests/integration/pipelex/plugins/test_openai_image_gen.py b/tests/integration/pipelex/plugins/test_openai_image_gen.py
index 7fc2af006..b8d5cd718 100644
--- a/tests/integration/pipelex/plugins/test_openai_image_gen.py
+++ b/tests/integration/pipelex/plugins/test_openai_image_gen.py
@@ -1,20 +1,20 @@
import pytest
-from pipelex.cogt.exceptions import ImggGenerationError
+from pipelex.cogt.exceptions import ImgGenGenerationError
from pipelex.hub import get_models_manager
from pipelex.plugins.openai.openai_factory import OpenAIFactory
from pipelex.plugins.plugin_sdk_registry import Plugin
from pipelex.tools.misc.base_64_utils import save_base64_to_binary_file
from pipelex.tools.misc.file_utils import ensure_path, get_incremental_file_path
from tests.conftest import TEST_OUTPUTS_DIR
-from tests.integration.pipelex.test_data import IMGGTestCases
+from tests.integration.pipelex.test_data import ImageGenTestCases
@pytest.mark.inference
@pytest.mark.asyncio(loop_scope="class")
class TestImggByOpenAIGpt:
- @pytest.mark.parametrize("topic, image_desc", IMGGTestCases.IMAGE_DESC)
+ @pytest.mark.parametrize("topic, image_desc", ImageGenTestCases.IMAGE_DESC)
async def test_gpt_image_generation(self, topic: str, image_desc: str):
backend = get_models_manager().get_required_inference_backend("openai")
client = OpenAIFactory.make_openai_client(
@@ -33,14 +33,14 @@ class TestImggByOpenAIGpt:
n=2,
)
if not result.data:
- raise ImggGenerationError("No result from OpenAI")
+ raise ImgGenGenerationError("No result from OpenAI")
for image_index, image_data in enumerate(result.data):
image_base64 = image_data.b64_json
if not image_base64:
- raise ImggGenerationError("No base64 image data received from OpenAI")
+ raise ImgGenGenerationError("No base64 image data received from OpenAI")
- folder_path = f"{TEST_OUTPUTS_DIR}/imgg_by_gpt_image"
+ folder_path = f"{TEST_OUTPUTS_DIR}/img_gen_by_gpt_image"
ensure_path(folder_path)
filename = f"{topic}_{image_index}"
img_path = get_incremental_file_path(
diff --git a/tests/integration/pipelex/test_data.py b/tests/integration/pipelex/test_data.py
index d098ce72c..63b3bcdbb 100644
--- a/tests/integration/pipelex/test_data.py
+++ b/tests/integration/pipelex/test_data.py
@@ -185,16 +185,261 @@ class PipeOcrTestCases:
PIPE_OCR_PDF_TEST_CASES: ClassVar[List[str]] = PDFTestCases.DOCUMENT_FILE_PATHS + PDFTestCases.DOCUMENT_URLS
-class IMGGTestCases:
- IMGG_PROMPT_1 = "woman wearing marino cargo pants"
- IMGG_PROMPT_2 = "wide legged denim pants with hippy addition"
- IMGG_PROMPT_3 = """
+class ImageGenTestCases:
+ IMG_GEN_PROMPT_1 = "woman wearing marino cargo pants"
+ IMG_GEN_PROMPT_2 = "wide legged denim pants with hippy addition"
+ IMG_GEN_PROMPT_3 = """
Woman typing on a laptop. On the laptop screen you see python code to generate code to write a prompt for an AI model.
"""
- IMAGE_DESC: ClassVar[List[Tuple[str, str]]] = [ # topic, imgg_prompt_text
- # (IMGG_PROMPT_1, IMGG_PROMPT_1),
- # (IMGG_PROMPT_2, IMGG_PROMPT_2),
- # (IMGG_PROMPT_3, IMGG_PROMPT_3),
- ("coding girl", "a girl with a dragon tatoo, coding in python"),
+ IMAGE_DESC: ClassVar[List[Tuple[str, str]]] = [ # topic, img_gen_prompt_text
+ # (IMG_GEN_PROMPT_1, IMG_GEN_PROMPT_1),
+ # (IMG_GEN_PROMPT_2, IMG_GEN_PROMPT_2),
+ # (IMG_GEN_PROMPT_3, IMG_GEN_PROMPT_3),
+ ("coding girl with dragon tatoo", "a girl with a dragon tatoo, coding in python"),
+ ]
+
+
+class BasicStructuredDataTestCases:
+ """Test cases for extracting basic structured data without union types."""
+
+ EXTRACTION_PROMPT = """
+Extract information from the following text:
+@data
+"""
+
+ # ConceptWithSimpleStructure test cases
+ SIMPLE_STRUCTURE_1 = """
+ - name: "Alice Johnson"
+ - age: 28
+ - is_active: true
+ """
+
+ SIMPLE_STRUCTURE_2 = """
+ The person's name is Bob Smith, they are 45 years old and their account is not active (is_active: false).
+ """
+
+ # ConceptWithOptionals test cases
+ OPTIONAL_ALL_PRESENT = """
+ - required_field: "This is required"
+ - optional_string: "This optional has a value"
+ - optional_number: 42
+ - optional_date: "2024-03-15T10:30:00"
+ """
+
+ OPTIONAL_SOME_MISSING = """
+ - required_field: "Only required field present"
+ - optional_string: NOT PROVIDED (should be null)
+ - optional_number: 100
+ - optional_date: NOT PROVIDED (should be null)
+ """
+
+ OPTIONAL_ALL_MISSING = """
+ - required_field: "Mandatory value here"
+ Note: All optional fields should be null/None as they are not provided.
+ """
+
+ # ConceptWithLists test cases
+ LISTS_WITH_DATA = """
+ - string_list: ["apple", "banana", "cherry", "date"]
+ - number_list: [1, 2, 3, 5, 8, 13]
+ - optional_list: ["first", "second", "third"]
+ """
+
+ LISTS_EMPTY = """
+ - string_list: [] (empty list)
+ - number_list: [] (empty list)
+ - optional_list: null (not provided, should be None)
+ """
+
+ LISTS_MIXED = """
+ - string_list: ["only", "one"]
+ - number_list: [42]
+ - optional_list: NOT PROVIDED (should be null)
+ """
+
+ # ConceptWithNestedStructures test cases
+ NESTED_FULL = """
+ - simple_nested: {
+ name: "Nested Person",
+ age: 30,
+ is_active: true
+ }
+ - optional_nested: {
+ required_field: "Nested required",
+ optional_string: "Has value",
+ optional_number: 55,
+ optional_date: "2024-01-20T14:00:00"
+ }
+ - list_of_nested: [
+ {name: "First", age: 20, is_active: true},
+ {name: "Second", age: 25, is_active: false}
+ ]
+ """
+
+ NESTED_PARTIAL = """
+ - simple_nested: {
+ name: "Main Structure",
+ age: 40,
+ is_active: false
+ }
+ - optional_nested: NOT PROVIDED (should be None)
+ - list_of_nested: [] (empty list)
+ """
+
+ NESTED_COMPLEX = """
+ - simple_nested: {
+ name: "Complex Example",
+ age: 35,
+ is_active: true
+ }
+ - optional_nested: {
+ required_field: "Required in optional",
+ optional_string: null,
+ optional_number: null,
+ optional_date: "2024-12-01T09:30:00"
+ }
+ - list_of_nested: [
+ {name: "Solo Entry", age: 50, is_active: true}
+ ]
+ """
+
+ # Combined test cases for parametrized tests
+ STRUCTURE_TEST_CASES: ClassVar[List[Tuple[str, str, str]]] = [ # topic, data, concept
+ ("Simple structure basic", SIMPLE_STRUCTURE_1, "ConceptWithSimpleStructure"),
+ ("Simple structure narrative", SIMPLE_STRUCTURE_2, "ConceptWithSimpleStructure"),
+ ("Optionals all present", OPTIONAL_ALL_PRESENT, "ConceptWithOptionals"),
+ ("Optionals some missing", OPTIONAL_SOME_MISSING, "ConceptWithOptionals"),
+ ("Optionals all missing", OPTIONAL_ALL_MISSING, "ConceptWithOptionals"),
+ ("Lists with data", LISTS_WITH_DATA, "ConceptWithLists"),
+ ("Lists empty", LISTS_EMPTY, "ConceptWithLists"),
+ ("Lists mixed", LISTS_MIXED, "ConceptWithLists"),
+ ("Nested full", NESTED_FULL, "ConceptWithNestedStructures"),
+ ("Nested partial", NESTED_PARTIAL, "ConceptWithNestedStructures"),
+ ("Nested complex", NESTED_COMPLEX, "ConceptWithNestedStructures"),
+ ]
+
+
+class ComplexStructuredDataTestCases:
+ """Test cases for extracting structured data with complex types (unions, dicts, etc.)."""
+
+ EXTRACTION_PROMPT = """
+Extract information from the following text:
+@data
+"""
+
+ # ConceptWithDicts test cases
+ DICTS_WITH_DATA = """
+ - string_dict: {"key1": "value1", "key2": "value2", "name": "test"}
+ - number_dict: {"count": 10, "score": 95, "id": 999}
+ - optional_dict: {"status": "active", "mode": "production"}
+ """
+
+ DICTS_EMPTY = """
+ - string_dict: {} (empty dictionary)
+ - number_dict: {"total": 100}
+ - optional_dict: null (not provided)
+ """
+
+ # ConceptWithUnions test cases
+ UNIONS_STRING_VARIANT = """
+ - string_or_int: "This is a string value"
+ - optional_union: "Also a string"
+ - list_of_unions: ["text1", "text2", 100, 200, "text3"]
+ """
+
+ UNIONS_INT_VARIANT = """
+ - string_or_int: 42 (as integer)
+ - optional_union: true (as boolean)
+ - list_of_unions: [1, 2, 3, 4, 5]
+ """
+
+ UNIONS_MIXED = """
+ - string_or_int: 999
+ - optional_union: NOT PROVIDED (should be None)
+ - list_of_unions: ["mixed", 123, "types", 456, "here"]
+ """
+
+ # ConceptWithComplexUnions test cases
+ COMPLEX_UNIONS_1 = """
+ - mixed_dict: {"status": "active", "count": 42, "enabled": true}
+ - union_or_list: "single string value"
+ - optional_complex_union: {"key1": "value1", "key2": "value2"}
+ - number_or_bool: 3.14
+ """
+
+ COMPLEX_UNIONS_2 = """
+ - mixed_dict: {"name": "test", "score": 100, "passed": false}
+ - union_or_list: [10, 20, 30, 40]
+ - optional_complex_union: ["item1", "item2", "item3"]
+ - number_or_bool: true
+ """
+
+ COMPLEX_UNIONS_3 = """
+ - mixed_dict: {} (empty dictionary)
+ - union_or_list: [] (empty list as list of integers)
+ - optional_complex_union: NOT PROVIDED (should be None)
+ - number_or_bool: 0
+ """
+
+ # ConceptWithNestedUnions test cases
+ NESTED_UNIONS_SIMPLE = """
+ - simple_or_complex: {
+ name: "John Doe",
+ age: 25,
+ is_active: true
+ }
+ - list_of_union_structures: []
+ - optional_nested_union: null
+ """
+
+ NESTED_UNIONS_COMPLEX = """
+ - simple_or_complex: {
+ required_field: "Complex choice",
+ optional_string: "Has a value",
+ optional_number: null,
+ optional_date: "2024-06-15T10:00:00"
+ }
+ - list_of_union_structures: [
+ {name: "Simple", age: 30, is_active: false},
+ {string_or_int: "text", optional_union: true, list_of_unions: [1, 2, 3]}
+ ]
+ - optional_nested_union: {
+ string_or_int: 42,
+ optional_union: null,
+ list_of_unions: ["a", 1, "b", 2]
+ }
+ """
+
+ NESTED_UNIONS_MIXED = """
+ - simple_or_complex: {
+ name: "Test User",
+ age: 40,
+ is_active: false
+ }
+ - list_of_union_structures: [
+ {string_or_int: 100, optional_union: "string", list_of_unions: []},
+ {name: "Another", age: 22, is_active: true}
+ ]
+ - optional_nested_union: {
+ mixed_dict: {"x": 1, "y": "two", "z": false},
+ union_or_list: "just a string",
+ optional_complex_union: ["list", "of", "strings"],
+ number_or_bool: false
+ }
+ """
+
+ # Combined test cases for parametrized tests
+ STRUCTURE_TEST_CASES: ClassVar[List[Tuple[str, str, str]]] = [ # topic, data, concept
+ ("Dicts with data", DICTS_WITH_DATA, "ConceptWithDicts"),
+ ("Dicts empty", DICTS_EMPTY, "ConceptWithDicts"),
+ ("Unions string variant", UNIONS_STRING_VARIANT, "ConceptWithUnions"),
+ ("Unions int variant", UNIONS_INT_VARIANT, "ConceptWithUnions"),
+ ("Unions mixed", UNIONS_MIXED, "ConceptWithUnions"),
+ ("Complex unions 1", COMPLEX_UNIONS_1, "ConceptWithComplexUnions"),
+ ("Complex unions 2", COMPLEX_UNIONS_2, "ConceptWithComplexUnions"),
+ ("Complex unions 3", COMPLEX_UNIONS_3, "ConceptWithComplexUnions"),
+ ("Nested unions simple", NESTED_UNIONS_SIMPLE, "ConceptWithNestedUnions"),
+ ("Nested unions complex", NESTED_UNIONS_COMPLEX, "ConceptWithNestedUnions"),
+ ("Nested unions mixed", NESTED_UNIONS_MIXED, "ConceptWithNestedUnions"),
]
diff --git a/tests/integration/pipelex/test_fundamentals.py b/tests/integration/pipelex/test_fundamentals.py
index 551cf042a..5e53fc616 100644
--- a/tests/integration/pipelex/test_fundamentals.py
+++ b/tests/integration/pipelex/test_fundamentals.py
@@ -1,6 +1,7 @@
import pytest
-from pipelex.pipe_works.pipe_dry import dry_run_all_pipes
+from pipelex.hub import get_pipe_provider
+from pipelex.pipe_works.pipe_dry import DryRunStatus, dry_run_pipes
from pipelex.pipelex import Pipelex
@@ -17,4 +18,12 @@ class TestFundamentals:
@pytest.mark.asyncio(loop_scope="class")
async def test_dry_run_all_pipes(self):
- await dry_run_all_pipes()
+ results = await dry_run_pipes(pipes=get_pipe_provider().get_pipes(), raise_on_failure=False)
+
+ # Check if there were any failures
+
+ failed_pipes = {pipe_code: output for pipe_code, output in results.items() if output.status == DryRunStatus.FAILURE}
+
+ if failed_pipes:
+ failure_details = "\n".join([f" - {pipe_code}: {output.error_message}" for pipe_code, output in failed_pipes.items()])
+ pytest.fail(f"Dry run failed for {len(failed_pipes)} pipes:\n{failure_details}")
diff --git a/tests/integration/pipelex/test_inline_structure_concepts.py b/tests/integration/pipelex/test_inline_structure_concepts.py
index e8deb5236..c6cbbde14 100644
--- a/tests/integration/pipelex/test_inline_structure_concepts.py
+++ b/tests/integration/pipelex/test_inline_structure_concepts.py
@@ -1,6 +1,6 @@
"""Integration tests for inline structure definitions in concepts."""
-from typing import Dict
+from typing import Dict, Union
import pytest
@@ -8,7 +8,6 @@ from pipelex.core.concepts.concept_blueprint import (
ConceptBlueprint,
ConceptStructureBlueprint,
ConceptStructureBlueprintFieldType,
- ConceptStructureBlueprintType,
)
from pipelex.core.concepts.concept_factory import ConceptFactory
from pipelex.core.stuffs.stuff_content import StructuredContent
@@ -22,7 +21,7 @@ class TestInlineStructureConcepts:
def test_inline_structure_definition_creation(self):
"""Test that concepts with inline structure definitions are created correctly."""
# Define inline structure with mixed syntax
- inline_structure: Dict[str, ConceptStructureBlueprintType] = {
+ inline_structure: Dict[str, Union[str, ConceptStructureBlueprint]] = {
"dominant_feature": ConceptStructureBlueprint(
type=ConceptStructureBlueprintFieldType.TEXT, definition="The most important feature", required=False
),
@@ -105,7 +104,7 @@ class TestInlineStructureConcepts:
def test_inline_structure_with_complex_types(self):
"""Test inline structure with complex field types."""
- inline_structure: Dict[str, ConceptStructureBlueprintType] = {
+ inline_structure: Dict[str, Union[str, ConceptStructureBlueprint]] = {
"title": ConceptStructureBlueprint(type=ConceptStructureBlueprintFieldType.TEXT, definition="Document title"),
"tags": ConceptStructureBlueprint(
type=ConceptStructureBlueprintFieldType.LIST,
@@ -175,7 +174,7 @@ not a registered subclass of StuffContent",
def test_multiple_inline_structures_do_not_conflict(self):
"""Test that multiple inline structures with same field names don't conflict."""
# First structure
- structure1: Dict[str, ConceptStructureBlueprintType] = {
+ structure1: Dict[str, Union[str, ConceptStructureBlueprint]] = {
"name": ConceptStructureBlueprint(type=ConceptStructureBlueprintFieldType.TEXT, definition="Person name"),
"age": ConceptStructureBlueprint(type=ConceptStructureBlueprintFieldType.INTEGER, definition="Person age", required=False),
}
@@ -186,7 +185,7 @@ not a registered subclass of StuffContent",
)
# Second structure with same field names but different context
- structure2: Dict[str, ConceptStructureBlueprintType] = {
+ structure2: Dict[str, Union[str, ConceptStructureBlueprint]] = {
"name": "Product name",
"age": ConceptStructureBlueprint(type=ConceptStructureBlueprintFieldType.INTEGER, definition="Product age in days", required=False),
}
diff --git a/tests/pipelex_unit_test.toml b/tests/pipelex_unit_test.toml
index 40ace2e55..54e968154 100644
--- a/tests/pipelex_unit_test.toml
+++ b/tests/pipelex_unit_test.toml
@@ -1,7 +1,4 @@
-[cogt.imgg_config]
-default_imgg_handle = "fal-ai/fast-lightning-sdxl"
-
[pipelex.feature_config]
-is_reporting_enabled = false
+is_reporting_enabled = true
diff --git a/tests/test_pipelines/misc_tests/test_image_inputs.plx b/tests/test_pipelines/misc_tests/test_image_inputs.plx
index 329418215..156e70882 100644
--- a/tests/test_pipelines/misc_tests/test_image_inputs.plx
+++ b/tests/test_pipelines/misc_tests/test_image_inputs.plx
@@ -14,7 +14,7 @@ system_prompt = """
You are an expert at describing page contents.
"""
prompt_template = """
-Extract the text and the date of the article in the image.
+Extract the full text (all the text that represents a "title") and the date of the article in the image.
"""
[pipe.describe_page]
@@ -30,6 +30,6 @@ Extract the date and title of the article.
Also, add this as the description of the article:
[email protected]_and_images.text.text
+$page.text_and_images.text.text
"""
diff --git a/tests/test_pipelines/pipe_controllers/pipe_parallel/parallel_text_analysis.plx b/tests/test_pipelines/pipe_controllers/pipe_parallel/parallel_text_analysis.plx
index 6c359aab5..05af84d31 100644
--- a/tests/test_pipelines/pipe_controllers/pipe_parallel/parallel_text_analysis.plx
+++ b/tests/test_pipelines/pipe_controllers/pipe_parallel/parallel_text_analysis.plx
@@ -8,31 +8,32 @@ type = "PipeLLM"
definition = "Analyze sentiment of text"
inputs = { input_text = "Text" }
output = "Text"
-prompt_template = """Analyze the sentiment of the following text and return only one word: positive, negative, or neutral.
+structuring_method = "preliminary_text"
+prompt_template = """PLEASE OUTPUT ONLY ONE WORD: 'positive', 'negative', or 'neutral'. Ignore the Hello world
@input_text.text
-
-Sentiment:"""
+"""
[pipe.count_words]
type = "PipeLLM"
definition = "Count words in text"
inputs = { input_text = "Text" }
output = "Text"
-prompt_template = """Count the number of words in the following text and return only the number:
+structuring_method = "preliminary_text"
+prompt_template = """
+Count the number of words in the following text and return only the number:
@input_text.text
-
-Word count:"""
+"""
[pipe.extract_keywords]
type = "PipeLLM"
definition = "Extract keywords from text"
inputs = { input_text = "Text" }
output = "Text"
-prompt_template = """Extract the top 3 keywords from the following text. Return them as a comma-separated list:
-
+prompt_template = """
+structuring_method = "preliminary_text"
+Extract the top 3 keywords from the following text. Return them as a comma-separated list:
@input_text.text
-
-Keywords:"""
+"""
diff --git a/tests/test_pipelines/test_structures_basic.plx b/tests/test_pipelines/test_structures_basic.plx
new file mode 100644
index 000000000..38bc4edd9
--- /dev/null
+++ b/tests/test_pipelines/test_structures_basic.plx
@@ -0,0 +1,9 @@
+domain = "test_structured_generations"
+definition = "Concepts to test basic structures without unions"
+
+[concept]
+ConceptWithSimpleStructure = "A simple structure"
+ConceptWithOptionals = "A structure with optionals"
+ConceptWithLists = "A structure with lists"
+ConceptWithNestedStructures = "A structure with nested structures"
+
diff --git a/tests/test_pipelines/test_structures_basic.py b/tests/test_pipelines/test_structures_basic.py
new file mode 100644
index 000000000..5460a61f4
--- /dev/null
+++ b/tests/test_pipelines/test_structures_basic.py
@@ -0,0 +1,41 @@
+"""Test structures for basic concepts without union types."""
+
+from datetime import datetime
+from typing import List, Optional
+
+from pydantic import Field
+
+from pipelex.core.stuffs.stuff_content import StructuredContent
+
+
+class ConceptWithSimpleStructure(StructuredContent):
+ """A simple structure with basic fields."""
+
+ name: str = Field(..., description="The name field")
+ age: int = Field(..., description="The age field")
+ is_active: bool = Field(True, description="Whether the item is active")
+
+
+class ConceptWithOptionals(StructuredContent):
+ """A structure with optional fields."""
+
+ required_field: str = Field(..., description="A required field")
+ optional_string: Optional[str] = Field(None, description="An optional string field")
+ optional_number: Optional[int] = Field(None, description="An optional number field")
+ optional_date: Optional[datetime] = Field(None, description="An optional date field")
+
+
+class ConceptWithLists(StructuredContent):
+ """A structure with list fields."""
+
+ string_list: List[str] = Field(default_factory=list, description="A list of strings")
+ number_list: List[int] = Field(default_factory=list, description="A list of numbers")
+ optional_list: Optional[List[str]] = Field(None, description="An optional list")
+
+
+class ConceptWithNestedStructures(StructuredContent):
+ """A structure with nested structures."""
+
+ simple_nested: ConceptWithSimpleStructure = Field(..., description="A nested simple structure")
+ optional_nested: Optional[ConceptWithOptionals] = Field(None, description="An optional nested structure")
+ list_of_nested: List[ConceptWithSimpleStructure] = Field(default_factory=list, description="A list of nested structures")
diff --git a/tests/test_pipelines/test_structures_complex.plx b/tests/test_pipelines/test_structures_complex.plx
new file mode 100644
index 000000000..e9ca731fa
--- /dev/null
+++ b/tests/test_pipelines/test_structures_complex.plx
@@ -0,0 +1,9 @@
+domain = "test_structured_generations"
+definition = "Concepts to test complex structures (dicts, unions, etc.)"
+
+[concept]
+ConceptWithDicts = "A structure with dicts"
+ConceptWithUnions = "A structure with union types"
+ConceptWithComplexUnions = "A structure with more complex union combinations"
+ConceptWithNestedUnions = "A structure with nested unions"
+
diff --git a/tests/test_pipelines/test_structures_complex.py b/tests/test_pipelines/test_structures_complex.py
new file mode 100644
index 000000000..06e13705d
--- /dev/null
+++ b/tests/test_pipelines/test_structures_complex.py
@@ -0,0 +1,45 @@
+"""Test structures for complex concepts (dicts, unions, etc.)."""
+
+from typing import Dict, List, Optional, Union
+
+from pydantic import Field
+
+from pipelex.core.stuffs.stuff_content import StructuredContent
+from tests.test_pipelines.test_structures_basic import ConceptWithOptionals, ConceptWithSimpleStructure
+
+
+class ConceptWithDicts(StructuredContent):
+ """A structure with dictionary fields."""
+
+ string_dict: Dict[str, str] = Field(default_factory=dict, description="A dictionary of strings")
+ number_dict: Dict[str, int] = Field(default_factory=dict, description="A dictionary of numbers")
+ optional_dict: Optional[Dict[str, str]] = Field(None, description="An optional dictionary")
+
+
+class ConceptWithUnions(StructuredContent):
+ """A structure with union types."""
+
+ string_or_int: Union[str, int] = Field(..., description="A field that can be string or int")
+ optional_union: Optional[Union[str, bool]] = Field(None, description="An optional union field")
+ list_of_unions: List[Union[str, int]] = Field(default_factory=list, description="A list of union types")
+
+
+class ConceptWithComplexUnions(StructuredContent):
+ """A structure with more complex union combinations."""
+
+ mixed_dict: Dict[str, Union[str, int, bool]] = Field(default_factory=dict, description="A dictionary with mixed value types")
+ union_or_list: Union[str, List[int]] = Field(..., description="Either a string or a list of integers")
+ optional_complex_union: Optional[Union[Dict[str, str], List[str]]] = Field(None, description="Optional dict or list")
+ number_or_bool: Union[int, float, bool] = Field(..., description="Number or boolean value")
+
+
+class ConceptWithNestedUnions(StructuredContent):
+ """A structure with nested unions including other structures."""
+
+ simple_or_complex: Union[ConceptWithSimpleStructure, ConceptWithOptionals] = Field(..., description="Simple or complex structure")
+ list_of_union_structures: List[Union[ConceptWithSimpleStructure, ConceptWithUnions]] = Field(
+ default_factory=list, description="List of different structure types"
+ )
+ optional_nested_union: Optional[Union[ConceptWithUnions, ConceptWithComplexUnions]] = Field(
+ None, description="Optional union of union structures"
+ )
diff --git a/tests/unit/pipelex/client/test_api_serialization.py b/tests/unit/pipelex/client/test_api_serialization.py
index 063b97993..ed9b40fee 100644
--- a/tests/unit/pipelex/client/test_api_serialization.py
+++ b/tests/unit/pipelex/client/test_api_serialization.py
@@ -148,7 +148,7 @@ class TestApiSerialization:
assert "sample_text" in compact_memory
text_blueprint = compact_memory["sample_text"]
- assert text_blueprint["concept_code"] == NativeConceptEnum.TEXT.value
+ assert text_blueprint["concept_code"] == NativeConceptEnum.TEXT
assert isinstance(text_blueprint["content"], str)
assert text_blueprint["content"] == "Sample text content"
@@ -160,6 +160,6 @@ class TestApiSerialization:
assert "pi_value" in compact_memory
number_blueprint = compact_memory["pi_value"]
- assert number_blueprint["concept_code"] == NativeConceptEnum.NUMBER.value
+ assert number_blueprint["concept_code"] == NativeConceptEnum.NUMBER
assert isinstance(number_blueprint["content"], dict)
assert number_blueprint["content"]["number"] == 3.14159
diff --git a/tests/unit/pipelex/cogt/models/test_model_deck.py b/tests/unit/pipelex/cogt/models/test_model_deck.py
index 2ff662841..0620618ce 100644
--- a/tests/unit/pipelex/cogt/models/test_model_deck.py
+++ b/tests/unit/pipelex/cogt/models/test_model_deck.py
@@ -42,6 +42,10 @@ class TestModelDeckGetOptionalInferenceModel:
for_text=LLMSetting(llm_handle="default_text", temperature=0.7, max_tokens=1000),
for_object=LLMSetting(llm_handle="default_object", temperature=0.1, max_tokens=1000),
),
+ ocr_presets={},
+ ocr_choice_default="base_ocr_mistral",
+ img_gen_presets={},
+ img_gen_choice_default="base_img_gen",
)
def test_direct_model_lookup_success(self):
diff --git a/tests/unit/pipelex/core/concepts/concept/test_concept.py b/tests/unit/pipelex/core/concepts/concept/test_concept.py
index 9e9a4ec00..905136eee 100644
--- a/tests/unit/pipelex/core/concepts/concept/test_concept.py
+++ b/tests/unit/pipelex/core/concepts/concept/test_concept.py
@@ -3,7 +3,7 @@ import pytest
from pipelex.core.concepts.concept import Concept
from pipelex.core.concepts.concept_blueprint import ConceptBlueprint
from pipelex.core.concepts.concept_factory import ConceptFactory
-from pipelex.core.concepts.concept_native import NATIVE_CONCEPTS_DATA, NativeConceptEnum
+from pipelex.core.concepts.concept_native import NATIVE_CONCEPTS_DATA, NativeConceptEnum, NativeConceptManager
from pipelex.core.concepts.exceptions import ConceptCodeError, ConceptStringError
from pipelex.core.domains.domain import SpecialDomain
from pipelex.core.domains.exceptions import DomainError
@@ -14,53 +14,53 @@ class TestConcept:
def test_is_native_concept_code(self):
"""Test is_native_concept_code method."""
- assert ConceptBlueprint.is_native_concept_code(NativeConceptEnum.TEXT.value) is True
- assert ConceptBlueprint.is_native_concept_code(NativeConceptEnum.IMAGE.value) is True
- assert ConceptBlueprint.is_native_concept_code(NativeConceptEnum.PDF.value) is True
- assert ConceptBlueprint.is_native_concept_code(NativeConceptEnum.TEXT_AND_IMAGES.value) is True
- assert ConceptBlueprint.is_native_concept_code(NativeConceptEnum.NUMBER.value) is True
- assert ConceptBlueprint.is_native_concept_code(NativeConceptEnum.LLM_PROMPT.value) is True
- assert ConceptBlueprint.is_native_concept_code(NativeConceptEnum.ANYTHING.value) is True
- assert ConceptBlueprint.is_native_concept_code(NativeConceptEnum.DYNAMIC.value) is True
+ assert ConceptBlueprint.is_native_concept_code(NativeConceptEnum.TEXT) is True
+ assert ConceptBlueprint.is_native_concept_code(NativeConceptEnum.IMAGE) is True
+ assert ConceptBlueprint.is_native_concept_code(NativeConceptEnum.PDF) is True
+ assert ConceptBlueprint.is_native_concept_code(NativeConceptEnum.TEXT_AND_IMAGES) is True
+ assert ConceptBlueprint.is_native_concept_code(NativeConceptEnum.NUMBER) is True
+ assert ConceptBlueprint.is_native_concept_code(NativeConceptEnum.LLM_PROMPT) is True
+ assert ConceptBlueprint.is_native_concept_code(NativeConceptEnum.ANYTHING) is True
+ assert ConceptBlueprint.is_native_concept_code(NativeConceptEnum.DYNAMIC) is True
with pytest.raises(ConceptCodeError):
- ConceptBlueprint.is_native_concept_code(f"{SpecialDomain.NATIVE.value}.{NativeConceptEnum.TEXT.value}")
+ ConceptBlueprint.is_native_concept_code(f"{SpecialDomain.NATIVE}.{NativeConceptEnum.TEXT}")
with pytest.raises(ConceptCodeError):
- ConceptBlueprint.is_native_concept_code(f"{SpecialDomain.NATIVE.value}.{NativeConceptEnum.IMAGE.value}")
+ ConceptBlueprint.is_native_concept_code(f"{SpecialDomain.NATIVE}.{NativeConceptEnum.IMAGE}")
with pytest.raises(ConceptCodeError):
- ConceptBlueprint.is_native_concept_code(f"{SpecialDomain.NATIVE.value}.{NativeConceptEnum.PDF.value}")
+ ConceptBlueprint.is_native_concept_code(f"{SpecialDomain.NATIVE}.{NativeConceptEnum.PDF}")
with pytest.raises(ConceptCodeError):
- ConceptBlueprint.is_native_concept_code(f"{SpecialDomain.NATIVE.value}.{NativeConceptEnum.TEXT_AND_IMAGES.value}")
+ ConceptBlueprint.is_native_concept_code(f"{SpecialDomain.NATIVE}.{NativeConceptEnum.TEXT_AND_IMAGES}")
with pytest.raises(ConceptCodeError):
- ConceptBlueprint.is_native_concept_code(f"{SpecialDomain.NATIVE.value}.{NativeConceptEnum.NUMBER.value}")
+ ConceptBlueprint.is_native_concept_code(f"{SpecialDomain.NATIVE}.{NativeConceptEnum.NUMBER}")
with pytest.raises(ConceptCodeError):
- ConceptBlueprint.is_native_concept_code(f"{SpecialDomain.NATIVE.value}.{NativeConceptEnum.LLM_PROMPT.value}")
+ ConceptBlueprint.is_native_concept_code(f"{SpecialDomain.NATIVE}.{NativeConceptEnum.LLM_PROMPT}")
with pytest.raises(ConceptCodeError):
- ConceptBlueprint.is_native_concept_code(f"not_native.{NativeConceptEnum.TEXT.value}")
+ ConceptBlueprint.is_native_concept_code(f"not_native.{NativeConceptEnum.TEXT}")
with pytest.raises(ConceptCodeError):
- ConceptBlueprint.is_native_concept_code(f"not_native.{NativeConceptEnum.IMAGE.value}")
+ ConceptBlueprint.is_native_concept_code(f"not_native.{NativeConceptEnum.IMAGE}")
with pytest.raises(ConceptCodeError):
- ConceptBlueprint.is_native_concept_code(f"not_native.{NativeConceptEnum.PDF.value}")
+ ConceptBlueprint.is_native_concept_code(f"not_native.{NativeConceptEnum.PDF}")
with pytest.raises(ConceptCodeError):
- ConceptBlueprint.is_native_concept_code(f"not_native.{NativeConceptEnum.TEXT_AND_IMAGES.value}")
+ ConceptBlueprint.is_native_concept_code(f"not_native.{NativeConceptEnum.TEXT_AND_IMAGES}")
with pytest.raises(ConceptCodeError):
- ConceptBlueprint.is_native_concept_code(f"not_native.{NativeConceptEnum.NUMBER.value}")
+ ConceptBlueprint.is_native_concept_code(f"not_native.{NativeConceptEnum.NUMBER}")
with pytest.raises(ConceptCodeError):
- ConceptBlueprint.is_native_concept_code(f"not_native.{NativeConceptEnum.LLM_PROMPT.value}")
+ ConceptBlueprint.is_native_concept_code(f"not_native.{NativeConceptEnum.LLM_PROMPT}")
with pytest.raises(ConceptCodeError):
- ConceptBlueprint.is_native_concept_code(f"not_native.{NativeConceptEnum.DYNAMIC.value}")
+ ConceptBlueprint.is_native_concept_code(f"not_native.{NativeConceptEnum.DYNAMIC}")
assert ConceptBlueprint.is_native_concept_code("RandomConcept") is False
with pytest.raises(ConceptCodeError):
@@ -68,46 +68,43 @@ class TestConcept:
def test_is_native_concept_string_or_concept_code(self):
"""Test is_native_concept_code method."""
- assert ConceptBlueprint.is_native_concept_string_or_concept_code(NativeConceptEnum.TEXT.value) is True
- assert ConceptBlueprint.is_native_concept_string_or_concept_code(NativeConceptEnum.IMAGE.value) is True
- assert ConceptBlueprint.is_native_concept_string_or_concept_code(NativeConceptEnum.PDF.value) is True
- assert ConceptBlueprint.is_native_concept_string_or_concept_code(NativeConceptEnum.TEXT_AND_IMAGES.value) is True
- assert ConceptBlueprint.is_native_concept_string_or_concept_code(NativeConceptEnum.NUMBER.value) is True
- assert ConceptBlueprint.is_native_concept_string_or_concept_code(NativeConceptEnum.LLM_PROMPT.value) is True
- assert ConceptBlueprint.is_native_concept_string_or_concept_code(NativeConceptEnum.ANYTHING.value) is True
- assert ConceptBlueprint.is_native_concept_string_or_concept_code(NativeConceptEnum.DYNAMIC.value) is True
- assert ConceptBlueprint.is_native_concept_string_or_concept_code(f"{SpecialDomain.NATIVE.value}.{NativeConceptEnum.TEXT.value}") is True
- assert ConceptBlueprint.is_native_concept_string_or_concept_code(f"{SpecialDomain.NATIVE.value}.{NativeConceptEnum.IMAGE.value}") is True
- assert ConceptBlueprint.is_native_concept_string_or_concept_code(f"{SpecialDomain.NATIVE.value}.{NativeConceptEnum.PDF.value}") is True
- assert (
- ConceptBlueprint.is_native_concept_string_or_concept_code(f"{SpecialDomain.NATIVE.value}.{NativeConceptEnum.TEXT_AND_IMAGES.value}")
- is True
- )
- assert ConceptBlueprint.is_native_concept_string_or_concept_code(f"{SpecialDomain.NATIVE.value}.{NativeConceptEnum.NUMBER.value}") is True
- assert ConceptBlueprint.is_native_concept_string_or_concept_code(f"{SpecialDomain.NATIVE.value}.{NativeConceptEnum.LLM_PROMPT.value}") is True
- assert ConceptBlueprint.is_native_concept_string_or_concept_code(f"not_native.{NativeConceptEnum.TEXT.value}") is False
- assert ConceptBlueprint.is_native_concept_string_or_concept_code(f"not_native.{NativeConceptEnum.IMAGE.value}") is False
- assert ConceptBlueprint.is_native_concept_string_or_concept_code(f"not_native.{NativeConceptEnum.PDF.value}") is False
- assert ConceptBlueprint.is_native_concept_string_or_concept_code(f"not_native.{NativeConceptEnum.TEXT_AND_IMAGES.value}") is False
- assert ConceptBlueprint.is_native_concept_string_or_concept_code(f"not_native.{NativeConceptEnum.NUMBER.value}") is False
- assert ConceptBlueprint.is_native_concept_string_or_concept_code(f"not_native.{NativeConceptEnum.LLM_PROMPT.value}") is False
- assert ConceptBlueprint.is_native_concept_string_or_concept_code(f"not_native.{NativeConceptEnum.ANYTHING.value}") is False
- assert ConceptBlueprint.is_native_concept_string_or_concept_code(f"not_native.{NativeConceptEnum.DYNAMIC.value}") is False
- assert ConceptBlueprint.is_native_concept_string_or_concept_code("RandomConcept") is False
- assert ConceptBlueprint.is_native_concept_string_or_concept_code("text") is False
+ assert NativeConceptManager.is_native_concept(NativeConceptEnum.TEXT) is True
+ assert NativeConceptManager.is_native_concept(NativeConceptEnum.IMAGE) is True
+ assert NativeConceptManager.is_native_concept(NativeConceptEnum.PDF) is True
+ assert NativeConceptManager.is_native_concept(NativeConceptEnum.TEXT_AND_IMAGES) is True
+ assert NativeConceptManager.is_native_concept(NativeConceptEnum.NUMBER) is True
+ assert NativeConceptManager.is_native_concept(NativeConceptEnum.LLM_PROMPT) is True
+ assert NativeConceptManager.is_native_concept(NativeConceptEnum.ANYTHING) is True
+ assert NativeConceptManager.is_native_concept(NativeConceptEnum.DYNAMIC) is True
+ assert NativeConceptManager.is_native_concept(f"{SpecialDomain.NATIVE}.{NativeConceptEnum.TEXT}") is True
+ assert NativeConceptManager.is_native_concept(f"{SpecialDomain.NATIVE}.{NativeConceptEnum.IMAGE}") is True
+ assert NativeConceptManager.is_native_concept(f"{SpecialDomain.NATIVE}.{NativeConceptEnum.PDF}") is True
+ assert NativeConceptManager.is_native_concept(f"{SpecialDomain.NATIVE}.{NativeConceptEnum.TEXT_AND_IMAGES}") is True
+ assert NativeConceptManager.is_native_concept(f"{SpecialDomain.NATIVE}.{NativeConceptEnum.NUMBER}") is True
+ assert NativeConceptManager.is_native_concept(f"{SpecialDomain.NATIVE}.{NativeConceptEnum.LLM_PROMPT}") is True
+ assert NativeConceptManager.is_native_concept(f"not_native.{NativeConceptEnum.TEXT}") is False
+ assert NativeConceptManager.is_native_concept(f"not_native.{NativeConceptEnum.IMAGE}") is False
+ assert NativeConceptManager.is_native_concept(f"not_native.{NativeConceptEnum.PDF}") is False
+ assert NativeConceptManager.is_native_concept(f"not_native.{NativeConceptEnum.TEXT_AND_IMAGES}") is False
+ assert NativeConceptManager.is_native_concept(f"not_native.{NativeConceptEnum.NUMBER}") is False
+ assert NativeConceptManager.is_native_concept(f"not_native.{NativeConceptEnum.LLM_PROMPT}") is False
+ assert NativeConceptManager.is_native_concept(f"not_native.{NativeConceptEnum.ANYTHING}") is False
+ assert NativeConceptManager.is_native_concept(f"not_native.{NativeConceptEnum.DYNAMIC}") is False
+ assert NativeConceptManager.is_native_concept("RandomConcept") is False
+ assert NativeConceptManager.is_native_concept("text") is False
def test_is_native_concept(self):
"""Test is_native_concept method."""
valid_domain = "valid_domain"
valid_definition = "Lorem Ipsum"
- for native_concept in NativeConceptEnum:
+ for native_concept in [native_concept for native_concept in NativeConceptEnum]:
assert Concept.is_native_concept(ConceptFactory.make_native_concept(native_concept_data=NATIVE_CONCEPTS_DATA[native_concept])) is True
assert (
Concept.is_native_concept(
ConceptFactory.make_from_blueprint(
- concept_code=NativeConceptEnum.TEXT.value,
+ concept_code=NativeConceptEnum.TEXT,
domain=valid_domain,
blueprint=ConceptBlueprint(definition=valid_definition),
concept_codes_from_the_same_domain=["RandomConcept"],
@@ -118,8 +115,8 @@ class TestConcept:
assert (
Concept.is_native_concept(
ConceptFactory.make_from_blueprint(
- concept_code=NativeConceptEnum.TEXT.value,
- domain=SpecialDomain.NATIVE.value,
+ concept_code=NativeConceptEnum.TEXT,
+ domain=SpecialDomain.NATIVE,
blueprint=ConceptBlueprint(definition=valid_definition),
)
)
@@ -128,7 +125,7 @@ class TestConcept:
assert (
Concept.is_native_concept(
ConceptFactory.make_from_blueprint(
- concept_code=NativeConceptEnum.IMAGE.value,
+ concept_code=NativeConceptEnum.IMAGE,
domain=valid_domain,
blueprint=ConceptBlueprint(definition=valid_definition),
concept_codes_from_the_same_domain=["RandomConcept"],
@@ -139,7 +136,7 @@ class TestConcept:
assert (
Concept.is_native_concept(
ConceptFactory.make_from_blueprint(
- concept_code=NativeConceptEnum.PDF.value,
+ concept_code=NativeConceptEnum.PDF,
domain=valid_domain,
blueprint=ConceptBlueprint(definition=valid_definition),
concept_codes_from_the_same_domain=["RandomConcept"],
@@ -150,7 +147,7 @@ class TestConcept:
assert (
Concept.is_native_concept(
ConceptFactory.make_from_blueprint(
- concept_code=NativeConceptEnum.TEXT_AND_IMAGES.value,
+ concept_code=NativeConceptEnum.TEXT_AND_IMAGES,
domain=valid_domain,
blueprint=ConceptBlueprint(definition=valid_definition),
concept_codes_from_the_same_domain=["RandomConcept"],
@@ -161,7 +158,7 @@ class TestConcept:
assert (
Concept.is_native_concept(
ConceptFactory.make_from_blueprint(
- concept_code=NativeConceptEnum.NUMBER.value,
+ concept_code=NativeConceptEnum.NUMBER,
domain=valid_domain,
blueprint=ConceptBlueprint(definition=valid_definition),
concept_codes_from_the_same_domain=["RandomConcept"],
@@ -172,7 +169,7 @@ class TestConcept:
assert (
Concept.is_native_concept(
ConceptFactory.make_from_blueprint(
- concept_code=NativeConceptEnum.LLM_PROMPT.value,
+ concept_code=NativeConceptEnum.LLM_PROMPT,
domain=valid_domain,
blueprint=ConceptBlueprint(definition=valid_definition),
concept_codes_from_the_same_domain=["RandomConcept"],
@@ -183,7 +180,7 @@ class TestConcept:
assert (
Concept.is_native_concept(
ConceptFactory.make_from_blueprint(
- concept_code=NativeConceptEnum.ANYTHING.value,
+ concept_code=NativeConceptEnum.ANYTHING,
domain=valid_domain,
blueprint=ConceptBlueprint(definition=valid_definition),
concept_codes_from_the_same_domain=["RandomConcept"],
@@ -207,8 +204,8 @@ class TestConcept:
"""Test construct_concept_string_with_domain method."""
valid_domain = "valid_domain"
assert (
- ConceptFactory.construct_concept_string_with_domain(domain=valid_domain, concept_code=NativeConceptEnum.TEXT.value)
- == f"{valid_domain}.{NativeConceptEnum.TEXT.value}"
+ ConceptFactory.construct_concept_string_with_domain(domain=valid_domain, concept_code=NativeConceptEnum.TEXT)
+ == f"{valid_domain}.{NativeConceptEnum.TEXT}"
)
def test_validate_concept_string(self):
@@ -221,7 +218,7 @@ class TestConcept:
assert ConceptBlueprint.validate_concept_string(f"snake_case_domain.{valid_concept_code}") is None
assert ConceptBlueprint.validate_concept_string(f"domain_123.{valid_concept_code}") is None
assert ConceptBlueprint.validate_concept_string(f"{valid_domain}.TEXT") is None
- assert ConceptBlueprint.validate_concept_string(f"{SpecialDomain.NATIVE.value}.{NativeConceptEnum.ANYTHING.value}") is None
+ assert ConceptBlueprint.validate_concept_string(f"{SpecialDomain.NATIVE}.{NativeConceptEnum.ANYTHING}") is None
# Invalid cases - should raise ConceptCodeError
@@ -260,20 +257,20 @@ class TestConcept:
# Invalid native concept
with pytest.raises(ConceptStringError):
- ConceptBlueprint.validate_concept_string(f"{SpecialDomain.NATIVE.value}.InvalidNativeConcept")
+ ConceptBlueprint.validate_concept_string(f"{SpecialDomain.NATIVE}.InvalidNativeConcept")
def test_are_concept_compatible(self):
"""Test are_concept_compatible method."""
concept1 = ConceptFactory.make_from_blueprint(
concept_code="Code1",
domain="domain1",
- blueprint=ConceptBlueprint(definition="Lorem Ipsum", refines=NativeConceptEnum.TEXT.value),
+ blueprint=ConceptBlueprint(definition="Lorem Ipsum", refines=NativeConceptEnum.TEXT),
concept_codes_from_the_same_domain=["Code1"],
)
concept2 = ConceptFactory.make_from_blueprint(
concept_code="Code2",
domain="domain1",
- blueprint=ConceptBlueprint(definition="Lorem Ipsum", refines=NativeConceptEnum.TEXT.value),
+ blueprint=ConceptBlueprint(definition="Lorem Ipsum", refines=NativeConceptEnum.TEXT),
concept_codes_from_the_same_domain=["Code1"],
)
concept3 = ConceptFactory.make_from_blueprint(
diff --git a/tests/unit/pipelex/core/concepts/concept_factory.py/data.py b/tests/unit/pipelex/core/concepts/concept_factory.py/data.py
index 659170052..9a5488618 100644
--- a/tests/unit/pipelex/core/concepts/concept_factory.py/data.py
+++ b/tests/unit/pipelex/core/concepts/concept_factory.py/data.py
@@ -14,22 +14,20 @@ from pipelex.core.domains.domain import SpecialDomain
class TestCases:
- """Test cases for ConceptFactory methods."""
-
# Test cases for make_refines method - only native concepts can be refined
MAKE_REFINES_TEST_CASES: ClassVar[List[Tuple[str, ConceptBlueprint, str]]] = [
(
"native_concept_string",
- ConceptBlueprint(definition="A concept that refines a native text concept", refines=NativeConceptEnum.TEXT.value),
- f"{SpecialDomain.NATIVE.value}.{NativeConceptEnum.TEXT.value}",
+ ConceptBlueprint(definition="A concept that refines a native text concept", refines=NativeConceptEnum.TEXT),
+ f"{SpecialDomain.NATIVE}.{NativeConceptEnum.TEXT}",
),
(
"fully_qualified_native_string",
ConceptBlueprint(
definition="A concept that refines a fully qualified native concept",
- refines=f"{SpecialDomain.NATIVE.value}.{NativeConceptEnum.TEXT.value}",
+ refines=f"{SpecialDomain.NATIVE}.{NativeConceptEnum.TEXT}",
),
- f"{SpecialDomain.NATIVE.value}.{NativeConceptEnum.TEXT.value}",
+ f"{SpecialDomain.NATIVE}.{NativeConceptEnum.TEXT}",
),
]
diff --git a/tests/unit/pipelex/core/concepts/concept_factory.py/test_concept_factory.py b/tests/unit/pipelex/core/concepts/concept_factory.py/test_concept_factory.py
index 790e5526d..d62bb71be 100644
--- a/tests/unit/pipelex/core/concepts/concept_factory.py/test_concept_factory.py
+++ b/tests/unit/pipelex/core/concepts/concept_factory.py/test_concept_factory.py
@@ -1,6 +1,4 @@
-"""Tests for ConceptFactory methods."""
-
-from typing import Dict, List, Optional
+from typing import Dict, List, Optional, Union
import pytest
@@ -9,7 +7,6 @@ from pipelex.core.concepts.concept_blueprint import (
ConceptBlueprint,
ConceptStructureBlueprint,
ConceptStructureBlueprintFieldType,
- ConceptStructureBlueprintType,
)
from pipelex.core.concepts.concept_factory import ConceptFactory
@@ -35,7 +32,7 @@ class TestConceptFactory:
def test_normalize_structure_blueprint(self):
"""Test that mixed structure blueprints are properly normalized."""
- mixed_structure_blueprint: Dict[str, ConceptStructureBlueprintType] = {
+ mixed_structure_blueprint: Dict[str, Union[str, ConceptStructureBlueprint]] = {
"name": "The name of the person",
"age": ConceptStructureBlueprint(definition="The age of the person", type=ConceptStructureBlueprintFieldType.NUMBER, required=True),
"active": ConceptStructureBlueprint(
@@ -53,7 +50,7 @@ class TestConceptFactory:
assert ConceptFactory.normalize_structure_blueprint(mixed_structure_blueprint) == expected_structure
- mixed_structure_blueprint2: Dict[str, ConceptStructureBlueprintType] = {
+ mixed_structure_blueprint2: Dict[str, Union[str, ConceptStructureBlueprint]] = {
"name": ConceptStructureBlueprint(definition="The name of the person", type=ConceptStructureBlueprintFieldType.TEXT, required=True),
"age": ConceptStructureBlueprint(definition="The age of the person", type=ConceptStructureBlueprintFieldType.NUMBER, required=True),
"active": ConceptStructureBlueprint(
@@ -72,20 +69,20 @@ class TestConceptFactory:
assert ConceptFactory.normalize_structure_blueprint(mixed_structure_blueprint2) == expected_structure2
@pytest.mark.parametrize(
- "domain,concept_string_or_concept_code,concept_codes_from_the_same_domain,expected_result",
+ "domain,concept_string_or_code,concept_codes_from_the_same_domain,expected_result",
TestCases.MAKE_DOMAIN_AND_CONCEPT_CODE_TEST_CASES,
)
def test_make_domain_and_concept_code_from_concept_string_or_concept_code(
self,
domain: str,
- concept_string_or_concept_code: str,
+ concept_string_or_code: str,
concept_codes_from_the_same_domain: Optional[List[str]],
expected_result: List[str],
):
"""Test make_domain_and_concept_code_from_concept_string_or_concept_code method with various inputs."""
result = ConceptFactory.make_domain_and_concept_code_from_concept_string_or_concept_code(
domain=domain,
- concept_string_or_concept_code=concept_string_or_concept_code,
+ concept_string_or_code=concept_string_or_code,
concept_codes_from_the_same_domain=concept_codes_from_the_same_domain,
)
assert result == expected_result
diff --git a/tests/unit/pipelex/core/concepts/test_concept_blueprint.py b/tests/unit/pipelex/core/concepts/test_concept_blueprint.py
index 613a8d9d0..49e486db5 100644
--- a/tests/unit/pipelex/core/concepts/test_concept_blueprint.py
+++ b/tests/unit/pipelex/core/concepts/test_concept_blueprint.py
@@ -42,19 +42,19 @@ class TestConceptStructureBlueprintValidation:
)
assert list_blueprint.default_value == ["item1", "item2"]
- # Valid dict field with default
- dict_blueprint = ConceptStructureBlueprint(
- definition="A dict field",
- type=ConceptStructureBlueprintFieldType.DICT,
- key_type="text",
- value_type="text",
- default_value={"key": "value"},
- )
- assert dict_blueprint.default_value == {"key": "value"}
-
- # Valid choice field with default
- choice_blueprint = ConceptStructureBlueprint(definition="A choice field", choices=["low", "medium", "high"], default_value="medium")
- assert choice_blueprint.default_value == "medium"
+ # Valid dict field with default
+ dict_blueprint = ConceptStructureBlueprint(
+ definition="A dict field",
+ type=ConceptStructureBlueprintFieldType.DICT,
+ key_type="text",
+ value_type="text",
+ default_value={"key": "value"},
+ )
+ assert dict_blueprint.default_value == {"key": "value"}
+
+ # Valid choice field with default
+ choice_blueprint = ConceptStructureBlueprint(definition="A choice field", choices=["low", "medium", "high"], default_value="medium")
+ assert choice_blueprint.default_value == "medium"
def test_default_value_type_mismatch(self):
"""Test that default_value type mismatches are caught."""
diff --git a/tests/unit/pipelex/core/interpreter/test_interpreter.py b/tests/unit/pipelex/core/interpreter/test_interpreter.py
index ce0a1e95b..fb2c2be72 100644
--- a/tests/unit/pipelex/core/interpreter/test_interpreter.py
+++ b/tests/unit/pipelex/core/interpreter/test_interpreter.py
@@ -9,8 +9,6 @@ from tests.unit.pipelex.core.test_data import InterpreterTestCases
class TestPipelexInterpreter:
- """Test the PipelexInterpreter class with various PLX configurations."""
-
def test_init_with_both_file_path_and_content(self, tmp_path: Path):
"""Test initialization with both file_path and file_content."""
test_file = tmp_path / "test.plx"
diff --git a/tests/unit/pipelex/core/interpreter/test_interpreter_serialize.py b/tests/unit/pipelex/core/interpreter/test_interpreter_serialize.py
index e41aee383..923b80aa1 100644
--- a/tests/unit/pipelex/core/interpreter/test_interpreter_serialize.py
+++ b/tests/unit/pipelex/core/interpreter/test_interpreter_serialize.py
@@ -4,7 +4,7 @@ from typing import Any, Dict
from pipelex.core.concepts.concept_blueprint import ConceptBlueprint, ConceptStructureBlueprint, ConceptStructureBlueprintFieldType
from pipelex.core.interpreter import PipelexInterpreter
-from pipelex.core.pipes.pipe_input_spec_blueprint import InputRequirementBlueprint
+from pipelex.core.pipes.pipe_input_blueprint import InputRequirementBlueprint
from pipelex.pipe_controllers.sequence.pipe_sequence_blueprint import PipeSequenceBlueprint
from pipelex.pipe_controllers.sub_pipe_blueprint import SubPipeBlueprint
from pipelex.pipe_operators.llm.pipe_llm_blueprint import PipeLLMBlueprint
@@ -287,10 +287,10 @@ class TestSerializePipeBlueprints:
def test_serialize_ocr_pipe_basic(self):
"""Test serializing basic PipeOcr blueprint."""
- pipe = PipeOcrBlueprint(type="PipeOcr", definition="Extract text from PDF", output="Page", ocr_model="pypdfium2-extract-text")
+ pipe = PipeOcrBlueprint(type="PipeOcr", definition="Extract text from PDF", output="Page", ocr="base_ocr_pypdfium2")
result = PipelexInterpreter.serialize_ocr_pipe(pipe, "test_domain")
- expected = {"type": "PipeOcr", "definition": "Extract text from PDF", "output": "Page", "ocr_model": "pypdfium2-extract-text"}
+ expected = {"type": "PipeOcr", "definition": "Extract text from PDF", "output": "Page", "ocr": "base_ocr_pypdfium2"}
assert result == expected
def test_serialize_sequence_pipe_basic(self):
diff --git a/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_img_gen_plx.py b/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_img_gen_plx.py
index b20b3f539..663f44187 100644
--- a/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_img_gen_plx.py
+++ b/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_img_gen_plx.py
@@ -1,6 +1,6 @@
import pytest
-from pipelex.cogt.imgg.imgg_job_components import AspectRatio
+from pipelex.cogt.img_gen.img_gen_job_components import AspectRatio
from pipelex.core.interpreter import PipelexInterpreter
from pipelex.pipe_operators.img_gen.pipe_img_gen_blueprint import PipeImgGenBlueprint
diff --git a/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_input_spec.py b/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_input_spec.py
index daa048a48..4a9f30432 100644
--- a/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_input_spec.py
+++ b/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_input_spec.py
@@ -3,7 +3,7 @@ from typing import Any
import pytest
from pipelex.core.interpreter import PipelexInterpreter
-from pipelex.core.pipes.pipe_input_spec_blueprint import InputRequirementBlueprint
+from pipelex.core.pipes.pipe_input_blueprint import InputRequirementBlueprint
class TestPipelexInterpreterInputSpec:
diff --git a/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_llm_plx.py b/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_llm_plx.py
index d7c2822cc..08d6641a7 100644
--- a/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_llm_plx.py
+++ b/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_llm_plx.py
@@ -1,7 +1,7 @@
import pytest
from pipelex.core.interpreter import PipelexInterpreter
-from pipelex.core.pipes.pipe_input_spec_blueprint import InputRequirementBlueprint
+from pipelex.core.pipes.pipe_input_blueprint import InputRequirementBlueprint
from pipelex.pipe_operators.llm.pipe_llm_blueprint import PipeLLMBlueprint
diff --git a/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_ocr_plx.py b/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_ocr_plx.py
index 4c46c5d46..54d83bf8d 100644
--- a/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_ocr_plx.py
+++ b/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_ocr_plx.py
@@ -17,13 +17,13 @@ class TestPipelexInterpreterOcrPLX:
type="PipeOcr",
definition="Extract text from document",
output="Page",
- ocr_model="pypdfium2-extract-text",
+ ocr="base_ocr_pypdfium2",
),
"""[pipe.extract_text]
type = "PipeOcr"
definition = "Extract text from document"
output = "Page"
-ocr_model = "pypdfium2-extract-text\"""",
+ocr = "base_ocr_pypdfium2\"""",
),
# OCR pipe with inputs
(
@@ -33,14 +33,14 @@ ocr_model = "pypdfium2-extract-text\"""",
definition="Extract text from PDF",
inputs={"ocr_input": "PDF"},
output="Page",
- ocr_model="pypdfium2-extract-text",
+ ocr="base_ocr_pypdfium2",
),
"""[pipe.extract_with_input]
type = "PipeOcr"
definition = "Extract text from PDF"
inputs = { ocr_input = "PDF" }
output = "Page"
-ocr_model = "pypdfium2-extract-text\"""",
+ocr = "base_ocr_pypdfium2\"""",
),
],
)
diff --git a/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_sequence_plx.py b/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_sequence_plx.py
index 3ec650001..1c0f95bdf 100644
--- a/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_sequence_plx.py
+++ b/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_sequence_plx.py
@@ -3,7 +3,7 @@
import pytest
from pipelex.core.interpreter import PipelexInterpreter
-from pipelex.core.pipes.pipe_input_spec_blueprint import InputRequirementBlueprint
+from pipelex.core.pipes.pipe_input_blueprint import InputRequirementBlueprint
from pipelex.pipe_controllers.sequence.pipe_sequence_blueprint import PipeSequenceBlueprint
from pipelex.pipe_controllers.sub_pipe_blueprint import SubPipeBlueprint
diff --git a/tests/unit/pipelex/core/memory/test_dry_working_memory.py b/tests/unit/pipelex/core/memory/test_dry_working_memory.py
index ea25c0708..83a6b5f53 100644
--- a/tests/unit/pipelex/core/memory/test_dry_working_memory.py
+++ b/tests/unit/pipelex/core/memory/test_dry_working_memory.py
@@ -9,7 +9,7 @@ from pipelex import log
from pipelex.core.concepts.concept_factory import ConceptFactory
from pipelex.core.concepts.concept_native import NativeConceptEnum
from pipelex.core.memory.working_memory_factory import WorkingMemoryFactory
-from pipelex.core.pipes.pipe_input_spec import TypedNamedInputRequirement
+from pipelex.core.pipes.pipe_input import TypedNamedInputRequirement
from pipelex.core.stuffs.stuff_content import PageContent, TextContent
from tests.test_pipelines.tricky_questions import ThoughtfulAnswer
@@ -31,7 +31,7 @@ class TestDryWorkingMemory:
TypedNamedInputRequirement(
variable_name="page",
concept=ConceptFactory.make(
- concept_code=NativeConceptEnum.PAGE.value, domain="test_tricky_questions", definition="Lorem Ipsum", structure_class_name="Page"
+ concept_code=NativeConceptEnum.PAGE, domain="test_tricky_questions", definition="Lorem Ipsum", structure_class_name="Page"
),
structure_class=PageContent,
),
@@ -44,7 +44,7 @@ class TestDryWorkingMemory:
# Verify concept code is correct
page_stuff = dry_memory.get_stuff("page")
- assert page_stuff.concept.code == NativeConceptEnum.PAGE.value
+ assert page_stuff.concept.code == NativeConceptEnum.PAGE
assert page_stuff.stuff_name == "page"
# Verify structured content was created properly
diff --git a/tests/unit/pipelex/core/memory/test_working_memory.py b/tests/unit/pipelex/core/memory/test_working_memory.py
index c68be5983..8158b9017 100644
--- a/tests/unit/pipelex/core/memory/test_working_memory.py
+++ b/tests/unit/pipelex/core/memory/test_working_memory.py
@@ -60,7 +60,7 @@ class TestWorkingMemory:
def single_text_memory(self) -> WorkingMemory:
"""Create WorkingMemory with single text content."""
return WorkingMemoryFactory.make_from_text(
- text=TestWorkingMemoryData.SAMPLE_TEXT, concept_string=SpecialDomain.NATIVE.value + "." + NativeConceptEnum.TEXT.value, name="sample_text"
+ text=TestWorkingMemoryData.SAMPLE_TEXT, concept_string=SpecialDomain.NATIVE + "." + NativeConceptEnum.TEXT, name="sample_text"
)
@pytest.fixture
@@ -74,7 +74,7 @@ class TestWorkingMemory:
def single_pdf_memory(self) -> WorkingMemory:
"""Create WorkingMemory with single PDF content."""
return WorkingMemoryFactory.make_from_pdf(
- pdf_url=TestWorkingMemoryData.SAMPLE_PDF_URL, concept_string=NativeConceptEnum.PDF.value, name="pdf_document"
+ pdf_url=TestWorkingMemoryData.SAMPLE_PDF_URL, concept_string=NativeConceptEnum.PDF, name="pdf_document"
)
@pytest.fixture
@@ -134,7 +134,7 @@ class TestWorkingMemory:
complex_stuff = StuffFactory.make_stuff(
concept=ConceptFactory.make(
- concept_code="List", domain=SpecialDomain.NATIVE.value, definition="Lorem Ipsum", structure_class_name="ListContent"
+ concept_code="List", domain=SpecialDomain.NATIVE, definition="Lorem Ipsum", structure_class_name="ListContent"
),
name="mixed_list",
content=complex_content,
@@ -168,7 +168,7 @@ class TestWorkingMemory:
stuff = StuffFactory.make_stuff(
concept=ConceptFactory.make(
- concept_code="Html", domain=SpecialDomain.NATIVE.value, definition="Lorem Ipsum", structure_class_name="HtmlContent"
+ concept_code="Html", domain=SpecialDomain.NATIVE, definition="Lorem Ipsum", structure_class_name="HtmlContent"
),
name="test_report",
content=html_content,
@@ -197,7 +197,7 @@ class TestWorkingMemory:
# Check stuff retrieval
stuff = single_text_memory.get_stuff("sample_text")
- assert stuff.concept.code == NativeConceptEnum.TEXT.value
+ assert stuff.concept.code == NativeConceptEnum.TEXT
assert isinstance(stuff.content, TextContent)
assert stuff.content.text == TestWorkingMemoryData.SAMPLE_TEXT
diff --git a/tests/unit/pipelex/core/stuffs/test_stuff_content_factory.py b/tests/unit/pipelex/core/stuffs/test_stuff_content_factory.py
index c26cb4bda..8e6574b9f 100644
--- a/tests/unit/pipelex/core/stuffs/test_stuff_content_factory.py
+++ b/tests/unit/pipelex/core/stuffs/test_stuff_content_factory.py
@@ -12,19 +12,19 @@ from pipelex.core.stuffs.stuff_factory import StuffContentFactory
class TestCases:
# Test cases for TextContent with string content
TEXT_STRING_BLUEPRINT: ClassVar[Dict[str, Any]] = {
- "concept_string": f"{SpecialDomain.NATIVE.value}.{NativeConceptEnum.TEXT.value}",
+ "concept_string": f"{SpecialDomain.NATIVE}.{NativeConceptEnum.TEXT}",
"content": "The Dawn of Ultra-Rapid Transit: NextGen High-Speed Trains Redefine Travel",
}
# Test cases for TextContent with dict content
TEXT_DICT_BLUEPRINT: ClassVar[Dict[str, Any]] = {
- "concept_string": f"{SpecialDomain.NATIVE.value}.{NativeConceptEnum.TEXT.value}",
+ "concept_string": f"{SpecialDomain.NATIVE}.{NativeConceptEnum.TEXT}",
"content": {"text": "Sample text content"},
}
# Test cases for native concept without prefix (should work)
TEXT_NO_PREFIX_BLUEPRINT: ClassVar[Dict[str, Any]] = {
- "concept_string": f"{NativeConceptEnum.TEXT.value}",
+ "concept_string": f"{NativeConceptEnum.TEXT}",
"content": {"text": "Text content without native prefix"},
}
diff --git a/tests/unit/pipelex/core/stuffs/test_stuff_factory_make_stuff_from_content.py b/tests/unit/pipelex/core/stuffs/test_stuff_factory_make_stuff_from_content.py
index 4d75b776b..5bbe352fd 100644
--- a/tests/unit/pipelex/core/stuffs/test_stuff_factory_make_stuff_from_content.py
+++ b/tests/unit/pipelex/core/stuffs/test_stuff_factory_make_stuff_from_content.py
@@ -4,7 +4,6 @@ import pytest
from pytest_mock import MockerFixture
from pipelex.core.concepts.concept_native import NATIVE_CONCEPTS_DATA, NativeConceptEnum
-from pipelex.core.domains.domain import SpecialDomain
from pipelex.core.stuffs.stuff import Stuff
from pipelex.core.stuffs.stuff_content import (
ListContent,
@@ -40,7 +39,7 @@ class TestData:
EMPTY_LIST_CONTENT: ClassVar[ListContent[TextContent]] = ListContent(items=[])
# Dictionary test data - native concept
- NATIVE_TEXT_DICT: ClassVar[Dict[str, Any]] = {"concept": NativeConceptEnum.TEXT.value, "content": {"text": "Native text content"}}
+ NATIVE_TEXT_DICT: ClassVar[Dict[str, Any]] = {"concept": NativeConceptEnum.TEXT, "content": {"text": "Native text content"}}
# Dictionary test data - custom concept with concept field
CUSTOM_CONCEPT_DICT: ClassVar[Dict[str, Any]] = {
@@ -111,7 +110,7 @@ class TestMakeStuffFromStuffContentUsingSearchDomains:
"""Test StuffContent with native concept class name."""
mock_provider = mocker.patch("pipelex.core.stuffs.stuff_factory.get_concept_provider").return_value
mock_concept = mocker.Mock()
- mock_provider.get_required_concept.return_value = mock_concept
+ mock_provider.get_native_concept.return_value = mock_concept
text_content = TextContent(text="test")
@@ -126,7 +125,7 @@ class TestMakeStuffFromStuffContentUsingSearchDomains:
)
assert result == mock_stuff
- mock_provider.get_required_concept.assert_called_once_with(concept_string=f"{SpecialDomain.NATIVE.value}.Text")
+ mock_provider.get_native_concept.assert_called_once_with(native_concept=NativeConceptEnum.TEXT)
def test_stuffcontent_with_non_native_concept(self, mocker: MockerFixture):
"""Test StuffContent with non-native concept properly extracts concept name."""
diff --git a/tests/unit/pipelex/core/test_data/pipes/operators/img_gen/pipe_img_gen.py b/tests/unit/pipelex/core/test_data/pipes/operators/img_gen/pipe_img_gen.py
index 90dbe87a4..51c194749 100644
--- a/tests/unit/pipelex/core/test_data/pipes/operators/img_gen/pipe_img_gen.py
+++ b/tests/unit/pipelex/core/test_data/pipes/operators/img_gen/pipe_img_gen.py
@@ -22,7 +22,7 @@ img_gen_prompt = "A beautiful landscape"
"generate_image": PipeImgGenBlueprint(
type="PipeImgGen",
definition="Generate an image from a prompt",
- output=NativeConceptEnum.IMAGE.value,
+ output=NativeConceptEnum.IMAGE,
img_gen_prompt="A beautiful landscape",
),
},
diff --git a/tests/unit/pipelex/core/test_data/pipes/operators/jinja2/pipe_jinja2.py b/tests/unit/pipelex/core/test_data/pipes/operators/jinja2/pipe_jinja2.py
index 0336a8624..8e5b3f984 100644
--- a/tests/unit/pipelex/core/test_data/pipes/operators/jinja2/pipe_jinja2.py
+++ b/tests/unit/pipelex/core/test_data/pipes/operators/jinja2/pipe_jinja2.py
@@ -22,7 +22,7 @@ jinja2 = "Hello {{ name }}!"
"process_template": PipeJinja2Blueprint(
type="PipeJinja2",
definition="Process a Jinja2 template",
- output=NativeConceptEnum.TEXT.value,
+ output=NativeConceptEnum.TEXT,
jinja2="Hello {{ name }}!",
),
},
diff --git a/tests/unit/pipelex/core/test_data/pipes/operators/llm/pipe_llm.py b/tests/unit/pipelex/core/test_data/pipes/operators/llm/pipe_llm.py
index 1fd99bc19..9e5eb9816 100644
--- a/tests/unit/pipelex/core/test_data/pipes/operators/llm/pipe_llm.py
+++ b/tests/unit/pipelex/core/test_data/pipes/operators/llm/pipe_llm.py
@@ -2,7 +2,7 @@
from pipelex.core.bundles.pipelex_bundle_blueprint import PipelexBundleBlueprint
from pipelex.core.concepts.concept_native import NativeConceptEnum
-from pipelex.core.pipes.pipe_input_spec_blueprint import InputRequirementBlueprint
+from pipelex.core.pipes.pipe_input_blueprint import InputRequirementBlueprint
from pipelex.pipe_operators.llm.pipe_llm_blueprint import PipeLLMBlueprint
# Basic PipeLLM with prompt_template
@@ -24,7 +24,7 @@ prompt_template = "Generate a story about a programmer"
"generate_text": PipeLLMBlueprint(
type="PipeLLM",
definition="Generate text using LLM",
- output=NativeConceptEnum.TEXT.value,
+ output=NativeConceptEnum.TEXT,
prompt_template="Generate a story about a programmer",
),
},
@@ -52,7 +52,7 @@ prompt_template = "Extract information about $topic from this text:\\n\\n@text"
type="PipeLLM",
definition="Extract information from text",
inputs={"text": "Text", "topic": "Text"},
- output=NativeConceptEnum.TEXT.value,
+ output=NativeConceptEnum.TEXT,
prompt_template="Extract information about $topic from this text:\n\n@text",
),
},
@@ -79,7 +79,7 @@ prompt_template = "Analyze the following data and provide insights"
"expert_analysis": PipeLLMBlueprint(
type="PipeLLM",
definition="Expert analysis with system prompt",
- output=NativeConceptEnum.TEXT.value,
+ output=NativeConceptEnum.TEXT,
system_prompt="You are a data analysis expert with 20 years of experience",
prompt_template="Analyze the following data and provide insights",
),
@@ -107,7 +107,7 @@ prompt_template = "Generate creative ideas for a mobile app"
"generate_ideas": PipeLLMBlueprint(
type="PipeLLM",
definition="Generate multiple ideas",
- output=NativeConceptEnum.TEXT.value,
+ output=NativeConceptEnum.TEXT,
nb_output=3,
prompt_template="Generate creative ideas for a mobile app",
),
@@ -137,7 +137,7 @@ prompt_template = "Brainstorm solutions for this problem: @problem"
type="PipeLLM",
definition="Brainstorm multiple solutions",
inputs={"problem": InputRequirementBlueprint(concept="Text")},
- output=NativeConceptEnum.TEXT.value,
+ output=NativeConceptEnum.TEXT,
multiple_output=True,
prompt_template="Brainstorm solutions for this problem: @problem",
),
@@ -166,7 +166,7 @@ prompt_template = "Describe what you see in this image in detail"
type="PipeLLM",
definition="Analyze image content",
inputs={"image": "Image"},
- output=NativeConceptEnum.TEXT.value,
+ output=NativeConceptEnum.TEXT,
prompt_template="Describe what you see in this image in detail",
),
},
@@ -194,7 +194,7 @@ template_name = "analysis_template"
type="PipeLLM",
definition="Use named template",
inputs={"data": "Text"},
- output=NativeConceptEnum.TEXT.value,
+ output=NativeConceptEnum.TEXT,
template_name="analysis_template",
),
},
@@ -220,7 +220,7 @@ prompt = "Write a haiku about programming"
"simple_generation": PipeLLMBlueprint(
type="PipeLLM",
definition="Simple text generation with fixed prompt",
- output=NativeConceptEnum.TEXT.value,
+ output=NativeConceptEnum.TEXT,
prompt="Write a haiku about programming",
),
},
@@ -249,7 +249,7 @@ prompt_template = "Analyze this content: @content"
type="PipeLLM",
definition="Analysis with dynamic system prompt",
inputs={"expertise_level": "Text", "content": "Text"},
- output=NativeConceptEnum.TEXT.value,
+ output=NativeConceptEnum.TEXT,
system_prompt_template="You are an expert with $expertise_level level knowledge",
prompt_template="Analyze this content: @content",
),
diff --git a/tests/unit/pipelex/core/test_data/pipes/operators/ocr/pipe_ocr.py b/tests/unit/pipelex/core/test_data/pipes/operators/ocr/pipe_ocr.py
index 810e2f5f7..72d099884 100644
--- a/tests/unit/pipelex/core/test_data/pipes/operators/ocr/pipe_ocr.py
+++ b/tests/unit/pipelex/core/test_data/pipes/operators/ocr/pipe_ocr.py
@@ -13,7 +13,7 @@ definition = "Domain with OCR pipe"
type = "PipeOcr"
definition = "Extract text from document"
output = "Page"
-ocr_model = "pypdfium2-extract-text"
+ocr = "base_ocr_pypdfium2"
""",
PipelexBundleBlueprint(
domain="test_pipes",
@@ -22,8 +22,8 @@ ocr_model = "pypdfium2-extract-text"
"extract_text": PipeOcrBlueprint(
type="PipeOcr",
definition="Extract text from document",
- output=NativeConceptEnum.PAGE.value,
- ocr_model="pypdfium2-extract-text",
+ output=NativeConceptEnum.PAGE,
+ ocr="base_ocr_pypdfium2",
),
},
),
diff --git a/tests/unit/pipelex/create/test_structured_output_generator.py b/tests/unit/pipelex/create/test_structured_output_generator.py
index 0798cfee7..38cce78f7 100644
--- a/tests/unit/pipelex/create/test_structured_output_generator.py
+++ b/tests/unit/pipelex/create/test_structured_output_generator.py
@@ -1,11 +1,11 @@
"""Tests for structured output generator."""
-from typing import Dict
+from typing import Dict, Union
from pipelex import pretty_print
-from pipelex.core.concepts.concept_blueprint import ConceptStructureBlueprint, ConceptStructureBlueprintFieldType, ConceptStructureBlueprintType
+from pipelex.core.concepts.concept_blueprint import ConceptStructureBlueprint, ConceptStructureBlueprintFieldType
from pipelex.core.concepts.concept_factory import ConceptFactory
-from pipelex.create.structured_output_generator import StructureGenerator
+from pipelex.core.concepts.structure_generator import StructureGenerator
class TestStructureGenerator:
@@ -407,9 +407,9 @@ class TestStructureGenerator:
def test_mixed_structure_blueprint_normalization(self):
"""Test that mixed structure blueprints (strings and ConceptStructureBlueprint objects) are properly normalized."""
- # Create a mixed structure blueprint similar to what would come from PLX parsing
- mixed_structure_blueprint: Dict[str, ConceptStructureBlueprintType] = {
- "name": "The name of the person",
+ # Create a mixed structure blueprint similar to what would come from TOML parsing
+ mixed_structure_blueprint: Dict[str, Union[str, ConceptStructureBlueprint]] = {
+ "name": "The name of the person", # Simple string definition
"age": ConceptStructureBlueprint(definition="The age of the person", type=ConceptStructureBlueprintFieldType.NUMBER, required=True),
"birthdate": ConceptStructureBlueprint(
definition="The birthdate of the person", type=ConceptStructureBlueprintFieldType.DATE, required=True
diff --git a/tests/unit/pipelex/libraries/pipelines/builder/concept/test_concept_blueprint.py b/tests/unit/pipelex/libraries/pipelines/builder/concept/test_concept_blueprint.py
new file mode 100644
index 000000000..b6c5a1bae
--- /dev/null
+++ b/tests/unit/pipelex/libraries/pipelines/builder/concept/test_concept_blueprint.py
@@ -0,0 +1,16 @@
+import pytest
+
+from pipelex.core.concepts.concept_blueprint import ConceptBlueprint
+from pipelex.libraries.pipelines.builder.concept.concept_spec import ConceptSpec
+
+from .test_data import ConceptBlueprintTestCases
+
+
+class TestConceptBlueprintConversion:
+ @pytest.mark.parametrize(
+ "test_name,concept_spec,expected_core",
+ ConceptBlueprintTestCases.TEST_CASES,
+ )
+ def test_concept_to_core_blueprint(self, test_name: str, concept_spec: ConceptSpec, expected_core: ConceptBlueprint):
+ result = concept_spec.to_blueprint()
+ assert result == expected_core
diff --git a/tests/unit/pipelex/libraries/pipelines/builder/concept/test_data.py b/tests/unit/pipelex/libraries/pipelines/builder/concept/test_data.py
new file mode 100644
index 000000000..1bc960393
--- /dev/null
+++ b/tests/unit/pipelex/libraries/pipelines/builder/concept/test_data.py
@@ -0,0 +1,157 @@
+from typing import ClassVar, List, Tuple
+
+from pipelex.core.concepts.concept_blueprint import ConceptBlueprint, ConceptStructureBlueprint, ConceptStructureBlueprintFieldType
+from pipelex.libraries.pipelines.builder.concept.concept_spec import ConceptSpec, ConceptStructureSpec, ConceptStructureSpecFieldType
+
+
+class ConceptBlueprintTestCases:
+ """Test cases for ConceptBlueprint.to_blueprint conversion."""
+
+ SIMPLE_CONCEPT = (
+ "simple_concept",
+ ConceptSpec(
+ the_concept_code="ConceptCode",
+ definition="A simple test concept",
+ refines=None,
+ structure=None,
+ ),
+ ConceptBlueprint(
+ definition="A simple test concept",
+ refines=None,
+ structure=None,
+ ),
+ )
+
+ CONCEPT_WITH_REFINES = (
+ "concept_with_refines",
+ ConceptSpec(
+ the_concept_code="ConceptCode",
+ definition="An enhanced text concept",
+ refines="Text",
+ structure=None,
+ ),
+ ConceptBlueprint(
+ definition="An enhanced text concept",
+ refines="Text",
+ structure=None,
+ ),
+ )
+
+ CONCEPT_WITH_TEXT_FIELD = (
+ "concept_with_text_field",
+ ConceptSpec(
+ the_concept_code="ConceptCode",
+ definition="Entity with text field",
+ structure={
+ "name": ConceptStructureSpec(
+ the_field_name="name",
+ definition="The name field",
+ type=ConceptStructureSpecFieldType.TEXT,
+ required=True,
+ ),
+ },
+ ),
+ ConceptBlueprint(
+ definition="Entity with text field",
+ refines=None,
+ structure={
+ "name": ConceptStructureBlueprint(
+ definition="The name field",
+ type=ConceptStructureBlueprintFieldType.TEXT,
+ required=True,
+ default_value=None,
+ ),
+ },
+ ),
+ )
+
+ CONCEPT_WITH_INTEGER_FIELD = (
+ "concept_with_integer_field",
+ ConceptSpec(
+ the_concept_code="ConceptCode",
+ definition="Entity with integer field",
+ structure={
+ "age": ConceptStructureSpec(
+ the_field_name="age",
+ definition="The age field",
+ type=ConceptStructureSpecFieldType.INTEGER,
+ required=False,
+ default_value=0,
+ ),
+ },
+ ),
+ ConceptBlueprint(
+ definition="Entity with integer field",
+ refines=None,
+ structure={
+ "age": ConceptStructureBlueprint(
+ definition="The age field",
+ type=ConceptStructureBlueprintFieldType.INTEGER,
+ required=False,
+ default_value=0,
+ ),
+ },
+ ),
+ )
+
+ CONCEPT_WITH_MULTIPLE_FIELDS = (
+ "concept_with_multiple_fields",
+ ConceptSpec(
+ the_concept_code="ConceptCode",
+ definition="Entity with multiple fields",
+ structure={
+ "name": ConceptStructureSpec(
+ the_field_name="name",
+ definition="Name",
+ type=ConceptStructureSpecFieldType.TEXT,
+ required=True,
+ ),
+ "age": ConceptStructureSpec(
+ the_field_name="age",
+ definition="Age",
+ type=ConceptStructureSpecFieldType.INTEGER,
+ required=True,
+ default_value=18,
+ ),
+ "active": ConceptStructureSpec(
+ the_field_name="active",
+ definition="Active status",
+ type=ConceptStructureSpecFieldType.BOOLEAN,
+ required=False,
+ default_value=True,
+ ),
+ },
+ ),
+ ConceptBlueprint(
+ definition="Entity with multiple fields",
+ refines=None,
+ structure={
+ "name": ConceptStructureBlueprint(
+ definition="Name",
+ type=ConceptStructureBlueprintFieldType.TEXT,
+ required=True,
+ default_value=None,
+ ),
+ "age": ConceptStructureBlueprint(
+ definition="Age",
+ type=ConceptStructureBlueprintFieldType.INTEGER,
+ required=True,
+ default_value=18,
+ ),
+ "active": ConceptStructureBlueprint(
+ definition="Active status",
+ type=ConceptStructureBlueprintFieldType.BOOLEAN,
+ required=False,
+ default_value=True,
+ ),
+ },
+ ),
+ )
+
+ TEST_CASES: ClassVar[List[Tuple[str, ConceptSpec, ConceptBlueprint]]] = [
+ SIMPLE_CONCEPT,
+ CONCEPT_WITH_REFINES,
+ CONCEPT_WITH_TEXT_FIELD,
+ CONCEPT_WITH_INTEGER_FIELD,
+ CONCEPT_WITH_MULTIPLE_FIELDS,
+ ]
diff --git a/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_controllers/pipe_batch/test_data.py b/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_controllers/pipe_batch/test_data.py
new file mode 100644
index 000000000..98b0a8016
--- /dev/null
+++ b/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_controllers/pipe_batch/test_data.py
@@ -0,0 +1,59 @@
+from typing import ClassVar, List, Tuple
+
+from pipelex.core.pipes.pipe_input_blueprint import InputRequirementBlueprint
+from pipelex.libraries.pipelines.builder.pipe.inputs_spec import InputRequirementSpec
+from pipelex.libraries.pipelines.builder.pipe.pipe_batch_spec import PipeBatchSpec
+from pipelex.pipe_controllers.batch.pipe_batch_blueprint import PipeBatchBlueprint
+
+
+class PipeBatchTestCases:
+ SIMPLE_BATCH = (
+ "simple_batch",
+ PipeBatchSpec(
+ the_pipe_code="batch_processor",
+ definition="Process items in batch",
+ inputs={"items": InputRequirementSpec(concept="ItemList")},
+ output="ProcessedItems",
+ branch_pipe_code="process_item",
+ ),
+ "test_domain",
+ PipeBatchBlueprint(
+ definition="Process items in batch",
+ inputs={"items": InputRequirementBlueprint(concept="ItemList")},
+ output="ProcessedItems",
+ type="PipeBatch",
+ category="PipeController",
+ branch_pipe_code="process_item",
+ input_list_name=None,
+ input_item_name=None,
+ ),
+ )
+
+ BATCH_WITH_NAMES = (
+ "batch_with_names",
+ PipeBatchSpec(
+ the_pipe_code="named_batch",
+ definition="Batch with custom names",
+ inputs={"data": InputRequirementSpec(concept="DataList")},
+ output="Results",
+ branch_pipe_code="transform_data",
+ input_list_name="data_list",
+ input_item_name="current_data",
+ ),
+ "test_domain",
+ PipeBatchBlueprint(
+ definition="Batch with custom names",
+ inputs={"data": InputRequirementBlueprint(concept="DataList")},
+ output="Results",
+ type="PipeBatch",
+ category="PipeController",
+ branch_pipe_code="transform_data",
+ input_list_name="data_list",
+ input_item_name="current_data",
+ ),
+ )
+
+ TEST_CASES: ClassVar[List[Tuple[str, PipeBatchSpec, str, PipeBatchBlueprint]]] = [
+ SIMPLE_BATCH,
+ BATCH_WITH_NAMES,
+ ]
diff --git a/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_controllers/pipe_batch/test_pipe_batch.py b/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_controllers/pipe_batch/test_pipe_batch.py
new file mode 100644
index 000000000..b469ef32e
--- /dev/null
+++ b/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_controllers/pipe_batch/test_pipe_batch.py
@@ -0,0 +1,22 @@
+import pytest
+
+from pipelex.libraries.pipelines.builder.pipe.pipe_batch_spec import PipeBatchSpec
+from pipelex.pipe_controllers.batch.pipe_batch_blueprint import PipeBatchBlueprint
+
+from .test_data import PipeBatchTestCases
+
+
+class TestPipeBatchBlueprintConversion:
+ @pytest.mark.parametrize(
+ "test_name,pipe_spec,domain,expected_blueprint",
+ PipeBatchTestCases.TEST_CASES,
+ )
+ def test_pipe_batch_spec_to_blueprint(
+ self,
+ test_name: str,
+ pipe_spec: PipeBatchSpec,
+ domain: str,
+ expected_blueprint: PipeBatchBlueprint,
+ ):
+ result = pipe_spec.to_blueprint()
+ assert result == expected_blueprint
diff --git a/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_controllers/pipe_condition/test_data.py b/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_controllers/pipe_condition/test_data.py
new file mode 100644
index 000000000..09f189046
--- /dev/null
+++ b/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_controllers/pipe_condition/test_data.py
@@ -0,0 +1,87 @@
+from typing import ClassVar, List, Tuple
+
+from pipelex.core.pipes.pipe_input_blueprint import InputRequirementBlueprint
+from pipelex.libraries.pipelines.builder.pipe.inputs_spec import InputRequirementSpec
+from pipelex.libraries.pipelines.builder.pipe.pipe_condition_spec import PipeConditionPipeMapSpec, PipeConditionSpec
+from pipelex.pipe_controllers.condition.pipe_condition_blueprint import PipeConditionBlueprint, PipeConditionPipeMapBlueprint
+
+
+class PipeConditionTestCases:
+ SIMPLE_CONDITION = (
+ "simple_condition",
+ PipeConditionSpec(
+ the_pipe_code="conditional_processor",
+ definition="Choose pipe based on condition",
+ inputs={"data": InputRequirementSpec(concept="Data")},
+ output="Result",
+ expression="data.status",
+ pipe_map=PipeConditionPipeMapSpec(
+ root={
+ "active": "process_active",
+ "inactive": "process_inactive",
+ }
+ ),
+ ),
+ "test_domain",
+ PipeConditionBlueprint(
+ definition="Choose pipe based on condition",
+ inputs={"data": InputRequirementBlueprint(concept="Data")},
+ output="Result",
+ type="PipeCondition",
+ category="PipeController",
+ expression="data.status",
+ expression_template=None,
+ pipe_map=PipeConditionPipeMapBlueprint(
+ root={
+ "active": "process_active",
+ "inactive": "process_inactive",
+ }
+ ),
+ default_pipe_code=None,
+ add_alias_from_expression_to=None,
+ ),
+ )
+
+ CONDITION_WITH_TEMPLATE = (
+ "condition_with_template",
+ PipeConditionSpec(
+ the_pipe_code="template_condition",
+ definition="Conditional with template",
+ inputs={"item": InputRequirementSpec(concept="Item")},
+ output="ProcessedItem",
+ expression_template="{{ item.category }}",
+ pipe_map=PipeConditionPipeMapSpec(
+ root={
+ "A": "process_a",
+ "B": "process_b",
+ "C": "process_c",
+ }
+ ),
+ default_pipe_code="process_default",
+ add_alias_from_expression_to="category_result",
+ ),
+ "test_domain",
+ PipeConditionBlueprint(
+ definition="Conditional with template",
+ inputs={"item": InputRequirementBlueprint(concept="Item")},
+ output="ProcessedItem",
+ type="PipeCondition",
+ category="PipeController",
+ expression=None,
+ expression_template="{{ item.category }}",
+ pipe_map=PipeConditionPipeMapBlueprint(
+ root={
+ "A": "process_a",
+ "B": "process_b",
+ "C": "process_c",
+ }
+ ),
+ default_pipe_code="process_default",
+ add_alias_from_expression_to="category_result",
+ ),
+ )
+
+ TEST_CASES: ClassVar[List[Tuple[str, PipeConditionSpec, str, PipeConditionBlueprint]]] = [
+ SIMPLE_CONDITION,
+ CONDITION_WITH_TEMPLATE,
+ ]
diff --git a/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_controllers/pipe_condition/test_pipe_condition.py b/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_controllers/pipe_condition/test_pipe_condition.py
new file mode 100644
index 000000000..ba32686fb
--- /dev/null
+++ b/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_controllers/pipe_condition/test_pipe_condition.py
@@ -0,0 +1,22 @@
+import pytest
+
+from pipelex.libraries.pipelines.builder.pipe.pipe_condition_spec import PipeConditionSpec
+from pipelex.pipe_controllers.condition.pipe_condition_blueprint import PipeConditionBlueprint
+
+from .test_data import PipeConditionTestCases
+
+
+class TestPipeConditionBlueprintConversion:
+ @pytest.mark.parametrize(
+ "test_name,pipe_spec,domain,expected_blueprint",
+ PipeConditionTestCases.TEST_CASES,
+ )
+ def test_pipe_condition_spec_to_blueprint(
+ self,
+ test_name: str,
+ pipe_spec: PipeConditionSpec,
+ domain: str,
+ expected_blueprint: PipeConditionBlueprint,
+ ):
+ result = pipe_spec.to_blueprint()
+ assert result == expected_blueprint
diff --git a/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_controllers/pipe_parallel/test_data.py b/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_controllers/pipe_parallel/test_data.py
new file mode 100644
index 000000000..ed5579415
--- /dev/null
+++ b/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_controllers/pipe_parallel/test_data.py
@@ -0,0 +1,75 @@
+from typing import ClassVar, List, Tuple
+
+from pipelex.core.pipes.pipe_input_blueprint import InputRequirementBlueprint
+from pipelex.libraries.pipelines.builder.pipe.inputs_spec import InputRequirementSpec
+from pipelex.libraries.pipelines.builder.pipe.pipe_parallel_spec import PipeParallelSpec
+from pipelex.libraries.pipelines.builder.pipe.sub_pipe_spec import SubPipeSpec
+from pipelex.pipe_controllers.parallel.pipe_parallel_blueprint import PipeParallelBlueprint
+from pipelex.pipe_controllers.sub_pipe_blueprint import SubPipeBlueprint
+
+
+class PipeParallelTestCases:
+ SIMPLE_PARALLEL = (
+ "simple_parallel",
+ PipeParallelSpec(
+ the_pipe_code="parallel_processor",
+ definition="Run pipes in parallel",
+ inputs={"data": InputRequirementSpec(concept="Data")},
+ output="Results",
+ parallels=[
+ SubPipeSpec(the_pipe_code="analyze_data", result="analysis"),
+ SubPipeSpec(the_pipe_code="transform_data", result="transformed"),
+ SubPipeSpec(the_pipe_code="validate_data", result="validation"),
+ ],
+ ),
+ "test_domain",
+ PipeParallelBlueprint(
+ definition="Run pipes in parallel",
+ inputs={"data": InputRequirementBlueprint(concept="Data")},
+ output="Results",
+ type="PipeParallel",
+ category="PipeController",
+ parallels=[
+ SubPipeBlueprint(pipe="analyze_data", result="analysis"),
+ SubPipeBlueprint(pipe="transform_data", result="transformed"),
+ SubPipeBlueprint(pipe="validate_data", result="validation"),
+ ],
+ add_each_output=True,
+ combined_output=None,
+ ),
+ )
+
+ PARALLEL_WITH_COMBINED = (
+ "parallel_with_combined",
+ PipeParallelSpec(
+ the_pipe_code="combined_parallel",
+ definition="Parallel with combined output",
+ inputs={"input": InputRequirementSpec(concept="Input")},
+ output="CombinedResult",
+ parallels=[
+ SubPipeSpec(the_pipe_code="pipe1", result="result1"),
+ SubPipeSpec(the_pipe_code="pipe2", result="result2"),
+ ],
+ add_each_output=False,
+ combined_output="MergedData",
+ ),
+ "test_domain",
+ PipeParallelBlueprint(
+ definition="Parallel with combined output",
+ inputs={"input": InputRequirementBlueprint(concept="Input")},
+ output="CombinedResult",
+ type="PipeParallel",
+ category="PipeController",
+ parallels=[
+ SubPipeBlueprint(pipe="pipe1", result="result1"),
+ SubPipeBlueprint(pipe="pipe2", result="result2"),
+ ],
+ add_each_output=False,
+ combined_output="MergedData",
+ ),
+ )
+
+ TEST_CASES: ClassVar[List[Tuple[str, PipeParallelSpec, str, PipeParallelBlueprint]]] = [
+ SIMPLE_PARALLEL,
+ PARALLEL_WITH_COMBINED,
+ ]
diff --git a/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_controllers/pipe_parallel/test_pipe_parallel.py b/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_controllers/pipe_parallel/test_pipe_parallel.py
new file mode 100644
index 000000000..cf3315971
--- /dev/null
+++ b/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_controllers/pipe_parallel/test_pipe_parallel.py
@@ -0,0 +1,22 @@
+import pytest
+
+from pipelex.libraries.pipelines.builder.pipe.pipe_parallel_spec import PipeParallelSpec
+from pipelex.pipe_controllers.parallel.pipe_parallel_blueprint import PipeParallelBlueprint
+
+from .test_data import PipeParallelTestCases
+
+
+class TestPipeParallelBlueprintConversion:
+ @pytest.mark.parametrize(
+ "test_name,pipe_spec,domain,expected_blueprint",
+ PipeParallelTestCases.TEST_CASES,
+ )
+ def test_pipe_parallel_spec_to_blueprint(
+ self,
+ test_name: str,
+ pipe_spec: PipeParallelSpec,
+ domain: str,
+ expected_blueprint: PipeParallelBlueprint,
+ ):
+ result = pipe_spec.to_blueprint()
+ assert result == expected_blueprint
diff --git a/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_controllers/pipe_sequence/test_data.py b/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_controllers/pipe_sequence/test_data.py
new file mode 100644
index 000000000..3448c51d9
--- /dev/null
+++ b/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_controllers/pipe_sequence/test_data.py
@@ -0,0 +1,79 @@
+from typing import ClassVar, List, Tuple
+
+from pipelex.core.pipes.pipe_input_blueprint import InputRequirementBlueprint
+from pipelex.libraries.pipelines.builder.pipe.inputs_spec import InputRequirementSpec
+from pipelex.libraries.pipelines.builder.pipe.pipe_sequence_spec import PipeSequenceSpec
+from pipelex.libraries.pipelines.builder.pipe.sub_pipe_spec import SubPipeSpec
+from pipelex.pipe_controllers.sequence.pipe_sequence_blueprint import PipeSequenceBlueprint
+from pipelex.pipe_controllers.sub_pipe_blueprint import SubPipeBlueprint
+
+
+class PipeSequenceTestCases:
+ SIMPLE_SEQUENCE = (
+ "simple_sequence",
+ PipeSequenceSpec(
+ the_pipe_code="sequence_processor",
+ definition="A sequence of operations",
+ inputs={"input_data": InputRequirementSpec(concept="Text")},
+ output="ProcessedData",
+ steps=[
+ SubPipeSpec(the_pipe_code="step1", result="result1"),
+ SubPipeSpec(the_pipe_code="step2", result="result2"),
+ SubPipeSpec(the_pipe_code="step3", result="final_result"),
+ ],
+ ),
+ "test_domain",
+ PipeSequenceBlueprint(
+ definition="A sequence of operations",
+ inputs={"input_data": InputRequirementBlueprint(concept="Text")},
+ output="ProcessedData",
+ type="PipeSequence",
+ category="PipeController",
+ steps=[
+ SubPipeBlueprint(pipe="step1", result="result1"),
+ SubPipeBlueprint(pipe="step2", result="result2"),
+ SubPipeBlueprint(pipe="step3", result="final_result"),
+ ],
+ ),
+ )
+
+ SEQUENCE_WITH_BATCH = (
+ "sequence_with_batch",
+ PipeSequenceSpec(
+ the_pipe_code="batch_sequence",
+ definition="Sequence with batch",
+ inputs={"items": InputRequirementSpec(concept="ItemList")},
+ output="ProcessedItems",
+ steps=[
+ SubPipeSpec(the_pipe_code="prepare", result="prepared_items"),
+ SubPipeSpec(
+ the_pipe_code="process_item",
+ result="processed_items",
+ batch_over="prepared_items",
+ batch_as="current_item",
+ ),
+ ],
+ ),
+ "test_domain",
+ PipeSequenceBlueprint(
+ definition="Sequence with batch",
+ inputs={"items": InputRequirementBlueprint(concept="ItemList")},
+ output="ProcessedItems",
+ type="PipeSequence",
+ category="PipeController",
+ steps=[
+ SubPipeBlueprint(pipe="prepare", result="prepared_items"),
+ SubPipeBlueprint(
+ pipe="process_item",
+ result="processed_items",
+ batch_over="prepared_items",
+ batch_as="current_item",
+ ),
+ ],
+ ),
+ )
+
+ TEST_CASES: ClassVar[List[Tuple[str, PipeSequenceSpec, str, PipeSequenceBlueprint]]] = [
+ SIMPLE_SEQUENCE,
+ SEQUENCE_WITH_BATCH,
+ ]
diff --git a/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_controllers/pipe_sequence/test_pipe_sequence.py b/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_controllers/pipe_sequence/test_pipe_sequence.py
new file mode 100644
index 000000000..4d57af598
--- /dev/null
+++ b/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_controllers/pipe_sequence/test_pipe_sequence.py
@@ -0,0 +1,22 @@
+import pytest
+
+from pipelex.libraries.pipelines.builder.pipe.pipe_sequence_spec import PipeSequenceSpec
+from pipelex.pipe_controllers.sequence.pipe_sequence_blueprint import PipeSequenceBlueprint
+
+from .test_data import PipeSequenceTestCases
+
+
+class TestPipeSequenceBlueprintConversion:
+ @pytest.mark.parametrize(
+ "test_name,pipe_spec,domain,expected_blueprint",
+ PipeSequenceTestCases.TEST_CASES,
+ )
+ def test_pipe_sequence_spec_to_blueprint(
+ self,
+ test_name: str,
+ pipe_spec: PipeSequenceSpec,
+ domain: str,
+ expected_blueprint: PipeSequenceBlueprint,
+ ):
+ result = pipe_spec.to_blueprint()
+ assert result == expected_blueprint
diff --git a/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_operators/pipe_func/test_data.py b/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_operators/pipe_func/test_data.py
new file mode 100644
index 000000000..b184452fa
--- /dev/null
+++ b/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_operators/pipe_func/test_data.py
@@ -0,0 +1,53 @@
+from typing import ClassVar, List, Tuple
+
+from pipelex.core.pipes.pipe_input_blueprint import InputRequirementBlueprint
+from pipelex.libraries.pipelines.builder.pipe.inputs_spec import InputRequirementSpec
+from pipelex.libraries.pipelines.builder.pipe.pipe_func_spec import PipeFuncSpec
+from pipelex.pipe_operators.func.pipe_func_blueprint import PipeFuncBlueprint
+
+
+class PipeFuncTestCases:
+ SIMPLE_FUNC = (
+ "simple_func",
+ PipeFuncSpec(
+ the_pipe_code="func_processor",
+ definition="Execute a function",
+ inputs={"data": InputRequirementSpec(concept="Data")},
+ output="ProcessedData",
+ function_name="process_data",
+ ),
+ "test_domain",
+ PipeFuncBlueprint(
+ definition="Execute a function",
+ inputs={"data": InputRequirementBlueprint(concept="Data")},
+ output="ProcessedData",
+ type="PipeFunc",
+ category="PipeOperator",
+ function_name="process_data",
+ ),
+ )
+
+ FUNC_NO_INPUTS = (
+ "func_no_inputs",
+ PipeFuncSpec(
+ the_pipe_code="generator_func",
+ definition="Generate data",
+ inputs=None,
+ output="GeneratedData",
+ function_name="generate_data",
+ ),
+ "test_domain",
+ PipeFuncBlueprint(
+ definition="Generate data",
+ inputs=None,
+ output="GeneratedData",
+ type="PipeFunc",
+ category="PipeOperator",
+ function_name="generate_data",
+ ),
+ )
+
+ TEST_CASES: ClassVar[List[Tuple[str, PipeFuncSpec, str, PipeFuncBlueprint]]] = [
+ SIMPLE_FUNC,
+ FUNC_NO_INPUTS,
+ ]
diff --git a/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_operators/pipe_func/test_pipe_func.py b/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_operators/pipe_func/test_pipe_func.py
new file mode 100644
index 000000000..b77d93727
--- /dev/null
+++ b/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_operators/pipe_func/test_pipe_func.py
@@ -0,0 +1,22 @@
+import pytest
+
+from pipelex.libraries.pipelines.builder.pipe.pipe_func_spec import PipeFuncSpec
+from pipelex.pipe_operators.func.pipe_func_blueprint import PipeFuncBlueprint
+
+from .test_data import PipeFuncTestCases
+
+
+class TestPipeFuncBlueprintConversion:
+ @pytest.mark.parametrize(
+ "test_name,pipe_spec,domain,expected_blueprint",
+ PipeFuncTestCases.TEST_CASES,
+ )
+ def test_pipe_func_spec_to_blueprint(
+ self,
+ test_name: str,
+ pipe_spec: PipeFuncSpec,
+ domain: str,
+ expected_blueprint: PipeFuncBlueprint,
+ ):
+ result = pipe_spec.to_blueprint()
+ assert result == expected_blueprint
diff --git a/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_operators/pipe_img/test_data.py b/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_operators/pipe_img/test_data.py
new file mode 100644
index 000000000..6920cc151
--- /dev/null
+++ b/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_operators/pipe_img/test_data.py
@@ -0,0 +1,71 @@
+from typing import ClassVar, List, Tuple
+
+from pipelex.cogt.img_gen.img_gen_job_components import AspectRatio
+from pipelex.core.pipes.pipe_input_blueprint import InputRequirementBlueprint
+from pipelex.libraries.pipelines.builder.pipe.inputs_spec import InputRequirementSpec
+from pipelex.libraries.pipelines.builder.pipe.pipe_img_spec import PipeImgGenSpec
+from pipelex.pipe_operators.img_gen.pipe_img_gen_blueprint import PipeImgGenBlueprint
+
+
+class PipeImgGenTestCases:
+ SIMPLE_IMG_GEN = (
+ "simple_img_gen",
+ PipeImgGenSpec(
+ the_pipe_code="img_generator",
+ definition="Generate an image",
+ inputs=None,
+ output="GeneratedImage",
+ img_gen_prompt="A beautiful sunset over mountains",
+ ),
+ PipeImgGenBlueprint(
+ definition="Generate an image",
+ inputs=None,
+ output="GeneratedImage",
+ type="PipeImgGen",
+ category="PipeOperator",
+ img_gen_prompt="A beautiful sunset over mountains",
+ img_gen=None,
+ aspect_ratio=None,
+ background=None,
+ output_format=None,
+ nb_output=None,
+ is_raw=None,
+ seed=None,
+ img_gen_prompt_var_name=None,
+ ),
+ )
+
+ IMG_GEN_WITH_OPTIONS = (
+ "img_gen_with_options",
+ PipeImgGenSpec(
+ the_pipe_code="advanced_img_gen",
+ definition="Generate image with options",
+ inputs={"description": InputRequirementSpec(concept="Text")},
+ output="Image",
+ img_gen="gpt-image-1",
+ aspect_ratio=AspectRatio.SQUARE,
+ seed=42,
+ nb_output=3,
+ ),
+ PipeImgGenBlueprint(
+ definition="Generate image with options",
+ inputs={"description": InputRequirementBlueprint(concept="Text")},
+ output="Image",
+ type="PipeImgGen",
+ category="PipeOperator",
+ img_gen_prompt=None,
+ img_gen="gpt-image-1",
+ aspect_ratio=AspectRatio.SQUARE,
+ background=None,
+ output_format=None,
+ is_raw=None,
+ seed=42,
+ nb_output=3,
+ img_gen_prompt_var_name=None,
+ ),
+ )
+
+ TEST_CASES: ClassVar[List[Tuple[str, PipeImgGenSpec, PipeImgGenBlueprint]]] = [
+ SIMPLE_IMG_GEN,
+ IMG_GEN_WITH_OPTIONS,
+ ]
diff --git a/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_operators/pipe_img/test_pipe_img.py b/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_operators/pipe_img/test_pipe_img.py
new file mode 100644
index 000000000..e54b5360f
--- /dev/null
+++ b/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_operators/pipe_img/test_pipe_img.py
@@ -0,0 +1,25 @@
+import pytest
+
+from pipelex.libraries.pipelines.builder.pipe.pipe_img_spec import PipeImgGenSpec
+from pipelex.pipe_operators.img_gen.pipe_img_gen_blueprint import PipeImgGenBlueprint
+
+from .test_data import PipeImgGenTestCases
+
+
+class TestPipeImgGenBlueprintConversion:
+ @pytest.mark.parametrize(
+ "test_name,pipe_spec,expected_blueprint",
+ PipeImgGenTestCases.TEST_CASES,
+ )
+ def test_pipe_img_gen_spec_to_blueprint(
+ self,
+ test_name: str,
+ pipe_spec: PipeImgGenSpec,
+ expected_blueprint: PipeImgGenBlueprint,
+ ):
+ result = pipe_spec.to_blueprint()
+ from pipelex import pretty_print
+
+ pretty_print(result, title=f"Result {test_name}")
+ pretty_print(expected_blueprint, title=f"Expected {test_name}")
+ assert result == expected_blueprint
diff --git a/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_operators/pipe_jinja2/test_data.py b/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_operators/pipe_jinja2/test_data.py
new file mode 100644
index 000000000..df261e843
--- /dev/null
+++ b/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_operators/pipe_jinja2/test_data.py
@@ -0,0 +1,72 @@
+from typing import ClassVar, List, Tuple
+
+from pipelex.core.pipes.pipe_input_blueprint import InputRequirementBlueprint
+from pipelex.libraries.pipelines.builder.pipe.inputs_spec import InputRequirementSpec
+from pipelex.libraries.pipelines.builder.pipe.pipe_jinja2_spec import PipeJinja2Spec, PromptingStyleSpec
+from pipelex.pipe_operators.jinja2.pipe_jinja2_blueprint import PipeJinja2Blueprint
+from pipelex.tools.templating.jinja2_template_category import Jinja2TemplateCategory
+from pipelex.tools.templating.templating_models import PromptingStyle, TagStyle, TextFormat
+
+
+class PipeJinja2TestCases:
+ SIMPLE_JINJA2 = (
+ "simple_jinja2",
+ PipeJinja2Spec(
+ the_pipe_code="template_renderer",
+ definition="Render a template",
+ inputs={"data": InputRequirementSpec(concept="Data")},
+ output="RenderedText",
+ jinja2="Hello {{ data.name }}!",
+ ),
+ "test_domain",
+ PipeJinja2Blueprint(
+ definition="Render a template",
+ inputs={"data": InputRequirementBlueprint(concept="Data")},
+ output="RenderedText",
+ type="PipeJinja2",
+ category="PipeOperator",
+ jinja2_name=None,
+ jinja2="Hello {{ data.name }}!",
+ prompting_style=None,
+ template_category=Jinja2TemplateCategory.LLM_PROMPT,
+ extra_context=None,
+ ),
+ )
+
+ JINJA2_WITH_STYLE = (
+ "jinja2_with_style",
+ PipeJinja2Spec(
+ the_pipe_code="styled_template",
+ definition="Template with prompting style",
+ inputs={"input": InputRequirementSpec(concept="Input")},
+ output="Output",
+ jinja2_name="custom_template",
+ prompting_style=PromptingStyleSpec(
+ tag_style=TagStyle.XML,
+ text_format=TextFormat.MARKDOWN,
+ ),
+ template_category=Jinja2TemplateCategory.MARKDOWN,
+ extra_context={"version": "1.0"},
+ ),
+ "test_domain",
+ PipeJinja2Blueprint(
+ definition="Template with prompting style",
+ inputs={"input": InputRequirementBlueprint(concept="Input")},
+ output="Output",
+ type="PipeJinja2",
+ category="PipeOperator",
+ jinja2_name="custom_template",
+ jinja2=None,
+ prompting_style=PromptingStyle(
+ tag_style=TagStyle.XML,
+ text_format=TextFormat.MARKDOWN,
+ ),
+ template_category=Jinja2TemplateCategory.MARKDOWN,
+ extra_context={"version": "1.0"},
+ ),
+ )
+
+ TEST_CASES: ClassVar[List[Tuple[str, PipeJinja2Spec, str, PipeJinja2Blueprint]]] = [
+ SIMPLE_JINJA2,
+ JINJA2_WITH_STYLE,
+ ]
diff --git a/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_operators/pipe_jinja2/test_pipe_jinja2.py b/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_operators/pipe_jinja2/test_pipe_jinja2.py
new file mode 100644
index 000000000..d471ae3e4
--- /dev/null
+++ b/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_operators/pipe_jinja2/test_pipe_jinja2.py
@@ -0,0 +1,22 @@
+import pytest
+
+from pipelex.libraries.pipelines.builder.pipe.pipe_jinja2_spec import PipeJinja2Spec
+from pipelex.pipe_operators.jinja2.pipe_jinja2_blueprint import PipeJinja2Blueprint
+
+from .test_data import PipeJinja2TestCases
+
+
+class TestPipeJinja2BlueprintConversion:
+ @pytest.mark.parametrize(
+ "test_name,pipe_spec,domain,expected_blueprint",
+ PipeJinja2TestCases.TEST_CASES,
+ )
+ def test_pipe_jinja2_spec_to_blueprint(
+ self,
+ test_name: str,
+ pipe_spec: PipeJinja2Spec,
+ domain: str,
+ expected_blueprint: PipeJinja2Blueprint,
+ ):
+ result = pipe_spec.to_blueprint()
+ assert result == expected_blueprint
diff --git a/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_operators/pipe_llm/test_data.py b/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_operators/pipe_llm/test_data.py
new file mode 100644
index 000000000..d0a1e5441
--- /dev/null
+++ b/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_operators/pipe_llm/test_data.py
@@ -0,0 +1,204 @@
+from typing import ClassVar, List, Tuple
+
+from pipelex.cogt.llm.llm_setting import LLMSetting
+from pipelex.core.pipes.pipe_input_blueprint import InputRequirementBlueprint
+from pipelex.libraries.pipelines.builder.pipe.inputs_spec import InputRequirementSpec
+from pipelex.libraries.pipelines.builder.pipe.pipe_llm_spec import LLMSettingSpec, PipeLLMSpec
+from pipelex.pipe_operators.llm.pipe_llm_blueprint import PipeLLMBlueprint, StructuringMethod
+
+
+class PipeLLMTestCases:
+ SIMPLE_LLM = (
+ "simple_llm",
+ PipeLLMSpec(
+ the_pipe_code="test_pipe",
+ definition="Generate text",
+ inputs={"topic": InputRequirementSpec(concept="Text")},
+ output="Text",
+ prompt_template="Write about $topic",
+ ),
+ "test_domain",
+ PipeLLMBlueprint(
+ type="PipeLLM",
+ definition="Generate text",
+ inputs={"topic": InputRequirementBlueprint(concept="Text")},
+ output="Text",
+ prompt_template="Write about $topic",
+ ),
+ )
+
+ LLM_NO_INPUTS = (
+ "llm_no_inputs",
+ PipeLLMSpec(
+ the_pipe_code="generate_pipe",
+ definition="Generate without inputs",
+ inputs=None,
+ output="Text",
+ prompt_template="Generate something interesting",
+ ),
+ "test_domain",
+ PipeLLMBlueprint(
+ type="PipeLLM",
+ definition="Generate without inputs",
+ output="Text",
+ prompt_template="Generate something interesting",
+ ),
+ )
+
+ LLM_WITH_PRESET = (
+ "llm_with_preset",
+ PipeLLMSpec(
+ the_pipe_code="generate",
+ definition="Generate with preset",
+ inputs=None,
+ output="Text",
+ prompt_template="Generate text",
+ llm="llm_to_reason",
+ ),
+ "test_domain",
+ PipeLLMBlueprint(
+ type="PipeLLM",
+ definition="Generate with preset",
+ llm="llm_to_reason",
+ output="Text",
+ prompt_template="Generate text",
+ ),
+ )
+
+ LLM_WITH_SETTINGS = (
+ "llm_with_settings",
+ PipeLLMSpec(
+ the_pipe_code="generate",
+ definition="Generate with settings",
+ inputs=None,
+ output="Text",
+ prompt_template="Generate text",
+ llm=LLMSettingSpec(
+ llm_handle="gpt-4o-mini",
+ temperature=0.7,
+ max_tokens=None, # "auto" is handled at conversion to core
+ ),
+ ),
+ "test_domain",
+ PipeLLMBlueprint(
+ type="PipeLLM",
+ definition="Generate with settings",
+ llm=LLMSetting(
+ llm_handle="gpt-4o-mini",
+ temperature=0.7,
+ max_tokens=None, # "auto" is handled at conversion to core
+ ),
+ output="Text",
+ prompt_template="Generate text",
+ ),
+ )
+
+ LLM_WITH_SYSTEM_PROMPT = (
+ "llm_with_system_prompt",
+ PipeLLMSpec(
+ the_pipe_code="analyze",
+ definition="Generate with system prompt",
+ inputs={"data": InputRequirementSpec(concept="Data")},
+ output="Analysis",
+ system_prompt="You are a data analyst",
+ prompt_template="Analyze: @data",
+ ),
+ "test_domain",
+ PipeLLMBlueprint(
+ type="PipeLLM",
+ definition="Generate with system prompt",
+ inputs={"data": InputRequirementBlueprint(concept="Data")},
+ system_prompt="You are a data analyst",
+ prompt_template="Analyze: @data",
+ output="Analysis",
+ ),
+ )
+
+ LLM_WITH_MULTIPLE_OUTPUT = (
+ "llm_with_multiple_output",
+ PipeLLMSpec(
+ the_pipe_code="generate_items",
+ definition="Generate multiple items",
+ inputs=None,
+ output="Item",
+ prompt_template="Generate items",
+ multiple_output=True,
+ ),
+ "test_domain",
+ PipeLLMBlueprint(
+ type="PipeLLM",
+ definition="Generate multiple items",
+ multiple_output=True,
+ nb_output=None,
+ output="Item",
+ prompt_template="Generate items",
+ ),
+ )
+
+ LLM_WITH_FIXED_OUTPUT = (
+ "llm_with_fixed_output",
+ PipeLLMSpec(
+ the_pipe_code="generate_items",
+ definition="Generate exactly 5 items",
+ inputs=None,
+ output="Item",
+ prompt_template="Generate items",
+ nb_output=5,
+ ),
+ "test_domain",
+ PipeLLMBlueprint(
+ type="PipeLLM",
+ definition="Generate exactly 5 items",
+ nb_output=5,
+ multiple_output=None,
+ output="Item",
+ prompt_template="Generate items",
+ ),
+ )
+
+ LLM_WITH_STRUCTURING = (
+ "llm_with_structuring",
+ PipeLLMSpec(
+ the_pipe_code="test_pipe",
+ definition="Extract structured data",
+ inputs=None,
+ output="PersonInfo",
+ prompt_template="Extract person info",
+ llm="llm_to_extract",
+ llm_to_structure=LLMSettingSpec(
+ llm_handle="claude-3-sonnet",
+ temperature=0.1,
+ max_tokens=None, # "auto" is handled at conversion to core
+ ),
+ structuring_method=StructuringMethod.PRELIMINARY_TEXT,
+ prompt_template_to_structure="Structure the output",
+ system_prompt_to_structure="You are a data structurer",
+ ),
+ "test_domain",
+ PipeLLMBlueprint(
+ type="PipeLLM",
+ definition="Extract structured data",
+ structuring_method=StructuringMethod.PRELIMINARY_TEXT,
+ prompt_template_to_structure="Structure the output",
+ system_prompt_to_structure="You are a data structurer",
+ llm="llm_to_extract",
+ llm_to_structure=LLMSetting(
+ llm_handle="claude-3-sonnet",
+ temperature=0.1,
+ max_tokens=None, # "auto" is handled at conversion to core
+ ),
+ output="PersonInfo",
+ prompt_template="Extract person info",
+ ),
+ )
+
+ TEST_CASES: ClassVar[List[Tuple[str, PipeLLMSpec, str, PipeLLMBlueprint]]] = [
+ SIMPLE_LLM,
+ LLM_NO_INPUTS,
+ LLM_WITH_PRESET,
+ LLM_WITH_SETTINGS,
+ LLM_WITH_SYSTEM_PROMPT,
+ LLM_WITH_MULTIPLE_OUTPUT,
+ LLM_WITH_FIXED_OUTPUT,
+ LLM_WITH_STRUCTURING,
+ ]
diff --git a/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_operators/pipe_llm/test_pipe_llm.py b/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_operators/pipe_llm/test_pipe_llm.py
new file mode 100644
index 000000000..48f075d7a
--- /dev/null
+++ b/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_operators/pipe_llm/test_pipe_llm.py
@@ -0,0 +1,16 @@
+import pytest
+
+from pipelex.libraries.pipelines.builder.pipe.pipe_llm_spec import PipeLLMSpec
+from pipelex.pipe_operators.llm.pipe_llm_blueprint import PipeLLMBlueprint
+
+from .test_data import PipeLLMTestCases
+
+
+class TestPipeLLMBlueprintConversion:
+ @pytest.mark.parametrize(
+ "test_name,pipe_spec,domain,expected_blueprint",
+ PipeLLMTestCases.TEST_CASES,
+ )
+ def test_pipe_llm_spec_to_blueprint(self, test_name: str, pipe_spec: PipeLLMSpec, domain: str, expected_blueprint: PipeLLMBlueprint):
+ result = pipe_spec.to_blueprint()
+ assert result == expected_blueprint
diff --git a/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_operators/pipe_ocr/test_data.py b/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_operators/pipe_ocr/test_data.py
new file mode 100644
index 000000000..2e5fc5b05
--- /dev/null
+++ b/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_operators/pipe_ocr/test_data.py
@@ -0,0 +1,57 @@
+from typing import ClassVar, List, Tuple
+
+from pipelex.core.pipes.pipe_input_blueprint import InputRequirementBlueprint
+from pipelex.libraries.pipelines.builder.pipe.inputs_spec import InputRequirementSpec
+from pipelex.libraries.pipelines.builder.pipe.pipe_ocr_spec import PipeOcrSpec
+from pipelex.pipe_operators.ocr.pipe_ocr_blueprint import PipeOcrBlueprint
+
+
+class PipeOcrTestCases:
+ SIMPLE_OCR = (
+ "simple_ocr",
+ PipeOcrSpec(
+ the_pipe_code="ocr_extractor",
+ definition="Extract text from image",
+ inputs={"ocr_input": InputRequirementSpec(concept="Image")},
+ output="ExtractedText",
+ ocr="mistral-pixtral",
+ ),
+ "test_domain",
+ PipeOcrBlueprint(
+ definition="Extract text from image",
+ inputs={"ocr_input": InputRequirementBlueprint(concept="Image")},
+ output="ExtractedText",
+ type="PipeOcr",
+ category="PipeOperator",
+ ocr="mistral-pixtral",
+ ),
+ )
+
+ OCR_WITH_OPTIONS = (
+ "ocr_with_options",
+ PipeOcrSpec(
+ the_pipe_code="advanced_ocr",
+ definition="OCR with page options",
+ inputs={"ocr_input": InputRequirementSpec(concept="PDF")},
+ output="PageContent",
+ ocr="tesseract",
+ page_images=True,
+ page_image_captions=True,
+ page_views=True,
+ page_views_dpi=300,
+ ),
+ "test_domain",
+ PipeOcrBlueprint(
+ definition="OCR with page options",
+ inputs={"ocr_input": InputRequirementBlueprint(concept="PDF")},
+ output="PageContent",
+ type="PipeOcr",
+ category="PipeOperator",
+ ocr="tesseract",
+ ),
+ )
+
+ TEST_CASES: ClassVar[List[Tuple[str, PipeOcrSpec, str, PipeOcrBlueprint]]] = [
+ SIMPLE_OCR,
+ OCR_WITH_OPTIONS,
+ ]
diff --git a/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_operators/pipe_ocr/test_pipe_ocr.py b/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_operators/pipe_ocr/test_pipe_ocr.py
new file mode 100644
index 000000000..b33043418
--- /dev/null
+++ b/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_operators/pipe_ocr/test_pipe_ocr.py
@@ -0,0 +1,22 @@
+import pytest
+
+from pipelex.libraries.pipelines.builder.pipe.pipe_ocr_spec import PipeOcrSpec
+from pipelex.pipe_operators.ocr.pipe_ocr_blueprint import PipeOcrBlueprint
+
+from .test_data import PipeOcrTestCases
+
+
+class TestPipeOcrBlueprintConversion:
+ @pytest.mark.parametrize(
+ "test_name,pipe_spec,domain,expected_blueprint",
+ PipeOcrTestCases.TEST_CASES,
+ )
+ def test_pipe_ocr_spec_to_blueprint(
+ self,
+ test_name: str,
+ pipe_spec: PipeOcrSpec,
+ domain: str,
+ expected_blueprint: PipeOcrBlueprint,
+ ):
+ result = pipe_spec.to_blueprint()
+ assert result == expected_blueprint
diff --git a/tests/unit/pipelex/libraries/pipelines/builder/pipe/test_data_inputs.py b/tests/unit/pipelex/libraries/pipelines/builder/pipe/test_data_inputs.py
new file mode 100644
index 000000000..dd84d0c7b
--- /dev/null
+++ b/tests/unit/pipelex/libraries/pipelines/builder/pipe/test_data_inputs.py
@@ -0,0 +1,41 @@
+from typing import ClassVar, List, Tuple
+
+from pipelex.core.pipes.pipe_input_blueprint import InputRequirementBlueprint
+from pipelex.libraries.pipelines.builder.pipe.inputs_spec import InputRequirementSpec
+
+
+class InputRequirementTestCases:
+ SIMPLE_TEXT_INPUT = (
+ "simple_text_input",
+ InputRequirementSpec(concept="Text"),
+ "test_domain",
+ InputRequirementBlueprint(concept="Text"),
+ )
+
+ IMAGE_INPUT = (
+ "image_input",
+ InputRequirementSpec(concept="Image"),
+ "test_domain",
+ InputRequirementBlueprint(concept="Image"),
+ )
+
+ CUSTOM_CONCEPT_INPUT = (
+ "custom_concept_input",
+ InputRequirementSpec(concept="CustomConcept"),
+ "test_domain",
+ InputRequirementBlueprint(concept="CustomConcept"),
+ )
+
+ DOMAIN_CONCEPT_INPUT = (
+ "domain_concept_input",
+ InputRequirementSpec(concept="domain.ConceptName"),
+ "test_domain",
+ InputRequirementBlueprint(concept="domain.ConceptName"),
+ )
+
+ TEST_CASES: ClassVar[List[Tuple[str, InputRequirementSpec, str, InputRequirementBlueprint]]] = [
+ SIMPLE_TEXT_INPUT,
+ IMAGE_INPUT,
+ CUSTOM_CONCEPT_INPUT,
+ DOMAIN_CONCEPT_INPUT,
+ ]
diff --git a/tests/unit/pipelex/libraries/pipelines/builder/pipe/test_data_pipe.py b/tests/unit/pipelex/libraries/pipelines/builder/pipe/test_data_pipe.py
new file mode 100644
index 000000000..0769cb741
--- /dev/null
+++ b/tests/unit/pipelex/libraries/pipelines/builder/pipe/test_data_pipe.py
@@ -0,0 +1,77 @@
+from typing import ClassVar, List, Tuple
+
+from pipelex.core.pipes.pipe_blueprint import PipeBlueprint
+from pipelex.core.pipes.pipe_input_blueprint import InputRequirementBlueprint
+from pipelex.libraries.pipelines.builder.pipe.inputs_spec import InputRequirementSpec
+from pipelex.libraries.pipelines.builder.pipe.pipe_signature import PipeSpec
+
+
+class PipeBlueprintTestCases:
+ SIMPLE_PIPE = (
+ "simple_pipe",
+ PipeSpec(
+ type="PipeLLM",
+ category="PipeOperator",
+ definition="A simple pipe",
+ inputs={"input": "Text"},
+ output="ProcessedText",
+ ),
+ "test_domain",
+ PipeBlueprint(
+ type="PipeLLM",
+ category="PipeOperator",
+ definition="A simple pipe",
+ inputs={"input": InputRequirementBlueprint(concept="Text")},
+ output="ProcessedText",
+ ),
+ )
+
+ PIPE_WITH_INPUT_REQUIREMENTS = (
+ "pipe_with_requirements",
+ PipeSpec(
+ type="PipeFunc",
+ category="PipeOperator",
+ definition="Pipe with input requirements",
+ inputs={
+ "data": InputRequirementSpec(concept="Data"),
+ "config": InputRequirementSpec(concept="Config"),
+ },
+ output="Result",
+ ),
+ "test_domain",
+ PipeBlueprint(
+ type="PipeFunc",
+ category="PipeOperator",
+ definition="Pipe with input requirements",
+ inputs={
+ "data": InputRequirementBlueprint(concept="Data"),
+ "config": InputRequirementBlueprint(concept="Config"),
+ },
+ output="Result",
+ ),
+ )
+
+ PIPE_NO_INPUTS = (
+ "pipe_no_inputs",
+ PipeSpec(
+ type="PipeFunc",
+ category="PipeOperator",
+ definition="Pipe without inputs",
+ inputs=None,
+ output="GeneratedData",
+ ),
+ "test_domain",
+ PipeBlueprint(
+ type="PipeFunc",
+ category="PipeOperator",
+ definition="Pipe without inputs",
+ inputs=None,
+ output="GeneratedData",
+ ),
+ )
+
+ TEST_CASES: ClassVar[List[Tuple[str, PipeSpec, str, PipeBlueprint]]] = [
+ SIMPLE_PIPE,
+ PIPE_WITH_INPUT_REQUIREMENTS,
+ PIPE_NO_INPUTS,
+ ]
diff --git a/tests/unit/pipelex/libraries/pipelines/builder/pipe/test_data_sub_pipe.py b/tests/unit/pipelex/libraries/pipelines/builder/pipe/test_data_sub_pipe.py
new file mode 100644
index 000000000..46bfdd378
--- /dev/null
+++ b/tests/unit/pipelex/libraries/pipelines/builder/pipe/test_data_sub_pipe.py
@@ -0,0 +1,37 @@
+from typing import ClassVar, List, Tuple
+
+from pipelex.libraries.pipelines.builder.pipe.sub_pipe_spec import SubPipeSpec
+from pipelex.pipe_controllers.sub_pipe_blueprint import SubPipeBlueprint
+
+
+class SubPipeTestCases:
+ SIMPLE_SUB_PIPE = (
+ "simple_sub_pipe",
+ SubPipeSpec(the_pipe_code="process_data", result="processed_data"),
+ SubPipeBlueprint(pipe="process_data", result="processed_data"),
+ )
+
+ SUB_PIPE_WITH_MULTIPLE_OUTPUT = (
+ "sub_pipe_with_multiple_output",
+ SubPipeSpec(the_pipe_code="generate_items", result="items", multiple_output=True),
+ SubPipeBlueprint(pipe="generate_items", result="items", multiple_output=True),
+ )
+
+ SUB_PIPE_WITH_FIXED_OUTPUT = (
+ "sub_pipe_with_fixed_output",
+ SubPipeSpec(the_pipe_code="generate_ideas", result="ideas", nb_output=3),
+ SubPipeBlueprint(pipe="generate_ideas", result="ideas", nb_output=3),
+ )
+
+ SUB_PIPE_WITH_BATCH = (
+ "sub_pipe_with_batch",
+ SubPipeSpec(the_pipe_code="process_item", result="processed_items", batch_over="input_list", batch_as="current_item"),
+ SubPipeBlueprint(pipe="process_item", result="processed_items", batch_over="input_list", batch_as="current_item"),
+ )
+
+ TEST_CASES: ClassVar[List[Tuple[str, SubPipeSpec, SubPipeBlueprint]]] = [
+ SIMPLE_SUB_PIPE,
+ SUB_PIPE_WITH_MULTIPLE_OUTPUT,
+ SUB_PIPE_WITH_FIXED_OUTPUT,
+ SUB_PIPE_WITH_BATCH,
+ ]
diff --git a/tests/unit/pipelex/libraries/pipelines/builder/pipe/test_inputs_blueprint.py b/tests/unit/pipelex/libraries/pipelines/builder/pipe/test_inputs_blueprint.py
new file mode 100644
index 000000000..4657f1d67
--- /dev/null
+++ b/tests/unit/pipelex/libraries/pipelines/builder/pipe/test_inputs_blueprint.py
@@ -0,0 +1,18 @@
+import pytest
+
+from pipelex.core.pipes.pipe_input_blueprint import InputRequirementBlueprint
+from pipelex.libraries.pipelines.builder.pipe.inputs_spec import InputRequirementSpec
+
+from .test_data_inputs import InputRequirementTestCases
+
+
+class TestInputRequirementBlueprintConversion:
+ @pytest.mark.parametrize(
+ "test_name,input_spec,domain,expected_blueprint",
+ InputRequirementTestCases.TEST_CASES,
+ )
+ def test_input_requirement_spec_to_blueprint(
+ self, test_name: str, input_spec: InputRequirementSpec, domain: str, expected_blueprint: InputRequirementBlueprint
+ ):
+ result = input_spec.to_blueprint()
+ assert result == expected_blueprint
diff --git a/tests/unit/pipelex/libraries/pipelines/builder/pipe/test_pipe_blueprint.py b/tests/unit/pipelex/libraries/pipelines/builder/pipe/test_pipe_blueprint.py
new file mode 100644
index 000000000..191b33820
--- /dev/null
+++ b/tests/unit/pipelex/libraries/pipelines/builder/pipe/test_pipe_blueprint.py
@@ -0,0 +1,22 @@
+import pytest
+
+from pipelex.core.pipes.pipe_blueprint import PipeBlueprint
+from pipelex.libraries.pipelines.builder.pipe.pipe_signature import PipeSpec
+
+from .test_data_pipe import PipeBlueprintTestCases
+
+
+class TestPipeBlueprintConversion:
+ @pytest.mark.parametrize(
+ "test_name,pipe_spec,domain,expected_blueprint",
+ PipeBlueprintTestCases.TEST_CASES,
+ )
+ def test_pipe_blueprint_to_core(
+ self,
+ test_name: str,
+ pipe_spec: PipeSpec,
+ domain: str,
+ expected_blueprint: PipeBlueprint,
+ ):
+ result = pipe_spec.to_blueprint()
+ assert result == expected_blueprint
diff --git a/tests/unit/pipelex/libraries/pipelines/builder/pipe/test_sub_pipe_blueprint.py b/tests/unit/pipelex/libraries/pipelines/builder/pipe/test_sub_pipe_blueprint.py
new file mode 100644
index 000000000..eb5418de6
--- /dev/null
+++ b/tests/unit/pipelex/libraries/pipelines/builder/pipe/test_sub_pipe_blueprint.py
@@ -0,0 +1,16 @@
+import pytest
+
+from pipelex.libraries.pipelines.builder.pipe.sub_pipe_spec import SubPipeSpec
+from pipelex.pipe_controllers.sub_pipe_blueprint import SubPipeBlueprint
+
+from .test_data_sub_pipe import SubPipeTestCases
+
+
+class TestSubPipeBlueprintConversion:
+ @pytest.mark.parametrize(
+ "test_name,sub_pipe_spec,expected_blueprint",
+ SubPipeTestCases.TEST_CASES,
+ )
+ def test_sub_pipe_to_core(self, test_name: str, sub_pipe_spec: SubPipeSpec, expected_blueprint: SubPipeBlueprint):
+ result = sub_pipe_spec.to_blueprint()
+ assert result == expected_blueprint
diff --git a/tests/unit/pipelex/libraries/pipelines/builder/test_pipelex_bundle_blueprint.py b/tests/unit/pipelex/libraries/pipelines/builder/test_pipelex_bundle_blueprint.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/tests/unit/pipelex/tools/misc/test_dict_utils.py b/tests/unit/pipelex/tools/misc/test_dict_utils.py
index 8347db504..f24765c01 100644
--- a/tests/unit/pipelex/tools/misc/test_dict_utils.py
+++ b/tests/unit/pipelex/tools/misc/test_dict_utils.py
@@ -41,7 +41,7 @@ class TestDictUtils:
def test_complex_nested_structure(self) -> None:
"""Test with complex nested dictionary structure."""
- original = {"type": "PipeLLM", "definition": "Test pipe", "output": NativeConceptEnum.TEXT.value, "system_prompt": "Test prompt"}
+ original = {"type": "PipeLLM", "definition": "Test pipe", "output": NativeConceptEnum.TEXT, "system_prompt": "Test prompt"}
# Insert inputs before output
result = insert_before(original, "output", "inputs", "InputText")
diff --git a/tests/unit/pipelex/tools/misc/test_placeholder.py b/tests/unit/pipelex/tools/misc/test_placeholder.py
new file mode 100644
index 000000000..9b1bf84f9
--- /dev/null
+++ b/tests/unit/pipelex/tools/misc/test_placeholder.py
@@ -0,0 +1,72 @@
+from pipelex.tools.misc.placeholder import PLACEHOLDER_PREFIX, make_placeholder_value, value_is_placeholder
+
+
+class TestPlaceholder:
+ def test_value_is_placeholder_valid_placeholder(self):
+ """Test that values starting with the prefix are identified as placeholders."""
+ result = value_is_placeholder(make_placeholder_value("TEST_VAR"))
+ assert result is True
+
+ def test_value_is_placeholder_exact_prefix_match(self):
+ """Test that exact prefix match is identified as a placeholder."""
+ result = value_is_placeholder(PLACEHOLDER_PREFIX)
+ assert result is True
+
+ def test_value_is_placeholder_non_placeholder(self):
+ """Test that regular values are not identified as placeholders."""
+ result = value_is_placeholder("regular_value")
+ assert result is False
+
+ def test_value_is_placeholder_prefix_in_middle(self):
+ """Test that values with prefix not at start are not identified as placeholders."""
+ result = value_is_placeholder(f"prefix-{PLACEHOLDER_PREFIX}-suffix")
+ assert result is False
+
+ def test_value_is_placeholder_none_value(self):
+ """Test that None values are not identified as placeholders."""
+ result = value_is_placeholder(None)
+ assert result is False
+
+ def test_value_is_placeholder_empty_string(self):
+ """Test that empty strings are not identified as placeholders."""
+ result = value_is_placeholder("")
+ assert result is False
+
+ def test_value_is_placeholder_partial_prefix_match(self):
+ """Test that partial prefix matches are not identified as placeholders."""
+ partial_prefix = PLACEHOLDER_PREFIX[:-3] # Remove last 3 characters
+ result = value_is_placeholder(f"{partial_prefix}-for-TEST")
+ assert result is False
+
+ def test_make_placeholder_value_basic(self):
+ """Test that make_placeholder_value creates the correct format."""
+ result = make_placeholder_value("TEST_VAR")
+ assert result == "placeholder-for-TEST_VAR"
+
+ def test_make_placeholder_value_different_keys(self):
+ """Test make_placeholder_value with different key formats."""
+ assert make_placeholder_value("OPENAI_API_KEY") == "placeholder-for-OPENAI_API_KEY"
+ assert make_placeholder_value("simple_key") == "placeholder-for-simple_key"
+ assert make_placeholder_value("Key123") == "placeholder-for-Key123"
+
+ def test_make_placeholder_value_empty_key(self):
+ """Test make_placeholder_value with empty key."""
+ result = make_placeholder_value("")
+ assert result == "placeholder-for-"
+
+ def test_make_placeholder_value_special_characters(self):
+ """Test make_placeholder_value with keys containing special characters."""
+ assert make_placeholder_value("KEY_WITH_UNDERSCORES") == "placeholder-for-KEY_WITH_UNDERSCORES"
+ assert make_placeholder_value("key-with-dashes") == "placeholder-for-key-with-dashes"
+ assert make_placeholder_value("key.with.dots") == "placeholder-for-key.with.dots"
+
+ def test_make_placeholder_value_integration_with_value_is_placeholder(self):
+ """Test that make_placeholder_value output is correctly identified by value_is_placeholder."""
+ test_keys = ["TEST1", "TEST2", "ANOTHER_KEY"]
+ for key in test_keys:
+ placeholder_value = make_placeholder_value(key)
+ assert value_is_placeholder(placeholder_value), f"Failed for key: {key}"
+
+ def test_placeholder_prefix_constant(self):
+ """Test that the placeholder prefix constant has the expected value."""
+ assert PLACEHOLDER_PREFIX == "placeholder"
diff --git a/tests/unit/pipelex/tools/test_environment.py b/tests/unit/pipelex/tools/test_environment.py
index 84316ccb4..e0f181f43 100644
--- a/tests/unit/pipelex/tools/test_environment.py
+++ b/tests/unit/pipelex/tools/test_environment.py
@@ -5,15 +5,17 @@ from pytest_mock import MockerFixture
from pipelex.tools.environment import (
EnvVarNotFoundError,
- get_env_rooted_path,
+ all_env_vars_are_set,
+ any_env_var_is_placeholder,
get_optional_env,
get_required_env,
- get_rooted_path,
+ is_env_var_set,
set_env,
)
+from pipelex.tools.misc.placeholder import PLACEHOLDER_PREFIX, make_placeholder_value
-class TestGetRequiredEnv:
+class TestEnvironment:
def test_get_required_env_success(self, mocker: MockerFixture):
mocker.patch.dict(os.environ, {"TEST_VAR": "test_value"})
result = get_required_env("TEST_VAR")
@@ -29,8 +31,6 @@ class TestGetRequiredEnv:
with pytest.raises(EnvVarNotFoundError, match="Environment variable 'EMPTY_VAR' is required but not set"):
get_required_env("EMPTY_VAR")
-
-class TestGetOptionalEnv:
def test_get_optional_env_success(self, mocker: MockerFixture):
mocker.patch.dict(os.environ, {"TEST_VAR": "test_value"})
result = get_optional_env("TEST_VAR")
@@ -46,8 +46,6 @@ class TestGetOptionalEnv:
result = get_optional_env("EMPTY_VAR")
assert result == ""
-
-class TestSetEnv:
def test_set_env_success(self):
set_env("TEST_SET_VAR", "new_value")
assert os.environ["TEST_SET_VAR"] == "new_value"
@@ -57,73 +55,103 @@ class TestSetEnv:
set_env("EXISTING_VAR", "new_value")
assert os.environ["EXISTING_VAR"] == "new_value"
+ def test_all_env_vars_are_set_all_keys_present(self, mocker: MockerFixture):
+ mocker.patch.dict(os.environ, {"VAR1": "value1", "VAR2": "value2", "VAR3": "value3"})
+ result = all_env_vars_are_set(keys=["VAR1", "VAR2", "VAR3"])
+ assert result is True
-class TestGetRootedPath:
- def test_get_rooted_path_none_path(self):
- result = get_rooted_path("/root")
- assert result == "/root"
-
- def test_get_rooted_path_empty_path(self):
- result = get_rooted_path("/root", "")
- assert result == "/root"
-
- def test_get_rooted_path_already_starts_with_root(self):
- result = get_rooted_path("/root", "/root/subdir")
- assert result == "/root/subdir"
-
- def test_get_rooted_path_absolute_path_different_root(self):
- result = get_rooted_path("/root", "/other/absolute/path")
- assert result == "/other/absolute/path"
-
- def test_get_rooted_path_relative_path(self):
- result = get_rooted_path("/root", "relative/path")
- assert result == "/root/relative/path"
-
- def test_get_rooted_path_removes_trailing_slash(self):
- result = get_rooted_path("/root", "subdir/")
- assert result == "/root/subdir"
+ def test_all_env_vars_are_set_some_keys_missing(self, mocker: MockerFixture):
+ mocker.patch.dict(os.environ, {"VAR1": "value1"}, clear=True)
+ result = all_env_vars_are_set(keys=["VAR1", "VAR2", "VAR3"])
+ assert result is False
- def test_get_rooted_path_keeps_root_without_trailing_slash(self):
- result = get_rooted_path("/root/", "")
- assert result == "/root"
-
- def test_get_rooted_path_absolute_check(self, mocker: MockerFixture):
- mock_isabs = mocker.patch("os.path.isabs", return_value=True)
- result = get_rooted_path("/root", "fake/absolute")
- assert result == "fake/absolute"
- mock_isabs.assert_called_once_with("fake/absolute")
-
- def test_get_rooted_path_join_called(self, mocker: MockerFixture):
- mock_join = mocker.patch("os.path.join", return_value="/root/relative")
- mocker.patch("os.path.isabs", return_value=False)
- result = get_rooted_path("/root", "relative")
- assert result == "/root/relative"
- mock_join.assert_called_once_with("/root", "relative")
-
-
-class TestGetEnvRootedPath:
- def test_get_env_rooted_path_with_env_var(self, mocker: MockerFixture):
- mocker.patch.dict(os.environ, {"ROOT_ENV": "/env/root"})
- result = get_env_rooted_path("ROOT_ENV", "subdir")
- assert result == "/env/root/subdir"
+ def test_all_env_vars_are_set_all_keys_missing(self, mocker: MockerFixture):
+ mocker.patch.dict(os.environ, {}, clear=True)
+ result = all_env_vars_are_set(keys=["VAR1", "VAR2", "VAR3"])
+ assert result is False
- def test_get_env_rooted_path_missing_env_var(self, mocker: MockerFixture):
+ def test_all_env_vars_are_set_empty_iterable(self, mocker: MockerFixture):
mocker.patch.dict(os.environ, {}, clear=True)
- result = get_env_rooted_path("MISSING_ROOT_ENV", "subdir")
- assert result == "subdir"
+ result = all_env_vars_are_set(keys=[])
+ assert result is True
- def test_get_env_rooted_path_empty_env_var(self, mocker: MockerFixture):
- mocker.patch.dict(os.environ, {"EMPTY_ROOT_ENV": ""})
- result = get_env_rooted_path("EMPTY_ROOT_ENV", "subdir")
- assert result == "subdir"
+ def test_all_env_vars_are_set_single_key_present(self, mocker: MockerFixture):
+ mocker.patch.dict(os.environ, {"SINGLE_VAR": "value"})
+ result = is_env_var_set(key="SINGLE_VAR")
+ assert result is True
- def test_get_env_rooted_path_none_path(self, mocker: MockerFixture):
- mocker.patch.dict(os.environ, {"ROOT_ENV": "/env/root"})
- result = get_env_rooted_path("ROOT_ENV")
- assert result == "/env/root"
+ def test_all_env_vars_are_set_single_key_missing(self, mocker: MockerFixture):
+ mocker.patch.dict(os.environ, {}, clear=True)
+ result = is_env_var_set(key="MISSING_VAR")
+ assert result is False
+ def test_all_env_vars_are_set_empty_string_value_counts_as_set(self, mocker: MockerFixture):
+ mocker.patch.dict(os.environ, {"EMPTY_VAR": ""})
+ result = is_env_var_set(key="EMPTY_VAR")
+ assert result is True
+
+ def test_any_env_var_is_placeholder_has_placeholder(self, mocker: MockerFixture):
+ mocker.patch.dict(os.environ, {"VAR1": "real_value", "VAR2": make_placeholder_value("VAR2"), "VAR3": "another_value"})
+ result = any_env_var_is_placeholder(["VAR1", "VAR2", "VAR3"])
+ assert result is True
+
+ def test_any_env_var_is_placeholder_no_placeholders(self, mocker: MockerFixture):
+ mocker.patch.dict(os.environ, {"VAR1": "real_value", "VAR2": "another_value", "VAR3": "third_value"})
+ result = any_env_var_is_placeholder(["VAR1", "VAR2", "VAR3"])
+ assert result is False
+
+ def test_any_env_var_is_placeholder_all_placeholders(self, mocker: MockerFixture):
+ mocker.patch.dict(
+ os.environ,
+ {
+ "VAR1": make_placeholder_value("VAR1"),
+ "VAR2": make_placeholder_value("VAR2"),
+ "VAR3": make_placeholder_value("VAR3"),
+ },
+ )
+ result = any_env_var_is_placeholder(["VAR1", "VAR2", "VAR3"])
+ assert result is True
+
+ def test_any_env_var_is_placeholder_missing_vars_no_placeholder(self, mocker: MockerFixture):
+ mocker.patch.dict(os.environ, {"VAR1": "real_value"}, clear=True)
+ result = any_env_var_is_placeholder(["VAR1", "MISSING_VAR"])
+ assert result is False
+
+ def test_any_env_var_is_placeholder_empty_iterable(self, mocker: MockerFixture):
+ mocker.patch.dict(os.environ, {})
+ result = any_env_var_is_placeholder([])
+ assert result is False
+
+ def test_any_env_var_is_placeholder_single_placeholder(self, mocker: MockerFixture):
+ mocker.patch.dict(os.environ, {"PLACEHOLDER_VAR": make_placeholder_value("PLACEHOLDER_VAR")})
+ result = any_env_var_is_placeholder(["PLACEHOLDER_VAR"])
+ assert result is True
+
+ def test_any_env_var_is_placeholder_single_non_placeholder(self, mocker: MockerFixture):
+ mocker.patch.dict(os.environ, {"NORMAL_VAR": "normal_value"})
+ result = any_env_var_is_placeholder(["NORMAL_VAR"])
+ assert result is False
+
+ def test_any_env_var_is_placeholder_prefix_at_start_only(self, mocker: MockerFixture):
+ """Test that only values starting with the prefix are considered placeholders."""
+ mocker.patch.dict(
+ os.environ,
+ {
+ "VAR1": f"not-{PLACEHOLDER_PREFIX}-suffix",
+ "VAR2": make_placeholder_value("VAR2"),
+ },
+ )
+ result = any_env_var_is_placeholder(["VAR1"])
+ assert result is False
+ result = any_env_var_is_placeholder(["VAR2"])
+ assert result is True
+
+ def test_any_env_var_is_placeholder_exact_prefix_match(self, mocker: MockerFixture):
+ """Test that exact prefix match (without suffix) is also considered a placeholder."""
+ mocker.patch.dict(os.environ, {"EXACT_PREFIX": PLACEHOLDER_PREFIX})
+ result = any_env_var_is_placeholder(["EXACT_PREFIX"])
+ assert result is True
-class TestEnvVarNotFoundError:
def test_env_var_not_found_error_inheritance(self):
error = EnvVarNotFoundError("test message")
assert isinstance(error, Exception)
diff --git a/tests/unit/pipelex/tools/test_func_registry.py b/tests/unit/pipelex/tools/test_func_registry.py
index 698701df0..a49068dab 100644
--- a/tests/unit/pipelex/tools/test_func_registry.py
+++ b/tests/unit/pipelex/tools/test_func_registry.py
@@ -1,9 +1,11 @@
import logging
-from typing import Any, Callable, Dict, List
+from typing import Any, Callable, Dict, List, Tuple
import pytest
from pytest import LogCaptureFixture
+from pipelex.core.memory.working_memory import WorkingMemory
+from pipelex.core.stuffs.stuff_content import TextContent
from pipelex.tools.func_registry import FuncRegistry, FuncRegistryError
@@ -15,6 +17,61 @@ def another_function(x: int):
return x * 2
+# Valid function for testing
+def valid_function(working_memory: WorkingMemory) -> TextContent:
+ return TextContent(text="test")
+
+
+# Valid async function for testing (should also be eligible)
+async def valid_async_function(working_memory: WorkingMemory) -> TextContent:
+ return TextContent(text="test")
+
+
+# Invalid functions for testing eligibility
+def wrong_param_name(other_param: WorkingMemory) -> TextContent:
+ return TextContent(text="test")
+
+
+def wrong_param_type(working_memory: str) -> TextContent: # type: ignore
+ return TextContent(text="test")
+
+
+def wrong_return_type(working_memory: WorkingMemory) -> str:
+ return "test"
+
+
+def too_many_params(working_memory: WorkingMemory, extra: str) -> TextContent:
+ return TextContent(text="test")
+
+
+def no_params() -> TextContent:
+ return TextContent(text="test")
+
+
+def no_type_hints(working_memory): # type: ignore
+ return TextContent(text="test")
+
+
+# Type alias for test parameters
+TestParams = Tuple[str, Callable[..., Any], bool]
+
+# Test data
+TEST_CASES: List[TestParams] = [
+ # Valid cases
+ ("valid_function", valid_function, True),
+ ("valid_async_function", valid_async_function, True),
+ # Invalid cases
+ ("wrong_param_name", wrong_param_name, False),
+ ("wrong_param_type", wrong_param_type, False),
+ ("wrong_return_type", wrong_return_type, False),
+ ("too_many_params", too_many_params, False),
+ ("no_params", no_params, False),
+ ("no_type_hints", no_type_hints, False),
+ ("sample_function", sample_function, False),
+ ("another_function", another_function, False),
+]
+
+
@pytest.fixture
def registry():
# Create a new registry for each test
@@ -25,40 +82,47 @@ def registry():
class TestFuncRegistry:
- def test_register_and_get_function(self, registry: FuncRegistry):
- registry.register_function(sample_function)
- assert registry.get_function("sample_function") is sample_function
+ @pytest.mark.parametrize("test_name, func, is_eligible", TEST_CASES)
+ def test_function_eligibility_and_registration(self, registry: FuncRegistry, test_name: str, func: Callable[..., Any], is_eligible: bool):
+ """Test function eligibility check and registration behavior."""
+
+ # Test eligibility check directly
+ actual_eligibility = registry.is_eligible_function(func)
+ assert actual_eligibility == is_eligible, f"Eligibility check failed for {test_name}: expected {is_eligible}, got {actual_eligibility}"
+
+ # Test registration behavior
+ if is_eligible:
+ # Should register successfully
+ registry.register_function(func)
+ assert registry.has_function(func.__name__), f"Eligible function {test_name} should be registered"
+ assert registry.get_function(func.__name__) is func, f"Should be able to retrieve registered function {test_name}"
+ else:
+ # Should be silently skipped (no error, no registration)
+ registry.register_function(func)
+ assert not registry.has_function(func.__name__), f"Ineligible function {test_name} should not be registered"
def test_register_function_with_custom_name(self, registry: FuncRegistry):
- registry.register_function(sample_function, name="custom_name")
- assert registry.get_function("custom_name") is sample_function
- assert registry.get_function("sample_function") is None
-
- def test_has_function(self, registry: FuncRegistry):
- registry.register_function(sample_function)
- assert registry.has_function("sample_function")
- assert not registry.has_function("non_existent_function")
-
- def test_get_required_function(self, registry: FuncRegistry):
- registry.register_function(sample_function)
- assert registry.get_required_function("sample_function") is sample_function
+ """Test registering a function with a custom name."""
+ registry.register_function(valid_function, name="custom_name")
+ assert registry.get_function("custom_name") is valid_function
+ assert registry.get_function("valid_function") is None
def test_get_required_function_not_found(self, registry: FuncRegistry):
with pytest.raises(FuncRegistryError, match="not found in registry"):
registry.get_required_function("non_existent_function")
def test_unregister_function(self, registry: FuncRegistry):
- registry.register_function(sample_function)
- assert registry.has_function("sample_function")
- registry.unregister_function(sample_function)
- assert not registry.has_function("sample_function")
+ registry.register_function(valid_function)
+ assert registry.has_function("valid_function")
+ registry.unregister_function(valid_function)
+ assert not registry.has_function("valid_function")
def test_unregister_function_not_found(self, registry: FuncRegistry):
with pytest.raises(FuncRegistryError, match="not found in registry"):
registry.unregister_function(sample_function)
def test_unregister_function_by_name(self, registry: FuncRegistry):
- registry.register_function(sample_function, name="custom")
+ registry.register_function(valid_function, name="custom")
registry.unregister_function_by_name("custom")
assert not registry.has_function("custom")
@@ -66,44 +130,42 @@ class TestFuncRegistry:
with pytest.raises(FuncRegistryError, match="not found in registry"):
registry.unregister_function_by_name("non_existent")
- def test_register_functions_dict(self, registry: FuncRegistry):
+ def test_register_functions_dict_with_ineligible_functions(self, registry: FuncRegistry):
functions: Dict[str, Callable[..., Any]] = {"func1": sample_function, "func2": another_function}
registry.register_functions_dict(functions)
- assert registry.get_function("func1") is sample_function
- assert registry.get_function("func2") is another_function
+ # Ineligible functions should be silently skipped
+ assert not registry.has_function("func1")
+ assert not registry.has_function("func2")
+ assert len(registry.root) == 0
- def test_register_functions_list(self, registry: FuncRegistry):
+ def test_register_functions_list_with_ineligible_functions(self, registry: FuncRegistry):
functions: List[Callable[..., Any]] = [sample_function, another_function]
registry.register_functions(functions)
- assert registry.get_function("sample_function") is sample_function
- assert registry.get_function("another_function") is another_function
+ # Ineligible functions should be silently skipped
+ assert not registry.has_function("sample_function")
+ assert not registry.has_function("another_function")
+ assert len(registry.root) == 0
def test_register_functions_empty_list(self, registry: FuncRegistry):
registry.register_functions([])
assert len(registry.root) == 0
def test_register_existing_function_with_warning(self, registry: FuncRegistry, caplog: LogCaptureFixture):
- registry.register_function(sample_function)
+ registry.register_function(valid_function)
with caplog.at_level("DEBUG"):
- registry.register_function(sample_function)
+ registry.register_function(valid_function)
assert "already exists in registry" in caplog.text
- def test_register_existing_function_no_warning(self, registry: FuncRegistry, caplog: LogCaptureFixture):
- registry.register_function(sample_function)
- with caplog.at_level("DEBUG"):
- registry.register_function(sample_function, should_warn_if_already_registered=False)
- assert "already exists in registry" not in caplog.text
-
def test_teardown(self, registry: FuncRegistry):
- registry.register_function(sample_function)
- assert registry.has_function("sample_function")
+ registry.register_function(valid_function)
+ assert registry.has_function("valid_function")
registry.teardown()
- assert not registry.has_function("sample_function")
+ assert not registry.has_function("valid_function")
def test_get_required_function_with_signature(self, registry: FuncRegistry):
- registry.register_function(another_function)
- func = registry.get_required_function_with_signature("another_function", another_function)
- assert func is another_function
+ registry.register_function(valid_function)
+ func = registry.get_required_function_with_signature("valid_function", valid_function)
+ assert func is valid_function
def test_get_required_function_with_signature_not_found(self, registry: FuncRegistry):
with pytest.raises(FuncRegistryError, match="not found in registry"):
@@ -114,52 +176,14 @@ class TestFuncRegistry:
with pytest.raises(FuncRegistryError, match="is not a callable function"):
registry.get_required_function_with_signature("not_a_function", sample_function)
- def test_register_function_without_warning_when_already_exists(self, registry: FuncRegistry, caplog: LogCaptureFixture):
- """Test that no 'already exists' log is generated when should_warn_if_already_registered=False and function doesn't exist yet"""
- with caplog.at_level("DEBUG"):
- registry.register_function(sample_function, should_warn_if_already_registered=False)
- # This covers line 28 - the else branch when function doesn't exist yet
- assert "Registered new single function" in caplog.text
-
- def test_register_functions_dict_single_function(self, registry: FuncRegistry, caplog: LogCaptureFixture):
- """Test registering a single function via dict - covers line 72"""
- functions: Dict[str, Callable[..., Any]] = {"single_func": sample_function}
- with caplog.at_level("DEBUG"):
- registry.register_functions_dict(functions)
- assert "Registered single function 'sample_function' in registry" in caplog.text
- assert registry.get_function("single_func") is sample_function
-
- def test_register_functions_skip_existing(self, registry: FuncRegistry, caplog: LogCaptureFixture):
- """Test that existing functions are skipped when registering multiple functions - covers lines 83-84"""
- # First register a function
- registry.register_function(sample_function)
-
- # Now try to register a list that includes the existing function
- functions: List[Callable[..., Any]] = [sample_function, another_function]
- with caplog.at_level("DEBUG"):
- registry.register_functions(functions)
-
- # Should log that existing function was skipped
- assert "already exists in registry, skipping" in caplog.text
- # But should still register the new function
- assert registry.get_function("another_function") is another_function
-
- def test_register_functions_single_function_in_list(self, registry: FuncRegistry, caplog: LogCaptureFixture):
- """Test registering a single function via list - covers line 93"""
- functions: List[Callable[..., Any]] = [sample_function]
- with caplog.at_level("DEBUG"):
- registry.register_functions(functions)
- assert "Registered single function 'sample_function' in registry" in caplog.text
- assert registry.get_function("sample_function") is sample_function
-
def test_set_logger(self, registry: FuncRegistry, caplog: LogCaptureFixture):
- """Test setting a custom logger - covers line 28"""
+ """Test setting a custom logger"""
custom_logger = logging.getLogger("custom_test_logger")
registry.set_logger(custom_logger)
# Test that the custom logger is being used by triggering a log message
with caplog.at_level("DEBUG", logger="custom_test_logger"):
- registry.register_function(sample_function)
+ registry.register_function(valid_function)
# Verify the log message was captured by our custom logger
assert len(caplog.records) > 0
diff --git a/tests/unit/pipelex/tools/test_func_registry_utils.py b/tests/unit/pipelex/tools/test_func_registry_utils.py
index e93b84b47..b1f55469d 100644
--- a/tests/unit/pipelex/tools/test_func_registry_utils.py
+++ b/tests/unit/pipelex/tools/test_func_registry_utils.py
@@ -3,7 +3,9 @@
import tempfile
from pathlib import Path
+from typing import ClassVar, List, Tuple
+import pytest
from pydantic import Field
from pipelex.core.stuffs.stuff_content import StructuredContent
@@ -24,33 +26,11 @@ class CodebaseFileContent(StructuredContent):
file_content: str = Field(description="Content of the codebase file")
-def test_func_registry_utils():
- """Test the FuncRegistryUtils implementation."""
+class TestCases:
+ """Test cases for function eligibility."""
- # Create a temporary directory and file with the example function
- with tempfile.TemporaryDirectory() as temp_dir:
- test_file = Path(temp_dir) / "test_functions.py"
-
- # Write the example function to the test file
- test_file.write_text("""
-from typing import List
-
-from pydantic import Field
-
-from pipelex.core.memory.working_memory import WorkingMemory
-from pipelex.core.stuffs.stuff_content import ListContent, StructuredContent
-
-
-class FilePath(StructuredContent):
- path: str = Field(description="Path to the file")
-
-
-class CodebaseFileContent(StructuredContent):
- file_path: str = Field(description="Path to the codebase file")
- file_content: str = Field(description="Content of the codebase file")
-
-
-def read_file_content(working_memory: WorkingMemory) -> ListContent[CodebaseFileContent]:
+ VALID_ASYNC_FUNCTION = """
+async def read_file_content(working_memory: WorkingMemory) -> ListContent[CodebaseFileContent]:
'''Read the content of related codebase files.'''
file_paths_list = working_memory.get_stuff_as_list("related_file_paths", item_type=FilePath)
@@ -67,49 +47,239 @@ def read_file_content(working_memory: WorkingMemory) -> ListContent[CodebaseFile
)
return ListContent[CodebaseFileContent](items=codebase_files)
+"""
+ VALID_SYNC_FUNCTION = """
+# Sync function - should be accepted
+def sync_function(working_memory: WorkingMemory) -> StructuredContent:
+ '''This should be registered - sync functions are now eligible.'''
+ pass
+"""
+ INVALID_FUNCTIONS = """
+# Wrong parameter name - should be rejected
def invalid_function_wrong_param_name(other_param: WorkingMemory) -> StructuredContent:
'''This should not be registered - wrong parameter name.'''
pass
-
+# Wrong parameter type - should be rejected
def invalid_function_wrong_param_type(working_memory: str) -> StructuredContent:
'''This should not be registered - wrong parameter type.'''
pass
-
+# Wrong return type - should be rejected
def invalid_function_wrong_return_type(working_memory: WorkingMemory) -> str:
'''This should not be registered - wrong return type.'''
return "test"
-
+# Too many parameters - should be rejected
def invalid_function_too_many_params(working_memory: WorkingMemory, extra_param: str) -> StructuredContent:
'''This should not be registered - too many parameters.'''
pass
+
+# No parameters - should be rejected
+def invalid_function_no_params() -> StructuredContent:
+ '''This should not be registered - no parameters.'''
+ pass
+
+# Missing type hints - should be rejected
+def invalid_function_no_type_hints(working_memory):
+ '''This should not be registered - missing type hints.'''
+ pass
+"""
+
+ TEST_CASES: ClassVar[List[Tuple[str, str, List[str], List[str]]]] = [
+ ("valid_async_function", VALID_ASYNC_FUNCTION, ["read_file_content"], []),
+ ("valid_sync_function", VALID_SYNC_FUNCTION, ["sync_function"], []),
+ (
+ "invalid_functions",
+ INVALID_FUNCTIONS,
+ [],
+ [
+ "invalid_function_wrong_param_name",
+ "invalid_function_wrong_param_type",
+ "invalid_function_wrong_return_type",
+ "invalid_function_too_many_params",
+ "invalid_function_no_params",
+ "invalid_function_no_type_hints",
+ ],
+ ),
+ ]
+
+
+class TestFuncRegistryUtils:
+ """Test class for FuncRegistryUtils."""
+
+ @pytest.mark.parametrize(
+ "test_name, function_code, expected_registered, expected_not_registered",
+ TestCases.TEST_CASES,
+ )
+ def test_function_registration_eligibility(
+ self,
+ test_name: str,
+ function_code: str,
+ expected_registered: List[str],
+ expected_not_registered: List[str],
+ ):
+ """Test that only eligible functions are registered (both sync and async are now eligible)."""
+
+ # Create a temporary directory and file with test functions
+ with tempfile.TemporaryDirectory() as temp_dir:
+ test_file = Path(temp_dir) / "test_functions.py"
+
+ # Write the complete test file with imports and function code
+ full_code = f"""
+from typing import List
+
+from pydantic import Field
+
+from pipelex.core.memory.working_memory import WorkingMemory
+from pipelex.core.stuffs.stuff_content import ListContent, StructuredContent
+
+
+class FilePath(StructuredContent):
+ path: str = Field(description="Path to the file")
+
+
+class CodebaseFileContent(StructuredContent):
+ file_path: str = Field(description="Path to the codebase file")
+ file_content: str = Field(description="Content of the codebase file")
+
+{function_code}
+"""
+
+ test_file.write_text(full_code)
+
+ # Clear the registry to start fresh
+ func_registry.teardown()
+
+ # Test the registration
+ FuncRegistryUtils.register_funcs_in_folder(folder_path=temp_dir, is_recursive=False)
+
+ # Check that expected functions were registered
+ for func_name in expected_registered:
+ assert func_registry.has_function(func_name), f"{func_name} should be registered"
+
+ # Verify we can get the function
+ registered_func = func_registry.get_function(func_name)
+ assert registered_func is not None, f"Registered function {func_name} should be retrievable"
+ assert registered_func.__name__ == func_name, f"Function name should match {func_name}"
+
+ # Check that expected functions were NOT registered
+ for func_name in expected_not_registered:
+ assert not func_registry.has_function(func_name), f"{func_name} should NOT be registered"
+
+ def test_recursive_folder_search(self):
+ """Test that recursive folder search works correctly."""
+
+ with tempfile.TemporaryDirectory() as temp_dir:
+ # Create nested directory structure
+ nested_dir = Path(temp_dir) / "subdir"
+ nested_dir.mkdir()
+
+ # Create valid async function in root
+ root_file = Path(temp_dir) / "root_functions.py"
+ root_file.write_text("""
+from pipelex.core.memory.working_memory import WorkingMemory
+from pipelex.core.stuffs.stuff_content import TextContent
+
+async def root_function(working_memory: WorkingMemory) -> TextContent:
+ return TextContent(text="root")
+""")
+
+ # Create valid async function in subdirectory
+ nested_file = nested_dir / "nested_functions.py"
+ nested_file.write_text("""
+from pipelex.core.memory.working_memory import WorkingMemory
+from pipelex.core.stuffs.stuff_content import TextContent
+
+async def nested_function(working_memory: WorkingMemory) -> TextContent:
+ return TextContent(text="nested")
""")
- # Clear the registry to start fresh
+ # Clear the registry
+ func_registry.teardown()
+
+ # Test recursive search
+ FuncRegistryUtils.register_funcs_in_folder(folder_path=temp_dir, is_recursive=True)
+
+ # Both functions should be registered
+ assert func_registry.has_function("root_function"), "root_function should be registered"
+ assert func_registry.has_function("nested_function"), "nested_function should be registered"
+
+ # Clear and test non-recursive search
+ func_registry.teardown()
+ FuncRegistryUtils.register_funcs_in_folder(folder_path=temp_dir, is_recursive=False)
+
+ # Only root function should be registered
+ assert func_registry.has_function("root_function"), "root_function should be registered"
+ assert not func_registry.has_function("nested_function"), "nested_function should NOT be registered"
+
+ def test_eligibility_check_directly(self):
+ """Test the eligibility check method directly."""
+
+ # Create test functions dynamically for direct testing
+ from pipelex.core.memory.working_memory import WorkingMemory
+ from pipelex.core.stuffs.stuff_content import TextContent
+
+ # Valid async function
+ async def valid_async_function(working_memory: WorkingMemory) -> TextContent:
+ return TextContent(text="test")
+
+ # Valid sync function
+ def valid_sync_function(working_memory: WorkingMemory) -> TextContent:
+ return TextContent(text="test")
+
+ # Test eligibility
+ assert func_registry.is_eligible_function(valid_async_function), "Valid async function should be eligible"
+ assert func_registry.is_eligible_function(valid_sync_function), "Valid sync function should be eligible"
+
+ def test_register_function_checks_eligibility(self):
+ """Test that register_function checks eligibility before registering."""
+
+ from pipelex.core.memory.working_memory import WorkingMemory
+ from pipelex.core.stuffs.stuff_content import TextContent
+
+ # Clear registry
func_registry.teardown()
- # Test the registration
- FuncRegistryUtils.register_funcs_in_folder(folder_path=temp_dir, is_recursive=False)
+ # Valid async function
+ async def valid_async_function(working_memory: WorkingMemory) -> TextContent:
+ return TextContent(text="valid")
+
+ # Valid sync function
+ def valid_sync_function(working_memory: WorkingMemory) -> TextContent:
+ return TextContent(text="valid")
+
+ # Try to register both functions
+ func_registry.register_function(valid_async_function)
+ func_registry.register_function(valid_sync_function)
+
+ # Both functions should be registered
+ assert func_registry.has_function("valid_async_function"), "Valid async function should be registered"
+ assert func_registry.has_function("valid_sync_function"), "Valid sync function should be registered"
+
+ # Test register_functions method as well
+ func_registry.teardown()
+
+ async def another_valid_async_function(working_memory: WorkingMemory) -> TextContent:
+ return TextContent(text="another_valid_async")
+
+ def another_valid_sync_function(working_memory: WorkingMemory) -> TextContent:
+ return TextContent(text="another_valid_sync")
- # Check that only the valid function was registered
- assert func_registry.has_function("read_file_content"), "read_file_content should be registered"
- assert not func_registry.has_function("invalid_function_wrong_param_name"), "invalid_function_wrong_param_name should not be registered"
- assert not func_registry.has_function("invalid_function_wrong_param_type"), "invalid_function_wrong_param_type should not be registered"
- assert not func_registry.has_function("invalid_function_wrong_return_type"), "invalid_function_wrong_return_type should not be registered"
- assert not func_registry.has_function("invalid_function_too_many_params"), "invalid_function_too_many_params should not be registered"
+ # Invalid function (wrong parameter name)
+ def invalid_function(other_param: WorkingMemory) -> TextContent:
+ return TextContent(text="invalid")
- # Verify we can get the function
- registered_func = func_registry.get_function("read_file_content")
- assert registered_func is not None, "Registered function should be retrievable"
- assert registered_func.__name__ == "read_file_content", "Function name should match"
+ # Register multiple functions at once - invalid functions should be silently skipped
+ func_registry.register_functions([invalid_function, another_valid_async_function, another_valid_sync_function])
- print("✅ All tests passed!")
- print(f"✅ Successfully registered function: {registered_func.__name__}")
+ # Valid functions should be registered, invalid function silently skipped
+ assert func_registry.has_function("another_valid_async_function"), "Valid async function should be registered"
+ assert func_registry.has_function("another_valid_sync_function"), "Valid sync function should be registered"
+ assert not func_registry.has_function("invalid_function"), "Invalid function should NOT be registered"
if __name__ == "__main__":
- test_func_registry_utils()
+ pytest.main([__file__])
diff --git a/tests/unit/pipelex/tools/test_testing.py b/tests/unit/pipelex/tools/test_testing.py
index e69d5331b..8866810d9 100644
--- a/tests/unit/pipelex/tools/test_testing.py
+++ b/tests/unit/pipelex/tools/test_testing.py
@@ -1,12 +1,12 @@
-from pipelex.tools.environment import is_env_set
+from pipelex.tools.environment import is_env_var_set
from pipelex.tools.runtime_manager import RunMode, runtime_manager
def test_testing():
match runtime_manager.run_mode:
case RunMode.CI_TEST:
- assert is_env_set("GITHUB_ACTIONS") or is_env_set("CI")
+ assert is_env_var_set(key="GITHUB_ACTIONS") or is_env_var_set(key="CI")
case RunMode.UNIT_TEST:
- assert not is_env_set("GITHUB_ACTIONS") and not is_env_set("CI")
+ assert not is_env_var_set(key="GITHUB_ACTIONS") and not is_env_var_set(key="CI")
case _:
raise RuntimeError(f"Invalid run mode: {runtime_manager.run_mode}")
diff --git a/tests/unit/pipelex/tools/typing/test_type_inspector.py b/tests/unit/pipelex/tools/typing/test_structure_printer.py
similarity index 93%
rename from tests/unit/pipelex/tools/typing/test_type_inspector.py
rename to tests/unit/pipelex/tools/typing/test_structure_printer.py
index 5ea82b6c8..de79872b0 100644
--- a/tests/unit/pipelex/tools/typing/test_type_inspector.py
+++ b/tests/unit/pipelex/tools/typing/test_structure_printer.py
@@ -5,7 +5,7 @@ from pydantic import BaseModel, Field, field_validator
from pytest import FixtureRequest
from pipelex.core.stuffs.stuff_content import ListContent, StructuredContent, TextContent
-from pipelex.tools.typing.type_inspector import get_type_structure
+from pipelex.tools.typing.structure_printer import StructurePrinter
from pipelex.types import StrEnum
@@ -121,23 +121,22 @@ class GanttChart(StructuredContent):
milestones: Optional[List[Milestone]] = None
-class TestTypeInspector:
+class TestStructurePrinter:
"""Tests for the type inspector functionality"""
def test_simple_text_content(self, request: FixtureRequest):
"""Test structure of simple text content"""
- result = get_type_structure(SimpleTextContent)
+ result = StructurePrinter().get_type_structure(SimpleTextContent)
expected = [
"class SimpleTextContent(TextContent):",
' """A simple text content class"""',
- " # Inherits from TextContent",
" # No additional fields",
]
assert result == expected, f"Expected:\n{''.join(expected)}\n\nGot:\n{''.join(result)}"
def test_simple_structured_content(self, request: FixtureRequest):
"""Test structure of simple structured content"""
- result = get_type_structure(SimpleStructuredContent)
+ result = StructurePrinter().get_type_structure(SimpleStructuredContent)
expected = [
"class SimpleStructuredContent(StructuredContent):",
' """A simple structured content with primitive types"""',
@@ -149,7 +148,7 @@ class TestTypeInspector:
def test_enum_content(self, request: FixtureRequest):
"""Test structure of content with enum"""
- result = get_type_structure(DocumentTypeContent)
+ result = StructurePrinter().get_type_structure(DocumentTypeContent)
expected = [
"class DocumentTypeContent(StructuredContent):",
' """Content with enum type"""',
@@ -163,7 +162,7 @@ class TestTypeInspector:
def test_nested_content(self, request: FixtureRequest):
"""Test structure of nested content"""
- result = get_type_structure(PersonContent)
+ result = StructurePrinter().get_type_structure(PersonContent)
expected = [
"class PersonContent(StructuredContent):",
' """Complex nested content with various types"""',
@@ -195,7 +194,7 @@ class TestTypeInspector:
def test_list_content(self, request: FixtureRequest):
"""Test structure of list content"""
- result = get_type_structure(ComplexListContent)
+ result = StructurePrinter().get_type_structure(ComplexListContent)
expected = [
"class ComplexListContent(ListContent[PersonContent]):",
' """List content with complex items"""',
@@ -239,7 +238,7 @@ class TestTypeInspector:
class Employee(Person):
job: str = Field(description="Job title, must be lowercase")
- result = get_type_structure(Employee)
+ result = StructurePrinter().get_type_structure(Employee)
expected = [
"class Employee(Person):",
" job: str # Job title, must be lowercase",
@@ -264,7 +263,7 @@ class TestTypeInspector:
description: str = Field(description="Detailed description of what needs to be done")
is_completed: bool = Field(False, description="Whether the task is completed")
- result = get_type_structure(TaskContent)
+ result = StructurePrinter().get_type_structure(TaskContent)
expected = [
"class TaskContent(StructuredContent):",
' """A task content model that represents a single task.',
@@ -280,7 +279,7 @@ class TestTypeInspector:
def test_literal_field_content(self, request: FixtureRequest):
"""Test structure of content with Literal field"""
- result = get_type_structure(MusicCategoryContent)
+ result = StructurePrinter().get_type_structure(MusicCategoryContent)
expected = [
"class MusicCategoryContent(StructuredContent):",
' """A content class with a Literal field for music genres."""',
@@ -305,7 +304,7 @@ class TestTypeInspector:
def test_gantt_chart_content(self, request: FixtureRequest):
"""Test structure of Gantt chart content with datetime validators"""
- result = get_type_structure(GanttChart, base_class=StructuredContent)
+ result = StructurePrinter().get_type_structure(GanttChart, base_class=StructuredContent)
expected = [
"class GanttChart(StructuredContent):",
" tasks: Optional[List[GanttTaskDetails]] = None",
diff --git a/uv.lock b/uv.lock
index ba0d573e2..0c38f6dff 100644
--- a/uv.lock
+++ b/uv.lock
@@ -182,20 +182,21 @@ wheels = [
[[package]]
name = "anthropic"
-version = "0.67.0"
+version = "0.68.0"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "anyio" },
{ name = "distro" },
+ { name = "docstring-parser" },
{ name = "httpx" },
{ name = "jiter" },
{ name = "pydantic" },
{ name = "sniffio" },
{ name = "typing-extensions" },
]
-sdist = { url = "https://files.pythonhosted.org/packages/09/08/ee91464cd821e6fca52d9a23be44815c95edd3c1cf1e844b2c5e85f0d57f/anthropic-0.67.0.tar.gz", hash = "sha256:d1531b210ea300c73423141d29bcee20fcd24ef9f426f6437c0a5d93fc98fb8e", size = 441639, upload-time = "2025-09-10T14:47:18.137Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/64/46/da44bf087ddaf3f7dbe4808c00c7cde466fe68c4fc9fbebdfc231f4ea205/anthropic-0.68.0.tar.gz", hash = "sha256:507e9b5f627d1b249128ff15b21855e718fa4ed8dabc787d0e68860a4b32a7a8", size = 471584, upload-time = "2025-09-17T15:20:19.509Z" }
wheels = [
- { url = "https://files.pythonhosted.org/packages/5c/9d/9adbda372710918cc8271d089a2ceae4d977a125f90bc3c4b456bca4f281/anthropic-0.67.0-py3-none-any.whl", hash = "sha256:f80a81ec1132c514215f33d25edeeab1c4691ad5361b391ebb70d528b0605b55", size = 317126, upload-time = "2025-09-10T14:47:16.351Z" },
+ { url = "https://files.pythonhosted.org/packages/60/32/2d7553184b05bdbec61dd600014a55b9028408aee6128b25cb6f20e3002c/anthropic-0.68.0-py3-none-any.whl", hash = "sha256:ac579ea5eca22a7165b1042e6af57c4bf556e51afae3ca80e24768d4756b78c0", size = 325199, upload-time = "2025-09-17T15:20:17.452Z" },
]
[[package]]
@@ -288,16 +289,16 @@ wheels = [
[[package]]
name = "boto3-stubs"
-version = "1.40.29"
+version = "1.40.33"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "botocore-stubs" },
{ name = "types-s3transfer" },
{ name = "typing-extensions", marker = "python_full_version < '3.12'" },
]
-sdist = { url = "https://files.pythonhosted.org/packages/dd/35/0cdc62641577e8a0a6d4191ecc803fee16adf18de1e81280eb3d87c7d9e8/boto3_stubs-1.40.29.tar.gz", hash = "sha256:9fc7d24dcbcc786093daf42487a9ed4a58a6be7f1ccf28f5be0b2bad4a3edb11", size = 100996, upload-time = "2025-09-11T19:48:28.487Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/6c/9a/79689aab277396c8d3b8392a63d74e0720d36c824f94a294f8a8ac1384e9/boto3_stubs-1.40.33.tar.gz", hash = "sha256:aaebac70e09d8c962378045a3c40a37769733df21aabd34211b4cf3d5041e0bc", size = 100833, upload-time = "2025-09-17T19:47:18.388Z" }
wheels = [
- { url = "https://files.pythonhosted.org/packages/7b/a2/e47bf7595fadc6154ff2941e9ab9bb68173fba95f5ccdb24e5c13d16e5e5/boto3_stubs-1.40.29-py3-none-any.whl", hash = "sha256:1ad373b68b1c9a5e8e5deb243ef3a4c5b1d2c25c3477559eba1089ed4a0ee94e", size = 69769, upload-time = "2025-09-11T19:48:20.453Z" },
+ { url = "https://files.pythonhosted.org/packages/a7/9b/79402dceace5ac0f59849197ad2bcc744ff33c0fee47489cac47ef20642e/boto3_stubs-1.40.33-py3-none-any.whl", hash = "sha256:a42518ad8dca7e5cb57299255fbbd2a7707b5b741fded7d69cc2067c2d67a478", size = 69688, upload-time = "2025-09-17T19:47:10.638Z" },
]
[[package]]
@@ -316,14 +317,14 @@ wheels = [
[[package]]
name = "botocore-stubs"
-version = "1.40.29"
+version = "1.40.33"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "types-awscrt" },
]
-sdist = { url = "https://files.pythonhosted.org/packages/32/5c/49b2860e2a26b7383d5915374e61d962a3853e3fd569e4370444f0b902c0/botocore_stubs-1.40.29.tar.gz", hash = "sha256:324669d5ed7b5f7271bf3c3ea7208191b1d183f17d7e73398f11fef4a31fdf6b", size = 42742, upload-time = "2025-09-11T20:22:35.451Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/ae/94/16f8e1f41feaa38f1350aa5a4c60c5724b6c8524ca0e6c28523bf5070e74/botocore_stubs-1.40.33.tar.gz", hash = "sha256:89c51ae0b28d9d79fde8c497cf908ddf872ce027d2737d4d4ba473fde9cdaa82", size = 42742, upload-time = "2025-09-17T20:25:56.388Z" }
wheels = [
- { url = "https://files.pythonhosted.org/packages/e2/3c/f901ca6c4d66e0bebbfc56e614fc214416db72c613f768ee2fc84ffdbff4/botocore_stubs-1.40.29-py3-none-any.whl", hash = "sha256:84cbcc6328dddaa1f825830f7dec8fa0dcd3bac8002211322e8529cbfb5eaddd", size = 66843, upload-time = "2025-09-11T20:22:32.576Z" },
+ { url = "https://files.pythonhosted.org/packages/af/7b/6d8fe12a955b16094460e89ea7c4e063f131f4b3bd461b96bcd625d0c79e/botocore_stubs-1.40.33-py3-none-any.whl", hash = "sha256:ad21fee32cbdc7ad4730f29baf88424c7086bf88a745f8e43660ca3e9a7e5f89", size = 66843, upload-time = "2025-09-17T20:25:54.052Z" },
]
[[package]]
@@ -426,15 +427,6 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/ae/3a/dbeec9d1ee0844c679f6bb5d6ad4e9f198b1224f4e7a32825f47f6192b0c/cffi-2.0.0-cp314-cp314t-win_arm64.whl", hash = "sha256:0a1527a803f0a659de1af2e1fd700213caba79377e27e4693648c2923da066f9", size = 184195, upload-time = "2025-09-08T23:23:43.004Z" },
]
-[[package]]
-name = "cfgv"
-version = "3.4.0"
-source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/11/74/539e56497d9bd1d484fd863dd69cbbfa653cd2aa27abfe35653494d85e94/cfgv-3.4.0.tar.gz", hash = "sha256:e52591d4c5f5dead8e0f673fb16db7949d2cfb3f7da4582893288f0ded8fe560", size = 7114, upload-time = "2023-08-12T20:38:17.776Z" }
-wheels = [
- { url = "https://files.pythonhosted.org/packages/c5/55/51844dd50c4fc7a33b653bfaba4c2456f06955289ca770a5dbd5fd267374/cfgv-3.4.0-py2.py3-none-any.whl", hash = "sha256:b7265b1f29fd3316bfcd2b330d63d024f2bfd8bcb8b0272f8e19a504856c48f9", size = 7249, upload-time = "2023-08-12T20:38:16.269Z" },
-]
-
[[package]]
name = "charset-normalizer"
version = "3.4.3"
@@ -625,49 +617,67 @@ toml = [
[[package]]
name = "cryptography"
-version = "45.0.7"
+version = "46.0.1"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "cffi", marker = "platform_python_implementation != 'PyPy'" },
+ { name = "typing-extensions", marker = "python_full_version < '3.11'" },
]
-sdist = { url = "https://files.pythonhosted.org/packages/a7/35/c495bffc2056f2dadb32434f1feedd79abde2a7f8363e1974afa9c33c7e2/cryptography-45.0.7.tar.gz", hash = "sha256:4b1654dfc64ea479c242508eb8c724044f1e964a47d1d1cacc5132292d851971", size = 744980, upload-time = "2025-09-01T11:15:03.146Z" }
-wheels = [
- { url = "https://files.pythonhosted.org/packages/0c/91/925c0ac74362172ae4516000fe877912e33b5983df735ff290c653de4913/cryptography-45.0.7-cp311-abi3-macosx_10_9_universal2.whl", hash = "sha256:3be4f21c6245930688bd9e162829480de027f8bf962ede33d4f8ba7d67a00cee", size = 7041105, upload-time = "2025-09-01T11:13:59.684Z" },
- { url = "https://files.pythonhosted.org/packages/fc/63/43641c5acce3a6105cf8bd5baeceeb1846bb63067d26dae3e5db59f1513a/cryptography-45.0.7-cp311-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:67285f8a611b0ebc0857ced2081e30302909f571a46bfa7a3cc0ad303fe015c6", size = 4205799, upload-time = "2025-09-01T11:14:02.517Z" },
- { url = "https://files.pythonhosted.org/packages/bc/29/c238dd9107f10bfde09a4d1c52fd38828b1aa353ced11f358b5dd2507d24/cryptography-45.0.7-cp311-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:577470e39e60a6cd7780793202e63536026d9b8641de011ed9d8174da9ca5339", size = 4430504, upload-time = "2025-09-01T11:14:04.522Z" },
- { url = "https://files.pythonhosted.org/packages/62/62/24203e7cbcc9bd7c94739428cd30680b18ae6b18377ae66075c8e4771b1b/cryptography-45.0.7-cp311-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:4bd3e5c4b9682bc112d634f2c6ccc6736ed3635fc3319ac2bb11d768cc5a00d8", size = 4209542, upload-time = "2025-09-01T11:14:06.309Z" },
- { url = "https://files.pythonhosted.org/packages/cd/e3/e7de4771a08620eef2389b86cd87a2c50326827dea5528feb70595439ce4/cryptography-45.0.7-cp311-abi3-manylinux_2_28_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:465ccac9d70115cd4de7186e60cfe989de73f7bb23e8a7aa45af18f7412e75bf", size = 3889244, upload-time = "2025-09-01T11:14:08.152Z" },
- { url = "https://files.pythonhosted.org/packages/96/b8/bca71059e79a0bb2f8e4ec61d9c205fbe97876318566cde3b5092529faa9/cryptography-45.0.7-cp311-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:16ede8a4f7929b4b7ff3642eba2bf79aa1d71f24ab6ee443935c0d269b6bc513", size = 4461975, upload-time = "2025-09-01T11:14:09.755Z" },
- { url = "https://files.pythonhosted.org/packages/58/67/3f5b26937fe1218c40e95ef4ff8d23c8dc05aa950d54200cc7ea5fb58d28/cryptography-45.0.7-cp311-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:8978132287a9d3ad6b54fcd1e08548033cc09dc6aacacb6c004c73c3eb5d3ac3", size = 4209082, upload-time = "2025-09-01T11:14:11.229Z" },
- { url = "https://files.pythonhosted.org/packages/0e/e4/b3e68a4ac363406a56cf7b741eeb80d05284d8c60ee1a55cdc7587e2a553/cryptography-45.0.7-cp311-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:b6a0e535baec27b528cb07a119f321ac024592388c5681a5ced167ae98e9fff3", size = 4460397, upload-time = "2025-09-01T11:14:12.924Z" },
- { url = "https://files.pythonhosted.org/packages/22/49/2c93f3cd4e3efc8cb22b02678c1fad691cff9dd71bb889e030d100acbfe0/cryptography-45.0.7-cp311-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:a24ee598d10befaec178efdff6054bc4d7e883f615bfbcd08126a0f4931c83a6", size = 4337244, upload-time = "2025-09-01T11:14:14.431Z" },
- { url = "https://files.pythonhosted.org/packages/04/19/030f400de0bccccc09aa262706d90f2ec23d56bc4eb4f4e8268d0ddf3fb8/cryptography-45.0.7-cp311-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:fa26fa54c0a9384c27fcdc905a2fb7d60ac6e47d14bc2692145f2b3b1e2cfdbd", size = 4568862, upload-time = "2025-09-01T11:14:16.185Z" },
- { url = "https://files.pythonhosted.org/packages/29/56/3034a3a353efa65116fa20eb3c990a8c9f0d3db4085429040a7eef9ada5f/cryptography-45.0.7-cp311-abi3-win32.whl", hash = "sha256:bef32a5e327bd8e5af915d3416ffefdbe65ed975b646b3805be81b23580b57b8", size = 2936578, upload-time = "2025-09-01T11:14:17.638Z" },
- { url = "https://files.pythonhosted.org/packages/b3/61/0ab90f421c6194705a99d0fa9f6ee2045d916e4455fdbb095a9c2c9a520f/cryptography-45.0.7-cp311-abi3-win_amd64.whl", hash = "sha256:3808e6b2e5f0b46d981c24d79648e5c25c35e59902ea4391a0dcb3e667bf7443", size = 3405400, upload-time = "2025-09-01T11:14:18.958Z" },
- { url = "https://files.pythonhosted.org/packages/63/e8/c436233ddf19c5f15b25ace33979a9dd2e7aa1a59209a0ee8554179f1cc0/cryptography-45.0.7-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:bfb4c801f65dd61cedfc61a83732327fafbac55a47282e6f26f073ca7a41c3b2", size = 7021824, upload-time = "2025-09-01T11:14:20.954Z" },
- { url = "https://files.pythonhosted.org/packages/bc/4c/8f57f2500d0ccd2675c5d0cc462095adf3faa8c52294ba085c036befb901/cryptography-45.0.7-cp37-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:81823935e2f8d476707e85a78a405953a03ef7b7b4f55f93f7c2d9680e5e0691", size = 4202233, upload-time = "2025-09-01T11:14:22.454Z" },
- { url = "https://files.pythonhosted.org/packages/eb/ac/59b7790b4ccaed739fc44775ce4645c9b8ce54cbec53edf16c74fd80cb2b/cryptography-45.0.7-cp37-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:3994c809c17fc570c2af12c9b840d7cea85a9fd3e5c0e0491f4fa3c029216d59", size = 4423075, upload-time = "2025-09-01T11:14:24.287Z" },
- { url = "https://files.pythonhosted.org/packages/b8/56/d4f07ea21434bf891faa088a6ac15d6d98093a66e75e30ad08e88aa2b9ba/cryptography-45.0.7-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:dad43797959a74103cb59c5dac71409f9c27d34c8a05921341fb64ea8ccb1dd4", size = 4204517, upload-time = "2025-09-01T11:14:25.679Z" },
- { url = "https://files.pythonhosted.org/packages/e8/ac/924a723299848b4c741c1059752c7cfe09473b6fd77d2920398fc26bfb53/cryptography-45.0.7-cp37-abi3-manylinux_2_28_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:ce7a453385e4c4693985b4a4a3533e041558851eae061a58a5405363b098fcd3", size = 3882893, upload-time = "2025-09-01T11:14:27.1Z" },
- { url = "https://files.pythonhosted.org/packages/83/dc/4dab2ff0a871cc2d81d3ae6d780991c0192b259c35e4d83fe1de18b20c70/cryptography-45.0.7-cp37-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:b04f85ac3a90c227b6e5890acb0edbaf3140938dbecf07bff618bf3638578cf1", size = 4450132, upload-time = "2025-09-01T11:14:28.58Z" },
- { url = "https://files.pythonhosted.org/packages/12/dd/b2882b65db8fc944585d7fb00d67cf84a9cef4e77d9ba8f69082e911d0de/cryptography-45.0.7-cp37-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:48c41a44ef8b8c2e80ca4527ee81daa4c527df3ecbc9423c41a420a9559d0e27", size = 4204086, upload-time = "2025-09-01T11:14:30.572Z" },
- { url = "https://files.pythonhosted.org/packages/5d/fa/1d5745d878048699b8eb87c984d4ccc5da4f5008dfd3ad7a94040caca23a/cryptography-45.0.7-cp37-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:f3df7b3d0f91b88b2106031fd995802a2e9ae13e02c36c1fc075b43f420f3a17", size = 4449383, upload-time = "2025-09-01T11:14:32.046Z" },
- { url = "https://files.pythonhosted.org/packages/36/8b/fc61f87931bc030598e1876c45b936867bb72777eac693e905ab89832670/cryptography-45.0.7-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:dd342f085542f6eb894ca00ef70236ea46070c8a13824c6bde0dfdcd36065b9b", size = 4332186, upload-time = "2025-09-01T11:14:33.95Z" },
- { url = "https://files.pythonhosted.org/packages/0b/11/09700ddad7443ccb11d674efdbe9a832b4455dc1f16566d9bd3834922ce5/cryptography-45.0.7-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:1993a1bb7e4eccfb922b6cd414f072e08ff5816702a0bdb8941c247a6b1b287c", size = 4561639, upload-time = "2025-09-01T11:14:35.343Z" },
- { url = "https://files.pythonhosted.org/packages/71/ed/8f4c1337e9d3b94d8e50ae0b08ad0304a5709d483bfcadfcc77a23dbcb52/cryptography-45.0.7-cp37-abi3-win32.whl", hash = "sha256:18fcf70f243fe07252dcb1b268a687f2358025ce32f9f88028ca5c364b123ef5", size = 2926552, upload-time = "2025-09-01T11:14:36.929Z" },
- { url = "https://files.pythonhosted.org/packages/bc/ff/026513ecad58dacd45d1d24ebe52b852165a26e287177de1d545325c0c25/cryptography-45.0.7-cp37-abi3-win_amd64.whl", hash = "sha256:7285a89df4900ed3bfaad5679b1e668cb4b38a8de1ccbfc84b05f34512da0a90", size = 3392742, upload-time = "2025-09-01T11:14:38.368Z" },
- { url = "https://files.pythonhosted.org/packages/13/3e/e42f1528ca1ea82256b835191eab1be014e0f9f934b60d98b0be8a38ed70/cryptography-45.0.7-pp310-pypy310_pp73-macosx_10_9_x86_64.whl", hash = "sha256:de58755d723e86175756f463f2f0bddd45cc36fbd62601228a3f8761c9f58252", size = 3572442, upload-time = "2025-09-01T11:14:39.836Z" },
- { url = "https://files.pythonhosted.org/packages/59/aa/e947693ab08674a2663ed2534cd8d345cf17bf6a1facf99273e8ec8986dc/cryptography-45.0.7-pp310-pypy310_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:a20e442e917889d1a6b3c570c9e3fa2fdc398c20868abcea268ea33c024c4083", size = 4142233, upload-time = "2025-09-01T11:14:41.305Z" },
- { url = "https://files.pythonhosted.org/packages/24/06/09b6f6a2fc43474a32b8fe259038eef1500ee3d3c141599b57ac6c57612c/cryptography-45.0.7-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:258e0dff86d1d891169b5af222d362468a9570e2532923088658aa866eb11130", size = 4376202, upload-time = "2025-09-01T11:14:43.047Z" },
- { url = "https://files.pythonhosted.org/packages/00/f2/c166af87e95ce6ae6d38471a7e039d3a0549c2d55d74e059680162052824/cryptography-45.0.7-pp310-pypy310_pp73-manylinux_2_34_aarch64.whl", hash = "sha256:d97cf502abe2ab9eff8bd5e4aca274da8d06dd3ef08b759a8d6143f4ad65d4b4", size = 4141900, upload-time = "2025-09-01T11:14:45.089Z" },
- { url = "https://files.pythonhosted.org/packages/16/b9/e96e0b6cb86eae27ea51fa8a3151535a18e66fe7c451fa90f7f89c85f541/cryptography-45.0.7-pp310-pypy310_pp73-manylinux_2_34_x86_64.whl", hash = "sha256:c987dad82e8c65ebc985f5dae5e74a3beda9d0a2a4daf8a1115f3772b59e5141", size = 4375562, upload-time = "2025-09-01T11:14:47.166Z" },
- { url = "https://files.pythonhosted.org/packages/36/d0/36e8ee39274e9d77baf7d0dafda680cba6e52f3936b846f0d56d64fec915/cryptography-45.0.7-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:c13b1e3afd29a5b3b2656257f14669ca8fa8d7956d509926f0b130b600b50ab7", size = 3322781, upload-time = "2025-09-01T11:14:48.747Z" },
- { url = "https://files.pythonhosted.org/packages/99/4e/49199a4c82946938a3e05d2e8ad9482484ba48bbc1e809e3d506c686d051/cryptography-45.0.7-pp311-pypy311_pp73-macosx_10_9_x86_64.whl", hash = "sha256:4a862753b36620af6fc54209264f92c716367f2f0ff4624952276a6bbd18cbde", size = 3584634, upload-time = "2025-09-01T11:14:50.593Z" },
- { url = "https://files.pythonhosted.org/packages/16/ce/5f6ff59ea9c7779dba51b84871c19962529bdcc12e1a6ea172664916c550/cryptography-45.0.7-pp311-pypy311_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:06ce84dc14df0bf6ea84666f958e6080cdb6fe1231be2a51f3fc1267d9f3fb34", size = 4149533, upload-time = "2025-09-01T11:14:52.091Z" },
- { url = "https://files.pythonhosted.org/packages/ce/13/b3cfbd257ac96da4b88b46372e662009b7a16833bfc5da33bb97dd5631ae/cryptography-45.0.7-pp311-pypy311_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:d0c5c6bac22b177bf8da7435d9d27a6834ee130309749d162b26c3105c0795a9", size = 4385557, upload-time = "2025-09-01T11:14:53.551Z" },
- { url = "https://files.pythonhosted.org/packages/1c/c5/8c59d6b7c7b439ba4fc8d0cab868027fd095f215031bc123c3a070962912/cryptography-45.0.7-pp311-pypy311_pp73-manylinux_2_34_aarch64.whl", hash = "sha256:2f641b64acc00811da98df63df7d59fd4706c0df449da71cb7ac39a0732b40ae", size = 4149023, upload-time = "2025-09-01T11:14:55.022Z" },
- { url = "https://files.pythonhosted.org/packages/55/32/05385c86d6ca9ab0b4d5bb442d2e3d85e727939a11f3e163fc776ce5eb40/cryptography-45.0.7-pp311-pypy311_pp73-manylinux_2_34_x86_64.whl", hash = "sha256:f5414a788ecc6ee6bc58560e85ca624258a55ca434884445440a810796ea0e0b", size = 4385722, upload-time = "2025-09-01T11:14:57.319Z" },
- { url = "https://files.pythonhosted.org/packages/23/87/7ce86f3fa14bc11a5a48c30d8103c26e09b6465f8d8e9d74cf7a0714f043/cryptography-45.0.7-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:1f3d56f73595376f4244646dd5c5870c14c196949807be39e79e7bd9bac3da63", size = 3332908, upload-time = "2025-09-01T11:14:58.78Z" },
+sdist = { url = "https://files.pythonhosted.org/packages/a9/62/e3664e6ffd7743e1694b244dde70b43a394f6f7fbcacf7014a8ff5197c73/cryptography-46.0.1.tar.gz", hash = "sha256:ed570874e88f213437f5cf758f9ef26cbfc3f336d889b1e592ee11283bb8d1c7", size = 749198, upload-time = "2025-09-17T00:10:35.797Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/4c/8c/44ee01267ec01e26e43ebfdae3f120ec2312aa72fa4c0507ebe41a26739f/cryptography-46.0.1-cp311-abi3-macosx_10_9_universal2.whl", hash = "sha256:1cd6d50c1a8b79af1a6f703709d8973845f677c8e97b1268f5ff323d38ce8475", size = 7285044, upload-time = "2025-09-17T00:08:36.807Z" },
+ { url = "https://files.pythonhosted.org/packages/22/59/9ae689a25047e0601adfcb159ec4f83c0b4149fdb5c3030cc94cd218141d/cryptography-46.0.1-cp311-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:0ff483716be32690c14636e54a1f6e2e1b7bf8e22ca50b989f88fa1b2d287080", size = 4308182, upload-time = "2025-09-17T00:08:39.388Z" },
+ { url = "https://files.pythonhosted.org/packages/c4/ee/ca6cc9df7118f2fcd142c76b1da0f14340d77518c05b1ebfbbabca6b9e7d/cryptography-46.0.1-cp311-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:9873bf7c1f2a6330bdfe8621e7ce64b725784f9f0c3a6a55c3047af5849f920e", size = 4572393, upload-time = "2025-09-17T00:08:41.663Z" },
+ { url = "https://files.pythonhosted.org/packages/7f/a3/0f5296f63815d8e985922b05c31f77ce44787b3127a67c0b7f70f115c45f/cryptography-46.0.1-cp311-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:0dfb7c88d4462a0cfdd0d87a3c245a7bc3feb59de101f6ff88194f740f72eda6", size = 4308400, upload-time = "2025-09-17T00:08:43.559Z" },
+ { url = "https://files.pythonhosted.org/packages/5d/8c/74fcda3e4e01be1d32775d5b4dd841acaac3c1b8fa4d0774c7ac8d52463d/cryptography-46.0.1-cp311-abi3-manylinux_2_28_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:e22801b61613ebdebf7deb18b507919e107547a1d39a3b57f5f855032dd7cfb8", size = 4015786, upload-time = "2025-09-17T00:08:45.758Z" },
+ { url = "https://files.pythonhosted.org/packages/dc/b8/85d23287baeef273b0834481a3dd55bbed3a53587e3b8d9f0898235b8f91/cryptography-46.0.1-cp311-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:757af4f6341ce7a1e47c326ca2a81f41d236070217e5fbbad61bbfe299d55d28", size = 4982606, upload-time = "2025-09-17T00:08:47.602Z" },
+ { url = "https://files.pythonhosted.org/packages/e5/d3/de61ad5b52433b389afca0bc70f02a7a1f074651221f599ce368da0fe437/cryptography-46.0.1-cp311-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:f7a24ea78de345cfa7f6a8d3bde8b242c7fac27f2bd78fa23474ca38dfaeeab9", size = 4604234, upload-time = "2025-09-17T00:08:49.879Z" },
+ { url = "https://files.pythonhosted.org/packages/dc/1f/dbd4d6570d84748439237a7478d124ee0134bf166ad129267b7ed8ea6d22/cryptography-46.0.1-cp311-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:9e8776dac9e660c22241b6587fae51a67b4b0147daa4d176b172c3ff768ad736", size = 4307669, upload-time = "2025-09-17T00:08:52.321Z" },
+ { url = "https://files.pythonhosted.org/packages/ec/fd/ca0a14ce7f0bfe92fa727aacaf2217eb25eb7e4ed513b14d8e03b26e63ed/cryptography-46.0.1-cp311-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:9f40642a140c0c8649987027867242b801486865277cbabc8c6059ddef16dc8b", size = 4947579, upload-time = "2025-09-17T00:08:54.697Z" },
+ { url = "https://files.pythonhosted.org/packages/89/6b/09c30543bb93401f6f88fce556b3bdbb21e55ae14912c04b7bf355f5f96c/cryptography-46.0.1-cp311-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:449ef2b321bec7d97ef2c944173275ebdab78f3abdd005400cc409e27cd159ab", size = 4603669, upload-time = "2025-09-17T00:08:57.16Z" },
+ { url = "https://files.pythonhosted.org/packages/23/9a/38cb01cb09ce0adceda9fc627c9cf98eb890fc8d50cacbe79b011df20f8a/cryptography-46.0.1-cp311-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:2dd339ba3345b908fa3141ddba4025568fa6fd398eabce3ef72a29ac2d73ad75", size = 4435828, upload-time = "2025-09-17T00:08:59.606Z" },
+ { url = "https://files.pythonhosted.org/packages/0f/53/435b5c36a78d06ae0bef96d666209b0ecd8f8181bfe4dda46536705df59e/cryptography-46.0.1-cp311-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:7411c910fb2a412053cf33cfad0153ee20d27e256c6c3f14d7d7d1d9fec59fd5", size = 4709553, upload-time = "2025-09-17T00:09:01.832Z" },
+ { url = "https://files.pythonhosted.org/packages/f5/c4/0da6e55595d9b9cd3b6eb5dc22f3a07ded7f116a3ea72629cab595abb804/cryptography-46.0.1-cp311-abi3-win32.whl", hash = "sha256:cbb8e769d4cac884bb28e3ff620ef1001b75588a5c83c9c9f1fdc9afbe7f29b0", size = 3058327, upload-time = "2025-09-17T00:09:03.726Z" },
+ { url = "https://files.pythonhosted.org/packages/95/0f/cd29a35e0d6e78a0ee61793564c8cff0929c38391cb0de27627bdc7525aa/cryptography-46.0.1-cp311-abi3-win_amd64.whl", hash = "sha256:92e8cfe8bd7dd86eac0a677499894862cd5cc2fd74de917daa881d00871ac8e7", size = 3523893, upload-time = "2025-09-17T00:09:06.272Z" },
+ { url = "https://files.pythonhosted.org/packages/f2/dd/eea390f3e78432bc3d2f53952375f8b37cb4d37783e626faa6a51e751719/cryptography-46.0.1-cp311-abi3-win_arm64.whl", hash = "sha256:db5597a4c7353b2e5fb05a8e6cb74b56a4658a2b7bf3cb6b1821ae7e7fd6eaa0", size = 2932145, upload-time = "2025-09-17T00:09:08.568Z" },
+ { url = "https://files.pythonhosted.org/packages/0a/fb/c73588561afcd5e24b089952bd210b14676c0c5bf1213376350ae111945c/cryptography-46.0.1-cp314-cp314t-macosx_10_9_universal2.whl", hash = "sha256:4c49eda9a23019e11d32a0eb51a27b3e7ddedde91e099c0ac6373e3aacc0d2ee", size = 7193928, upload-time = "2025-09-17T00:09:10.595Z" },
+ { url = "https://files.pythonhosted.org/packages/26/34/0ff0bb2d2c79f25a2a63109f3b76b9108a906dd2a2eb5c1d460b9938adbb/cryptography-46.0.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:9babb7818fdd71394e576cf26c5452df77a355eac1a27ddfa24096665a27f8fd", size = 4293515, upload-time = "2025-09-17T00:09:12.861Z" },
+ { url = "https://files.pythonhosted.org/packages/df/b7/d4f848aee24ecd1be01db6c42c4a270069a4f02a105d9c57e143daf6cf0f/cryptography-46.0.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:9f2c4cc63be3ef43c0221861177cee5d14b505cd4d4599a89e2cd273c4d3542a", size = 4545619, upload-time = "2025-09-17T00:09:15.397Z" },
+ { url = "https://files.pythonhosted.org/packages/44/a5/42fedefc754fd1901e2d95a69815ea4ec8a9eed31f4c4361fcab80288661/cryptography-46.0.1-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:41c281a74df173876da1dc9a9b6953d387f06e3d3ed9284e3baae3ab3f40883a", size = 4299160, upload-time = "2025-09-17T00:09:17.155Z" },
+ { url = "https://files.pythonhosted.org/packages/86/a1/cd21174f56e769c831fbbd6399a1b7519b0ff6280acec1b826d7b072640c/cryptography-46.0.1-cp314-cp314t-manylinux_2_28_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:0a17377fa52563d730248ba1f68185461fff36e8bc75d8787a7dd2e20a802b7a", size = 3994491, upload-time = "2025-09-17T00:09:18.971Z" },
+ { url = "https://files.pythonhosted.org/packages/8d/2f/a8cbfa1c029987ddc746fd966711d4fa71efc891d37fbe9f030fe5ab4eec/cryptography-46.0.1-cp314-cp314t-manylinux_2_28_ppc64le.whl", hash = "sha256:0d1922d9280e08cde90b518a10cd66831f632960a8d08cb3418922d83fce6f12", size = 4960157, upload-time = "2025-09-17T00:09:20.923Z" },
+ { url = "https://files.pythonhosted.org/packages/67/ae/63a84e6789e0d5a2502edf06b552bcb0fa9ff16147265d5c44a211942abe/cryptography-46.0.1-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:af84e8e99f1a82cea149e253014ea9dc89f75b82c87bb6c7242203186f465129", size = 4577263, upload-time = "2025-09-17T00:09:23.356Z" },
+ { url = "https://files.pythonhosted.org/packages/ef/8f/1b9fa8e92bd9cbcb3b7e1e593a5232f2c1e6f9bd72b919c1a6b37d315f92/cryptography-46.0.1-cp314-cp314t-manylinux_2_34_aarch64.whl", hash = "sha256:ef648d2c690703501714588b2ba640facd50fd16548133b11b2859e8655a69da", size = 4298703, upload-time = "2025-09-17T00:09:25.566Z" },
+ { url = "https://files.pythonhosted.org/packages/c3/af/bb95db070e73fea3fae31d8a69ac1463d89d1c084220f549b00dd01094a8/cryptography-46.0.1-cp314-cp314t-manylinux_2_34_ppc64le.whl", hash = "sha256:e94eb5fa32a8a9f9bf991f424f002913e3dd7c699ef552db9b14ba6a76a6313b", size = 4926363, upload-time = "2025-09-17T00:09:27.451Z" },
+ { url = "https://files.pythonhosted.org/packages/f5/3b/d8fb17ffeb3a83157a1cc0aa5c60691d062aceecba09c2e5e77ebfc1870c/cryptography-46.0.1-cp314-cp314t-manylinux_2_34_x86_64.whl", hash = "sha256:534b96c0831855e29fc3b069b085fd185aa5353033631a585d5cd4dd5d40d657", size = 4576958, upload-time = "2025-09-17T00:09:29.924Z" },
+ { url = "https://files.pythonhosted.org/packages/d9/46/86bc3a05c10c8aa88c8ae7e953a8b4e407c57823ed201dbcba55c4d655f4/cryptography-46.0.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:f9b55038b5c6c47559aa33626d8ecd092f354e23de3c6975e4bb205df128a2a0", size = 4422507, upload-time = "2025-09-17T00:09:32.222Z" },
+ { url = "https://files.pythonhosted.org/packages/a8/4e/387e5a21dfd2b4198e74968a541cfd6128f66f8ec94ed971776e15091ac3/cryptography-46.0.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:ec13b7105117dbc9afd023300fb9954d72ca855c274fe563e72428ece10191c0", size = 4683964, upload-time = "2025-09-17T00:09:34.118Z" },
+ { url = "https://files.pythonhosted.org/packages/25/a3/f9f5907b166adb8f26762071474b38bbfcf89858a5282f032899075a38a1/cryptography-46.0.1-cp314-cp314t-win32.whl", hash = "sha256:504e464944f2c003a0785b81668fe23c06f3b037e9cb9f68a7c672246319f277", size = 3029705, upload-time = "2025-09-17T00:09:36.381Z" },
+ { url = "https://files.pythonhosted.org/packages/12/66/4d3a4f1850db2e71c2b1628d14b70b5e4c1684a1bd462f7fffb93c041c38/cryptography-46.0.1-cp314-cp314t-win_amd64.whl", hash = "sha256:c52fded6383f7e20eaf70a60aeddd796b3677c3ad2922c801be330db62778e05", size = 3502175, upload-time = "2025-09-17T00:09:38.261Z" },
+ { url = "https://files.pythonhosted.org/packages/52/c7/9f10ad91435ef7d0d99a0b93c4360bea3df18050ff5b9038c489c31ac2f5/cryptography-46.0.1-cp314-cp314t-win_arm64.whl", hash = "sha256:9495d78f52c804b5ec8878b5b8c7873aa8e63db9cd9ee387ff2db3fffe4df784", size = 2912354, upload-time = "2025-09-17T00:09:40.078Z" },
+ { url = "https://files.pythonhosted.org/packages/98/e5/fbd632385542a3311915976f88e0dfcf09e62a3fc0aff86fb6762162a24d/cryptography-46.0.1-cp38-abi3-macosx_10_9_universal2.whl", hash = "sha256:d84c40bdb8674c29fa192373498b6cb1e84f882889d21a471b45d1f868d8d44b", size = 7255677, upload-time = "2025-09-17T00:09:42.407Z" },
+ { url = "https://files.pythonhosted.org/packages/56/3e/13ce6eab9ad6eba1b15a7bd476f005a4c1b3f299f4c2f32b22408b0edccf/cryptography-46.0.1-cp38-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:9ed64e5083fa806709e74fc5ea067dfef9090e5b7a2320a49be3c9df3583a2d8", size = 4301110, upload-time = "2025-09-17T00:09:45.614Z" },
+ { url = "https://files.pythonhosted.org/packages/a2/67/65dc233c1ddd688073cf7b136b06ff4b84bf517ba5529607c9d79720fc67/cryptography-46.0.1-cp38-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:341fb7a26bc9d6093c1b124b9f13acc283d2d51da440b98b55ab3f79f2522ead", size = 4562369, upload-time = "2025-09-17T00:09:47.601Z" },
+ { url = "https://files.pythonhosted.org/packages/17/db/d64ae4c6f4e98c3dac5bf35dd4d103f4c7c345703e43560113e5e8e31b2b/cryptography-46.0.1-cp38-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:6ef1488967e729948d424d09c94753d0167ce59afba8d0f6c07a22b629c557b2", size = 4302126, upload-time = "2025-09-17T00:09:49.335Z" },
+ { url = "https://files.pythonhosted.org/packages/3d/19/5f1eea17d4805ebdc2e685b7b02800c4f63f3dd46cfa8d4c18373fea46c8/cryptography-46.0.1-cp38-abi3-manylinux_2_28_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:7823bc7cdf0b747ecfb096d004cc41573c2f5c7e3a29861603a2871b43d3ef32", size = 4009431, upload-time = "2025-09-17T00:09:51.239Z" },
+ { url = "https://files.pythonhosted.org/packages/81/b5/229ba6088fe7abccbfe4c5edb96c7a5ad547fac5fdd0d40aa6ea540b2985/cryptography-46.0.1-cp38-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:f736ab8036796f5a119ff8211deda416f8c15ce03776db704a7a4e17381cb2ef", size = 4980739, upload-time = "2025-09-17T00:09:54.181Z" },
+ { url = "https://files.pythonhosted.org/packages/3a/9c/50aa38907b201e74bc43c572f9603fa82b58e831bd13c245613a23cff736/cryptography-46.0.1-cp38-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:e46710a240a41d594953012213ea8ca398cd2448fbc5d0f1be8160b5511104a0", size = 4592289, upload-time = "2025-09-17T00:09:56.731Z" },
+ { url = "https://files.pythonhosted.org/packages/5a/33/229858f8a5bb22f82468bb285e9f4c44a31978d5f5830bb4ea1cf8a4e454/cryptography-46.0.1-cp38-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:84ef1f145de5aee82ea2447224dc23f065ff4cc5791bb3b506615957a6ba8128", size = 4301815, upload-time = "2025-09-17T00:09:58.548Z" },
+ { url = "https://files.pythonhosted.org/packages/52/cb/b76b2c87fbd6ed4a231884bea3ce073406ba8e2dae9defad910d33cbf408/cryptography-46.0.1-cp38-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:9394c7d5a7565ac5f7d9ba38b2617448eba384d7b107b262d63890079fad77ca", size = 4943251, upload-time = "2025-09-17T00:10:00.475Z" },
+ { url = "https://files.pythonhosted.org/packages/94/0f/f66125ecf88e4cb5b8017ff43f3a87ede2d064cb54a1c5893f9da9d65093/cryptography-46.0.1-cp38-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:ed957044e368ed295257ae3d212b95456bd9756df490e1ac4538857f67531fcc", size = 4591247, upload-time = "2025-09-17T00:10:02.874Z" },
+ { url = "https://files.pythonhosted.org/packages/f6/22/9f3134ae436b63b463cfdf0ff506a0570da6873adb4bf8c19b8a5b4bac64/cryptography-46.0.1-cp38-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:f7de12fa0eee6234de9a9ce0ffcfa6ce97361db7a50b09b65c63ac58e5f22fc7", size = 4428534, upload-time = "2025-09-17T00:10:04.994Z" },
+ { url = "https://files.pythonhosted.org/packages/89/39/e6042bcb2638650b0005c752c38ea830cbfbcbb1830e4d64d530000aa8dc/cryptography-46.0.1-cp38-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:7fab1187b6c6b2f11a326f33b036f7168f5b996aedd0c059f9738915e4e8f53a", size = 4699541, upload-time = "2025-09-17T00:10:06.925Z" },
+ { url = "https://files.pythonhosted.org/packages/68/46/753d457492d15458c7b5a653fc9a84a1c9c7a83af6ebdc94c3fc373ca6e8/cryptography-46.0.1-cp38-abi3-win32.whl", hash = "sha256:45f790934ac1018adeba46a0f7289b2b8fe76ba774a88c7f1922213a56c98bc1", size = 3043779, upload-time = "2025-09-17T00:10:08.951Z" },
+ { url = "https://files.pythonhosted.org/packages/2f/50/b6f3b540c2f6ee712feeb5fa780bb11fad76634e71334718568e7695cb55/cryptography-46.0.1-cp38-abi3-win_amd64.whl", hash = "sha256:7176a5ab56fac98d706921f6416a05e5aff7df0e4b91516f450f8627cda22af3", size = 3517226, upload-time = "2025-09-17T00:10:10.769Z" },
+ { url = "https://files.pythonhosted.org/packages/ff/e8/77d17d00981cdd27cc493e81e1749a0b8bbfb843780dbd841e30d7f50743/cryptography-46.0.1-cp38-abi3-win_arm64.whl", hash = "sha256:efc9e51c3e595267ff84adf56e9b357db89ab2279d7e375ffcaf8f678606f3d9", size = 2923149, upload-time = "2025-09-17T00:10:13.236Z" },
+ { url = "https://files.pythonhosted.org/packages/14/b9/b260180b31a66859648cfed5c980544ee22b15f8bd20ef82a23f58c0b83e/cryptography-46.0.1-pp310-pypy310_pp73-macosx_10_9_x86_64.whl", hash = "sha256:fd4b5e2ee4e60425711ec65c33add4e7a626adef79d66f62ba0acfd493af282d", size = 3714683, upload-time = "2025-09-17T00:10:15.601Z" },
+ { url = "https://files.pythonhosted.org/packages/c5/5a/1cd3ef86e5884edcbf8b27c3aa8f9544e9b9fcce5d3ed8b86959741f4f8e/cryptography-46.0.1-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:48948940d0ae00483e85e9154bb42997d0b77c21e43a77b7773c8c80de532ac5", size = 3443784, upload-time = "2025-09-17T00:10:18.014Z" },
+ { url = "https://files.pythonhosted.org/packages/27/27/077e09fd92075dd1338ea0ffaf5cfee641535545925768350ad90d8c36ca/cryptography-46.0.1-pp311-pypy311_pp73-macosx_10_9_x86_64.whl", hash = "sha256:b9c79af2c3058430d911ff1a5b2b96bbfe8da47d5ed961639ce4681886614e70", size = 3722319, upload-time = "2025-09-17T00:10:20.273Z" },
+ { url = "https://files.pythonhosted.org/packages/db/32/6fc7250280920418651640d76cee34d91c1e0601d73acd44364570cf041f/cryptography-46.0.1-pp311-pypy311_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:0ca4be2af48c24df689a150d9cd37404f689e2968e247b6b8ff09bff5bcd786f", size = 4249030, upload-time = "2025-09-17T00:10:22.396Z" },
+ { url = "https://files.pythonhosted.org/packages/32/33/8d5398b2da15a15110b2478480ab512609f95b45ead3a105c9a9c76f9980/cryptography-46.0.1-pp311-pypy311_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:13e67c4d3fb8b6bc4ef778a7ccdd8df4cd15b4bcc18f4239c8440891a11245cc", size = 4528009, upload-time = "2025-09-17T00:10:24.418Z" },
+ { url = "https://files.pythonhosted.org/packages/fd/1c/4012edad2a8977ab386c36b6e21f5065974d37afa3eade83a9968cba4855/cryptography-46.0.1-pp311-pypy311_pp73-manylinux_2_34_aarch64.whl", hash = "sha256:15b5fd9358803b0d1cc42505a18d8bca81dabb35b5cfbfea1505092e13a9d96d", size = 4248902, upload-time = "2025-09-17T00:10:26.255Z" },
+ { url = "https://files.pythonhosted.org/packages/58/a3/257cd5ae677302de8fa066fca9de37128f6729d1e63c04dd6a15555dd450/cryptography-46.0.1-pp311-pypy311_pp73-manylinux_2_34_x86_64.whl", hash = "sha256:e34da95e29daf8a71cb2841fd55df0511539a6cdf33e6f77c1e95e44006b9b46", size = 4527150, upload-time = "2025-09-17T00:10:28.28Z" },
+ { url = "https://files.pythonhosted.org/packages/6a/cd/fe6b65e1117ec7631f6be8951d3db076bac3e1b096e3e12710ed071ffc3c/cryptography-46.0.1-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:34f04b7311174469ab3ac2647469743720f8b6c8b046f238e5cb27905695eb2a", size = 3448210, upload-time = "2025-09-17T00:10:30.145Z" },
]
[[package]]
@@ -683,12 +693,12 @@ wheels = [
]
[[package]]
-name = "distlib"
-version = "0.4.0"
+name = "diskcache"
+version = "5.6.3"
source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/96/8e/709914eb2b5749865801041647dc7f4e6d00b549cfe88b65ca192995f07c/distlib-0.4.0.tar.gz", hash = "sha256:feec40075be03a04501a973d81f633735b4b69f98b05450592310c0f401a4e0d", size = 614605, upload-time = "2025-07-17T16:52:00.465Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/3f/21/1c1ffc1a039ddcc459db43cc108658f32c57d271d7289a2794e401d0fdb6/diskcache-5.6.3.tar.gz", hash = "sha256:2c3a3fa2743d8535d832ec61c2054a1641f41775aa7c556758a109941e33e4fc", size = 67916, upload-time = "2023-08-31T06:12:00.316Z" }
wheels = [
- { url = "https://files.pythonhosted.org/packages/33/6b/e0547afaf41bf2c42e52430072fa5658766e3d65bd4b03a563d1b6336f57/distlib-0.4.0-py2.py3-none-any.whl", hash = "sha256:9659f7d87e46584a30b5780e43ac7a2143098441670ff0a49d5f9034c54a6c16", size = 469047, upload-time = "2025-07-17T16:51:58.613Z" },
+ { url = "https://files.pythonhosted.org/packages/3f/27/4570e78fc0bf5ea0ca45eb1de3818a23787af9b390c0b0a0033a1b8236f9/diskcache-5.6.3-py3-none-any.whl", hash = "sha256:5e31b2d5fbad117cc363ebaf6b689474db18a1f6438bc82358b024abd4c2ca19", size = 45550, upload-time = "2023-08-31T06:11:58.822Z" },
]
[[package]]
@@ -750,14 +760,14 @@ wheels = [
[[package]]
name = "faker"
-version = "37.6.0"
+version = "37.8.0"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "tzdata" },
]
-sdist = { url = "https://files.pythonhosted.org/packages/24/cd/f7679c20f07d9e2013123b7f7e13809a3450a18d938d58e86081a486ea15/faker-37.6.0.tar.gz", hash = "sha256:0f8cc34f30095184adf87c3c24c45b38b33ad81c35ef6eb0a3118f301143012c", size = 1907960, upload-time = "2025-08-26T15:56:27.419Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/3a/da/1336008d39e5d4076dddb4e0f3a52ada41429274bf558a3cc28030d324a3/faker-37.8.0.tar.gz", hash = "sha256:090bb5abbec2b30949a95ce1ba6b20d1d0ed222883d63483a0d4be4a970d6fb8", size = 1912113, upload-time = "2025-09-15T20:24:13.592Z" }
wheels = [
- { url = "https://files.pythonhosted.org/packages/61/7d/8b50e4ac772719777be33661f4bde320793400a706f5eb214e4de46f093c/faker-37.6.0-py3-none-any.whl", hash = "sha256:3c5209b23d7049d596a51db5d76403a0ccfea6fc294ffa2ecfef6a8843b1e6a7", size = 1949837, upload-time = "2025-08-26T15:56:25.33Z" },
+ { url = "https://files.pythonhosted.org/packages/f5/11/02ebebb09ff2104b690457cb7bc6ed700c9e0ce88cf581486bb0a5d3c88b/faker-37.8.0-py3-none-any.whl", hash = "sha256:b08233118824423b5fc239f7dd51f145e7018082b4164f8da6a9994e1f1ae793", size = 1953940, upload-time = "2025-09-15T20:24:11.482Z" },
]
[[package]]
@@ -773,15 +783,6 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/26/b0/4ede5c23e2e818ea7dad8dc0aedb97ea13f270c567f1c39c4d203c73408d/fal_client-0.7.0-py3-none-any.whl", hash = "sha256:f847ff76af5bd8f4f68541eff629481730656a374e6071dbb947a2ad9212f129", size = 10694, upload-time = "2025-05-20T21:00:46.528Z" },
]
-[[package]]
-name = "filelock"
-version = "3.19.1"
-source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/40/bb/0ab3e58d22305b6f5440629d20683af28959bf793d98d11950e305c1c326/filelock-3.19.1.tar.gz", hash = "sha256:66eda1888b0171c998b35be2bcc0f6d75c388a7ce20c3f3f37aa8e96c2dddf58", size = 17687, upload-time = "2025-08-14T16:56:03.016Z" }
-wheels = [
- { url = "https://files.pythonhosted.org/packages/42/14/42b2651a2f46b022ccd948bca9f2d5af0fd8929c4eec235b8d6d844fbe67/filelock-3.19.1-py3-none-any.whl", hash = "sha256:d38e30481def20772f5baf097c122c3babc4fcdb7e14e57049eb9d88c6dc017d", size = 15988, upload-time = "2025-08-14T16:56:01.633Z" },
-]
-
[[package]]
name = "filetype"
version = "1.2.0"
@@ -924,6 +925,25 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/ac/84/40ee070be95771acd2f4418981edb834979424565c3eec3cd88b6aa09d24/google_auth_oauthlib-1.2.2-py3-none-any.whl", hash = "sha256:fd619506f4b3908b5df17b65f39ca8d66ea56986e5472eb5978fd8f3786f00a2", size = 19072, upload-time = "2025-04-22T16:40:28.174Z" },
]
+[[package]]
+name = "google-genai"
+version = "1.38.0"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+ { name = "anyio" },
+ { name = "google-auth" },
+ { name = "httpx" },
+ { name = "pydantic" },
+ { name = "requests" },
+ { name = "tenacity" },
+ { name = "typing-extensions" },
+ { name = "websockets" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/b4/11/108ddd3aca8af6a9e2369e59b9646a3a4c64aefb39d154f6467ab8d79f34/google_genai-1.38.0.tar.gz", hash = "sha256:363272fc4f677d0be6a1aed7ebabe8adf45e1626a7011a7886a587e9464ca9ec", size = 244903, upload-time = "2025-09-16T23:25:42.577Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/53/6c/1de711bab3c118284904c3bedf870519e8c63a7a8e0905ac3833f1db9cbc/google_genai-1.38.0-py3-none-any.whl", hash = "sha256:95407425132d42b3fa11bc92b3f5cf61a0fbd8d9add1f0e89aac52c46fbba090", size = 245558, upload-time = "2025-09-16T23:25:41.141Z" },
+]
+
[[package]]
name = "h11"
version = "0.16.0"
@@ -970,15 +990,6 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/25/0a/6269e3473b09aed2dab8aa1a600c70f31f00ae1349bee30658f7e358a159/httpx_sse-0.4.1-py3-none-any.whl", hash = "sha256:cba42174344c3a5b06f255ce65b350880f962d99ead85e776f23c6618a377a37", size = 8054, upload-time = "2025-06-24T13:21:04.772Z" },
]
-[[package]]
-name = "identify"
-version = "2.6.14"
-source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/52/c4/62963f25a678f6a050fb0505a65e9e726996171e6dbe1547f79619eefb15/identify-2.6.14.tar.gz", hash = "sha256:663494103b4f717cb26921c52f8751363dc89db64364cd836a9bf1535f53cd6a", size = 99283, upload-time = "2025-09-06T19:30:52.938Z" }
-wheels = [
- { url = "https://files.pythonhosted.org/packages/e5/ae/2ad30f4652712c82f1c23423d79136fbce338932ad166d70c1efb86a5998/identify-2.6.14-py2.py3-none-any.whl", hash = "sha256:11a073da82212c6646b1f39bb20d4483bfb9543bd5566fec60053c4bb309bf2e", size = 99172, upload-time = "2025-09-06T19:30:51.759Z" },
-]
-
[[package]]
name = "idna"
version = "3.10"
@@ -999,17 +1010,15 @@ wheels = [
[[package]]
name = "instructor"
-version = "1.9.2"
+version = "1.10.0"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "aiohttp" },
+ { name = "diskcache" },
{ name = "docstring-parser" },
{ name = "jinja2" },
{ name = "jiter" },
- { name = "mkdocs" },
- { name = "mkdocs-material" },
{ name = "openai" },
- { name = "pre-commit" },
{ name = "pydantic" },
{ name = "pydantic-core" },
{ name = "requests" },
@@ -1017,9 +1026,15 @@ dependencies = [
{ name = "tenacity" },
{ name = "typer" },
]
-sdist = { url = "https://files.pythonhosted.org/packages/f7/d9/c9703cffc074f94d585c25a0221ae41f381cac7ae7f176455f051ec82579/instructor-1.9.2.tar.gz", hash = "sha256:8c6b9fb4f8404a1e8290b8c3e9d26f8e9a1aceb5392ea8728a21a0828fb5e7c6", size = 69352231, upload-time = "2025-07-07T21:17:32.747Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/a5/67/63c4b4d2cc3c7b4238920ad3388a6f5d67265ab7c09ee34012d6b591130e/instructor-1.10.0.tar.gz", hash = "sha256:887d33e058b913290dbf526b0096b1bb8d7ea1a07d75afecbf716161f959697b", size = 69388981, upload-time = "2025-07-18T15:28:52.386Z" }
wheels = [
- { url = "https://files.pythonhosted.org/packages/2e/9b/008fb5f9f376007f0ef25b0c6c5777d3c4cfe37065fcc32de995f3e53e40/instructor-1.9.2-py3-none-any.whl", hash = "sha256:0f0503baade2d0b7ec40a4711de3cd49b25702a746b3bc18e3d375510dabe049", size = 96633, upload-time = "2025-07-07T21:17:29.301Z" },
+ { url = "https://files.pythonhosted.org/packages/2c/fb/ffc1ade9779795a8dc8e2379b1bfb522161ee7df8df12722f50d348fb4ea/instructor-1.10.0-py3-none-any.whl", hash = "sha256:9c789f0fce915d5498059afb5314530c8a5b22b0283302679148ddae98f732b0", size = 119455, upload-time = "2025-07-18T15:28:48.785Z" },
+]
+
+[package.optional-dependencies]
+google-genai = [
+ { name = "google-genai" },
+ { name = "jsonref" },
]
[[package]]
@@ -1130,6 +1145,15 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/16/8a/d63959f4eff03893a00e6e63592e3a9f15b9266ed8e0275ab77f8c7dbc94/jsonpath_python-1.0.6-py3-none-any.whl", hash = "sha256:1e3b78df579f5efc23565293612decee04214609208a2335884b3ee3f786b575", size = 7552, upload-time = "2022-03-14T02:34:59.754Z" },
]
+[[package]]
+name = "jsonref"
+version = "1.1.0"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/aa/0d/c1f3277e90ccdb50d33ed5ba1ec5b3f0a242ed8c1b1a85d3afeb68464dca/jsonref-1.1.0.tar.gz", hash = "sha256:32fe8e1d85af0fdefbebce950af85590b22b60f9e95443176adbde4e1ecea552", size = 8814, upload-time = "2023-01-16T16:10:04.455Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/0c/ec/e1db9922bceb168197a558a2b8c03a7963f1afe93517ddd3cf99f202f996/jsonref-1.1.0-py3-none-any.whl", hash = "sha256:590dc7773df6c21cbf948b5dac07a72a251db28b0238ceecce0a2abfa8ec30a9", size = 9425, upload-time = "2023-01-16T16:10:02.255Z" },
+]
+
[[package]]
name = "kajson"
version = "0.3.0"
@@ -1702,7 +1726,7 @@ wheels = [
[[package]]
name = "openai"
-version = "1.107.1"
+version = "1.108.0"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "anyio" },
@@ -1714,9 +1738,9 @@ dependencies = [
{ name = "tqdm" },
{ name = "typing-extensions" },
]
-sdist = { url = "https://files.pythonhosted.org/packages/f3/e0/a62daa7ff769df969cc1b782852cace79615039630b297005356f5fb46fb/openai-1.107.1.tar.gz", hash = "sha256:7c51b6b8adadfcf5cada08a613423575258b180af5ad4bc2954b36ebc0d3ad48", size = 563671, upload-time = "2025-09-10T15:04:40.288Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/07/3c/3ea4c40c62d5f4b11690de13de35554d0d49b5e5780669fad5e83562d635/openai-1.108.0.tar.gz", hash = "sha256:e859c64e4202d7f5956f19280eee92bb281f211c41cdd5be9e63bf51a024ff72", size = 564659, upload-time = "2025-09-17T22:03:23.075Z" }
wheels = [
- { url = "https://files.pythonhosted.org/packages/d4/12/32c19999a58eec4a695e8ce334442b6135df949f0bb61b2ceaa4fa60d3a9/openai-1.107.1-py3-none-any.whl", hash = "sha256:168f9885b1b70d13ada0868a0d0adfd538c16a02f7fd9fe063851a2c9a025e72", size = 945177, upload-time = "2025-09-10T15:04:37.782Z" },
+ { url = "https://files.pythonhosted.org/packages/af/dc/0a007b7c5a079e13d66eecc5d521bbc67b53c135e2a3131160ef76b5db1f/openai-1.108.0-py3-none-any.whl", hash = "sha256:31f2e58230e2703f13ddbb50c285f39dacf7fca64ab19882fd8a7a0b2bccd781", size = 948114, upload-time = "2025-09-17T22:03:20.972Z" },
]
[[package]]
@@ -1925,7 +1949,7 @@ wheels = [
[[package]]
name = "pipelex"
-version = "0.10.1"
+version = "0.10.2"
source = { editable = "." }
dependencies = [
{ name = "aiofiles" },
@@ -2000,6 +2024,10 @@ fal = [
google = [
{ name = "google-auth-oauthlib" },
]
+google-genai = [
+ { name = "google-genai" },
+ { name = "instructor", extra = ["google-genai"] },
+]
mistralai = [
{ name = "mistralai" },
]
@@ -2016,8 +2044,10 @@ requires-dist = [
{ name = "fal-client", marker = "extra == 'fal'", specifier = ">=0.4.1" },
{ name = "filetype", specifier = ">=1.2.0" },
{ name = "google-auth-oauthlib", marker = "extra == 'google'", specifier = ">=1.2.1" },
+ { name = "google-genai", marker = "extra == 'google-genai'" },
{ name = "httpx", specifier = ">=0.27.0" },
- { name = "instructor", specifier = ">=1.8.3,<1.10.0" },
+ { name = "instructor", specifier = ">=1.8.3,<1.11.0" },
+ { name = "instructor", extras = ["google-genai"], marker = "extra == 'google-genai'" },
{ name = "jinja2", specifier = ">=3.1.4" },
{ name = "json2html", specifier = ">=1.3.0" },
{ name = "kajson", specifier = "==0.3.0" },
@@ -2064,7 +2094,7 @@ requires-dist = [
{ name = "typing-extensions", specifier = ">=4.13.2" },
{ name = "yattag", specifier = ">=1.15.2" },
]
-provides-extras = ["anthropic", "bedrock", "fal", "google", "mistralai", "docs", "dev"]
+provides-extras = ["anthropic", "bedrock", "fal", "google", "google-genai", "mistralai", "docs", "dev"]
[[package]]
name = "platformdirs"
@@ -2097,22 +2127,6 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/e7/fe/d52c90e07c458f38b26f9972a25cb011b2744813f76fcd6121dde64744fa/polyfactory-2.22.2-py3-none-any.whl", hash = "sha256:9bea58ac9a80375b4153cd60820f75e558b863e567e058794d28c6a52b84118a", size = 63715, upload-time = "2025-08-15T06:23:19.664Z" },
]
-[[package]]
-name = "pre-commit"
-version = "4.3.0"
-source = { registry = "https://pypi.org/simple" }
-dependencies = [
- { name = "cfgv" },
- { name = "identify" },
- { name = "nodeenv" },
- { name = "pyyaml" },
- { name = "virtualenv" },
-]
-sdist = { url = "https://files.pythonhosted.org/packages/ff/29/7cf5bbc236333876e4b41f56e06857a87937ce4bf91e117a6991a2dbb02a/pre_commit-4.3.0.tar.gz", hash = "sha256:499fe450cc9d42e9d58e606262795ecb64dd05438943c62b66f6a8673da30b16", size = 193792, upload-time = "2025-08-09T18:56:14.651Z" }
-wheels = [
- { url = "https://files.pythonhosted.org/packages/5b/a5/987a405322d78a73b66e39e4a90e4ef156fd7141bf71df987e50717c321b/pre_commit-4.3.0-py2.py3-none-any.whl", hash = "sha256:2b0747ad7e6e967169136edffee14c16e148a778a54e4f967921aa1ebf2308d8", size = 220965, upload-time = "2025-08-09T18:56:13.192Z" },
-]
-
[[package]]
name = "propcache"
version = "0.3.2"
@@ -2492,14 +2506,14 @@ wheels = [
[[package]]
name = "pytest-mock"
-version = "3.15.0"
+version = "3.15.1"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "pytest" },
]
-sdist = { url = "https://files.pythonhosted.org/packages/61/99/3323ee5c16b3637b4d941c362182d3e749c11e400bea31018c42219f3a98/pytest_mock-3.15.0.tar.gz", hash = "sha256:ab896bd190316b9d5d87b277569dfcdf718b2d049a2ccff5f7aca279c002a1cf", size = 33838, upload-time = "2025-09-04T20:57:48.679Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/68/14/eb014d26be205d38ad5ad20d9a80f7d201472e08167f0bb4361e251084a9/pytest_mock-3.15.1.tar.gz", hash = "sha256:1849a238f6f396da19762269de72cb1814ab44416fa73a8686deac10b0d87a0f", size = 34036, upload-time = "2025-09-16T16:37:27.081Z" }
wheels = [
- { url = "https://files.pythonhosted.org/packages/2b/b3/7fefc43fb706380144bcd293cc6e446e6f637ddfa8b83f48d1734156b529/pytest_mock-3.15.0-py3-none-any.whl", hash = "sha256:ef2219485fb1bd256b00e7ad7466ce26729b30eadfc7cbcdb4fa9a92ca68db6f", size = 10050, upload-time = "2025-09-04T20:57:47.274Z" },
+ { url = "https://files.pythonhosted.org/packages/5a/cc/06253936f4a7fa2e0f48dfe6d851d9c56df896a9ab09ac019d70b760619c/pytest_mock-3.15.1-py3-none-any.whl", hash = "sha256:0a25e2eb88fe5168d535041d09a4529a188176ae608a6d249ee65abc0949630d", size = 10095, upload-time = "2025-09-16T16:37:25.734Z" },
]
[[package]]
@@ -2935,11 +2949,11 @@ wheels = [
[[package]]
name = "types-html5lib"
-version = "1.1.11.20250809"
+version = "1.1.11.20250917"
source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/70/ab/6aa4c487ae6f4f9da5153143bdc9e9b4fbc2b105df7ef8127fb920dc1f21/types_html5lib-1.1.11.20250809.tar.gz", hash = "sha256:7976ec7426bb009997dc5e072bca3ed988dd747d0cbfe093c7dfbd3d5ec8bf57", size = 16793, upload-time = "2025-08-09T03:14:20.819Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/68/4b/a970718e8bd9324ee8fb8eaf02ff069f6d03c20d4523bb4232892ecc3d06/types_html5lib-1.1.11.20250917.tar.gz", hash = "sha256:7b52743377f33f9b4fd7385afbd2d457b8864ee51f90ff2a795ad9e8c053373a", size = 16868, upload-time = "2025-09-17T02:47:41.18Z" }
wheels = [
- { url = "https://files.pythonhosted.org/packages/9b/05/328a2d6ecbd8aa3e16512600da78b1fe4605125896794a21824f3cac6f14/types_html5lib-1.1.11.20250809-py3-none-any.whl", hash = "sha256:e5f48ab670ae4cdeafd88bbc47113d8126dcf08318e0b8d70df26ecc13eca9b6", size = 22867, upload-time = "2025-08-09T03:14:20.048Z" },
+ { url = "https://files.pythonhosted.org/packages/78/8a/da91a9c64dcb5e69beb567519857411996d8ecae9f6f128bcef8260e7a8d/types_html5lib-1.1.11.20250917-py3-none-any.whl", hash = "sha256:b294fd06d60da205daeb2f615485ca4d475088d2eff1009cf427f4a80fcd5346", size = 22908, upload-time = "2025-09-17T02:47:40.39Z" },
]
[[package]]
@@ -2953,24 +2967,24 @@ wheels = [
[[package]]
name = "types-networkx"
-version = "3.5.0.20250901"
+version = "3.5.0.20250918"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" },
{ name = "numpy", version = "2.3.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" },
]
-sdist = { url = "https://files.pythonhosted.org/packages/73/79/c00bf3c8881a929093abdff12ba530be59275edc1ec9f4477b77d1a21709/types_networkx-3.5.0.20250901.tar.gz", hash = "sha256:94f3a7e880cba0ba47406337f64711352e8cd4814ba9ed6db3c523b724028b4d", size = 71399, upload-time = "2025-09-01T03:13:53.878Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/ce/88/f758a96247da3f2a1de3a655b725471828762ae28c9591a4b27c518c8866/types_networkx-3.5.0.20250918.tar.gz", hash = "sha256:346cc347fc7670b5db29af9d4ba5fa077507ad7100ed7a02707f74fbf317be3c", size = 71507, upload-time = "2025-09-18T02:50:44.561Z" }
wheels = [
- { url = "https://files.pythonhosted.org/packages/f6/98/4f8917a34c7a040f81a56dc96c4839507c6bf31797f9f6b7b1c0605eb5f2/types_networkx-3.5.0.20250901-py3-none-any.whl", hash = "sha256:56bac40a51569b63f164a61b49c6c23fd0e09a0168b78261bc2968a780d8bab0", size = 159102, upload-time = "2025-09-01T03:13:52.614Z" },
+ { url = "https://files.pythonhosted.org/packages/0f/82/e48e099e41d8b5515dc2ee98f6336bc2c330e1d63018c3c6582271f1c3f1/types_networkx-3.5.0.20250918-py3-none-any.whl", hash = "sha256:c7d3951e747333a65c0fb3011461e2de990c7ed5b49f4292ffc0d3febc8e2536", size = 159162, upload-time = "2025-09-18T02:50:42.974Z" },
]
[[package]]
name = "types-openpyxl"
-version = "3.1.5.20250822"
+version = "3.1.5.20250914"
source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/fe/7f/ea358482217448deafdb9232f198603511d2efa99e429822256f2b38975a/types_openpyxl-3.1.5.20250822.tar.gz", hash = "sha256:c8704a163e3798290d182c13c75da85f68cd97ff9b35f0ebfb94cf72f8b67bb3", size = 100858, upload-time = "2025-08-22T03:03:31.835Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/1b/64/c83b8b6ef6d16dd9548fffb962c224ed0648e9d90388c6179348ff0cc848/types_openpyxl-3.1.5.20250914.tar.gz", hash = "sha256:74602c740d4bfe628b258f67d7173eca34aefa472eb74579487df834f93837c2", size = 100843, upload-time = "2025-09-14T02:56:11.942Z" }
wheels = [
- { url = "https://files.pythonhosted.org/packages/5e/e8/cac4728e8dcbeb69d6de7de26bb9edb508e9f5c82476ecda22b58b939e60/types_openpyxl-3.1.5.20250822-py3-none-any.whl", hash = "sha256:da7a430d99c48347acf2dc351695f9db6ff90ecb761fed577b4a98fef2d0f831", size = 166093, upload-time = "2025-08-22T03:03:30.686Z" },
+ { url = "https://files.pythonhosted.org/packages/11/79/c00552fe903089431e680ba8cf3cde60b23ac3c35e9c0c4a6175185b2818/types_openpyxl-3.1.5.20250914-py3-none-any.whl", hash = "sha256:61763fa44414011acd87a42d4058dd221a4fe6e6efd105b7d35f5d5a194fef26", size = 166083, upload-time = "2025-09-14T02:56:10.737Z" },
]
[[package]]
@@ -2984,23 +2998,23 @@ wheels = [
[[package]]
name = "types-pyyaml"
-version = "6.0.12.20250822"
+version = "6.0.12.20250915"
source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/49/85/90a442e538359ab5c9e30de415006fb22567aa4301c908c09f19e42975c2/types_pyyaml-6.0.12.20250822.tar.gz", hash = "sha256:259f1d93079d335730a9db7cff2bcaf65d7e04b4a56b5927d49a612199b59413", size = 17481, upload-time = "2025-08-22T03:02:16.209Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/7e/69/3c51b36d04da19b92f9e815be12753125bd8bc247ba0470a982e6979e71c/types_pyyaml-6.0.12.20250915.tar.gz", hash = "sha256:0f8b54a528c303f0e6f7165687dd33fafa81c807fcac23f632b63aa624ced1d3", size = 17522, upload-time = "2025-09-15T03:01:00.728Z" }
wheels = [
- { url = "https://files.pythonhosted.org/packages/32/8e/8f0aca667c97c0d76024b37cffa39e76e2ce39ca54a38f285a64e6ae33ba/types_pyyaml-6.0.12.20250822-py3-none-any.whl", hash = "sha256:1fe1a5e146aa315483592d292b72a172b65b946a6d98aa6ddd8e4aa838ab7098", size = 20314, upload-time = "2025-08-22T03:02:15.002Z" },
+ { url = "https://files.pythonhosted.org/packages/bd/e0/1eed384f02555dde685fff1a1ac805c1c7dcb6dd019c916fe659b1c1f9ec/types_pyyaml-6.0.12.20250915-py3-none-any.whl", hash = "sha256:e7d4d9e064e89a3b3cae120b4990cd370874d2bf12fa5f46c97018dd5d3c9ab6", size = 20338, upload-time = "2025-09-15T03:00:59.218Z" },
]
[[package]]
name = "types-requests"
-version = "2.32.4.20250809"
+version = "2.32.4.20250913"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "urllib3" },
]
-sdist = { url = "https://files.pythonhosted.org/packages/ed/b0/9355adb86ec84d057fea765e4c49cce592aaf3d5117ce5609a95a7fc3dac/types_requests-2.32.4.20250809.tar.gz", hash = "sha256:d8060de1c8ee599311f56ff58010fb4902f462a1470802cf9f6ed27bc46c4df3", size = 23027, upload-time = "2025-08-09T03:17:10.664Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/36/27/489922f4505975b11de2b5ad07b4fe1dca0bca9be81a703f26c5f3acfce5/types_requests-2.32.4.20250913.tar.gz", hash = "sha256:abd6d4f9ce3a9383f269775a9835a4c24e5cd6b9f647d64f88aa4613c33def5d", size = 23113, upload-time = "2025-09-13T02:40:02.309Z" }
wheels = [
- { url = "https://files.pythonhosted.org/packages/2b/6f/ec0012be842b1d888d46884ac5558fd62aeae1f0ec4f7a581433d890d4b5/types_requests-2.32.4.20250809-py3-none-any.whl", hash = "sha256:f73d1832fb519ece02c85b1f09d5f0dd3108938e7d47e7f94bbfa18a6782b163", size = 20644, upload-time = "2025-08-09T03:17:09.716Z" },
+ { url = "https://files.pythonhosted.org/packages/2a/20/9a227ea57c1285986c4cf78400d0a91615d25b24e257fd9e2969606bdfae/types_requests-2.32.4.20250913-py3-none-any.whl", hash = "sha256:78c9c1fffebbe0fa487a418e0fa5252017e9c60d1a2da394077f1780f655d7e1", size = 20658, upload-time = "2025-09-13T02:40:01.115Z" },
]
[[package]]
@@ -3061,21 +3075,6 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/a7/c2/fe1e52489ae3122415c51f387e221dd0773709bad6c6cdaa599e8a2c5185/urllib3-2.5.0-py3-none-any.whl", hash = "sha256:e6b01673c0fa6a13e374b50871808eb3bf7046c4b125b216f6bf1cc604cff0dc", size = 129795, upload-time = "2025-06-18T14:07:40.39Z" },
]
-[[package]]
-name = "virtualenv"
-version = "20.34.0"
-source = { registry = "https://pypi.org/simple" }
-dependencies = [
- { name = "distlib" },
- { name = "filelock" },
- { name = "platformdirs" },
- { name = "typing-extensions", marker = "python_full_version < '3.11'" },
-]
-sdist = { url = "https://files.pythonhosted.org/packages/1c/14/37fcdba2808a6c615681cd216fecae00413c9dab44fb2e57805ecf3eaee3/virtualenv-20.34.0.tar.gz", hash = "sha256:44815b2c9dee7ed86e387b842a84f20b93f7f417f95886ca1996a72a4138eb1a", size = 6003808, upload-time = "2025-08-13T14:24:07.464Z" }
-wheels = [
- { url = "https://files.pythonhosted.org/packages/76/06/04c8e804f813cf972e3262f3f8584c232de64f0cde9f703b46cf53a45090/virtualenv-20.34.0-py3-none-any.whl", hash = "sha256:341f5afa7eee943e4984a9207c025feedd768baff6753cd660c857ceb3e36026", size = 5983279, upload-time = "2025-08-13T14:24:05.111Z" },
-]
-
[[package]]
name = "watchdog"
version = "6.0.0"
@@ -3108,6 +3107,65 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/33/e8/e40370e6d74ddba47f002a32919d91310d6074130fe4e17dabcafc15cbf1/watchdog-6.0.0-py3-none-win_ia64.whl", hash = "sha256:a1914259fa9e1454315171103c6a30961236f508b9b623eae470268bbcc6a22f", size = 79067, upload-time = "2024-11-01T14:07:11.845Z" },
]
+[[package]]
+name = "websockets"
+version = "15.0.1"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/21/e6/26d09fab466b7ca9c7737474c52be4f76a40301b08362eb2dbc19dcc16c1/websockets-15.0.1.tar.gz", hash = "sha256:82544de02076bafba038ce055ee6412d68da13ab47f0c60cab827346de828dee", size = 177016, upload-time = "2025-03-05T20:03:41.606Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/1e/da/6462a9f510c0c49837bbc9345aca92d767a56c1fb2939e1579df1e1cdcf7/websockets-15.0.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:d63efaa0cd96cf0c5fe4d581521d9fa87744540d4bc999ae6e08595a1014b45b", size = 175423, upload-time = "2025-03-05T20:01:35.363Z" },
+ { url = "https://files.pythonhosted.org/packages/1c/9f/9d11c1a4eb046a9e106483b9ff69bce7ac880443f00e5ce64261b47b07e7/websockets-15.0.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:ac60e3b188ec7574cb761b08d50fcedf9d77f1530352db4eef1707fe9dee7205", size = 173080, upload-time = "2025-03-05T20:01:37.304Z" },
+ { url = "https://files.pythonhosted.org/packages/d5/4f/b462242432d93ea45f297b6179c7333dd0402b855a912a04e7fc61c0d71f/websockets-15.0.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:5756779642579d902eed757b21b0164cd6fe338506a8083eb58af5c372e39d9a", size = 173329, upload-time = "2025-03-05T20:01:39.668Z" },
+ { url = "https://files.pythonhosted.org/packages/6e/0c/6afa1f4644d7ed50284ac59cc70ef8abd44ccf7d45850d989ea7310538d0/websockets-15.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0fdfe3e2a29e4db3659dbd5bbf04560cea53dd9610273917799f1cde46aa725e", size = 182312, upload-time = "2025-03-05T20:01:41.815Z" },
+ { url = "https://files.pythonhosted.org/packages/dd/d4/ffc8bd1350b229ca7a4db2a3e1c482cf87cea1baccd0ef3e72bc720caeec/websockets-15.0.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4c2529b320eb9e35af0fa3016c187dffb84a3ecc572bcee7c3ce302bfeba52bf", size = 181319, upload-time = "2025-03-05T20:01:43.967Z" },
+ { url = "https://files.pythonhosted.org/packages/97/3a/5323a6bb94917af13bbb34009fac01e55c51dfde354f63692bf2533ffbc2/websockets-15.0.1-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ac1e5c9054fe23226fb11e05a6e630837f074174c4c2f0fe442996112a6de4fb", size = 181631, upload-time = "2025-03-05T20:01:46.104Z" },
+ { url = "https://files.pythonhosted.org/packages/a6/cc/1aeb0f7cee59ef065724041bb7ed667b6ab1eeffe5141696cccec2687b66/websockets-15.0.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:5df592cd503496351d6dc14f7cdad49f268d8e618f80dce0cd5a36b93c3fc08d", size = 182016, upload-time = "2025-03-05T20:01:47.603Z" },
+ { url = "https://files.pythonhosted.org/packages/79/f9/c86f8f7af208e4161a7f7e02774e9d0a81c632ae76db2ff22549e1718a51/websockets-15.0.1-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:0a34631031a8f05657e8e90903e656959234f3a04552259458aac0b0f9ae6fd9", size = 181426, upload-time = "2025-03-05T20:01:48.949Z" },
+ { url = "https://files.pythonhosted.org/packages/c7/b9/828b0bc6753db905b91df6ae477c0b14a141090df64fb17f8a9d7e3516cf/websockets-15.0.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:3d00075aa65772e7ce9e990cab3ff1de702aa09be3940d1dc88d5abf1ab8a09c", size = 181360, upload-time = "2025-03-05T20:01:50.938Z" },
+ { url = "https://files.pythonhosted.org/packages/89/fb/250f5533ec468ba6327055b7d98b9df056fb1ce623b8b6aaafb30b55d02e/websockets-15.0.1-cp310-cp310-win32.whl", hash = "sha256:1234d4ef35db82f5446dca8e35a7da7964d02c127b095e172e54397fb6a6c256", size = 176388, upload-time = "2025-03-05T20:01:52.213Z" },
+ { url = "https://files.pythonhosted.org/packages/1c/46/aca7082012768bb98e5608f01658ff3ac8437e563eca41cf068bd5849a5e/websockets-15.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:39c1fec2c11dc8d89bba6b2bf1556af381611a173ac2b511cf7231622058af41", size = 176830, upload-time = "2025-03-05T20:01:53.922Z" },
+ { url = "https://files.pythonhosted.org/packages/9f/32/18fcd5919c293a398db67443acd33fde142f283853076049824fc58e6f75/websockets-15.0.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:823c248b690b2fd9303ba00c4f66cd5e2d8c3ba4aa968b2779be9532a4dad431", size = 175423, upload-time = "2025-03-05T20:01:56.276Z" },
+ { url = "https://files.pythonhosted.org/packages/76/70/ba1ad96b07869275ef42e2ce21f07a5b0148936688c2baf7e4a1f60d5058/websockets-15.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:678999709e68425ae2593acf2e3ebcbcf2e69885a5ee78f9eb80e6e371f1bf57", size = 173082, upload-time = "2025-03-05T20:01:57.563Z" },
+ { url = "https://files.pythonhosted.org/packages/86/f2/10b55821dd40eb696ce4704a87d57774696f9451108cff0d2824c97e0f97/websockets-15.0.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:d50fd1ee42388dcfb2b3676132c78116490976f1300da28eb629272d5d93e905", size = 173330, upload-time = "2025-03-05T20:01:59.063Z" },
+ { url = "https://files.pythonhosted.org/packages/a5/90/1c37ae8b8a113d3daf1065222b6af61cc44102da95388ac0018fcb7d93d9/websockets-15.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d99e5546bf73dbad5bf3547174cd6cb8ba7273062a23808ffea025ecb1cf8562", size = 182878, upload-time = "2025-03-05T20:02:00.305Z" },
+ { url = "https://files.pythonhosted.org/packages/8e/8d/96e8e288b2a41dffafb78e8904ea7367ee4f891dafc2ab8d87e2124cb3d3/websockets-15.0.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:66dd88c918e3287efc22409d426c8f729688d89a0c587c88971a0faa2c2f3792", size = 181883, upload-time = "2025-03-05T20:02:03.148Z" },
+ { url = "https://files.pythonhosted.org/packages/93/1f/5d6dbf551766308f6f50f8baf8e9860be6182911e8106da7a7f73785f4c4/websockets-15.0.1-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8dd8327c795b3e3f219760fa603dcae1dcc148172290a8ab15158cf85a953413", size = 182252, upload-time = "2025-03-05T20:02:05.29Z" },
+ { url = "https://files.pythonhosted.org/packages/d4/78/2d4fed9123e6620cbf1706c0de8a1632e1a28e7774d94346d7de1bba2ca3/websockets-15.0.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:8fdc51055e6ff4adeb88d58a11042ec9a5eae317a0a53d12c062c8a8865909e8", size = 182521, upload-time = "2025-03-05T20:02:07.458Z" },
+ { url = "https://files.pythonhosted.org/packages/e7/3b/66d4c1b444dd1a9823c4a81f50231b921bab54eee2f69e70319b4e21f1ca/websockets-15.0.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:693f0192126df6c2327cce3baa7c06f2a117575e32ab2308f7f8216c29d9e2e3", size = 181958, upload-time = "2025-03-05T20:02:09.842Z" },
+ { url = "https://files.pythonhosted.org/packages/08/ff/e9eed2ee5fed6f76fdd6032ca5cd38c57ca9661430bb3d5fb2872dc8703c/websockets-15.0.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:54479983bd5fb469c38f2f5c7e3a24f9a4e70594cd68cd1fa6b9340dadaff7cf", size = 181918, upload-time = "2025-03-05T20:02:11.968Z" },
+ { url = "https://files.pythonhosted.org/packages/d8/75/994634a49b7e12532be6a42103597b71098fd25900f7437d6055ed39930a/websockets-15.0.1-cp311-cp311-win32.whl", hash = "sha256:16b6c1b3e57799b9d38427dda63edcbe4926352c47cf88588c0be4ace18dac85", size = 176388, upload-time = "2025-03-05T20:02:13.32Z" },
+ { url = "https://files.pythonhosted.org/packages/98/93/e36c73f78400a65f5e236cd376713c34182e6663f6889cd45a4a04d8f203/websockets-15.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:27ccee0071a0e75d22cb35849b1db43f2ecd3e161041ac1ee9d2352ddf72f065", size = 176828, upload-time = "2025-03-05T20:02:14.585Z" },
+ { url = "https://files.pythonhosted.org/packages/51/6b/4545a0d843594f5d0771e86463606a3988b5a09ca5123136f8a76580dd63/websockets-15.0.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:3e90baa811a5d73f3ca0bcbf32064d663ed81318ab225ee4f427ad4e26e5aff3", size = 175437, upload-time = "2025-03-05T20:02:16.706Z" },
+ { url = "https://files.pythonhosted.org/packages/f4/71/809a0f5f6a06522af902e0f2ea2757f71ead94610010cf570ab5c98e99ed/websockets-15.0.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:592f1a9fe869c778694f0aa806ba0374e97648ab57936f092fd9d87f8bc03665", size = 173096, upload-time = "2025-03-05T20:02:18.832Z" },
+ { url = "https://files.pythonhosted.org/packages/3d/69/1a681dd6f02180916f116894181eab8b2e25b31e484c5d0eae637ec01f7c/websockets-15.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:0701bc3cfcb9164d04a14b149fd74be7347a530ad3bbf15ab2c678a2cd3dd9a2", size = 173332, upload-time = "2025-03-05T20:02:20.187Z" },
+ { url = "https://files.pythonhosted.org/packages/a6/02/0073b3952f5bce97eafbb35757f8d0d54812b6174ed8dd952aa08429bcc3/websockets-15.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e8b56bdcdb4505c8078cb6c7157d9811a85790f2f2b3632c7d1462ab5783d215", size = 183152, upload-time = "2025-03-05T20:02:22.286Z" },
+ { url = "https://files.pythonhosted.org/packages/74/45/c205c8480eafd114b428284840da0b1be9ffd0e4f87338dc95dc6ff961a1/websockets-15.0.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0af68c55afbd5f07986df82831c7bff04846928ea8d1fd7f30052638788bc9b5", size = 182096, upload-time = "2025-03-05T20:02:24.368Z" },
+ { url = "https://files.pythonhosted.org/packages/14/8f/aa61f528fba38578ec553c145857a181384c72b98156f858ca5c8e82d9d3/websockets-15.0.1-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:64dee438fed052b52e4f98f76c5790513235efaa1ef7f3f2192c392cd7c91b65", size = 182523, upload-time = "2025-03-05T20:02:25.669Z" },
+ { url = "https://files.pythonhosted.org/packages/ec/6d/0267396610add5bc0d0d3e77f546d4cd287200804fe02323797de77dbce9/websockets-15.0.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:d5f6b181bb38171a8ad1d6aa58a67a6aa9d4b38d0f8c5f496b9e42561dfc62fe", size = 182790, upload-time = "2025-03-05T20:02:26.99Z" },
+ { url = "https://files.pythonhosted.org/packages/02/05/c68c5adbf679cf610ae2f74a9b871ae84564462955d991178f95a1ddb7dd/websockets-15.0.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:5d54b09eba2bada6011aea5375542a157637b91029687eb4fdb2dab11059c1b4", size = 182165, upload-time = "2025-03-05T20:02:30.291Z" },
+ { url = "https://files.pythonhosted.org/packages/29/93/bb672df7b2f5faac89761cb5fa34f5cec45a4026c383a4b5761c6cea5c16/websockets-15.0.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:3be571a8b5afed347da347bfcf27ba12b069d9d7f42cb8c7028b5e98bbb12597", size = 182160, upload-time = "2025-03-05T20:02:31.634Z" },
+ { url = "https://files.pythonhosted.org/packages/ff/83/de1f7709376dc3ca9b7eeb4b9a07b4526b14876b6d372a4dc62312bebee0/websockets-15.0.1-cp312-cp312-win32.whl", hash = "sha256:c338ffa0520bdb12fbc527265235639fb76e7bc7faafbb93f6ba80d9c06578a9", size = 176395, upload-time = "2025-03-05T20:02:33.017Z" },
+ { url = "https://files.pythonhosted.org/packages/7d/71/abf2ebc3bbfa40f391ce1428c7168fb20582d0ff57019b69ea20fa698043/websockets-15.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:fcd5cf9e305d7b8338754470cf69cf81f420459dbae8a3b40cee57417f4614a7", size = 176841, upload-time = "2025-03-05T20:02:34.498Z" },
+ { url = "https://files.pythonhosted.org/packages/cb/9f/51f0cf64471a9d2b4d0fc6c534f323b664e7095640c34562f5182e5a7195/websockets-15.0.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:ee443ef070bb3b6ed74514f5efaa37a252af57c90eb33b956d35c8e9c10a1931", size = 175440, upload-time = "2025-03-05T20:02:36.695Z" },
+ { url = "https://files.pythonhosted.org/packages/8a/05/aa116ec9943c718905997412c5989f7ed671bc0188ee2ba89520e8765d7b/websockets-15.0.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:5a939de6b7b4e18ca683218320fc67ea886038265fd1ed30173f5ce3f8e85675", size = 173098, upload-time = "2025-03-05T20:02:37.985Z" },
+ { url = "https://files.pythonhosted.org/packages/ff/0b/33cef55ff24f2d92924923c99926dcce78e7bd922d649467f0eda8368923/websockets-15.0.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:746ee8dba912cd6fc889a8147168991d50ed70447bf18bcda7039f7d2e3d9151", size = 173329, upload-time = "2025-03-05T20:02:39.298Z" },
+ { url = "https://files.pythonhosted.org/packages/31/1d/063b25dcc01faa8fada1469bdf769de3768b7044eac9d41f734fd7b6ad6d/websockets-15.0.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:595b6c3969023ecf9041b2936ac3827e4623bfa3ccf007575f04c5a6aa318c22", size = 183111, upload-time = "2025-03-05T20:02:40.595Z" },
+ { url = "https://files.pythonhosted.org/packages/93/53/9a87ee494a51bf63e4ec9241c1ccc4f7c2f45fff85d5bde2ff74fcb68b9e/websockets-15.0.1-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3c714d2fc58b5ca3e285461a4cc0c9a66bd0e24c5da9911e30158286c9b5be7f", size = 182054, upload-time = "2025-03-05T20:02:41.926Z" },
+ { url = "https://files.pythonhosted.org/packages/ff/b2/83a6ddf56cdcbad4e3d841fcc55d6ba7d19aeb89c50f24dd7e859ec0805f/websockets-15.0.1-cp313-cp313-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0f3c1e2ab208db911594ae5b4f79addeb3501604a165019dd221c0bdcabe4db8", size = 182496, upload-time = "2025-03-05T20:02:43.304Z" },
+ { url = "https://files.pythonhosted.org/packages/98/41/e7038944ed0abf34c45aa4635ba28136f06052e08fc2168520bb8b25149f/websockets-15.0.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:229cf1d3ca6c1804400b0a9790dc66528e08a6a1feec0d5040e8b9eb14422375", size = 182829, upload-time = "2025-03-05T20:02:48.812Z" },
+ { url = "https://files.pythonhosted.org/packages/e0/17/de15b6158680c7623c6ef0db361da965ab25d813ae54fcfeae2e5b9ef910/websockets-15.0.1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:756c56e867a90fb00177d530dca4b097dd753cde348448a1012ed6c5131f8b7d", size = 182217, upload-time = "2025-03-05T20:02:50.14Z" },
+ { url = "https://files.pythonhosted.org/packages/33/2b/1f168cb6041853eef0362fb9554c3824367c5560cbdaad89ac40f8c2edfc/websockets-15.0.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:558d023b3df0bffe50a04e710bc87742de35060580a293c2a984299ed83bc4e4", size = 182195, upload-time = "2025-03-05T20:02:51.561Z" },
+ { url = "https://files.pythonhosted.org/packages/86/eb/20b6cdf273913d0ad05a6a14aed4b9a85591c18a987a3d47f20fa13dcc47/websockets-15.0.1-cp313-cp313-win32.whl", hash = "sha256:ba9e56e8ceeeedb2e080147ba85ffcd5cd0711b89576b83784d8605a7df455fa", size = 176393, upload-time = "2025-03-05T20:02:53.814Z" },
+ { url = "https://files.pythonhosted.org/packages/1b/6c/c65773d6cab416a64d191d6ee8a8b1c68a09970ea6909d16965d26bfed1e/websockets-15.0.1-cp313-cp313-win_amd64.whl", hash = "sha256:e09473f095a819042ecb2ab9465aee615bd9c2028e4ef7d933600a8401c79561", size = 176837, upload-time = "2025-03-05T20:02:55.237Z" },
+ { url = "https://files.pythonhosted.org/packages/02/9e/d40f779fa16f74d3468357197af8d6ad07e7c5a27ea1ca74ceb38986f77a/websockets-15.0.1-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:0c9e74d766f2818bb95f84c25be4dea09841ac0f734d1966f415e4edfc4ef1c3", size = 173109, upload-time = "2025-03-05T20:03:17.769Z" },
+ { url = "https://files.pythonhosted.org/packages/bc/cd/5b887b8585a593073fd92f7c23ecd3985cd2c3175025a91b0d69b0551372/websockets-15.0.1-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:1009ee0c7739c08a0cd59de430d6de452a55e42d6b522de7aa15e6f67db0b8e1", size = 173343, upload-time = "2025-03-05T20:03:19.094Z" },
+ { url = "https://files.pythonhosted.org/packages/fe/ae/d34f7556890341e900a95acf4886833646306269f899d58ad62f588bf410/websockets-15.0.1-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:76d1f20b1c7a2fa82367e04982e708723ba0e7b8d43aa643d3dcd404d74f1475", size = 174599, upload-time = "2025-03-05T20:03:21.1Z" },
+ { url = "https://files.pythonhosted.org/packages/71/e6/5fd43993a87db364ec60fc1d608273a1a465c0caba69176dd160e197ce42/websockets-15.0.1-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f29d80eb9a9263b8d109135351caf568cc3f80b9928bccde535c235de55c22d9", size = 174207, upload-time = "2025-03-05T20:03:23.221Z" },
+ { url = "https://files.pythonhosted.org/packages/2b/fb/c492d6daa5ec067c2988ac80c61359ace5c4c674c532985ac5a123436cec/websockets-15.0.1-pp310-pypy310_pp73-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b359ed09954d7c18bbc1680f380c7301f92c60bf924171629c5db97febb12f04", size = 174155, upload-time = "2025-03-05T20:03:25.321Z" },
+ { url = "https://files.pythonhosted.org/packages/68/a1/dcb68430b1d00b698ae7a7e0194433bce4f07ded185f0ee5fb21e2a2e91e/websockets-15.0.1-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:cad21560da69f4ce7658ca2cb83138fb4cf695a2ba3e475e0559e05991aa8122", size = 176884, upload-time = "2025-03-05T20:03:27.934Z" },
+ { url = "https://files.pythonhosted.org/packages/fa/a8/5b41e0da817d64113292ab1f8247140aac61cbf6cfd085d6a0fa77f4984f/websockets-15.0.1-py3-none-any.whl", hash = "sha256:f7a866fbc1e97b5c617ee4116daaa09b722101d4a3c170c787450ba409f9736f", size = 169743, upload-time = "2025-03-05T20:03:39.41Z" },
+]
+
[[package]]
name = "wrapt"
version = "1.17.3"