Skip to content

Build AI methods with your coding agent, then run them anywhere: as an MCP for chatbots, as a webapp for people, or via API for your software.

Rework builder pipeline. Simplify some specs. Fix bug of generating enums in ConceptStructureSpec and other places.

Louis Choquel15b04388518e
diff --git a/.pipelex/inference/routing_profiles.toml b/.pipelex/inference/routing_profiles.toml
index 7290382e0..85a11c6e1 100644
--- a/.pipelex/inference/routing_profiles.toml
+++ b/.pipelex/inference/routing_profiles.toml
@@ -48,6 +48,7 @@ default = "google"
[profiles.all_azure_openai]
description = "Use Azure OpenAI backend for all its supported models"
default = "azure_openai"
+
[profiles.custom_routing]
description = "Custom routing"
default = "pipelex_inference"
diff --git a/pipelex/core/pipes/pipe_abstract.py b/pipelex/core/pipes/pipe_abstract.py
index 68705a742..ff94528b8 100644
--- a/pipelex/core/pipes/pipe_abstract.py
+++ b/pipelex/core/pipes/pipe_abstract.py
@@ -34,10 +34,6 @@ class PipeAbstract(ABC, BaseModel):
def validate_output(self):
"""Validate the output for the pipe."""
- @property
- def class_name(self) -> str:
- return self.__class__.__name__
-
def validate_with_libraries(self):
"""Validate the pipe with the libraries, after the static validation"""
diff --git a/pipelex/libraries/pipelines/builder/builder.plx b/pipelex/libraries/pipelines/builder/builder.plx
index e9ff04d39..91a4dc0f7 100644
--- a/pipelex/libraries/pipelines/builder/builder.plx
+++ b/pipelex/libraries/pipelines/builder/builder.plx
@@ -19,18 +19,17 @@ DomainInformation = "A domain information object."
type = "PipeSequence"
description = "This pipe is going to be the entry point for the builder. It will take a UserBrief and return a PipelexBundleSpec."
inputs = { brief = "UserBrief" }
-output = "Dynamic"
+output = "PipelexBundleSpec"
steps = [
{ pipe = "draft_the_plan", result = "plan_draft" },
{ pipe = "draft_the_concepts", result = "concept_drafts" },
{ pipe = "structure_concepts", result = "concept_specs" },
{ pipe = "design_pipe_signatures", result = "pipe_signatures" },
- # { pipe = "materialize_pipe_signatures", result = "pipe_signatures" },
- # { pipe = "pipe_builder_domain_information", result = "domain_information" },
- # { pipe = "build_concept_spec", batch_over = "concept_spec_drafts", batch_as = "concept_spec_draft", result = "concept_specs" },
- # { pipe = "create_pipes_from_signatures", batch_over = "pipe_signatures", batch_as = "pipe_signature", result = "pipe_specs" },
- # { pipe = "compile_in_pipelex_bundle_spec", result = "pipelex_bundle_spec" }
- # { pipe = "validate_pipelex_bundle_spec", result = "pipelex_bundle_spec" }
+ { pipe = "detail_pipe_spec", batch_over = "pipe_signatures", batch_as = "pipe_signature", result = "pipe_specs" },
+ { pipe = "pipe_builder_domain_information", result = "domain_information" },
+ { pipe = "create_pipes_from_signatures", batch_over = "pipe_signatures", batch_as = "pipe_signature", result = "pipe_specs" },
+ { pipe = "compile_in_pipelex_bundle_spec", result = "pipelex_bundle_spec" }
+ { pipe = "validate_pipelex_bundle_spec" }
]
[pipe.pipe_builder_domain_information]
@@ -87,6 +86,7 @@ Available pipe operators:
- PipeCompose: A pipe that uses Jinja2 to render a template.
Keep your style concise, no need to write tags such as "Description:", just write what you need to write.
+Do not write any intro or outro, just write the plan.
@brief
"""
@@ -128,9 +128,12 @@ Else, if you need structure for your concept, draft its structure:
- required: add required = true if the field is required (otherwise, leave it empty)
- default_value: the default value of the field
+@plan_draft
+
DO NOT redefine native concepts such as: Text, Image, PDF, Number, Page. if you need one of these, they already exist so you should NOT REDEFINE THEM.
-@plan_draft
+Do not write any intro or outro, do not mention the brief or the plan draft, just write the concept drafts.
+List the concept drafts in Markdown format with a heading 3 for each, e.g. `### Concept FooBar`.
"""
[pipe.structure_concepts]
@@ -140,8 +143,11 @@ inputs = { concept_drafts = "ConceptDrafts", brief = "UserBrief" }
output = "concept.ConceptSpec"
multiple_output = true
llm = "llm_to_engineer"
+system_prompt = """
+You are an expert at data extraction and json formatting.
+"""
prompt_template = """
-Structure the concept drafts.
+Your job is to extract a list of ConceptSpec from these concept drafts:
@concept_drafts
"""
@@ -194,120 +200,10 @@ You can use the native concepts for inputs/outputs as required: Text, Image, PDF
{% endif %}
"""
-# """
-# Here are the ESSENTIAL features for each pipe type that should be included in important_features (only include these key ones):
-
-# **PipeLLM**: A pipe that uses an LLM to generate a text, or a structured object. It is a vision LLM that can read images.
-# The inputs of the PipeLLM should be:
-# The variables tagged in the prompt template (with $ or @). If there are no variables, the inputs should be empty.
-# The ouput should be the concept code of the output
-# - prompt_template: The prompt template with variable substitution ($ for inline, @ for blocks)
-# - multiple_output: true if generating multiple number of outputs: That means it will output a LIST of the CONCEPT!
-
-# CRITICAL RULE FOR PIPELLM:
-# - If extracting MULTIPLE items (like multiple articles, employees, products), use multiple_output = true
-# - The concept should represent ONE SINGLE item (Article, Employee, Product)
-# - DO NOT create concepts with plural field names like "item_names", "quantities"
-# - Instead: use multiple_output = true with singular concept fields like "item_name", "quantity"
-# - Example: To extract multiple articles, create concept "Article" with fields "item_name", "quantity", then use multiple_output = true
-
-# **PipeSequence**: A pipe that executes a sequence of pipes: It needs to reference the pipes it will execute.
-# The inputs of the PipeSequence should be all the necessary inputs in the below steps, and the inputs that are NOT generated by intermediate steps.
-# The output should be the concept code of the output of the last step.
-# - steps: List of pipe codes to execute in order, with result names
-# - Each step format: {"pipe": "pipe_code", "result": "result_name"}
-# - Can include batch operations: {"pipe": "pipe_code", "batch_over": "list_input", "batch_as": "item_name", "result": "result_name"}
-
-# **PipeParallel**: A pipe that executes a few pipes in parallel. It needs to reference the pipes it will execute.
-# The inputs of the PipeParallel should be all the necessary inputs in the below steps
-# The output should be the concept code of the output of the last step.
-# The results of each pipe will be in the working memory.
-# - parallels: List of pipes to execute in parallel
-# - Each parallel format: {"pipe": "pipe_code", "result": "result_name"}
-
-# **PipeCondition**: A pipe that based on a specific condition, branches to a specific pipe. You have to explain what the expression of the condition is,
-# and what the different pipes are that can be executed based on the condition. It needs to reference the pipes it will execute.
-# The inputs of the PipeCondition should be all the necessary inputs in the below steps
-# The output should be the concept code of the output of all the steps, except if the outputs are different, then its "Dynamic"
-# - expression: Direct expression to evaluate (e.g., "task_result.status")
-# - pipe_map: Dictionary mapping condition results to pipe codes (e.g., {"completed": "success_pipe", "failed": "failure_pipe"})
-# - default_pipe_code: Fallback pipe when no conditions match
-
-# **PipeBatch**: A pipe that executes a batch of pipes in parallel. It needs to reference the pipe it will execute.
-# - branch_pipe_code: The pipe code to execute for each item
-# - input_list_name: Name of the list to iterate over
-# - input_item_name: Name for individual items within each execution
-
-# **PipeImgGen**: A pipe that uses an LLM to generate an image.
-# The inputs of the PipeImgGen should be: {prompt: ImgGenPrompt}
-# The output should be the concept code that refines Image.
-# - img_gen_prompt: Static prompt for image generation (if using static prompt)
-# - nb_output: Number of images to generate (default 1)
-# VERY IMPORTANT: IF YOU DECIDE TO CREATE A PIPEIMGEN, YOU ALSO HAVE TO CREATE A PIPELLM THAT WILL WRITE THE PROMPT, AND THAT NEEDS TO PRECEED THE PIPEIMGEN, based on the necessary elements.
-# THERFORE, the OUTPUT OF THIS PIPELLM should be a VARIABLE NAMED "prompt" that will be used as input for the PipeImgGen.
-# That means that in the MAIN pipeline, the prompt should NOT be an input. It should be a step that generates the prompt.
-
-# **PipeOcr**: A pipe that uses an LLM to extract text from an image.
-# - The INPUTS of PipeOcr must be either an image or a pdf or a concept which refines one of them.
-
-# **PipeFunc**: A pipe that executes a custom Python function.
-# - function_name: Name of the Python function to call
-
-# **PipeCompose**: A pipe that uses Jinja2 to render a template.
-# - jinja2: Raw Jinja2 template string OR
-# - jinja2_name: Name reference to a template (use one or the other)
-
-# Plan:
-# @plan_draft
-
-# Brief:
-# @brief
-
-# No more than 10 PipeSignatures
-# """
-
-[pipe.materialize_pipe_signatures]
-type = "PipeLLM"
-description = "Turn PipeSignaturesText into PipeSignature objects that reference the ConceptSpec objects."
-inputs = { pipe_signatures_text = "Text", concept_spec_drafts = "concept.ConceptSpecDraft", brief = "UserBrief" }
-output = "pipe.PipeSignature"
-multiple_output = true
-llm = "llm_to_engineer"
-prompt_template = """
-Materialize PipeSignature objects from the PipeSignaturesText.
-- pipe_code MUST be snake_case
-- inputs must be a Dict[str, ConceptSpecDraft] referencing the provided ConceptSpecDraft objects. If Its the concept itself, use the concept code in PascalCase.
-- output must be a ConceptSpec from the provided set. If Its the concept itself, use the concept code in PascalCase.
-- important_features must be a Dict containing the pipe-specific features mentioned in the text
-
-VERY IMPORTANT: A pipe has inputs, and an output. The inputs are a dict of keys in snake_case, corresponding to the variables names in the working memory, and the values are the concept codes in PascalCase.
-The output is a concept code in PascalCase.
-The field "result" is corresponding to the name of the result of the pipe. It will be used in the inputs of the next pipes.
-It is important that they link each other in the right way.
-
-The output concept should be a concepts should be in PascalCase
-
-IMPORTANT:
-- THE MAIN PIPE SHOULD CONTAIN IN ITS NAME "main_pipeline"
-- IF THERE IS A PipeIMG, VERIFIES THAT THE INPUT PROMPT IS ACTUALLY GENERATED BY A PIPELLM BEFORE THE PIPEIMG.
-THIS PIPELLM SHOULD NAME THE RESULT OF ITS PIPE "prompt".
-
-PipeSignatures:
-@pipe_signatures_text
-
-ConceptSpecDrafts:
-@concept_spec_drafts
-
-Brief:
-@brief
-
-No more than 10 PipeSignatures
-"""
-
[pipe.compile_in_pipelex_bundle_spec]
type = "PipeFunc"
description = "Compile the pipelex bundle spec."
-inputs = { pipe_specs = "PipeSpec", concept_specs = "ConceptSpec" }
+inputs = { pipe_specs = "PipeSpec", concept_specs = "ConceptSpec", domain_information = "DomainInformation" }
output = "PipelexBundleSpec"
function_name = "compile_in_pipelex_bundle_spec"
diff --git a/pipelex/libraries/pipelines/builder/concept/concept_spec.py b/pipelex/libraries/pipelines/builder/concept/concept_spec.py
index 00e3827ec..e826b4277 100644
--- a/pipelex/libraries/pipelines/builder/concept/concept_spec.py
+++ b/pipelex/libraries/pipelines/builder/concept/concept_spec.py
@@ -1,8 +1,9 @@
from datetime import datetime
-from typing import Any, cast
+from typing import Any
from pydantic import ConfigDict, Field, field_validator, model_validator
+from pipelex import log
from pipelex.core.concepts.concept_blueprint import (
ConceptBlueprint,
ConceptBlueprintError,
@@ -10,13 +11,12 @@ from pipelex.core.concepts.concept_blueprint import (
ConceptStructureBlueprintError,
ConceptStructureBlueprintFieldType,
)
-from pipelex.core.concepts.concept_native import NativeConceptEnum, NativeConceptManager
-from pipelex.core.concepts.exceptions import ConceptCodeError, ConceptStringError, ConceptStringOrConceptCodeError
-from pipelex.core.domains.domain import SpecialDomain
+from pipelex.core.concepts.concept_native import NativeConceptManager
+from pipelex.core.concepts.exceptions import ConceptCodeError, ConceptStringOrConceptCodeError
from pipelex.core.domains.domain_blueprint import DomainBlueprint
from pipelex.core.memory.working_memory import WorkingMemory
from pipelex.core.stuffs.stuff_content import StructuredContent
-from pipelex.tools.misc.string_utils import is_pascal_case
+from pipelex.tools.misc.string_utils import is_pascal_case, snake_to_pascal_case
from pipelex.types import Self, StrEnum
@@ -33,11 +33,10 @@ class ConceptSpecError(Exception):
class ConceptStructureSpec(StructuredContent):
- """Spec defining a field in the structure of a concept, used as a Pydantic V2 model.
+ """Spec defining a field in the structure of a concept.
This class represents the schema for a single field in a concept's structure. It supports
- various field types including text, list, dict, integer, boolean, number, and date, as well
- as choice-based fields (enums).
+ various field types including text, integer, boolean, number, and date.
Attributes:
the_field_name: Field name. Must be snake_case.
@@ -58,15 +57,20 @@ class ConceptStructureSpec(StructuredContent):
the_field_name: str = Field(description="Field name. Must be snake_case.")
definition: str
- type: ConceptStructureSpecFieldType | None = Field(default=None, description="The type of the field.")
+ type: ConceptStructureSpecFieldType = Field(description="The type of the field.")
required: bool | None = None
default_value: Any | None = None
+ @field_validator("type", mode="before")
+ @classmethod
+ def validate_type(cls, type_value: str) -> ConceptStructureSpecFieldType:
+ return ConceptStructureSpecFieldType(type_value)
+
@model_validator(mode="after")
def validate_structure_blueprint(self) -> Self:
"""Validate the structure blueprint according to type rules."""
# Check default_value type is the same as type
- if self.default_value is not None and self.type is not None:
+ if self.default_value is not None:
self._validate_default_value_type()
return self
@@ -76,8 +80,6 @@ class ConceptStructureSpec(StructuredContent):
return
match self.type:
- case None:
- return
case ConceptStructureSpecFieldType.TEXT:
if not isinstance(self.default_value, str):
self._raise_type_mismatch_error("str", type(self.default_value).__name__)
@@ -101,10 +103,8 @@ class ConceptStructureSpec(StructuredContent):
def to_blueprint(self) -> ConceptStructureBlueprint:
# Convert the type enum value - self.type is already a ConceptStructureBlueprintFieldType enum
# We need to get the corresponding value in the core enum
- core_type = None
- if self.type is not None:
- # Get the string value and use it to get the core enum value
- core_type = ConceptStructureBlueprintFieldType(self.type)
+ # Get the string value and use it to get the core enum value
+ core_type = ConceptStructureBlueprintFieldType(self.type)
return ConceptStructureBlueprint(
definition=self.definition,
@@ -144,11 +144,13 @@ class ConceptSpec(StructuredContent):
model_config = ConfigDict(extra="forbid")
- the_concept_code: str = Field(description="Concept code. Must be PascalCase.")
+ the_concept_code: str = Field(description="Name of the concept. Must be PascalCase.")
definition: str = Field(description="Description of the concept, in natural language.")
- structure: str | dict[str, str | ConceptStructureSpec] | None = Field(
+ structure: dict[str, ConceptStructureSpec] | None = Field(
default=None,
- description=("Definition of the concept's structure. Each attribute (snake_case) specifies: definition, type, required, default_value"),
+ description=(
+ "Definition of the concept's structure. Each attribute (snake_case) specifies: definition, type, and required or default_value if needed"
+ ),
)
refines: str | None = Field(
default=None,
@@ -159,12 +161,7 @@ class ConceptSpec(StructuredContent):
)
@classmethod
- def is_native_concept_code(cls, concept_code: str) -> bool:
- ConceptSpec.validate_concept_code(concept_code=concept_code)
- return concept_code in NativeConceptEnum.values_list()
-
- @classmethod
- def validate_concept_code(cls, concept_code: str) -> None:
+ def _post_validate_concept_code(cls, concept_code: str) -> None:
if not is_pascal_case(concept_code):
msg = f"Concept code '{concept_code}' must be PascalCase (letters and numbers only, starting with uppercase, without `.`)"
raise ConceptCodeError(msg)
@@ -182,43 +179,26 @@ class ConceptSpec(StructuredContent):
if concept_string_or_code.count(".") == 1:
domain, concept_code = concept_string_or_code.split(".")
DomainBlueprint.validate_domain_code(code=domain)
- cls.validate_concept_code(concept_code=concept_code)
+ cls._post_validate_concept_code(concept_code=concept_code)
else:
- cls.validate_concept_code(concept_code=concept_string_or_code)
+ cls._post_validate_concept_code(concept_code=concept_string_or_code)
- @staticmethod
- def validate_concept_string(concept_string: str) -> None:
- """Validate that a concept code follows PascalCase convention."""
- if "." not in concept_string or concept_string.count(".") > 1:
- msg = (
- f"Concept string '{concept_string}' is invalid. It should contain a domain in snake_case "
- "and a concept code in PascalCase separated by one dot."
- )
- raise ConceptStringError(msg)
- domain, concept_code = concept_string.split(".", 1)
-
- DomainBlueprint.validate_domain_code(domain)
- if not is_pascal_case(concept_code):
- msg = f"Concept code '{concept_code}' must be PascalCase (letters and numbers only, starting with uppercase, without `.`)"
- raise ConceptCodeError(msg)
-
- # Validate that if the concept code is among the native concepts, the domain MUST be native.
- if not SpecialDomain.is_native(domain=domain) and concept_code in NativeConceptEnum.values_list():
- msg = (
- f"Concept string '{concept_string}' is invalid. "
- f"Concept code '{concept_code}' is a native concept, so the domain must be '{SpecialDomain.NATIVE}', "
- f"or nothing, but not '{domain}'"
- )
- raise ConceptStringError(msg)
-
- # Validate that if the domain is native, the concept code is a native concept
- if SpecialDomain.is_native(domain=domain):
- if concept_code not in NativeConceptEnum.values_list():
- msg = (
- f"Concept string '{concept_string}' is invalid. "
- f"Concept code '{concept_code}' is not a native concept, so the domain must not be '{SpecialDomain.NATIVE}'."
- )
- raise ConceptStringError(msg)
+ @field_validator("the_concept_code", mode="before")
+ @classmethod
+ def validate_concept_code(cls, concept_code: str) -> str:
+ if "." in concept_code:
+ domain, concept_code = concept_code.split(".")
+ if not is_pascal_case(concept_code):
+ log.warning(f"Concept code '{concept_code}' is not PascalCase, converting to PascalCase")
+ pascal_cased = snake_to_pascal_case(concept_code)
+ return f"{domain}.{pascal_cased}"
+ else:
+ return concept_code
+ elif not is_pascal_case(concept_code):
+ log.warning(f"Concept code '{concept_code}' is not PascalCase, converting to PascalCase")
+ return snake_to_pascal_case(concept_code)
+ else:
+ return concept_code
@field_validator("refines", mode="before")
@classmethod
@@ -247,11 +227,8 @@ class ConceptSpec(StructuredContent):
converted_structure: str | dict[str, str | ConceptStructureBlueprint] | None = None
if self.structure:
converted_structure = {}
- if isinstance(self.structure, str):
- converted_structure = self.structure
- else:
- for field_name, field_spec in cast("dict[str, ConceptStructureSpec]", self.structure).items():
- converted_structure[field_name] = field_spec.to_blueprint()
+ for field_name, field_spec in self.structure.items():
+ converted_structure[field_name] = field_spec.to_blueprint()
return ConceptBlueprint(definition=self.definition, structure=converted_structure, refines=self.refines)
@@ -260,7 +237,7 @@ async def create_concept_spec(working_memory: WorkingMemory) -> ConceptSpec:
concept_spec_draft = working_memory.get_stuff_as(name="concept_spec_draft", content_type=ConceptSpecDraft)
concept_spec_structures_stuff = working_memory.get_stuff_as_list(name="concept_spec_structures", item_type=ConceptStructureSpec)
- structure_dict: dict[str, str | ConceptStructureSpec] = {}
+ structure_dict: dict[str, ConceptStructureSpec] = {}
for structure_item in concept_spec_structures_stuff.items:
structure_spec = ConceptStructureSpec(
the_field_name=structure_item.the_field_name,
diff --git a/pipelex/libraries/pipelines/builder/pipe/pipe_design.plx b/pipelex/libraries/pipelines/builder/pipe/pipe_design.plx
new file mode 100644
index 000000000..27f28287c
--- /dev/null
+++ b/pipelex/libraries/pipelines/builder/pipe/pipe_design.plx
@@ -0,0 +1,217 @@
+domain = "pipe_design"
+definition = "Build and process pipes."
+
+[concept]
+PipeSignature = "A pipe contract which says what the pipe does, not how it does it: code (the pipe code in snake_case), type, description, inputs, output."
+PipeSpec = "A structured spec for a pipe (union)."
+# Pipe controllers
+PipeBatchSpec = "A structured spec for a pipe batch."
+PipeConditionSpec = "A structured spec for a pipe condition."
+PipeParallelSpec = "A structured spec for a pipe parallel."
+PipeSequenceSpec = "A structured spec for a pipe sequence."
+# Pipe operators
+PipeFuncSpec = "A structured spec for a pipe func."
+PipeImgGenSpec = "A structured spec for a pipe img gen."
+PipeComposeSpec = "A structured spec for a pipe jinja2."
+PipeLLMSpec = "A structured spec for a pipe llm."
+PipeOcrSpec = "A structured spec for a pipe ocr."
+PipeFailure = "Details of a single pipe failure during dry run."
+
+[pipe]
+
+# ────────────────────────────────────────────────────────────────────────────────
+# CORE: signature → route to spec emitter
+# ────────────────────────────────────────────────────────────────────────────────
+[pipe.detail_pipe_spec]
+type = "PipeCondition"
+description = "Route by signature.type to the correct spec emitter."
+inputs = { pipe_signature = "PipeSignature", concept_specs = "ConceptSpec" }
+output = "Dynamic"
+expression = "pipe_signature.type"
+
+[pipe.detail_pipe_spec.pipe_map]
+PipeSequence = "detail_pipe_sequence"
+PipeParallel = "detail_pipe_parallel"
+PipeCondition = "detail_pipe_condition"
+PipeLLM = "detail_pipe_llm"
+PipeOcr = "detail_pipe_ocr"
+PipeImgGen = "detail_pipe_img_gen"
+PipeCompose = "detail_pipe_jinja"
+PipeFunc = "detail_pipe_func"
+
+# ────────────────────────────────────────────────────────────────────────────────
+# EMITTERS — one per pipe type, all using the same minimal contract
+# (Optionally, emitters could be extended later to read from concept_index.)
+# ────────────────────────────────────────────────────────────────────────────────
+
+[pipe.detail_pipe_sequence]
+type = "PipeLLM"
+description = "Build a PipeSequenceSpec from the signature (children referenced by code)."
+inputs = { pipe_signature = "PipeSignature", concept_specs = "concept.ConceptSpec" }
+output = "PipeSequenceSpec"
+llm = "llm_to_engineer"
+prompt_template = """
+Return a PipeSequenceSpec for this signature.
+The Pipe sequence NEEDS to have at least one step.
+Orchestrate all the necessary steps to achieve the goal of the pipe.
+
+@pipe_signature
+
+{% if concept_specs %}
+We have already defined the concepts you can use for inputs/outputs:
+@concept_specs
+And of course you still have the native concepts if required: Text, Image, PDF, Number, Page.
+{% else %}
+You can use the native concepts for inputs/outputs as required: Text, Image, PDF, Number, Page.
+{% endif %}
+"""
+
+[pipe.detail_pipe_parallel]
+type = "PipeLLM"
+description = "Build a PipeParallelSpec from the signature."
+inputs = { pipe_signature = "PipeSignature", concept_specs = "concept.ConceptSpec" }
+output = "PipeParallelSpec"
+llm = "llm_to_engineer"
+prompt_template = """
+Return a PipeParallelSpec for this signature.
+
+@pipe_signature
+
+{% if concept_specs %}
+We have already defined the concepts you can use for inputs/outputs:
+@concept_specs
+And of course you still have the native concepts if required: Text, Image, PDF, Number, Page.
+{% else %}
+You can use the native concepts for inputs/outputs as required: Text, Image, PDF, Number, Page.
+{% endif %}
+"""
+
+[pipe.detail_pipe_condition]
+type = "PipeLLM"
+description = "Build a PipeConditionSpec from the signature (provide expression/pipe_map consistent with children)."
+inputs = { pipe_signature = "PipeSignature", concept_specs = "concept.ConceptSpec" }
+output = "PipeConditionSpec"
+llm = "llm_to_engineer"
+prompt_template = """
+Return a PipeConditionSpec for this signature.
+
+@pipe_signature
+
+{% if concept_specs %}
+We have already defined the concepts you can use for inputs/outputs:
+@concept_specs
+And of course you still have the native concepts if required: Text, Image, PDF, Number, Page.
+{% else %}
+You can use the native concepts for inputs/outputs as required: Text, Image, PDF, Number, Page.
+{% endif %}
+"""
+
+[pipe.detail_pipe_llm]
+type = "PipeLLM"
+description = "Build a PipeLLMSpec from the signature."
+inputs = { pipe_signature = "PipeSignature", concept_specs = "concept.ConceptSpec" }
+output = "PipeLLMSpec"
+llm = "llm_to_engineer"
+prompt_template = """
+Return a PipeLLMSpec for this signature.
+
+THe prompt is the field "prompt_template" in the PipeLLMSpec.
+@pipe_signature
+
+{% if concept_specs %}
+We have already defined the concepts you can use for inputs/outputs:
+@concept_specs
+And of course you still have the native concepts if required: Text, Image, PDF, Number, Page.
+{% else %}
+You can use the native concepts for inputs/outputs as required: Text, Image, PDF, Number, Page.
+{% endif %}
+"""
+
+[pipe.detail_pipe_ocr]
+type = "PipeLLM"
+description = "Build a PipeOcrSpec from the signature."
+inputs = { pipe_signature = "PipeSignature", concept_specs = "concept.ConceptSpec" }
+output = "PipeOcrSpec"
+prompt_template = """
+Return a PipeOcrSpec for this signature.
+
+VERY IMPORTANT: THE INPUT OF THE PIPEOCR MUST BE either an image or a pdf or a concept which refines one of them.
+@pipe_signature
+
+{% if concept_specs %}
+We have already defined the concepts you can use for inputs/outputs:
+@concept_specs
+And of course you still have the native concepts if required: Text, Image, PDF, Number, Page.
+{% else %}
+You can use the native concepts for inputs/outputs as required: Text, Image, PDF, Number, Page.
+{% endif %}
+"""
+
+[pipe.detail_pipe_img_gen]
+type = "PipeLLM"
+description = "Build a PipeImgGenSpec from the signature."
+inputs = { pipe_signature = "PipeSignature", concept_specs = "concept.ConceptSpec" }
+output = "PipeImgGenSpec"
+prompt_template = """
+Return a PipeImgGenSpec for this signature.
+
+The inputs for the image has to be a single input named "prompt" with a concept that is Text or refines Text.
+The output concept should be a concept in PascalCase that refines Image.
+For example:
+```
+inputs = { prompt: ImgGenPrompt }
+```
+if ImgGenPrompt is a text concept.
+
+The prompt will need to be be generated by a pipe with the necessary elements.
+
+@pipe_signature
+
+{% if concept_specs %}
+We have already defined the concepts you can use for inputs/outputs:
+@concept_specs
+And of course you still have the native concepts if required: Text, Image, PDF, Number, Page.
+{% else %}
+You can use the native concepts for inputs/outputs as required: Text, Image, PDF, Number, Page.
+{% endif %}
+"""
+
+[pipe.detail_pipe_jinja]
+type = "PipeLLM"
+description = "Build a PipeComposeSpec from the signature."
+inputs = { pipe_signature = "PipeSignature", concept_specs = "concept.ConceptSpec" }
+output = "PipeComposeSpec"
+prompt_template = """
+Return a PipeComposeSpec for this signature.
+
+@pipe_signature
+
+{% if concept_specs %}
+We have already defined the concepts you can use for inputs/outputs:
+@concept_specs
+And of course you still have the native concepts if required: Text, Image, PDF, Number, Page.
+{% else %}
+You can use the native concepts for inputs/outputs as required: Text, Image, PDF, Number, Page.
+{% endif %}
+"""
+
+[pipe.detail_pipe_func]
+type = "PipeLLM"
+description = "Build a PipeFuncSpec from the signature."
+inputs = { pipe_signature = "PipeSignature", concept_specs = "concept.ConceptSpec" }
+output = "PipeFuncSpec"
+llm = "llm_to_engineer"
+prompt_template = """
+Return a PipeFuncSpec for this signature.
+
+@pipe_signature
+
+{% if concept_specs %}
+We have already defined the concepts you can use for inputs/outputs:
+@concept_specs
+And of course you still have the native concepts if required: Text, Image, PDF, Number, Page.
+{% else %}
+You can use the native concepts for inputs/outputs as required: Text, Image, PDF, Number, Page.
+{% endif %}
+"""
+
diff --git a/pipelex/libraries/pipelines/builder/pipe/pipe_img_spec.py b/pipelex/libraries/pipelines/builder/pipe/pipe_img_spec.py
index a0c4f83b3..be400c516 100644
--- a/pipelex/libraries/pipelines/builder/pipe/pipe_img_spec.py
+++ b/pipelex/libraries/pipelines/builder/pipe/pipe_img_spec.py
@@ -1,12 +1,17 @@
from typing import Literal
-from pydantic import Field
+from pydantic import Field, field_validator
from typing_extensions import override
-from pipelex.cogt.img_gen.img_gen_job_components import AspectRatio, Background, OutputFormat
-from pipelex.cogt.img_gen.img_gen_setting import ImgGenChoice
from pipelex.libraries.pipelines.builder.pipe.pipe_signature import PipeSpec
from pipelex.pipe_operators.img_gen.pipe_img_gen_blueprint import PipeImgGenBlueprint
+from pipelex.types import StrEnum
+
+
+class RecommendedImgGen(StrEnum):
+ BASE_IMG_GEN = "base_img_gen"
+ FAST_IMG_GEN = "fast_img_gen"
+ HIGH_QUALITY_IMG_GEN = "high_quality_img_gen"
class PipeImgGenSpec(PipeSpec):
@@ -15,54 +20,21 @@ class PipeImgGenSpec(PipeSpec):
PipeImgGen enables AI-powered image generation using various models like DALL-E or
diffusion models. Supports static and dynamic prompts with configurable generation
parameters.
-
- Attributes:
- the_pipe_code: Pipe code. Must be snake_case.
- type: Fixed to "PipeImgGen" for this pipe type.
- img_gen_prompt: Static text prompt for image generation. Use this or dynamic input.
- img_gen: Image generation choice (e.g., 'gpt-image-1'). Defaults to global config.
- aspect_ratio: Desired image aspect ratio (e.g., '16:9', '1:1').
- quality: Generated image quality setting (e.g., 'standard', 'hd').
- nb_steps: Number of diffusion steps for diffusion models. More steps increase detail
- but take longer. Must be > 0.
- guidance_scale: Prompt adherence strength. Higher values mean closer adherence to prompt.
- Must be > 0.
- is_moderated: Whether to apply content moderation to generated images.
- safety_tolerance: Content moderation tolerance level. Must be between 1 and 6.
- is_raw: Whether to return raw image data instead of processed format.
- seed: Random seed for reproducibility. Use integer value or 'auto' for random seed.
- nb_output: Number of images to generate. Defaults to single image. Must be >= 1.
- img_gen_prompt_var_name: Variable name for dynamic prompt generation from inputs. Do not assign anything
-
- Validation Rules:
- 1. Quality and nb_steps are mutually exclusive (cannot specify both).
- 2. nb_steps must be greater than 0 when specified.
- 3. guidance_scale must be greater than 0 when specified.
- 4. safety_tolerance must be between 1 and 6 inclusive.
- 5. nb_output must be at least 1 when specified.
-
- Raises:
- PipeDefinitionError: When validation rules are violated or mutually exclusive
- fields are set simultaneously.
-
"""
type: Literal["PipeImgGen"] = "PipeImgGen"
category: Literal["PipeOperator"] = "PipeOperator"
the_pipe_code: str = Field(description="Pipe code. Must be snake_case.")
- img_gen_prompt: str | None = None
- img_gen_prompt_var_name: str | None = None
-
- # New ImgGenChoice pattern (like LLM)
- img_gen: ImgGenChoice | None = None
-
- # One-time settings (not in ImgGenSetting)
- aspect_ratio: AspectRatio | None = Field(default=None, strict=False)
- is_raw: bool | None = None
- seed: int | Literal["auto"] | None = None
+ img_gen: RecommendedImgGen | None = None
nb_output: int | None = Field(default=None, ge=1)
- background: Background | None = Field(default=None, strict=False)
- output_format: OutputFormat | None = Field(default=None, strict=False)
+
+ @field_validator("img_gen", mode="before")
+ @classmethod
+ def validate_img_gen(cls, img_gen_value: str | None) -> RecommendedImgGen | None:
+ if img_gen_value is None:
+ return None
+ else:
+ return RecommendedImgGen(img_gen_value)
@override
def to_blueprint(self) -> PipeImgGenBlueprint:
@@ -74,13 +46,13 @@ class PipeImgGenSpec(PipeSpec):
output=base_blueprint.output,
type=self.type,
category=self.category,
- img_gen_prompt=self.img_gen_prompt,
+ img_gen_prompt=None,
+ img_gen_prompt_var_name=None,
img_gen=self.img_gen,
- aspect_ratio=self.aspect_ratio,
- background=self.background,
- output_format=self.output_format,
- is_raw=self.is_raw,
- seed=self.seed,
+ aspect_ratio=None,
+ background=None,
+ output_format=None,
+ is_raw=None,
+ seed=None,
nb_output=self.nb_output,
- img_gen_prompt_var_name=None, # Core expects None, builder uses "prompt" as default
)
diff --git a/pipelex/libraries/pipelines/builder/pipe/pipe_ocr_spec.py b/pipelex/libraries/pipelines/builder/pipe/pipe_ocr_spec.py
index 8f2ce3723..41167bd3a 100644
--- a/pipelex/libraries/pipelines/builder/pipe/pipe_ocr_spec.py
+++ b/pipelex/libraries/pipelines/builder/pipe/pipe_ocr_spec.py
@@ -1,6 +1,6 @@
from typing import Literal
-from pydantic import Field
+from pydantic import Field, field_validator
from typing_extensions import override
from pipelex.libraries.pipelines.builder.pipe.pipe_signature import PipeSpec
@@ -46,6 +46,14 @@ class PipeOcrSpec(PipeSpec):
page_views: bool | None = None
page_views_dpi: int | None = None
+ @field_validator("ocr", mode="before")
+ @classmethod
+ def validate_ocr(cls, ocr_value: str | None) -> RecommendedOcrPreset | None:
+ if ocr_value is None:
+ return None
+ else:
+ return RecommendedOcrPreset(ocr_value)
+
@override
def to_blueprint(self) -> PipeOcrBlueprint:
base_blueprint = super().to_blueprint()
diff --git a/pipelex/libraries/pipelines/builder/pipe/pipe_sequence_spec.py b/pipelex/libraries/pipelines/builder/pipe/pipe_sequence_spec.py
index 2921c0a2e..54d1ce512 100644
--- a/pipelex/libraries/pipelines/builder/pipe/pipe_sequence_spec.py
+++ b/pipelex/libraries/pipelines/builder/pipe/pipe_sequence_spec.py
@@ -18,7 +18,7 @@ class PipeSequenceSpec(PipeSpec):
Attributes:
the_pipe_code: Pipe code. Must be snake_case.
type: Fixed to "PipeSequence" for this pipe type.
- steps: Ordered list of SubPipeBlueprint instances defining the pipes
+ steps: Ordered list of SubPipeSpec instances referencing the pipes
to execute. Each step runs after the previous one completes,
with access to all prior outputs in the context.
diff --git a/pipelex/libraries/pipelines/builder/pipe/pipe_signature.py b/pipelex/libraries/pipelines/builder/pipe/pipe_signature.py
index ca2bf07bb..06dcff190 100644
--- a/pipelex/libraries/pipelines/builder/pipe/pipe_signature.py
+++ b/pipelex/libraries/pipelines/builder/pipe/pipe_signature.py
@@ -6,7 +6,7 @@ from pipelex.core.pipes.exceptions import PipeBlueprintError
from pipelex.core.pipes.pipe_blueprint import AllowedPipeCategories, AllowedPipeTypes, PipeBlueprint
from pipelex.core.pipes.pipe_input_blueprint import InputRequirementBlueprint
from pipelex.core.stuffs.stuff_content import StructuredContent
-from pipelex.libraries.pipelines.builder.concept.concept_spec import ConceptSpec, ConceptSpecDraft
+from pipelex.libraries.pipelines.builder.concept.concept_spec import ConceptSpec
from pipelex.libraries.pipelines.builder.pipe.inputs_spec import InputRequirementSpec
from pipelex.tools.misc.string_utils import is_snake_case
@@ -27,41 +27,20 @@ class PipeSignature(StructuredContent):
class PipeSpec(StructuredContent):
- """Spec defining a pipe component in the Pipelex framework.
-
- Pipes are the fundamental processing units in Pipelex workflows. They transform
- input concepts into output concepts through various operations like LLM processing,
- image generation, OCR, or custom functions.
-
- Attributes:
- type: The pipe type (PipeFunc, PipeLLM, PipeImgGen, PipeOcr, PipeBatch,
- PipeCondition, PipeParallel, PipeSequence). Uses Any type to avoid
- type override conflicts but validated at runtime.
- category: The pipe category (PipeOperator, PipeController). Uses Any type to avoid
- category override conflicts but validated at runtime.
- The pipe controllers are PipeSequence, PipeParallel, PipeCondition, PipeBatch.
- The pipe operators are PipeFunc, PipeLLM, PipeImgGen, PipeOcr, PipeCompose.
- definition: Natural language description of what the pipe does.
- inputs: Input concept specifications. should be an InputRequirementBlueprint
- Dictionary keys are input names in snake_case, values are concept specifications in PascalCase.
- output: Output concept code in PascalCase format.
-
- Validation Rules:
- 1. Pipe type: Must be one of the AllowedPipeTypes enum values.
- 2. Output concept: Must be valid concept string or code in PascalCase.
- 3. Input concepts: When provided, must use PascalCase for concept references.
- 4. Pipe codes: When validating pipe codes, must be in snake_case format.
-
+ """Spec defining a pipe: an executable component with a clear contract defined by its inputs and output.
+ There are two categories of pipes: controllers and operators.
+ Controllers are used to control the flow of the pipeline, and operators are used to perform specific tasks.
"""
- type: Any = Field(description=f"Pipe type. Must be one of: {AllowedPipeTypes}")
- category: Any = Field(description=f"Pipe category. Must be one of: {AllowedPipeCategories}")
+ type: Any = Field(description=f"Pipe type. It is defined with type `Any` but validated at runtime and it must be one of: {AllowedPipeTypes}")
+ category: Any = Field(
+ description=f"Pipe category. It is defined with type `Any` but validated at runtime and it must be one of: {AllowedPipeCategories}"
+ )
definition: str | None = Field(description="Natural language description of what the pipe does.")
inputs: dict[str, str | InputRequirementSpec] | None = Field(
description=(
- "Input concept specifications. Can be either: "
- "InputRequirementSpec with additional constraints"
- "Dictionary keys are input names, values are concept specifications. If Its the concept itself, use the concept code in PascalCase."
+ "Input concept specifications. The keys are input names in snake_case. "
+ "Each value is aither the ConceptCode in PascalCase or an InputRequirementSpec with additional constraints"
),
)
output: str = Field(description="Output concept code in PascalCase format!! Very important")
diff --git a/pipelex/pipe_controllers/pipe_controller.py b/pipelex/pipe_controllers/pipe_controller.py
index 8211f7449..bab1babb7 100644
--- a/pipelex/pipe_controllers/pipe_controller.py
+++ b/pipelex/pipe_controllers/pipe_controller.py
@@ -14,6 +14,10 @@ from pipelex.pipeline.job_metadata import JobMetadata
class PipeController(PipeAbstract):
category: Literal["PipeController"] = "PipeController"
+ @property
+ def class_name(self) -> str:
+ return self.__class__.__name__
+
@override
async def run_pipe(
self,
diff --git a/pipelex/pipe_operators/llm/pipe_llm.py b/pipelex/pipe_operators/llm/pipe_llm.py
index fa2d4c4a5..da33d67d0 100644
--- a/pipelex/pipe_operators/llm/pipe_llm.py
+++ b/pipelex/pipe_operators/llm/pipe_llm.py
@@ -315,7 +315,7 @@ class PipeLLM(PipeOperator[PipeLLMOutput]):
log.debug(f"TextContent.__class__.__name__: {TextContent.__class__.__name__}")
log.debug(f"is_multiple_output: {is_multiple_output}")
if output_concept.structure_class_name == "TextContent" and not is_multiple_output:
- log.debug(f"PipeLLM generating a single text output: {self.class_name}_gen_text")
+ log.debug(f"PipeLLM generating a single text output: {self.__class__.__name__}_gen_text")
llm_prompt_1_for_text = await self.llm_prompt_spec.make_llm_prompt(
output_concept_string=output_concept.concept_string,
context_provider=working_memory,
@@ -445,9 +445,9 @@ class PipeLLM(PipeOperator[PipeLLMOutput]):
if is_multiple_output:
# We're generating a list of (possibly multiple) objects
if fixed_nb_output:
- task_desc = f"{self.class_name}_gen_{fixed_nb_output}x{content_class.__class__.__name__}"
+ task_desc = f"{self.__class__.__name__}_gen_{fixed_nb_output}x{content_class.__class__.__name__}"
else:
- task_desc = f"{self.class_name}_gen_list_{content_class.__class__.__name__}"
+ task_desc = f"{self.__class__.__name__}_gen_list_{content_class.__class__.__name__}"
log.dev(task_desc)
generated_objects: list[StuffContent]
if llm_prompt_2_factory is not None:
@@ -480,7 +480,7 @@ class PipeLLM(PipeOperator[PipeLLMOutput]):
the_content = ListContent(items=generated_objects)
else:
# We're generating a single object
- task_desc = f"{self.class_name}_gen_single_{content_class.__name__}"
+ task_desc = f"{self.__class__.__name__}_gen_single_{content_class.__name__}"
log.verbose(task_desc)
if llm_prompt_2_factory is not None:
# We're generating a single object using preliminary text
@@ -528,6 +528,7 @@ class PipeLLM(PipeOperator[PipeLLMOutput]):
@staticmethod
def get_output_structure_prompt(concept_string: str, is_with_preliminary_text: bool) -> str | None:
+ return None
concept = get_required_concept(concept_string=concept_string)
class_name = concept.structure_class_name
output_class = get_class_registry().get_class(class_name)
diff --git a/pipelex/pipe_operators/pipe_operator.py b/pipelex/pipe_operators/pipe_operator.py
index d35a1f411..569cacb29 100644
--- a/pipelex/pipe_operators/pipe_operator.py
+++ b/pipelex/pipe_operators/pipe_operator.py
@@ -19,6 +19,10 @@ PipeOperatorOutputType = TypeVar("PipeOperatorOutputType", bound=PipeOutput)
class PipeOperator(PipeAbstract, Generic[PipeOperatorOutputType]):
category: Literal["PipeOperator"] = "PipeOperator"
+ @property
+ def class_name(self) -> str:
+ return self.__class__.__name__
+
@override
async def run_pipe(
self,
diff --git a/pipelex/pipe_works/pipe_dry.py b/pipelex/pipe_works/pipe_dry.py
index f5285278a..1c99944ba 100644
--- a/pipelex/pipe_works/pipe_dry.py
+++ b/pipelex/pipe_works/pipe_dry.py
@@ -48,10 +48,10 @@ class DryRunOutput(BaseModel):
async def dry_run_pipe(pipe: PipeAbstract, raise_on_failure: bool = False) -> DryRunOutput:
"""Dry run a single pipe directly without parallelization."""
allowed_to_fail_pipes = get_config().pipelex.dry_run_config.allowed_to_fail_pipes
- needed_inputs_for_factory = _convert_to_working_memory_format(needed_inputs_spec=pipe.needed_inputs())
-
- working_memory = WorkingMemoryFactory.make_for_dry_run(needed_inputs=needed_inputs_for_factory)
try:
+ needed_inputs_for_factory = _convert_to_working_memory_format(needed_inputs_spec=pipe.needed_inputs())
+
+ working_memory = WorkingMemoryFactory.make_for_dry_run(needed_inputs=needed_inputs_for_factory)
pipe.validate_with_libraries()
await pipe.run_pipe(
job_metadata=JobMetadata(job_name=f"dry_run_{pipe.code}"),
diff --git a/pipelex/plugins/anthropic/anthropic_llm_worker.py b/pipelex/plugins/anthropic/anthropic_llm_worker.py
index 77b67798b..af0ffcf57 100644
--- a/pipelex/plugins/anthropic/anthropic_llm_worker.py
+++ b/pipelex/plugins/anthropic/anthropic_llm_worker.py
@@ -4,7 +4,7 @@ import instructor
from anthropic import AsyncAnthropic, AsyncAnthropicBedrock, omit
from typing_extensions import override
-from pipelex import log
+from pipelex import log, pretty_print
from pipelex.cogt.exceptions import LLMCompletionError, SdkTypeError
from pipelex.cogt.llm.llm_job import LLMJob
from pipelex.cogt.llm.llm_worker_internal_abstract import LLMWorkerInternalAbstract
@@ -17,6 +17,10 @@ from pipelex.tools.typing.pydantic_utils import BaseModelTypeVar
from pipelex.types import StrEnum
+def dump_kwargs(*_: Any, **kwargs: Any) -> None:
+ pretty_print(kwargs, title="Instructor about to send to Anthropic")
+
+
class AnthropicExtraField(StrEnum):
CLAUDE_4_TOKENS_LIMIT = "claude_4_tokens_limit"
@@ -75,6 +79,8 @@ class AnthropicLLMWorker(LLMWorkerInternalAbstract):
else:
self.instructor_for_objects = instructor.from_anthropic(client=sdk_instance)
+ self.instructor_for_objects.on(hook_name="completion:kwargs", handler=dump_kwargs)
+
#########################################################
# Instance methods
#########################################################
diff --git a/pipelex/plugins/openai/openai_llm_worker.py b/pipelex/plugins/openai/openai_llm_worker.py
index f3e6cd9bb..26907e0e0 100644
--- a/pipelex/plugins/openai/openai_llm_worker.py
+++ b/pipelex/plugins/openai/openai_llm_worker.py
@@ -9,7 +9,7 @@ if TYPE_CHECKING:
from openai.types.chat import ChatCompletionMessage
from typing_extensions import override
-from pipelex import log
+from pipelex import log, pretty_print
from pipelex.cogt.exceptions import LLMCompletionError, LLMModelNotFoundError, SdkTypeError
from pipelex.cogt.llm.llm_job import LLMJob
from pipelex.cogt.llm.llm_worker_internal_abstract import LLMWorkerInternalAbstract
@@ -21,6 +21,18 @@ from pipelex.reporting.reporting_protocol import ReportingProtocol
from pipelex.tools.typing.pydantic_utils import BaseModelTypeVar
+def dump_kwargs(*_: Any, **kwargs: Any) -> None:
+ pretty_print(kwargs, title="Instructor about to send to OpenAI")
+
+
+def dump_response(response: Any) -> None:
+ pretty_print(response, title="Instructor response from OpenAI")
+
+
+def dump_error(error: Exception) -> None:
+ pretty_print(error, title="Instructor error from OpenAI")
+
+
class OpenAILLMWorker(LLMWorkerInternalAbstract):
def __init__(
self,
@@ -48,6 +60,10 @@ class OpenAILLMWorker(LLMWorkerInternalAbstract):
else:
self.instructor_for_objects = instructor.from_openai(client=sdk_instance)
+ self.instructor_for_objects.on(hook_name="completion:kwargs", handler=dump_kwargs)
+ self.instructor_for_objects.on(hook_name="completion:response", handler=dump_response)
+ self.instructor_for_objects.on(hook_name="completion:error", handler=dump_error)
+
#########################################################
@override
def setup(self):
diff --git a/pipelex/tools/typing/pydantic_utils.py b/pipelex/tools/typing/pydantic_utils.py
index 8190dc636..411f40150 100644
--- a/pipelex/tools/typing/pydantic_utils.py
+++ b/pipelex/tools/typing/pydantic_utils.py
@@ -40,6 +40,14 @@ def format_pydantic_validation_error(exc: ValidationError) -> str:
enum_errors = [
f"{'.'.join(map(str, err['loc']))}: invalid enum value '{err.get('input', 'unknown')}'" for err in exc.errors() if err["type"] == "enum"
]
+ union_tag_errors: list[str] = []
+ for err in exc.errors():
+ if err["type"] == "union_tag_not_found":
+ field_path = ".".join(map(str, err["loc"]))
+ # Extract discriminator field name from context
+ discriminator = err.get("ctx", {}).get("discriminator", "type")
+ union_tag_errors.append(f"{field_path}: missing required discriminator field '{discriminator}'")
+
model_type_errors: list[str] = []
for err in exc.errors():
if err["type"] == "model_type":
@@ -61,11 +69,13 @@ def format_pydantic_validation_error(exc: ValidationError) -> str:
error_msg += f"\nValue errors: {value_errors}"
if enum_errors:
error_msg += f"\nEnum errors: {enum_errors}"
+ if union_tag_errors:
+ error_msg += f"\nUnion discriminator errors: {union_tag_errors}"
if model_type_errors:
error_msg += f"\nModel type errors: {model_type_errors}"
# If none of the specific error types were found, add the raw error messages
- if not any([missing_fields, extra_fields, type_errors, value_errors, enum_errors, model_type_errors]):
+ if not any([missing_fields, extra_fields, type_errors, value_errors, enum_errors, union_tag_errors, model_type_errors]):
error_msg += "\nOther validation errors:"
for err in exc.errors():
error_msg += f"\n{'.'.join(map(str, err['loc']))}: {err['type']}: {err['msg']}"
diff --git a/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_operators/pipe_img/test_data.py b/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_operators/pipe_img/test_data.py
index e239f230b..70ee1729a 100644
--- a/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_operators/pipe_img/test_data.py
+++ b/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_operators/pipe_img/test_data.py
@@ -3,7 +3,7 @@ from typing import ClassVar
from pipelex.cogt.img_gen.img_gen_job_components import AspectRatio
from pipelex.core.pipes.pipe_input_blueprint import InputRequirementBlueprint
from pipelex.libraries.pipelines.builder.pipe.inputs_spec import InputRequirementSpec
-from pipelex.libraries.pipelines.builder.pipe.pipe_img_spec import PipeImgGenSpec
+from pipelex.libraries.pipelines.builder.pipe.pipe_img_spec import PipeImgGenSpec, RecommendedImgGen
from pipelex.pipe_operators.img_gen.pipe_img_gen_blueprint import PipeImgGenBlueprint
@@ -15,7 +15,6 @@ class PipeImgGenTestCases:
definition="Generate an image",
inputs=None,
output="GeneratedImage",
- img_gen_prompt="A beautiful sunset over mountains",
),
PipeImgGenBlueprint(
definition="Generate an image",
@@ -23,7 +22,8 @@ class PipeImgGenTestCases:
output="GeneratedImage",
type="PipeImgGen",
category="PipeOperator",
- img_gen_prompt="A beautiful sunset over mountains",
+ img_gen_prompt=None,
+ img_gen_prompt_var_name=None,
img_gen=None,
aspect_ratio=None,
background=None,
@@ -31,7 +31,6 @@ class PipeImgGenTestCases:
nb_output=None,
is_raw=None,
seed=None,
- img_gen_prompt_var_name=None,
),
)
@@ -42,9 +41,7 @@ class PipeImgGenTestCases:
definition="Generate image with options",
inputs={"description": InputRequirementSpec(concept="Text")},
output="Image",
- img_gen="gpt-image-1",
- aspect_ratio=AspectRatio.SQUARE,
- seed=42,
+ img_gen=RecommendedImgGen.BASE_IMG_GEN,
nb_output=3,
),
PipeImgGenBlueprint(
@@ -54,12 +51,12 @@ class PipeImgGenTestCases:
type="PipeImgGen",
category="PipeOperator",
img_gen_prompt=None,
- img_gen="gpt-image-1",
- aspect_ratio=AspectRatio.SQUARE,
+ img_gen=RecommendedImgGen.BASE_IMG_GEN,
+ aspect_ratio=None,
background=None,
output_format=None,
is_raw=None,
- seed=42,
+ seed=None,
nb_output=3,
img_gen_prompt_var_name=None,
),