Skip to content

Build AI methods with your coding agent, then run them anywhere: as an MCP for chatbots, as a webapp for people, or via API for your software.

fix/remove-useless-functiona (#332)

Thomas Hebrard7457072dc22c
diff --git a/.badges/tests.json b/.badges/tests.json
index 04f36c790..38adf4c62 100644
--- a/.badges/tests.json
+++ b/.badges/tests.json
@@ -1,7 +1,7 @@
{
"schemaVersion": 1,
"label": "tests",
- "message": "1095",
+ "message": "1112",
"color": "blue",
"cacheSeconds": 300
}
\ No newline at end of file
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 0a929ea6e..7c71e77d4 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -20,6 +20,10 @@
- Removed `gpt-5-mini` from the possible models in the `pipe-builder`.
+### Removed
+
+- Removed useless functions in `LLMJobFactory`.
+
## [v0.11.0] - 2025-10-01
### Highlights
@@ -330,11 +334,11 @@ Simplified input memory:
- New method `create_mock_content` in `WorkingMemoryFactory` for creating mock content for requirements.
### Changed
-- Refactored `PipeInputSpec` to use `InputRequirement` and `TypedNamedInputRequirement` classes instead of plain strings for input specifications.
+- Refactored `PipeInput` to use `InputRequirement` and `TypedNamedInputRequirement` classes instead of plain strings for input specifications.
- Updated `WorkingMemoryFactory` to handle `ImplicitMemory` instead of `CompactMemory`.
- Replaced `ExecutePipelineException` with `PipelineInputError` in `execute_pipeline` function.
- Updated `PipeBatch`, `PipeCondition`, `PipeParallel`, `PipeSequence`, `PipeFunc`, `PipeImgGen`, `PipeCompose`, `PipeLLM`, and `PipeOcr` classes to use `InputRequirement` for input handling.
-- Updated `PipeInputSpec` creation in various test files to use `make_from_dict` method.
+- Updated `PipeInput` creation in various test files to use `make_from_dict` method.
- Updated `pyproject.toml` to exclude `pypdfium2` version `4.30.1`.
- Updated `Jinja2TemplateCategory` to handle HTML and Markdown templates differently.
diff --git a/pipelex/cogt/llm/llm_job_factory.py b/pipelex/cogt/llm/llm_job_factory.py
index 930a3b3b5..468d6cffd 100644
--- a/pipelex/cogt/llm/llm_job_factory.py
+++ b/pipelex/cogt/llm/llm_job_factory.py
@@ -1,18 +1,11 @@
-from typing import Any
-
-from pipelex import log
-from pipelex.cogt.image.prompt_image import PromptImage
from pipelex.cogt.llm.llm_job import LLMJob
from pipelex.cogt.llm.llm_job_components import LLMJobConfig, LLMJobParams
from pipelex.cogt.llm.llm_prompt import LLMPrompt
-from pipelex.cogt.llm.llm_prompt_factory_abstract import LLMPromptFactoryAbstract
-from pipelex.cogt.llm.llm_prompt_template import LLMPromptTemplate
from pipelex.config import get_config
from pipelex.pipeline.job_metadata import JobCategory, JobMetadata
class LLMJobFactory:
- # straightforward: the prompt is provided
@classmethod
def make_llm_job(
cls,
@@ -42,81 +35,3 @@ class LLMJobFactory:
job_params=job_params,
job_config=job_config,
)
-
- # powerful: the prompt is generated by a prompt factory using prompt arguments
- # but this one is using the generic llm_prompt_factory.make_llm_prompt_from_args()
- @classmethod
- async def make_llm_job_from_prompt_factory(
- cls,
- llm_prompt_factory: LLMPromptFactoryAbstract,
- llm_prompt_arguments: dict[str, Any],
- llm_job_params: LLMJobParams,
- llm_job_config: LLMJobConfig | None = None,
- job_metadata: JobMetadata | None = None,
- ) -> LLMJob:
- log.debug(llm_prompt_arguments, title="llm_prompt_arguments")
- llm_prompt = await llm_prompt_factory.make_llm_prompt_from_args(**llm_prompt_arguments)
-
- return cls.make_llm_job(
- job_metadata=job_metadata,
- llm_prompt=llm_prompt,
- llm_job_params=llm_job_params,
- llm_job_config=llm_job_config,
- )
-
- # powerful: the prompt is generated by a prompt template using prompt arguments
- @classmethod
- async def make_llm_job_from_prompt_template(
- cls,
- llm_prompt_template: LLMPromptTemplate,
- llm_prompt_arguments: dict[str, Any],
- llm_job_params: LLMJobParams,
- llm_job_config: LLMJobConfig | None = None,
- job_metadata: JobMetadata | None = None,
- ) -> LLMJob:
- log.debug(llm_prompt_arguments, title="llm_prompt_arguments")
- llm_prompt = await llm_prompt_template.make_llm_prompt_from_args(**llm_prompt_arguments)
-
- return cls.make_llm_job(
- job_metadata=job_metadata,
- llm_prompt=llm_prompt,
- llm_job_params=llm_job_params,
- llm_job_config=llm_job_config,
- )
-
- @classmethod
- def make_llm_job_from_prompt_contents(
- cls,
- llm_job_params: LLMJobParams,
- user_text: str | None = None,
- system_text: str | None = None,
- user_images: list[PromptImage] | None = None,
- llm_job_config: LLMJobConfig | None = None,
- job_metadata: JobMetadata | None = None,
- ) -> LLMJob:
- config = get_config()
- llm_config = config.cogt.llm_config
- if job_metadata:
- job_metadata.update(
- JobMetadata(
- job_category=JobCategory.LLM_JOB,
- ),
- )
- else:
- job_metadata = JobMetadata(
- job_category=JobCategory.LLM_JOB,
- )
- llm_prompt = LLMPrompt(
- system_text=system_text,
- user_text=user_text,
- user_images=user_images or [],
- )
- job_params = llm_job_params
- job_config = llm_job_config or llm_config.llm_job_config
-
- return LLMJob(
- job_metadata=job_metadata,
- llm_prompt=llm_prompt,
- job_params=job_params,
- job_config=job_config,
- )
diff --git a/pipelex/cogt/llm/llm_prompt_spec.py b/pipelex/cogt/llm/llm_prompt_spec.py
index 5b3c17e1d..08e09c96f 100644
--- a/pipelex/cogt/llm/llm_prompt_spec.py
+++ b/pipelex/cogt/llm/llm_prompt_spec.py
@@ -11,6 +11,7 @@ from pipelex.hub import get_content_generator, get_template, get_template_provid
from pipelex.tools.misc.context_provider_abstract import ContextProviderAbstract, ContextProviderException
from pipelex.tools.templating.jinja2_blueprint import Jinja2Blueprint
from pipelex.tools.templating.jinja2_required_variables import detect_jinja2_required_variables
+from pipelex.tools.templating.template_preprocessor import preprocess_template
from pipelex.tools.templating.templating_models import PromptingStyle
from pipelex.tools.typing.validation_utils import has_exactly_one_among_attributes_from_list, has_more_than_one_among_attributes_from_list
from pipelex.types import Self
@@ -75,12 +76,12 @@ class LLMPromptSpec(BaseModel):
user_images_top_object_name = [user_image.split(".", 1)[0] for user_image in self.user_images]
required_variables.update(user_images_top_object_name)
- if self.user_text_jinja2_blueprint:
+ if self.user_text_jinja2_blueprint and self.user_text_jinja2_blueprint.jinja2:
+ template_source = preprocess_template(self.user_text_jinja2_blueprint.jinja2)
required_variables = detect_jinja2_required_variables(
template_category=self.user_text_jinja2_blueprint.template_category,
template_provider=get_template_provider(),
- jinja2_name=self.user_text_jinja2_blueprint.jinja2_name,
- jinja2=self.user_text_jinja2_blueprint.jinja2,
+ jinja2=template_source,
)
return {
variable_name
diff --git a/pipelex/cogt/llm/llm_prompt_template.py b/pipelex/cogt/llm/llm_prompt_template.py
index 94a67c3eb..9f3f4ea3c 100644
--- a/pipelex/cogt/llm/llm_prompt_template.py
+++ b/pipelex/cogt/llm/llm_prompt_template.py
@@ -41,7 +41,6 @@ class LLMPromptTemplate(LLMPromptFactoryAbstract):
user_text: str | None = arguments_dict.pop("user_text", None)
if not user_text:
user_text = self.proto_prompt.user_text
-
# user_images is Optional here: None means the template is not altering the user_images field
user_images: list[PromptImage] | None = None
if "user_images" in arguments_dict:
@@ -50,7 +49,7 @@ class LLMPromptTemplate(LLMPromptFactoryAbstract):
user_images = [arguments_dict.pop("user_image")]
is_user_images_append: bool | None = arguments_dict.pop("is_user_images_append", None)
- return self.make_llm_prompt(
+ return self._make_llm_prompt(
system_text=system_text,
user_text=user_text,
user_images=user_images,
@@ -58,7 +57,7 @@ class LLMPromptTemplate(LLMPromptFactoryAbstract):
template_inputs=LLMPromptTemplateInputs(root=arguments_dict),
)
- def make_llm_prompt(
+ def _make_llm_prompt(
self,
system_text: str | None = None,
user_text: str | None = None,
diff --git a/pipelex/core/concepts/concept_library.py b/pipelex/core/concepts/concept_library.py
index 95684a3d6..f564f2309 100644
--- a/pipelex/core/concepts/concept_library.py
+++ b/pipelex/core/concepts/concept_library.py
@@ -1,5 +1,8 @@
+import types
+import typing
from typing import Any
+from kajson.kajson_manager import KajsonManager
from pydantic import Field, RootModel
from typing_extensions import override
@@ -9,7 +12,7 @@ from pipelex.core.concepts.concept_factory import ConceptFactory
from pipelex.core.concepts.concept_native import NATIVE_CONCEPTS_DATA, NativeConceptEnum
from pipelex.core.concepts.concept_provider_abstract import ConceptProviderAbstract
from pipelex.core.domains.domain import SpecialDomain
-from pipelex.core.stuffs.stuff_content import ImageContent
+from pipelex.core.stuffs.stuff_content import ImageContent, ListContent, StuffContent
from pipelex.exceptions import ConceptLibraryConceptNotFoundError, ConceptLibraryError
from pipelex.hub import get_class_registry
from pipelex.types import Self
@@ -125,6 +128,112 @@ class ConceptLibrary(RootModel[ConceptLibraryRoot], ConceptProviderAbstract):
)
return is_image_class or refines_image
+ @override
+ # TODO: Refactor this function. Codesmell, it is not a proper way to do this.
+ def find_image_field_paths(self, concept: Concept) -> list[str]:
+ """Find all field paths in the concept's structure that are strictly compatible with Image concept.
+
+ Args:
+ concept: The concept to analyze for image field paths
+
+ Returns:
+ List of dotted field paths (e.g., ["field1", "field2.subfield"]) that contain Image content.
+ """
+ # Get the structure class
+ structure_class = KajsonManager.get_class_registry().get_class(name=concept.structure_class_name)
+ if structure_class is None or not hasattr(structure_class, "model_fields"):
+ return []
+
+ image_concept = self.get_native_concept(NativeConceptEnum.IMAGE)
+ paths: list[str] = []
+
+ def find_image_fields_in_class(cls: type[StuffContent], current_path: str = "") -> None:
+ """Recursively find image fields in a structure class."""
+ # Check if the class has model_fields (is a Pydantic model)
+ if not hasattr(cls, "model_fields"):
+ return
+
+ # Iterate through all fields
+ for field_name, field_info in cls.model_fields.items():
+ # Build the path for this field
+ field_path = f"{current_path}.{field_name}" if current_path else field_name
+
+ # Get the field type annotation
+ field_type = field_info.annotation
+
+ # Handle Optional types (Union with None)
+ is_union = False
+ union_args = None
+
+ # Check for typing.Union (typing.Optional)
+ is_typing_union = hasattr(field_type, "__origin__") and field_type.__origin__ is typing.Union # type: ignore[union-attr] # pyright: ignore[reportOptionalMemberAccess]
+ is_types_union = hasattr(types, "UnionType") and isinstance(field_type, types.UnionType) # pyright: ignore[reportUnnecessaryIsInstance]
+ if is_typing_union or is_types_union:
+ is_union = True
+ union_args = field_type.__args__ # type: ignore[union-attr]
+
+ if is_union and union_args:
+ # Get non-None types from the Union
+ args = [arg for arg in union_args if arg is not type(None)]
+ if len(args) == 1:
+ field_type = args[0]
+ elif len(args) == 0:
+ continue # All args were None, skip this field
+
+ # Skip if field type is not a class
+ if not isinstance(field_type, type):
+ continue
+
+ # Check if it's a ListContent - skip it
+ try:
+ if issubclass(field_type, ListContent):
+ continue
+ except (TypeError, AttributeError):
+ pass
+
+ # Get the class name
+ field_class_name = field_type.__name__
+
+ # Check if it's a direct ImageContent first
+ try:
+ if issubclass(field_type, ImageContent):
+ paths.append(field_path)
+ continue
+ except TypeError:
+ pass
+
+ # Try to find a concept for this field type
+ try:
+ # Look for a concept with this structure class
+ matching_concept = None
+ for existing_concept in self.list_concepts():
+ if existing_concept.structure_class_name == field_class_name:
+ matching_concept = existing_concept
+ break
+
+ if matching_concept:
+ # Check if this concept is strictly compatible with Image
+ if Concept.are_concept_compatible(
+ concept_1=matching_concept,
+ concept_2=image_concept,
+ strict=True,
+ ):
+ paths.append(field_path)
+ continue # Found an image field, no need to recurse deeper
+
+ except Exception: # noqa: S110
+ pass # If we can't find a concept, continue with recursive search
+
+ # If it's a StuffContent subclass, recurse into it
+ try:
+ if issubclass(field_type, StuffContent):
+ find_image_fields_in_class(field_type, field_path) # pyright: ignore[reportUnknownArgumentType]
+ except TypeError:
+ pass
+
+ find_image_fields_in_class(structure_class)
+ return paths
+
@override
def search_for_concept_in_domains(self, concept_code: str, search_domains: list[str]) -> Concept | None:
ConceptBlueprint.validate_concept_code(concept_code=concept_code)
diff --git a/pipelex/core/concepts/concept_provider_abstract.py b/pipelex/core/concepts/concept_provider_abstract.py
index cbfbb404c..aa973b456 100644
--- a/pipelex/core/concepts/concept_provider_abstract.py
+++ b/pipelex/core/concepts/concept_provider_abstract.py
@@ -59,3 +59,7 @@ class ConceptProviderAbstract(ABC):
@abstractmethod
def get_native_concept(self, native_concept: NativeConceptEnum) -> Concept:
pass
+
+ @abstractmethod
+ def find_image_field_paths(self, concept: Concept) -> list[str]:
+ pass
diff --git a/pipelex/core/pipes/pipe_abstract.py b/pipelex/core/pipes/pipe_abstract.py
index 9ef22ec8f..22d0b6987 100644
--- a/pipelex/core/pipes/pipe_abstract.py
+++ b/pipelex/core/pipes/pipe_abstract.py
@@ -6,7 +6,7 @@ from pydantic import BaseModel, ConfigDict, Field, field_validator
from pipelex.core.concepts.concept import Concept
from pipelex.core.memory.working_memory import WorkingMemory
from pipelex.core.pipes.pipe_blueprint import PipeBlueprint
-from pipelex.core.pipes.pipe_input import PipeInputSpec
+from pipelex.core.pipes.pipe_input import PipeInput
from pipelex.core.pipes.pipe_output import PipeOutput
from pipelex.core.pipes.pipe_run_params import PipeRunParams
from pipelex.exceptions import PipeStackOverflowError
@@ -21,7 +21,7 @@ class PipeAbstract(ABC, BaseModel):
code: str
domain: str
description: str | None = None
- inputs: PipeInputSpec = Field(default_factory=PipeInputSpec)
+ inputs: PipeInput = Field(default_factory=PipeInput)
output: Concept
@field_validator("code", mode="before")
@@ -49,7 +49,7 @@ class PipeAbstract(ABC, BaseModel):
"""
@abstractmethod
- def needed_inputs(self, visited_pipes: set[str] | None = None) -> PipeInputSpec:
+ def needed_inputs(self, visited_pipes: set[str] | None = None) -> PipeInput:
"""Return the inputs that are needed for the pipe to run.
Args:
@@ -57,7 +57,7 @@ class PipeAbstract(ABC, BaseModel):
If None, starts recursion detection with an empty set.
Returns:
- PipeInputSpec containing all needed inputs for this pipe
+ PipeInput containing all needed inputs for this pipe
"""
diff --git a/pipelex/core/pipes/pipe_input.py b/pipelex/core/pipes/pipe_input.py
index 75bab0438..e0726dce0 100644
--- a/pipelex/core/pipes/pipe_input.py
+++ b/pipelex/core/pipes/pipe_input.py
@@ -31,11 +31,11 @@ class TypedNamedInputRequirement(NamedInputRequirement):
return cls(**named.model_dump(), structure_class=structure_class)
-PipeInputSpecRoot = dict[str, InputRequirement]
+PipeInputRoot = dict[str, InputRequirement]
-class PipeInputSpec(RootModel[PipeInputSpecRoot]):
- root: PipeInputSpecRoot = Field(default_factory=dict)
+class PipeInput(RootModel[PipeInputRoot]):
+ root: PipeInputRoot = Field(default_factory=dict)
@field_validator("root", mode="wrap")
@classmethod
diff --git a/pipelex/core/pipes/pipe_input_factory.py b/pipelex/core/pipes/pipe_input_factory.py
index 7f4f8123e..6546a51a2 100644
--- a/pipelex/core/pipes/pipe_input_factory.py
+++ b/pipelex/core/pipes/pipe_input_factory.py
@@ -1,14 +1,14 @@
from pipelex.core.concepts.concept_blueprint import ConceptBlueprint
from pipelex.core.concepts.concept_factory import ConceptFactory
-from pipelex.core.pipes.pipe_input import InputRequirement, PipeInputSpec
+from pipelex.core.pipes.pipe_input import InputRequirement, PipeInput
from pipelex.core.pipes.pipe_input_blueprint import InputRequirementBlueprint
from pipelex.hub import get_required_concept
-class PipeInputSpecFactory:
+class PipeInputFactory:
@classmethod
- def make_empty(cls) -> PipeInputSpec:
- return PipeInputSpec(root={})
+ def make_empty(cls) -> PipeInput:
+ return PipeInput(root={})
@classmethod
def make_from_blueprint(
@@ -16,7 +16,7 @@ class PipeInputSpecFactory:
domain: str,
blueprint: dict[str, str | InputRequirementBlueprint],
concept_codes_from_the_same_domain: list[str] | None = None,
- ) -> PipeInputSpec:
+ ) -> PipeInput:
inputs: dict[str, InputRequirement] = {}
for var_name, input_requirement_blueprint in blueprint.items():
if isinstance(input_requirement_blueprint, str):
@@ -34,4 +34,4 @@ class PipeInputSpecFactory:
concept=get_required_concept(concept_string=concept_string_with_domain),
multiplicity=input_requirement_blueprint.multiplicity,
)
- return PipeInputSpec(root=inputs)
+ return PipeInput(root=inputs)
diff --git a/pipelex/core/stuffs/stuff_factory.py b/pipelex/core/stuffs/stuff_factory.py
index 18a33be98..eec0a1fd2 100644
--- a/pipelex/core/stuffs/stuff_factory.py
+++ b/pipelex/core/stuffs/stuff_factory.py
@@ -15,7 +15,7 @@ from pipelex.core.stuffs.stuff_content import (
TextContent,
)
from pipelex.exceptions import PipelexError
-from pipelex.hub import get_class_registry, get_concept_provider
+from pipelex.hub import get_class_registry, get_concept_provider, get_native_concept
from pipelex.tools.typing.pydantic_utils import format_pydantic_validation_error
@@ -100,10 +100,10 @@ class StuffFactory:
concept_library = get_concept_provider()
if isinstance(blueprint.content, str) and concept_library.is_compatible(
tested_concept=concept_library.get_required_concept(concept_string=blueprint.concept_string),
- wanted_concept=concept_library.get_native_concept(native_concept=NativeConceptEnum.TEXT),
+ wanted_concept=get_native_concept(native_concept=NativeConceptEnum.TEXT),
):
the_stuff = cls.make_stuff(
- concept=concept_library.get_native_concept(native_concept=NativeConceptEnum.TEXT),
+ concept=get_native_concept(native_concept=NativeConceptEnum.TEXT),
content=TextContent(text=blueprint.content),
name=blueprint.stuff_name,
)
@@ -174,7 +174,7 @@ class StuffFactory:
native_concept_class_names = [data.content_class_name for data in NATIVE_CONCEPTS_DATA.values()]
if concept_class_name in native_concept_class_names:
- concept = get_concept_provider().get_native_concept(native_concept=NativeConceptEnum(concept_class_name.split("Content")[0]))
+ concept = get_native_concept(native_concept=NativeConceptEnum(concept_class_name.split("Content")[0]))
return cls.make_stuff(
concept=concept,
content=content,
diff --git a/pipelex/hub.py b/pipelex/hub.py
index 8f1364d1e..9272c6914 100644
--- a/pipelex/hub.py
+++ b/pipelex/hub.py
@@ -13,6 +13,7 @@ from pipelex.cogt.models.model_deck import ModelDeck
from pipelex.cogt.models.model_manager_abstract import ModelManagerAbstract
from pipelex.cogt.ocr.ocr_worker_abstract import OcrWorkerAbstract
from pipelex.core.concepts.concept import Concept
+from pipelex.core.concepts.concept_native import NativeConceptEnum
from pipelex.core.concepts.concept_provider_abstract import ConceptProviderAbstract
from pipelex.core.domains.domain import Domain
from pipelex.core.domains.domain_provider_abstract import DomainProviderAbstract
@@ -460,3 +461,7 @@ def get_library_manager() -> LibraryManagerAbstract:
def get_observer_provider() -> ObserverProtocol:
return get_pipelex_hub().get_observer_provider()
+
+
+def get_native_concept(native_concept: NativeConceptEnum) -> Concept:
+ return get_pipelex_hub().get_required_concept_provider().get_native_concept(native_concept=native_concept)
diff --git a/pipelex/libraries/pipelines/builder/builder.plx b/pipelex/libraries/pipelines/builder/builder.plx
index 274fa8da5..8a276aefa 100644
--- a/pipelex/libraries/pipelines/builder/builder.plx
+++ b/pipelex/libraries/pipelines/builder/builder.plx
@@ -150,8 +150,11 @@ system_prompt = """
You are an expert at data extraction and json formatting.
"""
prompt_template = """
-Your job is to extract a list of ConceptSpec from these concept drafts:
+You are on a big journey to construct a pipeline, and this is one of the steps.
+Here is the overalle mission of the user:
+@brief
+Your task here is to extract a list of ConceptSpec from these concept drafts:
@concept_drafts
"""
diff --git a/pipelex/libraries/pipelines/builder/pipe/pipe_design.plx b/pipelex/libraries/pipelines/builder/pipe/pipe_design.plx
index 333210b46..17cda1433 100644
--- a/pipelex/libraries/pipelines/builder/pipe/pipe_design.plx
+++ b/pipelex/libraries/pipelines/builder/pipe/pipe_design.plx
@@ -53,6 +53,9 @@ This PipeSequence is part of a larger pipeline:
You will specifically generate the PipeSequence related to this signature:
@pipe_signature
+
+Here are the concepts you can use for inputs/outputs:
+@concept_specs
"""
[pipe.detail_pipe_parallel]
@@ -69,6 +72,9 @@ This PipeParallel is part of a larger pipeline:
You will specifically generate the PipeParallel related to this signature:
@pipe_signature
+
+Here are the concepts you can use for inputs/outputs:
+@concept_specs
"""
[pipe.detail_pipe_condition]
@@ -85,6 +91,9 @@ This PipeCondition is part of a larger pipeline:
You will specifically generate the PipeCondition related to this signature:
@pipe_signature
+
+Here are the concepts you can use for inputs/outputs:
+@concept_specs
"""
# ────────────────────────────────────────────────────────────────────────────────
@@ -110,6 +119,9 @@ You will specifically generate the PipeLLM related to this signature:
If it's a structured generation, indicate it in the system_prompt to clarify the task.
If it's to generate free form text, the prompt_template should indicate to be concise.
If it's to generate an image generation, the prompt_template should indicate to be VERY concise and focus and apply the best practice for image generation.
+
+Here are the concepts you can use for inputs/outputs:
+@concept_specs
"""
[pipe.detail_pipe_ocr]
@@ -126,6 +138,9 @@ This PipeOcr is part of a larger pipeline:
You will specifically generate the PipeOcr related to this signature:
@pipe_signature
+
+Here are the concepts you can use for inputs/outputs:
+@concept_specs
"""
[pipe.detail_pipe_img_gen]
@@ -144,6 +159,9 @@ You will specifically generate the PipeImgGen related to this signature:
@pipe_signature
The inputs for the image has to be a single input which must be a Text or another concept which refines Text.
+
+Here are the concepts you can use for inputs/outputs:
+@concept_specs
"""
[pipe.detail_pipe_compose]
@@ -170,4 +188,7 @@ format - Apply the given values to a printf-style format string, like string % v
length — Returns the number of items (alias: count).
upper — Converts a string to uppercase.
lower — Converts a string to lowercase.
+
+Here are the concepts you can use for inputs/outputs:
+@concept_specs
"""
diff --git a/pipelex/pipe_controllers/batch/pipe_batch.py b/pipelex/pipe_controllers/batch/pipe_batch.py
index 5bdb437c5..2f2dba520 100644
--- a/pipelex/pipe_controllers/batch/pipe_batch.py
+++ b/pipelex/pipe_controllers/batch/pipe_batch.py
@@ -8,7 +8,7 @@ from typing_extensions import override
from pipelex import log
from pipelex.config import get_config
from pipelex.core.memory.working_memory import MAIN_STUFF_NAME, WorkingMemory
-from pipelex.core.pipes.pipe_input import PipeInputSpec
+from pipelex.core.pipes.pipe_input import PipeInput
from pipelex.core.pipes.pipe_output import PipeOutput
from pipelex.core.pipes.pipe_run_params import BatchParams, PipeRunMode, PipeRunParams
from pipelex.core.stuffs.stuff_content import ListContent, StuffContent
@@ -74,7 +74,7 @@ class PipeBatch(PipeController):
return required_variables
@override
- def needed_inputs(self, visited_pipes: set[str] | None = None) -> PipeInputSpec:
+ def needed_inputs(self, visited_pipes: set[str] | None = None) -> PipeInput:
return self.inputs
async def _run_batch_pipe(
diff --git a/pipelex/pipe_controllers/batch/pipe_batch_factory.py b/pipelex/pipe_controllers/batch/pipe_batch_factory.py
index 4c8122d3c..d6f4c6b3c 100644
--- a/pipelex/pipe_controllers/batch/pipe_batch_factory.py
+++ b/pipelex/pipe_controllers/batch/pipe_batch_factory.py
@@ -2,7 +2,7 @@ from typing_extensions import override
from pipelex.core.concepts.concept_factory import ConceptFactory
from pipelex.core.pipes.pipe_factory import PipeFactoryProtocol
-from pipelex.core.pipes.pipe_input_factory import PipeInputSpecFactory
+from pipelex.core.pipes.pipe_input_factory import PipeInputFactory
from pipelex.core.pipes.pipe_run_params import BatchParams
from pipelex.hub import get_concept_provider
from pipelex.pipe_controllers.batch.pipe_batch import PipeBatch
@@ -28,7 +28,7 @@ class PipeBatchFactory(PipeFactoryProtocol[PipeBatchBlueprint, PipeBatch]):
domain=domain,
code=pipe_code,
description=blueprint.description,
- inputs=PipeInputSpecFactory.make_from_blueprint(
+ inputs=PipeInputFactory.make_from_blueprint(
domain=domain,
blueprint=blueprint.inputs or {},
concept_codes_from_the_same_domain=concept_codes_from_the_same_domain,
diff --git a/pipelex/pipe_controllers/condition/pipe_condition.py b/pipelex/pipe_controllers/condition/pipe_condition.py
index 4ac2a4441..ab35e288b 100644
--- a/pipelex/pipe_controllers/condition/pipe_condition.py
+++ b/pipelex/pipe_controllers/condition/pipe_condition.py
@@ -9,9 +9,9 @@ from pipelex.config import StaticValidationReaction, get_config
from pipelex.core.concepts.concept_factory import ConceptFactory
from pipelex.core.concepts.concept_native import NATIVE_CONCEPTS_DATA, NativeConceptEnum, NativeConceptManager
from pipelex.core.memory.working_memory import WorkingMemory
-from pipelex.core.pipes.pipe_input import PipeInputSpec
+from pipelex.core.pipes.pipe_input import PipeInput
from pipelex.core.pipes.pipe_input_blueprint import InputRequirementBlueprint
-from pipelex.core.pipes.pipe_input_factory import PipeInputSpecFactory
+from pipelex.core.pipes.pipe_input_factory import PipeInputFactory
from pipelex.core.pipes.pipe_output import PipeOutput
from pipelex.core.pipes.pipe_run_params import PipeRunParams
from pipelex.core.pipes.specific_pipe import SpecificPipe, SpecificPipeCodesEnum
@@ -127,18 +127,18 @@ class PipeCondition(PipeController):
return self
@override
- def needed_inputs(self, visited_pipes: set[str] | None = None) -> PipeInputSpec:
+ def needed_inputs(self, visited_pipes: set[str] | None = None) -> PipeInput:
if visited_pipes is None:
visited_pipes = set()
# If we've already visited this pipe, stop recursion
if self.code in visited_pipes:
- return PipeInputSpecFactory.make_empty()
+ return PipeInputFactory.make_empty()
# Add this pipe to visited set for recursive calls
visited_pipes_with_current = visited_pipes | {self.code}
- needed_inputs = PipeInputSpecFactory.make_empty()
+ needed_inputs = PipeInputFactory.make_empty()
# 1. Add the variables from the expression/expression_template
required_variables = detect_jinja2_required_variables(
@@ -288,7 +288,7 @@ class PipeCondition(PipeController):
# TODO: restore pipe_layer feature
# pipe_run_params.push_pipe_code(pipe_code=pipe_code)
- # Convert PipeInputSpec to blueprint format
+ # Convert PipeInput to blueprint format
inputs_blueprint: dict[str, str | InputRequirementBlueprint] = {}
for var_name, requirement in self.inputs.root.items():
inputs_blueprint[var_name] = InputRequirementBlueprint(
diff --git a/pipelex/pipe_controllers/condition/pipe_condition_factory.py b/pipelex/pipe_controllers/condition/pipe_condition_factory.py
index 97c560a54..025251b3c 100644
--- a/pipelex/pipe_controllers/condition/pipe_condition_factory.py
+++ b/pipelex/pipe_controllers/condition/pipe_condition_factory.py
@@ -2,7 +2,7 @@ from typing_extensions import override
from pipelex.core.concepts.concept_factory import ConceptFactory
from pipelex.core.pipes.pipe_factory import PipeFactoryProtocol
-from pipelex.core.pipes.pipe_input_factory import PipeInputSpecFactory
+from pipelex.core.pipes.pipe_input_factory import PipeInputFactory
from pipelex.hub import get_concept_provider
from pipelex.pipe_controllers.condition.pipe_condition import PipeCondition
from pipelex.pipe_controllers.condition.pipe_condition_blueprint import PipeConditionBlueprint
@@ -27,7 +27,7 @@ class PipeConditionFactory(PipeFactoryProtocol[PipeConditionBlueprint, PipeCondi
domain=domain,
code=pipe_code,
description=blueprint.description,
- inputs=PipeInputSpecFactory.make_from_blueprint(
+ inputs=PipeInputFactory.make_from_blueprint(
domain=domain,
blueprint=blueprint.inputs or {},
concept_codes_from_the_same_domain=concept_codes_from_the_same_domain,
diff --git a/pipelex/pipe_controllers/parallel/pipe_parallel.py b/pipelex/pipe_controllers/parallel/pipe_parallel.py
index 7afd7d605..1a0b518c6 100644
--- a/pipelex/pipe_controllers/parallel/pipe_parallel.py
+++ b/pipelex/pipe_controllers/parallel/pipe_parallel.py
@@ -8,8 +8,8 @@ from pipelex import log
from pipelex.config import StaticValidationReaction, get_config
from pipelex.core.concepts.concept import Concept
from pipelex.core.memory.working_memory import WorkingMemory
-from pipelex.core.pipes.pipe_input import PipeInputSpec
-from pipelex.core.pipes.pipe_input_factory import PipeInputSpecFactory
+from pipelex.core.pipes.pipe_input import PipeInput
+from pipelex.core.pipes.pipe_input_factory import PipeInputFactory
from pipelex.core.pipes.pipe_output import PipeOutput
from pipelex.core.pipes.pipe_run_params import PipeRunMode, PipeRunParams
from pipelex.core.stuffs.stuff_factory import StuffFactory
@@ -39,18 +39,18 @@ class PipeParallel(PipeController):
return set()
@override
- def needed_inputs(self, visited_pipes: set[str] | None = None) -> PipeInputSpec:
+ def needed_inputs(self, visited_pipes: set[str] | None = None) -> PipeInput:
if visited_pipes is None:
visited_pipes = set()
# If we've already visited this pipe, stop recursion
if self.code in visited_pipes:
- return PipeInputSpecFactory.make_empty()
+ return PipeInputFactory.make_empty()
# Add this pipe to visited set for recursive calls
visited_pipes_with_current = visited_pipes | {self.code}
- needed_inputs = PipeInputSpecFactory.make_empty()
+ needed_inputs = PipeInputFactory.make_empty()
for sub_pipe in self.parallel_sub_pipes:
pipe = get_required_pipe(pipe_code=sub_pipe.pipe_code)
diff --git a/pipelex/pipe_controllers/parallel/pipe_parallel_factory.py b/pipelex/pipe_controllers/parallel/pipe_parallel_factory.py
index 4f6b7b266..ec07f8da4 100644
--- a/pipelex/pipe_controllers/parallel/pipe_parallel_factory.py
+++ b/pipelex/pipe_controllers/parallel/pipe_parallel_factory.py
@@ -4,7 +4,7 @@ from typing_extensions import override
from pipelex.core.concepts.concept_factory import ConceptFactory
from pipelex.core.pipes.pipe_factory import PipeFactoryProtocol
-from pipelex.core.pipes.pipe_input_factory import PipeInputSpecFactory
+from pipelex.core.pipes.pipe_input_factory import PipeInputFactory
from pipelex.exceptions import PipeDefinitionError
from pipelex.hub import get_required_concept
from pipelex.pipe_controllers.parallel.pipe_parallel import PipeParallel
@@ -67,7 +67,7 @@ class PipeParallelFactory(PipeFactoryProtocol[PipeParallelBlueprint, PipeParalle
domain=domain,
code=pipe_code,
description=blueprint.description,
- inputs=PipeInputSpecFactory.make_from_blueprint(
+ inputs=PipeInputFactory.make_from_blueprint(
domain=domain,
blueprint=blueprint.inputs or {},
concept_codes_from_the_same_domain=concept_codes_from_the_same_domain,
diff --git a/pipelex/pipe_controllers/sequence/pipe_sequence.py b/pipelex/pipe_controllers/sequence/pipe_sequence.py
index 7b6e157c6..d297d23c2 100644
--- a/pipelex/pipe_controllers/sequence/pipe_sequence.py
+++ b/pipelex/pipe_controllers/sequence/pipe_sequence.py
@@ -6,8 +6,8 @@ from typing_extensions import override
from pipelex import log
from pipelex.config import StaticValidationReaction, get_config
from pipelex.core.memory.working_memory import WorkingMemory
-from pipelex.core.pipes.pipe_input import PipeInputSpec
-from pipelex.core.pipes.pipe_input_factory import PipeInputSpecFactory
+from pipelex.core.pipes.pipe_input import PipeInput
+from pipelex.core.pipes.pipe_input_factory import PipeInputFactory
from pipelex.core.pipes.pipe_output import PipeOutput
from pipelex.core.pipes.pipe_run_params import PipeRunMode, PipeRunParams
from pipelex.exceptions import (
@@ -29,18 +29,18 @@ class PipeSequence(PipeController):
sequential_sub_pipes: list[SubPipe]
@override
- def needed_inputs(self, visited_pipes: set[str] | None = None) -> PipeInputSpec:
+ def needed_inputs(self, visited_pipes: set[str] | None = None) -> PipeInput:
if visited_pipes is None:
visited_pipes = set()
# If we've already visited this pipe, stop recursion
if self.code in visited_pipes:
- return PipeInputSpecFactory.make_empty()
+ return PipeInputFactory.make_empty()
# Add this pipe to visited set for recursive calls
visited_pipes_with_current = visited_pipes | {self.code}
- needed_inputs = PipeInputSpecFactory.make_empty()
+ needed_inputs = PipeInputFactory.make_empty()
generated_outputs: set[str] = set()
for sequential_sub_pipe in self.sequential_sub_pipes:
diff --git a/pipelex/pipe_controllers/sequence/pipe_sequence_factory.py b/pipelex/pipe_controllers/sequence/pipe_sequence_factory.py
index 9148b5584..59af06785 100644
--- a/pipelex/pipe_controllers/sequence/pipe_sequence_factory.py
+++ b/pipelex/pipe_controllers/sequence/pipe_sequence_factory.py
@@ -2,7 +2,7 @@ from typing_extensions import override
from pipelex.core.concepts.concept_factory import ConceptFactory
from pipelex.core.pipes.pipe_factory import PipeFactoryProtocol
-from pipelex.core.pipes.pipe_input_factory import PipeInputSpecFactory
+from pipelex.core.pipes.pipe_input_factory import PipeInputFactory
from pipelex.hub import get_concept_provider
from pipelex.pipe_controllers.sequence.pipe_sequence import PipeSequence
from pipelex.pipe_controllers.sequence.pipe_sequence_blueprint import PipeSequenceBlueprint
@@ -28,7 +28,7 @@ class PipeSequenceFactory(PipeFactoryProtocol[PipeSequenceBlueprint, PipeSequenc
domain=domain,
code=pipe_code,
description=blueprint.description,
- inputs=PipeInputSpecFactory.make_from_blueprint(
+ inputs=PipeInputFactory.make_from_blueprint(
domain=domain,
blueprint=blueprint.inputs or {},
concept_codes_from_the_same_domain=concept_codes_from_the_same_domain,
diff --git a/pipelex/pipe_operators/compose/pipe_compose.py b/pipelex/pipe_operators/compose/pipe_compose.py
index 5d41bd1b9..73b764dc8 100644
--- a/pipelex/pipe_operators/compose/pipe_compose.py
+++ b/pipelex/pipe_operators/compose/pipe_compose.py
@@ -12,8 +12,8 @@ from pipelex.core.concepts.concept import Concept
from pipelex.core.concepts.concept_factory import ConceptFactory
from pipelex.core.concepts.concept_native import NATIVE_CONCEPTS_DATA, NativeConceptEnum
from pipelex.core.memory.working_memory import WorkingMemory
-from pipelex.core.pipes.pipe_input import PipeInputSpec
-from pipelex.core.pipes.pipe_input_factory import PipeInputSpecFactory
+from pipelex.core.pipes.pipe_input import PipeInput
+from pipelex.core.pipes.pipe_input_factory import PipeInputFactory
from pipelex.core.pipes.pipe_output import PipeOutput
from pipelex.core.pipes.pipe_run_params import PipeRunMode, PipeRunParams
from pipelex.core.pipes.pipe_run_params_factory import PipeRunParamsFactory
@@ -89,8 +89,8 @@ class PipeCompose(PipeOperator[PipeComposeOutput]):
log.debug(f"Validated jinja2 template '{self.jinja2_name}':\n{the_template}")
@override
- def needed_inputs(self, visited_pipes: set[str] | None = None) -> PipeInputSpec:
- needed_inputs = PipeInputSpecFactory.make_empty()
+ def needed_inputs(self, visited_pipes: set[str] | None = None) -> PipeInput:
+ needed_inputs = PipeInputFactory.make_empty()
for input_name, requirement in self.inputs.root.items():
needed_inputs.add_requirement(variable_name=input_name, concept=requirement.concept)
return needed_inputs
diff --git a/pipelex/pipe_operators/compose/pipe_compose_factory.py b/pipelex/pipe_operators/compose/pipe_compose_factory.py
index dc526d3a4..cd6b10999 100644
--- a/pipelex/pipe_operators/compose/pipe_compose_factory.py
+++ b/pipelex/pipe_operators/compose/pipe_compose_factory.py
@@ -2,8 +2,8 @@ from typing_extensions import override
from pipelex.core.concepts.concept_factory import ConceptFactory
from pipelex.core.pipes.pipe_factory import PipeFactoryProtocol
-from pipelex.core.pipes.pipe_input import PipeInputSpec
-from pipelex.core.pipes.pipe_input_factory import PipeInputSpecFactory
+from pipelex.core.pipes.pipe_input import PipeInput
+from pipelex.core.pipes.pipe_input_factory import PipeInputFactory
from pipelex.exceptions import PipeDefinitionError
from pipelex.hub import get_concept_provider
from pipelex.pipe_operators.compose.pipe_compose import PipeCompose
@@ -42,7 +42,7 @@ class PipeComposeFactory(PipeFactoryProtocol[PipeComposeBlueprint, PipeCompose])
domain=domain,
code=pipe_code,
description=blueprint.description,
- inputs=PipeInputSpecFactory.make_from_blueprint(
+ inputs=PipeInputFactory.make_from_blueprint(
domain=domain,
blueprint=blueprint.inputs or {},
concept_codes_from_the_same_domain=concept_codes_from_the_same_domain,
@@ -64,7 +64,7 @@ class PipeComposeFactory(PipeFactoryProtocol[PipeComposeBlueprint, PipeCompose])
def make_pipe_compose_from_template_str(
cls,
domain: str,
- inputs: PipeInputSpec | None = None,
+ inputs: PipeInput | None = None,
template_str: str | None = None,
template_name: str | None = None,
) -> PipeCompose:
@@ -78,14 +78,14 @@ class PipeComposeFactory(PipeFactoryProtocol[PipeComposeBlueprint, PipeCompose])
domain=domain,
code="adhoc_pipe_compose_from_template_str",
jinja2=preprocessed_template,
- inputs=inputs or PipeInputSpecFactory.make_empty(),
+ inputs=inputs or PipeInputFactory.make_empty(),
)
elif template_name:
return PipeCompose(
domain=domain,
code="adhoc_pipe_compose_from_template_name",
jinja2_name=template_name,
- inputs=inputs or PipeInputSpecFactory.make_empty(),
+ inputs=inputs or PipeInputFactory.make_empty(),
)
else:
msg = "Could not make a PipeCompose because neither template_str nor template_name were provided"
diff --git a/pipelex/pipe_operators/func/pipe_func.py b/pipelex/pipe_operators/func/pipe_func.py
index 39b4b3eda..4f58533fd 100644
--- a/pipelex/pipe_operators/func/pipe_func.py
+++ b/pipelex/pipe_operators/func/pipe_func.py
@@ -7,7 +7,7 @@ from pipelex import log
from pipelex.core.concepts.concept_factory import ConceptFactory
from pipelex.core.memory.working_memory import WorkingMemory
from pipelex.core.memory.working_memory_factory import WorkingMemoryFactory
-from pipelex.core.pipes.pipe_input import PipeInputSpec, TypedNamedInputRequirement
+from pipelex.core.pipes.pipe_input import PipeInput, TypedNamedInputRequirement
from pipelex.core.pipes.pipe_output import PipeOutput
from pipelex.core.pipes.pipe_run_params import PipeRunParams
from pipelex.core.stuffs.stuff_content import ListContent, StuffContent, TextContent
@@ -31,7 +31,7 @@ class PipeFunc(PipeOperator[PipeFuncOutput]):
return set()
@override
- def needed_inputs(self, visited_pipes: set[str] | None = None) -> PipeInputSpec:
+ def needed_inputs(self, visited_pipes: set[str] | None = None) -> PipeInput:
return self.inputs
@override
diff --git a/pipelex/pipe_operators/func/pipe_func_factory.py b/pipelex/pipe_operators/func/pipe_func_factory.py
index 515af5eac..e47322791 100644
--- a/pipelex/pipe_operators/func/pipe_func_factory.py
+++ b/pipelex/pipe_operators/func/pipe_func_factory.py
@@ -2,7 +2,7 @@ from typing_extensions import override
from pipelex.core.concepts.concept_factory import ConceptFactory
from pipelex.core.pipes.pipe_factory import PipeFactoryProtocol
-from pipelex.core.pipes.pipe_input_factory import PipeInputSpecFactory
+from pipelex.core.pipes.pipe_input_factory import PipeInputFactory
from pipelex.hub import get_concept_provider
from pipelex.pipe_operators.func.pipe_func import PipeFunc
from pipelex.pipe_operators.func.pipe_func_blueprint import PipeFuncBlueprint
@@ -27,7 +27,7 @@ class PipeFuncFactory(PipeFactoryProtocol[PipeFuncBlueprint, PipeFunc]):
domain=domain,
code=pipe_code,
description=blueprint.description,
- inputs=PipeInputSpecFactory.make_from_blueprint(
+ inputs=PipeInputFactory.make_from_blueprint(
domain=domain,
blueprint=blueprint.inputs or {},
concept_codes_from_the_same_domain=concept_codes_from_the_same_domain,
diff --git a/pipelex/pipe_operators/img_gen/pipe_img_gen.py b/pipelex/pipe_operators/img_gen/pipe_img_gen.py
index d4d3684c8..0424b79b7 100644
--- a/pipelex/pipe_operators/img_gen/pipe_img_gen.py
+++ b/pipelex/pipe_operators/img_gen/pipe_img_gen.py
@@ -14,8 +14,8 @@ from pipelex.config import StaticValidationReaction, get_config
from pipelex.core.concepts.concept_factory import ConceptFactory
from pipelex.core.concepts.concept_native import NATIVE_CONCEPTS_DATA, NativeConceptEnum
from pipelex.core.memory.working_memory import WorkingMemory
-from pipelex.core.pipes.pipe_input import PipeInputSpec
-from pipelex.core.pipes.pipe_input_factory import PipeInputSpecFactory
+from pipelex.core.pipes.pipe_input import PipeInput
+from pipelex.core.pipes.pipe_input_factory import PipeInputFactory
from pipelex.core.pipes.pipe_output import PipeOutput
from pipelex.core.pipes.pipe_run_params import PipeOutputMultiplicity, PipeRunMode, PipeRunParams, output_multiplicity_to_apply
from pipelex.core.pipes.pipe_run_params_factory import PipeRunParamsFactory
@@ -30,7 +30,7 @@ from pipelex.exceptions import (
UnexpectedPipeDefinitionError,
WorkingMemoryStuffNotFoundError,
)
-from pipelex.hub import get_concept_provider, get_content_generator, get_model_deck
+from pipelex.hub import get_concept_provider, get_content_generator, get_model_deck, get_native_concept
from pipelex.pipe_operators.pipe_operator import PipeOperator
from pipelex.pipeline.job_metadata import JobMetadata
from pipelex.types import Self
@@ -102,7 +102,7 @@ class PipeImgGen(PipeOperator[PipeImgGenOutput]):
def validate_output(self):
if not get_concept_provider().is_compatible(
tested_concept=self.output,
- wanted_concept=get_concept_provider().get_native_concept(native_concept=NativeConceptEnum.IMAGE),
+ wanted_concept=get_native_concept(native_concept=NativeConceptEnum.IMAGE),
strict=True,
):
msg = (
@@ -112,8 +112,8 @@ class PipeImgGen(PipeOperator[PipeImgGenOutput]):
raise PipeDefinitionError(msg)
@override
- def needed_inputs(self, visited_pipes: set[str] | None = None) -> PipeInputSpec:
- needed_inputs = PipeInputSpecFactory.make_empty()
+ def needed_inputs(self, visited_pipes: set[str] | None = None) -> PipeInput:
+ needed_inputs = PipeInputFactory.make_empty()
if self.img_gen_prompt:
needed_inputs.add_requirement(
variable_name="img_gen_prompt",
diff --git a/pipelex/pipe_operators/img_gen/pipe_img_gen_factory.py b/pipelex/pipe_operators/img_gen/pipe_img_gen_factory.py
index 847a90d10..1896b1a0f 100644
--- a/pipelex/pipe_operators/img_gen/pipe_img_gen_factory.py
+++ b/pipelex/pipe_operators/img_gen/pipe_img_gen_factory.py
@@ -2,7 +2,7 @@ from typing_extensions import override
from pipelex.core.concepts.concept_factory import ConceptFactory
from pipelex.core.pipes.pipe_factory import PipeFactoryProtocol
-from pipelex.core.pipes.pipe_input_factory import PipeInputSpecFactory
+from pipelex.core.pipes.pipe_input_factory import PipeInputFactory
from pipelex.hub import get_concept_provider
from pipelex.pipe_operators.img_gen.pipe_img_gen import PipeImgGen
from pipelex.pipe_operators.img_gen.pipe_img_gen_blueprint import PipeImgGenBlueprint
@@ -27,7 +27,7 @@ class PipeImgGenFactory(PipeFactoryProtocol[PipeImgGenBlueprint, PipeImgGen]):
domain=domain,
code=pipe_code,
description=blueprint.description,
- inputs=PipeInputSpecFactory.make_from_blueprint(
+ inputs=PipeInputFactory.make_from_blueprint(
domain=domain,
blueprint=blueprint.inputs or {},
concept_codes_from_the_same_domain=concept_codes_from_the_same_domain,
diff --git a/pipelex/pipe_operators/llm/pipe_llm.py b/pipelex/pipe_operators/llm/pipe_llm.py
index b6773d2f9..8bffea50d 100644
--- a/pipelex/pipe_operators/llm/pipe_llm.py
+++ b/pipelex/pipe_operators/llm/pipe_llm.py
@@ -17,8 +17,8 @@ from pipelex.core.concepts.concept_factory import ConceptFactory
from pipelex.core.concepts.concept_native import NativeConceptEnum
from pipelex.core.domains.domain import Domain, SpecialDomain
from pipelex.core.memory.working_memory import WorkingMemory
-from pipelex.core.pipes.pipe_input import PipeInputSpec
-from pipelex.core.pipes.pipe_input_factory import PipeInputSpecFactory
+from pipelex.core.pipes.pipe_input import PipeInput
+from pipelex.core.pipes.pipe_input_factory import PipeInputFactory
from pipelex.core.pipes.pipe_output import PipeOutput
from pipelex.core.pipes.pipe_run_params import (
PipeOutputMultiplicity,
@@ -26,12 +26,10 @@ from pipelex.core.pipes.pipe_run_params import (
PipeRunParams,
output_multiplicity_to_apply,
)
-from pipelex.core.stuffs.stuff_content import ListContent, StructuredContent, StuffContent, TextContent
+from pipelex.core.stuffs.stuff_content import ListContent, StuffContent, TextContent
from pipelex.core.stuffs.stuff_factory import StuffFactory
from pipelex.exceptions import (
PipeDefinitionError,
- PipeInputError,
- PipeInputNotFoundError,
StaticValidationError,
StaticValidationErrorType,
)
@@ -40,6 +38,7 @@ from pipelex.hub import (
get_concept_provider,
get_content_generator,
get_model_deck,
+ get_native_concept,
get_optional_pipe,
get_required_concept,
get_required_domain,
@@ -68,9 +67,8 @@ class PipeLLM(PipeOperator[PipeLLMOutput]):
output_multiplicity: PipeOutputMultiplicity | None = None
@model_validator(mode="after")
- def validate_inputs(self) -> Self:
- self._validate_required_variables()
- self._validate_inputs()
+ def _validate_inputs(self) -> Self:
+ self.validate_inputs()
return self
@model_validator(mode="after")
@@ -85,7 +83,7 @@ class PipeLLM(PipeOperator[PipeLLMOutput]):
@override
def validate_with_libraries(self):
- self._validate_inputs()
+ self.validate_inputs()
self.llm_prompt_spec.validate_with_libraries()
if self.prompt_template_to_structure:
get_template(template_name=self.prompt_template_to_structure)
@@ -99,7 +97,7 @@ class PipeLLM(PipeOperator[PipeLLMOutput]):
def validate_output(self):
if get_concept_provider().is_compatible(
tested_concept=self.output,
- wanted_concept=get_concept_provider().get_native_concept(native_concept=NativeConceptEnum.IMAGE),
+ wanted_concept=get_native_concept(native_concept=NativeConceptEnum.IMAGE),
):
msg = (
f"The output of a LLM pipe cannot be compatible with the Image concept. In the "
@@ -108,55 +106,34 @@ class PipeLLM(PipeOperator[PipeLLMOutput]):
raise PipeDefinitionError(msg)
@override
- def needed_inputs(self, visited_pipes: set[str] | None = None) -> PipeInputSpec:
+ def needed_inputs(self, visited_pipes: set[str] | None = None) -> PipeInput:
"""Needed inputs are the inputs needed to run the pipe, specified in the inputs attribute of the pipe"""
- # The images are not tagged in the prompt_template.
- # Therefore if an image is provided in the inputs, it becomes a needed input.
- needed_inputs = PipeInputSpecFactory.make_empty()
- concept_provider = get_concept_provider()
+ needed_inputs = PipeInputFactory.make_empty()
for input_name, requirement in self.inputs.items:
- if concept_provider.is_image_concept(concept=requirement.concept):
- needed_inputs.add_requirement(
- variable_name=input_name,
- concept=concept_provider.get_native_concept(native_concept=NativeConceptEnum.IMAGE),
- )
- else:
- needed_inputs.add_requirement(variable_name=input_name, concept=requirement.concept)
-
+ needed_inputs.add_requirement(variable_name=input_name, concept=requirement.concept)
return needed_inputs
@override
def required_variables(self) -> set[str]:
"""Required variables are the variables that are used in the current prompt template or system prompt"""
- required_variables: set[str] = set()
- required_variables.update(self.llm_prompt_spec.required_variables())
- return {variable_name for variable_name in required_variables if not variable_name.startswith("_")}
+ return {variable_name for variable_name in self.llm_prompt_spec.required_variables() if not variable_name.startswith("_")}
- def _validate_required_variables(self) -> Self:
- """This method checks that all required variables are in the inputs"""
- required_variables = self.required_variables()
- for required_variable_name in required_variables:
- if required_variable_name not in self.inputs.variables:
- msg = f"Required variable '{required_variable_name}' is not in the inputs of pipe {self.code}"
- raise PipeDefinitionError(msg)
- return self
-
- def _validate_inputs(self):
- concept_provider = get_concept_provider()
+ def validate_inputs(self):
static_validation_config = get_config().pipelex.static_validation_config
default_reaction = static_validation_config.default_reaction
reactions = static_validation_config.reactions
+ # Those are the variables required in the prompt template or system prompt
+ required_variables = self.required_variables()
- the_needed_inputs = self.needed_inputs()
- # check all required variables are in the inputs
- for named_input_requirement in the_needed_inputs.named_input_requirements:
- if named_input_requirement.variable_name not in self.inputs.variables:
+ # 1: Check that all the required variables are actually in the inputs
+ for required_variable_name in required_variables:
+ if required_variable_name not in self.needed_inputs().variables:
missing_input_var_error = StaticValidationError(
error_type=StaticValidationErrorType.MISSING_INPUT_VARIABLE,
domain=self.domain,
pipe_code=self.code,
- variable_names=[named_input_requirement.variable_name],
+ variable_names=[required_variable_name],
)
match reactions.get(StaticValidationErrorType.MISSING_INPUT_VARIABLE, default_reaction):
case StaticValidationReaction.IGNORE:
@@ -166,53 +143,9 @@ class PipeLLM(PipeOperator[PipeLLMOutput]):
case StaticValidationReaction.RAISE:
raise missing_input_var_error
- # There is one case where the needed input is of specific concept: the user_images
- if named_input_requirement.concept == concept_provider.get_native_concept(native_concept=NativeConceptEnum.IMAGE):
- try:
- input_requirement_of_declared_input = self.inputs.get_required_input_requirement(
- variable_name=named_input_requirement.variable_name,
- )
- except PipeInputNotFoundError as exc:
- msg = f"Input variable '{named_input_requirement.variable_name}' is not in this PipeLLM '{self.code}' input spec: {self.inputs}"
- raise PipeInputError(msg) from exc
- if not concept_provider.is_compatible(
- tested_concept=input_requirement_of_declared_input.concept,
- wanted_concept=named_input_requirement.concept,
- ):
- if named_input_requirement.variable_name != named_input_requirement.requirement_expression:
- # the required_input is a sub-attribute of the required variable
- # TODO: check that the sub-attribute is compatible with the concept code
- # let's check at least that the input is a structured concept
- input_concept = input_requirement_of_declared_input.concept
- input_concept_class_name = input_concept.structure_class_name
- input_concept_class = get_class_registry().get_required_subclass(name=input_concept_class_name, base_class=StuffContent)
- if issubclass(input_concept_class, StructuredContent):
- continue
- explanation = "The input provided for LLM Vision must be an image or a concept that refines image"
- if inadequate_concept := input_requirement_of_declared_input.concept:
- explanation += f",\nconcept = {inadequate_concept}"
- else:
- explanation += ",\nconcept not found"
-
- inadequate_input_concept_error = StaticValidationError(
- error_type=StaticValidationErrorType.INADEQUATE_INPUT_CONCEPT,
- domain=self.domain,
- pipe_code=self.code,
- variable_names=[named_input_requirement.variable_name],
- provided_concept_code=input_requirement_of_declared_input.concept.code,
- explanation=explanation,
- )
- match reactions.get(StaticValidationErrorType.INADEQUATE_INPUT_CONCEPT, default_reaction):
- case StaticValidationReaction.IGNORE:
- pass
- case StaticValidationReaction.LOG:
- log.error(inadequate_input_concept_error.desc())
- case StaticValidationReaction.RAISE:
- raise inadequate_input_concept_error
-
- # Check that all inputs are in the required variables
- for input_name in self.inputs.variables:
- if input_name not in the_needed_inputs.required_names:
+ # 2: Check that all inputs are in the required variables
+ for input_name in self.needed_inputs().variables:
+ if input_name not in required_variables:
extraneous_input_var_error = StaticValidationError(
error_type=StaticValidationErrorType.EXTRANEOUS_INPUT_VARIABLE,
domain=self.domain,
diff --git a/pipelex/pipe_operators/llm/pipe_llm_factory.py b/pipelex/pipe_operators/llm/pipe_llm_factory.py
index 6a16ee08b..6d0d9c012 100644
--- a/pipelex/pipe_operators/llm/pipe_llm_factory.py
+++ b/pipelex/pipe_operators/llm/pipe_llm_factory.py
@@ -2,13 +2,15 @@ from typing_extensions import override
from pipelex.cogt.llm.llm_prompt_spec import LLMPromptSpec
from pipelex.cogt.llm.llm_setting import LLMSettingChoices
+from pipelex.core.concepts.concept import Concept
from pipelex.core.concepts.concept_factory import ConceptFactory
+from pipelex.core.concepts.concept_native import NativeConceptEnum
from pipelex.core.pipes.pipe_factory import PipeFactoryProtocol
from pipelex.core.pipes.pipe_input_blueprint import InputRequirementBlueprint
-from pipelex.core.pipes.pipe_input_factory import PipeInputSpecFactory
+from pipelex.core.pipes.pipe_input_factory import PipeInputFactory
from pipelex.core.pipes.pipe_run_params import make_output_multiplicity
from pipelex.exceptions import PipeDefinitionError
-from pipelex.hub import get_concept_provider, get_optional_domain
+from pipelex.hub import get_concept_provider, get_native_concept, get_optional_domain
from pipelex.pipe_operators.llm.pipe_llm import PipeLLM
from pipelex.pipe_operators.llm.pipe_llm_blueprint import PipeLLMBlueprint
from pipelex.tools.templating.jinja2_blueprint import Jinja2Blueprint
@@ -91,8 +93,14 @@ class PipeLLMFactory(PipeFactoryProtocol[PipeLLMBlueprint, PipeLLM]):
),
)
- if get_concept_provider().is_image_concept(concept=concept):
+ if Concept.are_concept_compatible(concept_1=concept, concept_2=get_native_concept(NativeConceptEnum.IMAGE), strict=True):
user_images.append(stuff_name)
+ elif Concept.are_concept_compatible(concept_1=concept, concept_2=get_native_concept(NativeConceptEnum.IMAGE), strict=False):
+ # Get image field paths relative to the concept
+ image_field_paths = get_concept_provider().find_image_field_paths(concept=concept)
+ # Prefix each path with the stuff_name to make them absolute
+ for field_path in image_field_paths:
+ user_images.append(f"{stuff_name}.{field_path}")
llm_prompt_spec = LLMPromptSpec(
system_prompt_jinja2_blueprint=system_prompt_jinja2_blueprint,
@@ -126,7 +134,7 @@ class PipeLLMFactory(PipeFactoryProtocol[PipeLLMBlueprint, PipeLLM]):
domain=domain,
code=pipe_code,
description=blueprint.description,
- inputs=PipeInputSpecFactory.make_from_blueprint(
+ inputs=PipeInputFactory.make_from_blueprint(
domain=domain,
blueprint=blueprint.inputs or {},
concept_codes_from_the_same_domain=concept_codes_from_the_same_domain,
diff --git a/pipelex/pipe_operators/ocr/pipe_ocr.py b/pipelex/pipe_operators/ocr/pipe_ocr.py
index 9b2c94c47..14175c028 100644
--- a/pipelex/pipe_operators/ocr/pipe_ocr.py
+++ b/pipelex/pipe_operators/ocr/pipe_ocr.py
@@ -13,7 +13,7 @@ from pipelex.cogt.ocr.ocr_setting import OcrChoice, OcrSetting
from pipelex.config import StaticValidationReaction, get_config
from pipelex.core.concepts.concept_native import NativeConceptEnum
from pipelex.core.memory.working_memory import WorkingMemory
-from pipelex.core.pipes.pipe_input import PipeInputSpec
+from pipelex.core.pipes.pipe_input import PipeInput
from pipelex.core.pipes.pipe_output import PipeOutput
from pipelex.core.pipes.pipe_run_params import PipeRunMode, PipeRunParams
from pipelex.core.stuffs.stuff_content import ImageContent, ListContent, PageContent, TextAndImagesContent, TextContent
@@ -27,6 +27,7 @@ from pipelex.hub import (
get_concept_provider,
get_content_generator,
get_model_deck,
+ get_native_concept,
)
from pipelex.pipe_operators.pipe_operator import PipeOperator
from pipelex.pipeline.job_metadata import JobMetadata
@@ -66,7 +67,7 @@ class PipeOcr(PipeOperator[PipeOcrOutput]):
@override
def validate_output(self):
- if self.output != get_concept_provider().get_native_concept(native_concept=NativeConceptEnum.PAGE):
+ if self.output != get_native_concept(native_concept=NativeConceptEnum.PAGE):
msg = f"PipeOcr output should be a Page concept, but is {self.output.concept_string}"
raise PipeDefinitionError(msg)
@@ -114,13 +115,13 @@ class PipeOcr(PipeOperator[PipeOcrOutput]):
log.debug(f"Validating input '{input_name}' with concept code '{requirement.concept.code}'")
if concept_provider.is_compatible(
tested_concept=requirement.concept,
- wanted_concept=concept_provider.get_native_concept(native_concept=NativeConceptEnum.IMAGE),
+ wanted_concept=get_native_concept(native_concept=NativeConceptEnum.IMAGE),
strict=True,
):
self.image_stuff_name = input_name
elif concept_provider.is_compatible(
tested_concept=requirement.concept,
- wanted_concept=concept_provider.get_native_concept(native_concept=NativeConceptEnum.PDF),
+ wanted_concept=get_native_concept(native_concept=NativeConceptEnum.PDF),
strict=True,
):
self.pdf_stuff_name = input_name
@@ -142,7 +143,7 @@ class PipeOcr(PipeOperator[PipeOcrOutput]):
raise inadequate_input_concept_error
@override
- def needed_inputs(self, visited_pipes: set[str] | None = None) -> PipeInputSpec:
+ def needed_inputs(self, visited_pipes: set[str] | None = None) -> PipeInput:
return self.inputs
@override
diff --git a/pipelex/pipe_operators/ocr/pipe_ocr_factory.py b/pipelex/pipe_operators/ocr/pipe_ocr_factory.py
index 159fe9022..3531f9d4d 100644
--- a/pipelex/pipe_operators/ocr/pipe_ocr_factory.py
+++ b/pipelex/pipe_operators/ocr/pipe_ocr_factory.py
@@ -3,7 +3,7 @@ from typing_extensions import override
from pipelex.config import get_config
from pipelex.core.concepts.concept_factory import ConceptFactory
from pipelex.core.pipes.pipe_factory import PipeFactoryProtocol
-from pipelex.core.pipes.pipe_input_factory import PipeInputSpecFactory
+from pipelex.core.pipes.pipe_input_factory import PipeInputFactory
from pipelex.hub import get_concept_provider
from pipelex.pipe_operators.ocr.pipe_ocr import PipeOcr
from pipelex.pipe_operators.ocr.pipe_ocr_blueprint import PipeOcrBlueprint
@@ -35,7 +35,7 @@ class PipeOcrFactory(PipeFactoryProtocol[PipeOcrBlueprint, PipeOcr]):
concept_code=output_domain_and_code.concept_code,
),
),
- inputs=PipeInputSpecFactory.make_from_blueprint(
+ inputs=PipeInputFactory.make_from_blueprint(
domain=domain,
blueprint=blueprint.inputs or {},
concept_codes_from_the_same_domain=concept_codes_from_the_same_domain,
diff --git a/pipelex/pipe_works/pipe_dry.py b/pipelex/pipe_works/pipe_dry.py
index 45136ff76..9ce536a24 100644
--- a/pipelex/pipe_works/pipe_dry.py
+++ b/pipelex/pipe_works/pipe_dry.py
@@ -9,7 +9,7 @@ from pipelex import log
from pipelex.config import get_config
from pipelex.core.memory.working_memory_factory import WorkingMemoryFactory
from pipelex.core.pipes.pipe_abstract import PipeAbstract
-from pipelex.core.pipes.pipe_input import PipeInputSpec, TypedNamedInputRequirement
+from pipelex.core.pipes.pipe_input import PipeInput, TypedNamedInputRequirement
from pipelex.core.pipes.pipe_run_params import PipeRunMode
from pipelex.core.pipes.pipe_run_params_factory import PipeRunParamsFactory
from pipelex.core.stuffs.stuff_content import StuffContent, TextContent
@@ -136,11 +136,11 @@ async def dry_run_pipes(pipes: list[PipeAbstract], run_in_parallel: bool = True,
return results
-def _convert_to_working_memory_format(needed_inputs_spec: PipeInputSpec) -> list[TypedNamedInputRequirement]:
- """Convert PipeInputSpec to the format needed by WorkingMemoryFactory.make_for_dry_run.
+def _convert_to_working_memory_format(needed_inputs_spec: PipeInput) -> list[TypedNamedInputRequirement]:
+ """Convert PipeInput to the format needed by WorkingMemoryFactory.make_for_dry_run.
Args:
- needed_inputs_spec: PipeInputSpec with detailed_requirements
+ needed_inputs_spec: PipeInput with detailed_requirements
Returns:
List of tuples (variable_name, concept_code, structure_class)
diff --git a/pipelex/tools/templating/jinja2_rendering.py b/pipelex/tools/templating/jinja2_rendering.py
index fd2cc23cf..be00123aa 100644
--- a/pipelex/tools/templating/jinja2_rendering.py
+++ b/pipelex/tools/templating/jinja2_rendering.py
@@ -63,7 +63,6 @@ async def render_jinja2(
explanation = make_jinja2_error_explanation(jinja2_name=jinja2_name, template_text=jinja2)
msg = f"Jinja2 render error: '{exc}' {explanation}"
raise Jinja2RenderError(msg) from exc
-
template_source = preprocess_template(template_source)
check_jinja2_parsing(
jinja2_template_source=template_source,
diff --git a/pipelex/tools/templating/jinja2_required_variables.py b/pipelex/tools/templating/jinja2_required_variables.py
index e901a79d6..6de93c39c 100644
--- a/pipelex/tools/templating/jinja2_required_variables.py
+++ b/pipelex/tools/templating/jinja2_required_variables.py
@@ -44,7 +44,6 @@ def detect_jinja2_required_variables(
else:
msg = "No jinja2 or jinja2_name provided"
raise Jinja2StuffError(msg)
-
try:
parsed_ast = jinja2_env.parse(template_source)
undeclared_variables = meta.find_undeclared_variables(parsed_ast)
diff --git a/tests/integration/pipelex/cogt/test_llm_gen_object.py b/tests/integration/pipelex/cogt/test_llm_gen_object.py
index d8caafdb4..cef867563 100644
--- a/tests/integration/pipelex/cogt/test_llm_gen_object.py
+++ b/tests/integration/pipelex/cogt/test_llm_gen_object.py
@@ -4,8 +4,9 @@ import pytest
from pydantic import BaseModel
from pipelex import pretty_print
-from pipelex.cogt.llm.llm_job_components import LLMJobParams
+from pipelex.cogt.llm.llm_job_components import LLMJobConfig, LLMJobParams
from pipelex.cogt.llm.llm_job_factory import LLMJobFactory
+from pipelex.cogt.llm.llm_prompt import LLMPrompt
from pipelex.hub import get_llm_worker, get_model_deck
from tests.integration.pipelex.cogt.test_data import LLMTestCases
@@ -16,9 +17,15 @@ def get_async_worker_and_job(llm_preset_id: str, user_text: str):
pretty_print(user_text)
llm_worker = get_llm_worker(llm_handle=llm_setting.llm_handle)
llm_job_params = llm_setting.make_llm_job_params()
- llm_job = LLMJobFactory.make_llm_job_from_prompt_contents(
- user_text=user_text,
+ llm_job = LLMJobFactory.make_llm_job(
+ llm_prompt=LLMPrompt(
+ user_text=user_text,
+ ),
llm_job_params=llm_job_params,
+ llm_job_config=LLMJobConfig(
+ is_streaming_enabled=False,
+ max_retries=3,
+ ),
)
return llm_worker, llm_job
@@ -32,9 +39,15 @@ class TestLLMGenObject:
llm_worker = get_llm_worker(llm_handle=llm_handle)
if not llm_worker.is_gen_object_supported:
pytest.skip(f"LLM worker '{llm_worker.desc}' does not support object generation")
- llm_job = LLMJobFactory.make_llm_job_from_prompt_contents(
- user_text=user_text,
+ llm_job = LLMJobFactory.make_llm_job(
+ llm_prompt=LLMPrompt(
+ user_text=user_text,
+ ),
llm_job_params=llm_job_params,
+ llm_job_config=LLMJobConfig(
+ is_streaming_enabled=False,
+ max_retries=3,
+ ),
)
expected_class = expected_instance.__class__
output = await llm_worker.gen_object(llm_job=llm_job, schema=expected_class)
@@ -67,9 +80,15 @@ class TestLLMGenObject:
for case_tuple in case_tuples:
user_text, expected_instance = case_tuple
expected_class = expected_instance.__class__
- llm_job = LLMJobFactory.make_llm_job_from_prompt_contents(
- user_text=user_text,
+ llm_job = LLMJobFactory.make_llm_job(
+ llm_prompt=LLMPrompt(
+ user_text=user_text,
+ ),
llm_job_params=llm_job_params,
+ llm_job_config=LLMJobConfig(
+ is_streaming_enabled=False,
+ max_retries=3,
+ ),
)
task: asyncio.Task[BaseModel] = asyncio.create_task(llm_worker.gen_object(llm_job=llm_job, schema=expected_class))
tasks.append(task)
@@ -91,9 +110,15 @@ class TestLLMGenObject:
for case_tuple in case_tuples:
user_text, expected_instance = case_tuple
expected_class = expected_instance.__class__
- llm_job = LLMJobFactory.make_llm_job_from_prompt_contents(
- user_text=user_text,
+ llm_job = LLMJobFactory.make_llm_job(
+ llm_prompt=LLMPrompt(
+ user_text=user_text,
+ ),
llm_job_params=llm_job.job_params,
+ llm_job_config=LLMJobConfig(
+ is_streaming_enabled=False,
+ max_retries=3,
+ ),
)
task: asyncio.Task[BaseModel] = asyncio.create_task(llm_worker.gen_object(llm_job=llm_job, schema=expected_class))
tasks.append(task)
diff --git a/tests/integration/pipelex/cogt/test_llm_gen_text.py b/tests/integration/pipelex/cogt/test_llm_gen_text.py
index 63b8a91ba..5905bbf6f 100644
--- a/tests/integration/pipelex/cogt/test_llm_gen_text.py
+++ b/tests/integration/pipelex/cogt/test_llm_gen_text.py
@@ -4,8 +4,9 @@ import pytest
from pipelex import log, pretty_print
from pipelex.cogt.llm.llm_job import LLMJob
-from pipelex.cogt.llm.llm_job_components import LLMJobParams
+from pipelex.cogt.llm.llm_job_components import LLMJobConfig, LLMJobParams
from pipelex.cogt.llm.llm_job_factory import LLMJobFactory
+from pipelex.cogt.llm.llm_prompt import LLMPrompt
from pipelex.cogt.llm.llm_worker_abstract import LLMWorkerAbstract
from pipelex.cogt.llm.llm_worker_internal_abstract import LLMWorkerInternalAbstract
from pipelex.cogt.model_backends.model_constraints import ModelConstraints
@@ -19,9 +20,15 @@ def get_worker_and_job(llm_preset_id: str, user_text: str) -> tuple[LLMWorkerAbs
pretty_print(user_text)
llm_worker = get_llm_worker(llm_handle=llm_setting.llm_handle)
llm_job_params = llm_setting.make_llm_job_params()
- llm_job = LLMJobFactory.make_llm_job_from_prompt_contents(
- user_text=user_text,
+ llm_job = LLMJobFactory.make_llm_job(
+ llm_prompt=LLMPrompt(
+ user_text=user_text,
+ ),
llm_job_params=llm_job_params,
+ llm_job_config=LLMJobConfig(
+ is_streaming_enabled=False,
+ max_retries=3,
+ ),
)
return llm_worker, llm_job
@@ -34,9 +41,15 @@ class TestLLMGenText:
async def test_gen_text_using_handle(self, llm_job_params: LLMJobParams, llm_handle: str, topic: str, prompt_text: str):
pretty_print(prompt_text, title=f"Generating text about '{topic}' using '{llm_handle}'")
llm_worker = get_llm_worker(llm_handle=llm_handle)
- llm_job = LLMJobFactory.make_llm_job_from_prompt_contents(
- user_text=prompt_text,
+ llm_job = LLMJobFactory.make_llm_job(
+ llm_prompt=LLMPrompt(
+ user_text=prompt_text,
+ ),
llm_job_params=llm_job_params,
+ llm_job_config=LLMJobConfig(
+ is_streaming_enabled=False,
+ max_retries=3,
+ ),
)
generated_text = await llm_worker.gen_text(llm_job=llm_job)
assert generated_text
diff --git a/tests/integration/pipelex/cogt/test_llm_inference.py b/tests/integration/pipelex/cogt/test_llm_inference.py
index ae96d2c91..d89a79c6f 100644
--- a/tests/integration/pipelex/cogt/test_llm_inference.py
+++ b/tests/integration/pipelex/cogt/test_llm_inference.py
@@ -5,6 +5,7 @@ from pipelex.cogt.exceptions import LLMCapabilityError, PromptImageFormatError
from pipelex.cogt.image.prompt_image import PromptImagePath
from pipelex.cogt.llm.llm_job_components import LLMJobParams
from pipelex.cogt.llm.llm_job_factory import LLMJobFactory
+from pipelex.cogt.llm.llm_prompt import LLMPrompt
from pipelex.hub import get_inference_manager
from tests.integration.pipelex.cogt.test_data import LLMTestConstants, LLMVisionTestCases, Person
@@ -17,9 +18,11 @@ class TestLLMInference:
log.info(f"test_simple_gen_text_from_text: Testing llm_handle '{llm_handle}'")
llm_worker = get_inference_manager().get_llm_worker(llm_handle=llm_handle)
log.info(f"Using llm_worker: {llm_worker.desc}")
- llm_job = LLMJobFactory.make_llm_job_from_prompt_contents(
- system_text=None,
- user_text=LLMTestConstants.USER_TEXT_SHORT,
+ llm_job = LLMJobFactory.make_llm_job(
+ llm_prompt=LLMPrompt(
+ system_text=None,
+ user_text=LLMTestConstants.USER_TEXT_SHORT,
+ ),
llm_job_params=llm_job_params,
)
generated_text = await llm_worker.gen_text(llm_job=llm_job)
@@ -30,9 +33,11 @@ class TestLLMInference:
log.info(f"test_simple_gen_object_from_text: Testing llm_handle '{llm_handle}'")
llm_worker = get_inference_manager().get_llm_worker(llm_handle=llm_handle)
log.info(f"Using llm_worker: {llm_worker.desc}")
- llm_job = LLMJobFactory.make_llm_job_from_prompt_contents(
- system_text=None,
- user_text=LLMTestConstants.USER_TEXT_SHORT,
+ llm_job = LLMJobFactory.make_llm_job(
+ llm_prompt=LLMPrompt(
+ system_text=None,
+ user_text=LLMTestConstants.USER_TEXT_SHORT,
+ ),
llm_job_params=llm_job_params,
)
if not llm_worker.is_gen_object_supported:
@@ -46,9 +51,11 @@ class TestLLMInference:
log.info(f"test_gen_text_from_image: Testing llm_handle '{llm_handle}'")
prompt_image = PromptImagePath(file_path=image_path)
llm_worker = get_inference_manager().get_llm_worker(llm_handle=llm_handle)
- llm_job = LLMJobFactory.make_llm_job_from_prompt_contents(
- user_text=LLMVisionTestCases.VISION_USER_TEXT_2,
- user_images=[prompt_image],
+ llm_job = LLMJobFactory.make_llm_job(
+ llm_prompt=LLMPrompt(
+ user_text=LLMVisionTestCases.VISION_USER_TEXT_2,
+ user_images=[prompt_image],
+ ),
llm_job_params=llm_job_params,
)
try:
diff --git a/tests/integration/pipelex/cogt/test_llm_report.py b/tests/integration/pipelex/cogt/test_llm_report.py
index 49fe562e5..2f26b0c66 100644
--- a/tests/integration/pipelex/cogt/test_llm_report.py
+++ b/tests/integration/pipelex/cogt/test_llm_report.py
@@ -3,7 +3,9 @@ import asyncio
import pytest
from pipelex import pretty_print
+from pipelex.cogt.llm.llm_job_components import LLMJobConfig
from pipelex.cogt.llm.llm_job_factory import LLMJobFactory
+from pipelex.cogt.llm.llm_prompt import LLMPrompt
from pipelex.hub import get_llm_worker, get_model_deck, get_report_delegate
from tests.integration.pipelex.cogt.test_data import LLMTestCases
@@ -30,8 +32,14 @@ class TestLLMReport:
pretty_print(prompt_text)
llm_worker = get_llm_worker(llm_handle=llm_setting.llm_handle)
llm_job_params = llm_setting.make_llm_job_params()
- llm_job = LLMJobFactory.make_llm_job_from_prompt_contents(
- user_text=prompt_text,
+ llm_job = LLMJobFactory.make_llm_job(
+ llm_prompt=LLMPrompt(
+ user_text=prompt_text,
+ ),
+ llm_job_config=LLMJobConfig(
+ is_streaming_enabled=False,
+ max_retries=3,
+ ),
llm_job_params=llm_job_params,
)
return llm_worker, llm_job
diff --git a/tests/integration/pipelex/cogt/test_llm_vision.py b/tests/integration/pipelex/cogt/test_llm_vision.py
index 43d60db4c..d4c8ea51c 100644
--- a/tests/integration/pipelex/cogt/test_llm_vision.py
+++ b/tests/integration/pipelex/cogt/test_llm_vision.py
@@ -4,8 +4,9 @@ from pipelex import log, pretty_print
from pipelex.cogt.exceptions import LLMCapabilityError, PromptImageFormatError
from pipelex.cogt.image.prompt_image import PromptImageBase64, PromptImagePath
from pipelex.cogt.image.prompt_image_factory import PromptImageFactory
-from pipelex.cogt.llm.llm_job import LLMJobParams
+from pipelex.cogt.llm.llm_job_components import LLMJobParams
from pipelex.cogt.llm.llm_job_factory import LLMJobFactory
+from pipelex.cogt.llm.llm_prompt import LLMPrompt
from pipelex.hub import get_llm_worker
from pipelex.tools.misc.base_64_utils import load_binary_as_base64
from tests.integration.pipelex.cogt.test_data import LLMVisionTestCases
@@ -20,11 +21,14 @@ class TestLLMVision:
prompt_image = PromptImageFactory.make_prompt_image_from_uri(uri=image_uri)
llm_worker = get_llm_worker(llm_handle=llm_handle_for_vision)
log.info(f"Using llm_worker: {llm_worker.desc}")
- llm_job = LLMJobFactory.make_llm_job_from_prompt_contents(
- user_text=LLMVisionTestCases.VISION_USER_TEXT_2,
- user_images=[prompt_image],
+ llm_job = LLMJobFactory.make_llm_job(
+ llm_prompt=LLMPrompt(
+ user_text=LLMVisionTestCases.VISION_USER_TEXT_2,
+ user_images=[prompt_image],
+ ),
llm_job_params=LLMJobParams(temperature=0.5, max_tokens=1000, seed=None),
)
+
try:
generated_text = await llm_worker.gen_text(llm_job=llm_job)
assert generated_text
@@ -39,9 +43,11 @@ class TestLLMVision:
image_bytes = load_binary_as_base64(path=image_path)
prompt_image = PromptImageBase64(base_64=image_bytes)
llm_worker = get_llm_worker(llm_handle=llm_handle_for_vision)
- llm_job = LLMJobFactory.make_llm_job_from_prompt_contents(
- user_text=LLMVisionTestCases.VISION_USER_TEXT_2,
- user_images=[prompt_image],
+ llm_job = LLMJobFactory.make_llm_job(
+ llm_prompt=LLMPrompt(
+ user_text=LLMVisionTestCases.VISION_USER_TEXT_2,
+ user_images=[prompt_image],
+ ),
llm_job_params=LLMJobParams(temperature=0.5, max_tokens=1000, seed=None),
)
try:
@@ -57,9 +63,11 @@ class TestLLMVision:
async def test_gen_text_from_vision_by_path(self, llm_handle_for_vision: str, topic: str, image_path: str):
prompt_image = PromptImagePath(file_path=image_path)
llm_worker = get_llm_worker(llm_handle=llm_handle_for_vision)
- llm_job = LLMJobFactory.make_llm_job_from_prompt_contents(
- user_text=LLMVisionTestCases.VISION_USER_TEXT_2,
- user_images=[prompt_image],
+ llm_job = LLMJobFactory.make_llm_job(
+ llm_prompt=LLMPrompt(
+ user_text=LLMVisionTestCases.VISION_USER_TEXT_2,
+ user_images=[prompt_image],
+ ),
llm_job_params=LLMJobParams(temperature=0.5, max_tokens=1000, seed=None),
)
try:
@@ -76,9 +84,11 @@ class TestLLMVision:
prompt_image1 = PromptImagePath(file_path=image_pair[0])
prompt_image2 = PromptImagePath(file_path=image_pair[1])
llm_worker = get_llm_worker(llm_handle=llm_handle_for_vision)
- llm_job = LLMJobFactory.make_llm_job_from_prompt_contents(
- user_text=LLMVisionTestCases.VISION_IMAGES_COMPARE_PROMPT,
- user_images=[prompt_image1, prompt_image2],
+ llm_job = LLMJobFactory.make_llm_job(
+ llm_prompt=LLMPrompt(
+ user_text=LLMVisionTestCases.VISION_IMAGES_COMPARE_PROMPT,
+ user_images=[prompt_image1, prompt_image2],
+ ),
llm_job_params=LLMJobParams(temperature=0.5, max_tokens=2000, seed=None),
)
try:
diff --git a/tests/integration/pipelex/libraries/pipelines/builder/concept/test_concept_spec_generation.py b/tests/integration/pipelex/libraries/pipelines/builder/concept/test_concept_spec_generation.py
index a72c37a25..d2c538e4e 100644
--- a/tests/integration/pipelex/libraries/pipelines/builder/concept/test_concept_spec_generation.py
+++ b/tests/integration/pipelex/libraries/pipelines/builder/concept/test_concept_spec_generation.py
@@ -1,8 +1,9 @@
import pytest
from pipelex import log, pretty_print
-from pipelex.cogt.llm.llm_job_components import LLMJobParams
+from pipelex.cogt.llm.llm_job_components import LLMJobConfig, LLMJobParams
from pipelex.cogt.llm.llm_job_factory import LLMJobFactory
+from pipelex.cogt.llm.llm_prompt import LLMPrompt
from pipelex.hub import get_llm_worker
from pipelex.libraries.pipelines.builder.concept.concept_spec import ConceptSpec
from tests.integration.pipelex.libraries.pipelines.builder.concept.integration_test_data import ConceptSpecGenerationTestCases
@@ -38,9 +39,15 @@ class TestConceptSpecGeneration:
log.info(f"Using llm_worker: {llm_worker.desc}")
# Create the LLM job with the prompt
- llm_job = LLMJobFactory.make_llm_job_from_prompt_contents(
- system_text="You are an expert at generating concept specifications for a data modeling system.",
- user_text=user_prompt,
+ llm_job = LLMJobFactory.make_llm_job(
+ llm_prompt=LLMPrompt(
+ system_text="You are an expert at generating concept specifications for a data modeling system.",
+ user_text=user_prompt,
+ ),
+ llm_job_config=LLMJobConfig(
+ is_streaming_enabled=False,
+ max_retries=3,
+ ),
llm_job_params=llm_job_params,
)
diff --git a/tests/integration/pipelex/pipes/controller/pipe_parallel/test_pipe_parallel_validation.py b/tests/integration/pipelex/pipes/controller/pipe_parallel/test_pipe_parallel_validation.py
index 1fe054c60..1d06ef249 100644
--- a/tests/integration/pipelex/pipes/controller/pipe_parallel/test_pipe_parallel_validation.py
+++ b/tests/integration/pipelex/pipes/controller/pipe_parallel/test_pipe_parallel_validation.py
@@ -1,6 +1,6 @@
from pipelex.core.concepts.concept_blueprint import ConceptBlueprint
from pipelex.core.concepts.concept_factory import ConceptFactory
-from pipelex.core.pipes.pipe_input import PipeInputSpec
+from pipelex.core.pipes.pipe_input import PipeInput
from pipelex.core.pipes.pipe_input_blueprint import InputRequirementBlueprint
from pipelex.hub import get_concept_provider, get_pipe_provider
from pipelex.pipe_controllers.parallel.pipe_parallel_blueprint import PipeParallelBlueprint
@@ -194,8 +194,8 @@ class TestPipeParallelValidation:
# Test that needed_inputs method can be called
needed_inputs = pipe_parallel.needed_inputs()
- # Verify it returns a PipeInputSpec object
- assert isinstance(needed_inputs, PipeInputSpec)
+ # Verify it returns a PipeInput object
+ assert isinstance(needed_inputs, PipeInput)
assert hasattr(needed_inputs, "root")
assert isinstance(needed_inputs.root, dict)
# With no sub-pipes, should return empty inputs
diff --git a/tests/integration/pipelex/pipes/controller/pipe_sequence/test_pipe_sequence_simple.py b/tests/integration/pipelex/pipes/controller/pipe_sequence/test_pipe_sequence_simple.py
index a475280ef..220706667 100644
--- a/tests/integration/pipelex/pipes/controller/pipe_sequence/test_pipe_sequence_simple.py
+++ b/tests/integration/pipelex/pipes/controller/pipe_sequence/test_pipe_sequence_simple.py
@@ -15,7 +15,7 @@ from pipelex.core.pipes.pipe_run_params import PipeRunMode
from pipelex.core.pipes.pipe_run_params_factory import PipeRunParamsFactory
from pipelex.core.stuffs.stuff_content import TextContent
from pipelex.core.stuffs.stuff_factory import StuffFactory
-from pipelex.hub import get_concept_provider
+from pipelex.hub import get_concept_provider, get_native_concept
from pipelex.pipe_controllers.sequence.pipe_sequence_blueprint import PipeSequenceBlueprint
from pipelex.pipe_controllers.sequence.pipe_sequence_factory import PipeSequenceFactory
from pipelex.pipe_controllers.sub_pipe_blueprint import SubPipeBlueprint
@@ -39,7 +39,7 @@ class TestPipeSequenceSimple:
)
concept_library = get_concept_provider()
concept_library.add_concepts([concept_1])
- concept_2 = concept_library.get_native_concept(native_concept=NativeConceptEnum.TEXT)
+ concept_2 = get_native_concept(native_concept=NativeConceptEnum.TEXT)
# Create PipeSequence instance - pipes are loaded from PLX files
pipe_sequence_blueprint = PipeSequenceBlueprint(
diff --git a/tests/integration/pipelex/pipes/test_image_inputs.py b/tests/integration/pipelex/pipes/test_image_inputs.py
index ed5e18879..741a84f39 100644
--- a/tests/integration/pipelex/pipes/test_image_inputs.py
+++ b/tests/integration/pipelex/pipes/test_image_inputs.py
@@ -9,6 +9,8 @@ from pipelex.core.pipes.pipe_run_params_factory import PipeRunParamsFactory
from pipelex.core.stuffs.stuff_content import ImageContent, PageContent, TextAndImagesContent, TextContent
from pipelex.core.stuffs.stuff_factory import StuffFactory
from pipelex.hub import get_pipe_router, get_required_pipe
+from pipelex.pipe_operators.llm.pipe_llm_blueprint import PipeLLMBlueprint
+from pipelex.pipe_operators.llm.pipe_llm_factory import PipeLLMFactory
from pipelex.pipe_works.pipe_job_factory import PipeJobFactory
from pipelex.pipeline.job_metadata import JobMetadata
from tests.cases import ImageTestCases
@@ -93,3 +95,23 @@ class TestImageInputs:
assert pipe_output is not None
assert pipe_output.working_memory is not None
assert pipe_output.main_stuff is not None
+
+ @pytest.mark.usefixtures("request")
+ async def test_image_input_within_concept_with_text(self) -> None:
+ """Test that a pipe can accept a PageContent input, give to the LLM the image via subattributes,
+ But also accepts basic objects
+ """
+ pipe_llm_blueprint = PipeLLMBlueprint(
+ description="Test that a pipe can accept a PageContent input, give to the LLM the image via subattributes",
+ inputs={"page": "Page"},
+ output="Text",
+ prompt_template="Describe the page: @page",
+ )
+
+ pipe_llm = PipeLLMFactory.make_from_blueprint(
+ domain="test_pipes",
+ pipe_code="test_image_input_within_concept_with_text",
+ blueprint=pipe_llm_blueprint,
+ )
+
+ assert pipe_llm.llm_prompt_spec.user_images == ["page.page_view"]
diff --git a/tests/integration/pipelex/plugins/test_external_plugin.py b/tests/integration/pipelex/plugins/test_external_plugin.py
index 8e5b62047..72c909e31 100644
--- a/tests/integration/pipelex/plugins/test_external_plugin.py
+++ b/tests/integration/pipelex/plugins/test_external_plugin.py
@@ -6,6 +6,7 @@ from pipelex import log, pretty_print
from pipelex.cogt.llm.llm_job import LLMJob
from pipelex.cogt.llm.llm_job_components import LLMJobParams
from pipelex.cogt.llm.llm_job_factory import LLMJobFactory
+from pipelex.cogt.llm.llm_prompt import LLMPrompt
from pipelex.cogt.llm.llm_setting import LLMSetting
from pipelex.cogt.llm.llm_worker_abstract import LLMWorkerAbstract
from pipelex.cogt.usage.token_category import NbTokensByCategoryDict, TokenCategory
@@ -61,9 +62,11 @@ class MockExternalLLMWorker(LLMWorkerAbstract):
class TestExternalPlugin:
async def test_external_llm_worker(self):
llm_worker = MockExternalLLMWorker(reporting_delegate=get_report_delegate())
- llm_job = LLMJobFactory.make_llm_job_from_prompt_contents(
- system_text=None,
- user_text=LLMTestConstants.USER_TEXT_SHORT,
+ llm_job = LLMJobFactory.make_llm_job(
+ llm_prompt=LLMPrompt(
+ system_text=None,
+ user_text=LLMTestConstants.USER_TEXT_SHORT,
+ ),
llm_job_params=LLMJobParams(
temperature=0.5,
max_tokens=None,
diff --git a/tests/test_pipelines/misc_tests/test_image_inputs.plx b/tests/test_pipelines/misc_tests/test_image_inputs.plx
index 800a5264f..7e2a53e04 100644
--- a/tests/test_pipelines/misc_tests/test_image_inputs.plx
+++ b/tests/test_pipelines/misc_tests/test_image_inputs.plx
@@ -15,6 +15,7 @@ You are an expert at describing page contents.
"""
prompt_template = """
Extract the full text (all the text that represents a "title") and the date of the article in the image.
+@image
"""
[pipe.describe_page]
@@ -28,6 +29,7 @@ You are an expert at describing page contents.
prompt_template = """
Extract the date and title of the article.
Also, add this as the description of the article:
$page.text_and_images.text.text
diff --git a/tests/test_pipelines/misc_tests/tests.plx b/tests/test_pipelines/misc_tests/tests.plx
index 7ce0effee..8d2b047e7 100644
--- a/tests/test_pipelines/misc_tests/tests.plx
+++ b/tests/test_pipelines/misc_tests/tests.plx
@@ -15,6 +15,7 @@ inputs = { image = "Image" }
output = "Text"
prompt_template = """
Describe the using rap lyrics, including puns and references to the image.
+@image
"""
[pipe.test_no_input]
diff --git a/tests/unit/pipelex/core/concepts/data.py b/tests/unit/pipelex/core/concepts/data.py
new file mode 100644
index 000000000..22a0656ff
--- /dev/null
+++ b/tests/unit/pipelex/core/concepts/data.py
@@ -0,0 +1,67 @@
+"""Test data and structures for stuff tests."""
+
+from typing import ClassVar
+
+from pipelex.core.stuffs.stuff_content import ImageContent, StructuredContent, TextContent
+
+
+# Test structures
+class ProfilePhoto(ImageContent):
+ """A profile photo that refines Image."""
+
+
+class PersonWithDirectImage(StructuredContent):
+ """A person with a direct image field."""
+
+ name: TextContent
+ photo: ImageContent
+
+
+class PersonWithRefinedImage(StructuredContent):
+ """A person with a refined image field."""
+
+ name: TextContent
+ profile_photo: ProfilePhoto
+
+
+class PersonWithText(StructuredContent):
+ """A person with only text, no images."""
+
+ name: TextContent
+ bio: TextContent
+
+
+class CompanyInfo(StructuredContent):
+ """Company info with nested person."""
+
+ company_name: TextContent
+ ceo: PersonWithDirectImage
+
+
+class NestedComplex(StructuredContent):
+ """Complex nested structure with multiple levels."""
+
+ title: TextContent
+ company: CompanyInfo
+ logo: ImageContent
+
+
+class PersonWithOptionalImage(StructuredContent):
+ """A person with an optional image field."""
+
+ name: TextContent
+ photo: ImageContent | None = None
+
+
+class TestData:
+ """Test data for find_image_field_paths tests."""
+
+ DOMAIN: ClassVar[str] = "test_images"
+
+ # Test content instances
+ SAMPLE_IMAGE: ClassVar[ImageContent] = ImageContent(url="https://example.com/photo.jpg", base_64="base64data")
+ SAMPLE_TEXT: ClassVar[TextContent] = TextContent(text="John Doe")
+ SAMPLE_BIO: ClassVar[TextContent] = TextContent(text="Software engineer")
+ COMPANY_NAME: ClassVar[TextContent] = TextContent(text="Tech Corp")
+ LOGO_IMAGE: ClassVar[ImageContent] = ImageContent(url="https://example.com/logo.png", base_64="logobase64")
+ TITLE_TEXT: ClassVar[TextContent] = TextContent(text="Company Profile")
diff --git a/tests/unit/pipelex/core/concepts/test_concept_find_image_field_paths.py b/tests/unit/pipelex/core/concepts/test_concept_find_image_field_paths.py
new file mode 100644
index 000000000..b6e2804e3
--- /dev/null
+++ b/tests/unit/pipelex/core/concepts/test_concept_find_image_field_paths.py
@@ -0,0 +1,199 @@
+import pytest
+
+from pipelex.core.concepts.concept import Concept # noqa: TC001
+from pipelex.core.concepts.concept_factory import ConceptFactory
+from pipelex.core.concepts.concept_native import NativeConceptEnum
+from pipelex.core.stuffs.stuff_content import StructuredContent
+from pipelex.hub import get_concept_provider, get_native_concept
+from pipelex.tools.class_registry_utils import ClassRegistryUtils
+from tests.unit.pipelex.core.concepts import data
+from tests.unit.pipelex.core.concepts.data import TestData
+
+
[email protected](scope="module", autouse=True)
+def register_test_concepts():
+ """Register test concepts for the module."""
+ concept_provider = get_concept_provider()
+
+ # Register the test structure classes
+ ClassRegistryUtils.register_classes_in_file(
+ file_path=data.__file__,
+ base_class=StructuredContent,
+ is_include_imported=False,
+ )
+
+ # Create and register concepts
+ concepts_to_register: list[Concept] = []
+
+ # ProfilePhoto concept that refines Image
+ profile_photo_concept = ConceptFactory.make(
+ domain=TestData.DOMAIN,
+ concept_code="ProfilePhoto",
+ description="A profile photo",
+ structure_class_name="ProfilePhoto",
+ refines=f"native.{NativeConceptEnum.IMAGE}",
+ )
+ concepts_to_register.append(profile_photo_concept)
+
+ # PersonWithDirectImage concept
+ person_direct_concept = ConceptFactory.make(
+ domain=TestData.DOMAIN,
+ concept_code="PersonWithDirectImage",
+ description="A person with a direct image field",
+ structure_class_name="PersonWithDirectImage",
+ )
+ concepts_to_register.append(person_direct_concept)
+
+ # PersonWithRefinedImage concept
+ person_refined_concept = ConceptFactory.make(
+ domain=TestData.DOMAIN,
+ concept_code="PersonWithRefinedImage",
+ description="A person with a refined image field",
+ structure_class_name="PersonWithRefinedImage",
+ )
+ concepts_to_register.append(person_refined_concept)
+
+ # PersonWithText concept
+ person_text_concept = ConceptFactory.make(
+ domain=TestData.DOMAIN,
+ concept_code="PersonWithText",
+ description="A person with only text",
+ structure_class_name="PersonWithText",
+ )
+ concepts_to_register.append(person_text_concept)
+
+ # CompanyInfo concept
+ company_concept = ConceptFactory.make(
+ domain=TestData.DOMAIN,
+ concept_code="CompanyInfo",
+ description="Company information",
+ structure_class_name="CompanyInfo",
+ )
+ concepts_to_register.append(company_concept)
+
+ # NestedComplex concept
+ nested_concept = ConceptFactory.make(
+ domain=TestData.DOMAIN,
+ concept_code="NestedComplex",
+ description="Complex nested structure",
+ structure_class_name="NestedComplex",
+ )
+ concepts_to_register.append(nested_concept)
+
+ # PersonWithOptionalImage concept
+ person_optional_concept = ConceptFactory.make(
+ domain=TestData.DOMAIN,
+ concept_code="PersonWithOptionalImage",
+ description="A person with optional image",
+ structure_class_name="PersonWithOptionalImage",
+ )
+ concepts_to_register.append(person_optional_concept)
+
+ # Add all concepts to the provider
+ concept_provider.add_concepts(concepts_to_register)
+
+ # Cleanup after tests (optional)
+
+
+class TestConceptFindImageFieldPaths:
+ """Test ConceptLibrary.find_image_field_paths() method."""
+
+ def test_direct_image_field(self):
+ """Test finding a direct image field."""
+ # Get concept
+ concept = get_concept_provider().get_required_concept(f"{TestData.DOMAIN}.PersonWithDirectImage")
+
+ # Find image paths
+ image_paths = get_concept_provider().find_image_field_paths(concept=concept)
+
+ # Assert
+ assert len(image_paths) == 1
+ assert "photo" in image_paths
+
+ def test_refined_image_field(self):
+ """Test finding an image field that uses a concept refining Image."""
+ # Get concept
+ concept = get_concept_provider().get_required_concept(f"{TestData.DOMAIN}.PersonWithRefinedImage")
+
+ # Find image paths
+ image_paths = get_concept_provider().find_image_field_paths(concept=concept)
+
+ # Assert
+ assert len(image_paths) == 1
+ assert "profile_photo" in image_paths
+
+ def test_no_image_fields(self):
+ """Test with content that has no image fields."""
+ # Get concept
+ concept = get_concept_provider().get_required_concept(f"{TestData.DOMAIN}.PersonWithText")
+
+ # Find image paths
+ image_paths = get_concept_provider().find_image_field_paths(concept=concept)
+
+ # Assert
+ assert len(image_paths) == 0
+
+ def test_nested_image_field(self):
+ """Test finding image fields in nested structures."""
+ # Get concept
+ concept = get_concept_provider().get_required_concept(f"{TestData.DOMAIN}.CompanyInfo")
+
+ # Find image paths
+ image_paths = get_concept_provider().find_image_field_paths(concept=concept)
+
+ # Assert
+ assert len(image_paths) == 1
+ assert "ceo.photo" in image_paths
+
+ def test_multiple_nested_levels_with_multiple_images(self):
+ """Test finding multiple image fields at different nesting levels."""
+ # Get concept
+ concept = get_concept_provider().get_required_concept(f"{TestData.DOMAIN}.NestedComplex")
+
+ # Find image paths
+ image_paths = get_concept_provider().find_image_field_paths(concept=concept)
+
+ # Assert - should find both the logo and the nested CEO photo
+ assert len(image_paths) == 2
+ assert "logo" in image_paths
+ assert "company.ceo.photo" in image_paths
+
+ def test_optional_image_field_with_value(self):
+ """Test finding an optional image field that has a value."""
+ # Get concept
+ concept = get_concept_provider().get_required_concept(f"{TestData.DOMAIN}.PersonWithOptionalImage")
+
+ # Find image paths
+ image_paths = get_concept_provider().find_image_field_paths(concept=concept)
+
+ # Assert
+ assert len(image_paths) == 1
+ assert "photo" in image_paths
+
+ def test_optional_image_field_without_value(self):
+ """Test finding an optional image field that is None.
+
+ Note: Since find_image_field_paths() works at the concept/class level (not instance level),
+ it returns all fields typed as Images, regardless of whether they have values in a specific instance.
+ """
+ # Get concept
+ concept = get_concept_provider().get_required_concept(f"{TestData.DOMAIN}.PersonWithOptionalImage")
+
+ # Find image paths
+ image_paths = get_concept_provider().find_image_field_paths(concept=concept)
+
+ # Assert - should find the photo field even though it's None in this instance
+ # because we're analyzing the class structure, not instance values
+ assert len(image_paths) == 1
+ assert "photo" in image_paths
+
+ def test_direct_image_concept(self):
+ """Test with a concept that is directly an Image."""
+ # Get concept
+ concept = get_native_concept(NativeConceptEnum.IMAGE)
+
+ # Find image paths
+ image_paths = get_concept_provider().find_image_field_paths(concept=concept)
+
+ # Assert - should return empty because the concept itself is an image, not a structured type with image fields
+ assert len(image_paths) == 0
diff --git a/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_controllers/pipe_batch/__init__.py b/tests/unit/pipelex/core/pipes/pipe_operator/__init__.py
similarity index 100%
rename from tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_controllers/pipe_batch/__init__.py
rename to tests/unit/pipelex/core/pipes/pipe_operator/__init__.py
diff --git a/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_controllers/pipe_condition/__init__.py b/tests/unit/pipelex/core/pipes/pipe_operator/pipe_llm/__init__.py
similarity index 100%
rename from tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_controllers/pipe_condition/__init__.py
rename to tests/unit/pipelex/core/pipes/pipe_operator/pipe_llm/__init__.py
diff --git a/tests/unit/pipelex/core/pipes/pipe_operator/pipe_llm/data.py b/tests/unit/pipelex/core/pipes/pipe_operator/pipe_llm/data.py
new file mode 100644
index 000000000..6ee8592d8
--- /dev/null
+++ b/tests/unit/pipelex/core/pipes/pipe_operator/pipe_llm/data.py
@@ -0,0 +1,117 @@
+from typing import ClassVar
+
+from pipelex.pipe_operators.llm.pipe_llm_blueprint import PipeLLMBlueprint
+
+
+class PipeLLMInputTestCases:
+ """Test cases for PipeLLM input validation."""
+
+ # Valid test cases: (test_id, blueprint)
+ VALID_TEXT_INPUT: ClassVar[tuple[str, PipeLLMBlueprint]] = (
+ "valid_text_input",
+ PipeLLMBlueprint(
+ description="Test case: valid_text_input",
+ inputs={"user_text": "native.Text"},
+ output="native.Text",
+ prompt_template="Process this text: @user_text",
+ ),
+ )
+
+ VALID_IMAGE_INPUT: ClassVar[tuple[str, PipeLLMBlueprint]] = (
+ "valid_image_input",
+ PipeLLMBlueprint(
+ description="Test case: valid_image_input",
+ inputs={"user_image": "native.Image"},
+ output="native.Text",
+ prompt_template="Describe this image: @user_image",
+ ),
+ )
+
+ VALID_TWO_INPUTS: ClassVar[tuple[str, PipeLLMBlueprint]] = (
+ "valid_two_inputs",
+ PipeLLMBlueprint(
+ description="Test case: valid_two_inputs",
+ inputs={"text_input": "native.Text", "another_text": "native.Text"},
+ output="native.Text",
+ prompt_template="Compare @text_input with @another_text",
+ ),
+ )
+
+ VALID_TEXT_AND_IMAGE: ClassVar[tuple[str, PipeLLMBlueprint]] = (
+ "valid_text_and_image",
+ PipeLLMBlueprint(
+ description="Test case: valid_text_and_image",
+ inputs={"user_text": "native.Text", "user_image": "native.Image"},
+ output="native.Text",
+ prompt_template="Analyze this text: @user_text with the image: @user_image",
+ ),
+ )
+
+ VALID_IMAGE_MENTIONED_IN_PROMPT: ClassVar[tuple[str, PipeLLMBlueprint]] = (
+ "image_mentioned_in_prompt",
+ PipeLLMBlueprint(
+ description="Test case: image_mentioned_in_prompt",
+ inputs={"user_image": "native.Image"},
+ output="native.Text",
+ prompt_template="Describe @user_image",
+ ),
+ )
+
+ VALID_TEXT_AND_IMAGE_BOTH_IN_PROMPT: ClassVar[tuple[str, PipeLLMBlueprint]] = (
+ "text_and_image_both_in_prompt",
+ PipeLLMBlueprint(
+ description="Test case: text_and_image_both_in_prompt",
+ inputs={"user_text": "native.Text", "user_image": "native.Image"},
+ output="native.Text",
+ prompt_template="Analyze @user_text and describe @user_image",
+ ),
+ )
+
+ VALID_CASES: ClassVar[list[tuple[str, PipeLLMBlueprint]]] = [
+ VALID_TEXT_INPUT,
+ VALID_IMAGE_INPUT,
+ VALID_TWO_INPUTS,
+ VALID_TEXT_AND_IMAGE,
+ VALID_IMAGE_MENTIONED_IN_PROMPT,
+ VALID_TEXT_AND_IMAGE_BOTH_IN_PROMPT,
+ ]
+
+ # Error test cases: (test_id, blueprint, expected_error_message_fragment)
+ ERROR_MISSING_INPUT_IN_PROMPT: ClassVar[tuple[str, PipeLLMBlueprint, str]] = (
+ "missing_input_in_prompt",
+ PipeLLMBlueprint(
+ description="Test case: missing_input_in_prompt",
+ inputs={},
+ output="native.Text",
+ prompt_template="Process this text: @user_text",
+ ),
+ "missing_input_variable",
+ )
+
+ ERROR_EXTRANEOUS_INPUT: ClassVar[tuple[str, PipeLLMBlueprint, str]] = (
+ "extraneous_input",
+ PipeLLMBlueprint(
+ description="Test case: extraneous_input",
+ inputs={"user_text": "native.Text", "extra_input": "native.Text"},
+ output="native.Text",
+ prompt_template="Process this: @user_text",
+ ),
+ "extraneous_input",
+ )
+
+ ERROR_TWO_INPUTS_THREE_VARIABLES: ClassVar[tuple[str, PipeLLMBlueprint, str]] = (
+ "two_inputs_three_variables",
+ PipeLLMBlueprint(
+ description="Test case: two_inputs_three_variables",
+ inputs={"input_one": "native.Text", "input_two": "native.Text"},
+ output="native.Text",
+ prompt_template="Process @input_one, @input_two, and @input_three",
+ ),
+ "missing_input_variable",
+ )
+
+ ERROR_CASES: ClassVar[list[tuple[str, PipeLLMBlueprint, str]]] = [
+ ERROR_MISSING_INPUT_IN_PROMPT,
+ ERROR_EXTRANEOUS_INPUT,
+ ERROR_TWO_INPUTS_THREE_VARIABLES,
+ ]
diff --git a/tests/unit/pipelex/core/pipes/pipe_operator/pipe_llm/test_pipe_llm_input.py b/tests/unit/pipelex/core/pipes/pipe_operator/pipe_llm/test_pipe_llm_input.py
new file mode 100644
index 000000000..48de06393
--- /dev/null
+++ b/tests/unit/pipelex/core/pipes/pipe_operator/pipe_llm/test_pipe_llm_input.py
@@ -0,0 +1,52 @@
+import pytest
+
+from pipelex import log
+from pipelex.exceptions import StaticValidationError
+from pipelex.pipe_operators.llm.pipe_llm_blueprint import PipeLLMBlueprint
+from pipelex.pipe_operators.llm.pipe_llm_factory import PipeLLMFactory
+from tests.unit.pipelex.core.pipes.pipe_operator.pipe_llm.data import PipeLLMInputTestCases
+
+
+class TestPipeLLMValidateInputs:
+ @pytest.mark.parametrize(
+ ("test_id", "blueprint"),
+ PipeLLMInputTestCases.VALID_CASES,
+ )
+ def test_validate_inputs_valid_cases(
+ self,
+ test_id: str,
+ blueprint: PipeLLMBlueprint,
+ ):
+ log.debug(f"Testing valid case: {test_id}")
+
+ pipe_llm = PipeLLMFactory.make_from_blueprint(
+ domain="test_domain",
+ pipe_code=f"test_pipe_{test_id}",
+ blueprint=blueprint,
+ )
+
+ pipe_llm.validate_inputs()
+
+ @pytest.mark.parametrize(
+ ("test_id", "blueprint", "expected_error_message_fragment"),
+ PipeLLMInputTestCases.ERROR_CASES,
+ )
+ def test_validate_inputs_error_cases(
+ self,
+ test_id: str,
+ blueprint: PipeLLMBlueprint,
+ expected_error_message_fragment: str,
+ ):
+ log.debug(f"Testing error case: {test_id}")
+
+ with pytest.raises(StaticValidationError) as exc_info:
+ PipeLLMFactory.make_from_blueprint(
+ domain="test_domain",
+ pipe_code=f"test_pipe_{test_id}",
+ blueprint=blueprint,
+ )
+
+ error_str = str(exc_info.value)
+ assert expected_error_message_fragment in error_str, (
+ f"Expected fragment '{expected_error_message_fragment}' not found in error message: {error_str}"
+ )
diff --git a/tests/unit/pipelex/core/stuffs/test_stuff_factory_make_stuff_from_content.py b/tests/unit/pipelex/core/stuffs/test_stuff_factory_make_stuff_from_content.py
index 2fecaa0c6..95507fcb3 100644
--- a/tests/unit/pipelex/core/stuffs/test_stuff_factory_make_stuff_from_content.py
+++ b/tests/unit/pipelex/core/stuffs/test_stuff_factory_make_stuff_from_content.py
@@ -101,9 +101,8 @@ class TestMakeStuffFromStuffContentUsingSearchDomains:
def test_stuffcontent_with_native_concept(self, mocker: MockerFixture):
"""Test StuffContent with native concept class name."""
- mock_provider = mocker.patch("pipelex.core.stuffs.stuff_factory.get_concept_provider").return_value
mock_concept = mocker.Mock()
- mock_provider.get_native_concept.return_value = mock_concept
+ mock_get_native_concept = mocker.patch("pipelex.core.stuffs.stuff_factory.get_native_concept", return_value=mock_concept)
text_content = TextContent(text="test")
@@ -118,7 +117,7 @@ class TestMakeStuffFromStuffContentUsingSearchDomains:
)
assert result == mock_stuff
- mock_provider.get_native_concept.assert_called_once_with(native_concept=NativeConceptEnum.TEXT)
+ mock_get_native_concept.assert_called_once_with(native_concept=NativeConceptEnum.TEXT)
def test_stuffcontent_with_non_native_concept(self, mocker: MockerFixture):
"""Test StuffContent with non-native concept properly extracts concept name."""
diff --git a/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_controllers/pipe_parallel/__init__.py b/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_controller/pipe_batch/__init__.py
similarity index 100%
rename from tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_controllers/pipe_parallel/__init__.py
rename to tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_controller/pipe_batch/__init__.py
diff --git a/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_controllers/pipe_batch/test_data.py b/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_controller/pipe_batch/test_data.py
similarity index 100%
rename from tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_controllers/pipe_batch/test_data.py
rename to tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_controller/pipe_batch/test_data.py
diff --git a/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_controllers/pipe_batch/test_pipe_batch.py b/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_controller/pipe_batch/test_pipe_batch.py
similarity index 92%
rename from tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_controllers/pipe_batch/test_pipe_batch.py
rename to tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_controller/pipe_batch/test_pipe_batch.py
index 265852a89..d24b48b48 100644
--- a/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_controllers/pipe_batch/test_pipe_batch.py
+++ b/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_controller/pipe_batch/test_pipe_batch.py
@@ -2,7 +2,7 @@ import pytest
from pipelex.libraries.pipelines.builder.pipe.pipe_batch_spec import PipeBatchSpec
from pipelex.pipe_controllers.batch.pipe_batch_blueprint import PipeBatchBlueprint
-from tests.unit.pipelex.libraries.pipelines.builder.pipe.pipe_controllers.pipe_batch.test_data import PipeBatchTestCases
+from tests.unit.pipelex.libraries.pipelines.builder.pipe.pipe_controller.pipe_batch.test_data import PipeBatchTestCases
class TestPipeBatchBlueprintConversion:
diff --git a/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_controllers/pipe_sequence/__init__.py b/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_controller/pipe_condition/__init__.py
similarity index 100%
rename from tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_controllers/pipe_sequence/__init__.py
rename to tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_controller/pipe_condition/__init__.py
diff --git a/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_controllers/pipe_condition/test_data.py b/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_controller/pipe_condition/test_data.py
similarity index 100%
rename from tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_controllers/pipe_condition/test_data.py
rename to tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_controller/pipe_condition/test_data.py
diff --git a/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_controllers/pipe_condition/test_pipe_condition.py b/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_controller/pipe_condition/test_pipe_condition.py
similarity index 91%
rename from tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_controllers/pipe_condition/test_pipe_condition.py
rename to tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_controller/pipe_condition/test_pipe_condition.py
index be6340f0b..3ec34ceb7 100644
--- a/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_controllers/pipe_condition/test_pipe_condition.py
+++ b/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_controller/pipe_condition/test_pipe_condition.py
@@ -2,7 +2,7 @@ import pytest
from pipelex.libraries.pipelines.builder.pipe.pipe_condition_spec import PipeConditionSpec
from pipelex.pipe_controllers.condition.pipe_condition_blueprint import PipeConditionBlueprint
-from tests.unit.pipelex.libraries.pipelines.builder.pipe.pipe_controllers.pipe_condition.test_data import PipeConditionTestCases
+from tests.unit.pipelex.libraries.pipelines.builder.pipe.pipe_controller.pipe_condition.test_data import PipeConditionTestCases
class TestPipeConditionBlueprintConversion:
diff --git a/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_operators/pipe_func/__init__.py b/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_controller/pipe_parallel/__init__.py
similarity index 100%
rename from tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_operators/pipe_func/__init__.py
rename to tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_controller/pipe_parallel/__init__.py
diff --git a/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_controllers/pipe_parallel/test_data.py b/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_controller/pipe_parallel/test_data.py
similarity index 100%
rename from tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_controllers/pipe_parallel/test_data.py
rename to tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_controller/pipe_parallel/test_data.py
diff --git a/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_controllers/pipe_parallel/test_pipe_parallel.py b/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_controller/pipe_parallel/test_pipe_parallel.py
similarity index 92%
rename from tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_controllers/pipe_parallel/test_pipe_parallel.py
rename to tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_controller/pipe_parallel/test_pipe_parallel.py
index 8496b3545..64db9b946 100644
--- a/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_controllers/pipe_parallel/test_pipe_parallel.py
+++ b/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_controller/pipe_parallel/test_pipe_parallel.py
@@ -3,7 +3,7 @@ import pytest
from pipelex import log
from pipelex.libraries.pipelines.builder.pipe.pipe_parallel_spec import PipeParallelSpec
from pipelex.pipe_controllers.parallel.pipe_parallel_blueprint import PipeParallelBlueprint
-from tests.unit.pipelex.libraries.pipelines.builder.pipe.pipe_controllers.pipe_parallel.test_data import PipeParallelTestCases
+from tests.unit.pipelex.libraries.pipelines.builder.pipe.pipe_controller.pipe_parallel.test_data import PipeParallelTestCases
class TestPipeParallelBlueprintConversion:
diff --git a/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_operators/pipe_img/__init__.py b/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_controller/pipe_sequence/__init__.py
similarity index 100%
rename from tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_operators/pipe_img/__init__.py
rename to tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_controller/pipe_sequence/__init__.py
diff --git a/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_controllers/pipe_sequence/test_data.py b/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_controller/pipe_sequence/test_data.py
similarity index 100%
rename from tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_controllers/pipe_sequence/test_data.py
rename to tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_controller/pipe_sequence/test_data.py
diff --git a/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_controllers/pipe_sequence/test_pipe_sequence.py b/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_controller/pipe_sequence/test_pipe_sequence.py
similarity index 91%
rename from tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_controllers/pipe_sequence/test_pipe_sequence.py
rename to tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_controller/pipe_sequence/test_pipe_sequence.py
index bac489590..3acda38a6 100644
--- a/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_controllers/pipe_sequence/test_pipe_sequence.py
+++ b/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_controller/pipe_sequence/test_pipe_sequence.py
@@ -2,7 +2,7 @@ import pytest
from pipelex.libraries.pipelines.builder.pipe.pipe_sequence_spec import PipeSequenceSpec
from pipelex.pipe_controllers.sequence.pipe_sequence_blueprint import PipeSequenceBlueprint
-from tests.unit.pipelex.libraries.pipelines.builder.pipe.pipe_controllers.pipe_sequence.test_data import PipeSequenceTestCases
+from tests.unit.pipelex.libraries.pipelines.builder.pipe.pipe_controller.pipe_sequence.test_data import PipeSequenceTestCases
class TestPipeSequenceBlueprintConversion:
diff --git a/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_operators/pipe_jinja2/__init__.py b/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_operator/pipe_func/__init__.py
similarity index 100%
rename from tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_operators/pipe_jinja2/__init__.py
rename to tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_operator/pipe_func/__init__.py
diff --git a/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_operators/pipe_func/test_data.py b/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_operator/pipe_func/test_data.py
similarity index 100%
rename from tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_operators/pipe_func/test_data.py
rename to tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_operator/pipe_func/test_data.py
diff --git a/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_operators/pipe_func/test_pipe_func.py b/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_operator/pipe_func/test_pipe_func.py
similarity index 92%
rename from tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_operators/pipe_func/test_pipe_func.py
rename to tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_operator/pipe_func/test_pipe_func.py
index 9023f33b7..063c4ef0c 100644
--- a/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_operators/pipe_func/test_pipe_func.py
+++ b/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_operator/pipe_func/test_pipe_func.py
@@ -2,7 +2,7 @@ import pytest
from pipelex.libraries.pipelines.builder.pipe.pipe_func_spec import PipeFuncSpec
from pipelex.pipe_operators.func.pipe_func_blueprint import PipeFuncBlueprint
-from tests.unit.pipelex.libraries.pipelines.builder.pipe.pipe_operators.pipe_func.test_data import PipeFuncTestCases
+from tests.unit.pipelex.libraries.pipelines.builder.pipe.pipe_operator.pipe_func.test_data import PipeFuncTestCases
class TestPipeFuncBlueprintConversion:
diff --git a/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_operators/pipe_llm/__init__.py b/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_operator/pipe_img/__init__.py
similarity index 100%
rename from tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_operators/pipe_llm/__init__.py
rename to tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_operator/pipe_img/__init__.py
diff --git a/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_operators/pipe_img/test_data.py b/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_operator/pipe_img/test_data.py
similarity index 100%
rename from tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_operators/pipe_img/test_data.py
rename to tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_operator/pipe_img/test_data.py
diff --git a/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_operators/pipe_img/test_pipe_img.py b/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_operator/pipe_img/test_pipe_img.py
similarity index 92%
rename from tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_operators/pipe_img/test_pipe_img.py
rename to tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_operator/pipe_img/test_pipe_img.py
index 3210fa212..bc2b1a001 100644
--- a/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_operators/pipe_img/test_pipe_img.py
+++ b/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_operator/pipe_img/test_pipe_img.py
@@ -2,7 +2,7 @@ import pytest
from pipelex.libraries.pipelines.builder.pipe.pipe_img_spec import PipeImgGenSpec
from pipelex.pipe_operators.img_gen.pipe_img_gen_blueprint import PipeImgGenBlueprint
-from tests.unit.pipelex.libraries.pipelines.builder.pipe.pipe_operators.pipe_img.test_data import PipeImgGenTestCases
+from tests.unit.pipelex.libraries.pipelines.builder.pipe.pipe_operator.pipe_img.test_data import PipeImgGenTestCases
class TestPipeImgGenBlueprintConversion:
diff --git a/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_operators/pipe_ocr/__init__.py b/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_operator/pipe_jinja2/__init__.py
similarity index 100%
rename from tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_operators/pipe_ocr/__init__.py
rename to tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_operator/pipe_jinja2/__init__.py
diff --git a/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_operators/pipe_jinja2/test_data.py b/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_operator/pipe_jinja2/test_data.py
similarity index 100%
rename from tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_operators/pipe_jinja2/test_data.py
rename to tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_operator/pipe_jinja2/test_data.py
diff --git a/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_operators/pipe_jinja2/test_pipe_jinja2.py b/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_operator/pipe_jinja2/test_pipe_jinja2.py
similarity index 92%
rename from tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_operators/pipe_jinja2/test_pipe_jinja2.py
rename to tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_operator/pipe_jinja2/test_pipe_jinja2.py
index f44a7a8ef..bfb6552c7 100644
--- a/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_operators/pipe_jinja2/test_pipe_jinja2.py
+++ b/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_operator/pipe_jinja2/test_pipe_jinja2.py
@@ -2,7 +2,7 @@ import pytest
from pipelex.libraries.pipelines.builder.pipe.pipe_compose_spec import PipeComposeSpec
from pipelex.pipe_operators.compose.pipe_compose_blueprint import PipeComposeBlueprint
-from tests.unit.pipelex.libraries.pipelines.builder.pipe.pipe_operators.pipe_jinja2.test_data import PipeComposeTestCases
+from tests.unit.pipelex.libraries.pipelines.builder.pipe.pipe_operator.pipe_jinja2.test_data import PipeComposeTestCases
class TestPipeComposeBlueprintConversion:
diff --git a/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_operator/pipe_llm/__init__.py b/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_operator/pipe_llm/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_operators/pipe_llm/test_data.py b/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_operator/pipe_llm/test_data.py
similarity index 100%
rename from tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_operators/pipe_llm/test_data.py
rename to tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_operator/pipe_llm/test_data.py
diff --git a/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_operators/pipe_llm/test_pipe_llm.py b/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_operator/pipe_llm/test_pipe_llm.py
similarity index 92%
rename from tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_operators/pipe_llm/test_pipe_llm.py
rename to tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_operator/pipe_llm/test_pipe_llm.py
index c3c451c3d..f19a7f583 100644
--- a/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_operators/pipe_llm/test_pipe_llm.py
+++ b/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_operator/pipe_llm/test_pipe_llm.py
@@ -2,7 +2,7 @@ import pytest
from pipelex.libraries.pipelines.builder.pipe.pipe_llm_spec import PipeLLMSpec
from pipelex.pipe_operators.llm.pipe_llm_blueprint import PipeLLMBlueprint
-from tests.unit.pipelex.libraries.pipelines.builder.pipe.pipe_operators.pipe_llm.test_data import PipeLLMTestCases
+from tests.unit.pipelex.libraries.pipelines.builder.pipe.pipe_operator.pipe_llm.test_data import PipeLLMTestCases
class TestPipeLLMBlueprintConversion:
diff --git a/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_operator/pipe_ocr/__init__.py b/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_operator/pipe_ocr/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_operators/pipe_ocr/test_data.py b/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_operator/pipe_ocr/test_data.py
similarity index 100%
rename from tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_operators/pipe_ocr/test_data.py
rename to tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_operator/pipe_ocr/test_data.py
diff --git a/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_operators/pipe_ocr/test_pipe_ocr.py b/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_operator/pipe_ocr/test_pipe_ocr.py
similarity index 93%
rename from tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_operators/pipe_ocr/test_pipe_ocr.py
rename to tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_operator/pipe_ocr/test_pipe_ocr.py
index 1e741f5a5..f3fad9ef8 100644
--- a/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_operators/pipe_ocr/test_pipe_ocr.py
+++ b/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_operator/pipe_ocr/test_pipe_ocr.py
@@ -3,7 +3,7 @@ import pytest
from pipelex import log
from pipelex.libraries.pipelines.builder.pipe.pipe_ocr_spec import PipeOcrSpec
from pipelex.pipe_operators.ocr.pipe_ocr_blueprint import PipeOcrBlueprint
-from tests.unit.pipelex.libraries.pipelines.builder.pipe.pipe_operators.pipe_ocr.test_data import PipeOcrTestCases
+from tests.unit.pipelex.libraries.pipelines.builder.pipe.pipe_operator.pipe_ocr.test_data import PipeOcrTestCases
class TestPipeOcrBlueprintConversion: