Build AI methods with your coding agent, then run them anywhere: as an MCP for chatbots, as a webapp for people, or via API for your software.
Refactor/remove pipe compose in pipe condition (#303)
Thomas Hebrard33bf9b78cb1a
diff --git a/.badges/tests.json b/.badges/tests.jsonindex 056566a85..a2aa3e5b9 100644--- a/.badges/tests.json+++ b/.badges/tests.json@@ -1,7 +1,7 @@{"schemaVersion": 1,"label": "tests",- "message": "1045",+ "message": "1052","color": "blue","cacheSeconds": 300}\ No newline at end of filediff --git a/.github/workflows/check-test-count-badge.yml b/.github/workflows/check-test-count-badge.ymlindex ee6e243c1..bc4ddb7c7 100644--- a/.github/workflows/check-test-count-badge.yml+++ b/.github/workflows/check-test-count-badge.yml@@ -2,10 +2,6 @@ name: Check test count matches badgeon:pull_request:- branches:- - main- - dev- - "release/v[0-9]+.[0-9]+.[0-9]+"jobs:check:diff --git a/.github/workflows/doc-check.yml b/.github/workflows/doc-check.ymlindex 54a70d29a..362c2d833 100644--- a/.github/workflows/doc-check.yml+++ b/.github/workflows/doc-check.yml@@ -2,10 +2,6 @@ name: Documentation Checkon:pull_request:- branches:- - main- - dev- - "release/v[0-9]+.[0-9]+.[0-9]+"paths:- 'docs/**'- 'mkdocs.yml'diff --git a/.github/workflows/lint-check.yml b/.github/workflows/lint-check.ymlindex 2eaed36dd..c4e82aa3d 100644--- a/.github/workflows/lint-check.yml+++ b/.github/workflows/lint-check.yml@@ -2,10 +2,6 @@ name: Lint checkon:pull_request:- branches:- - main- - dev- - "release/v[0-9]+.[0-9]+.[0-9]+"jobs:# --------------------------------------------------------------------------diff --git a/.github/workflows/tests-check.yml b/.github/workflows/tests-check.ymlindex 8e7229e63..532b4c43f 100644--- a/.github/workflows/tests-check.yml+++ b/.github/workflows/tests-check.yml@@ -2,10 +2,6 @@ name: Tests checkon:pull_request:- branches:- - main- - dev- - "release/v[0-9]+.[0-9]+.[0-9]+"jobs:matrix-test:diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 95b03a0cf..bea021d52 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -26,6 +26,7 @@- Modified `any_env_var_is_placeholder()` to use new placeholder detection logic- Updated test environment setup to use dynamic placeholder generation instead of hardcoded values- Restructured test classes and methods in environment tests+ - Lint and tests GHA should run on every PR### Fixed- Fixed logic error in `any_env_var_is_placeholder()` function - now correctly returns False when no placeholders are found@@ -35,6 +36,8 @@- Removed `get_rooted_path()` and `get_env_rooted_path()` utility functions- Removed hardcoded placeholder dictionary in test setup- Removed function `run_pipe_code` in pipe router because it was not relevant (used mostly in tests)+ - Remove the use of `PipeCompose` in `PipeCondition`, to only use jinja2 with the `ContentGenerator`+ - Remove the template libraries from the pipelex libraries.## [v0.10.2] - 2025-09-18diff --git a/pipelex/cogt/content_generation/jinja2_generate.py b/pipelex/cogt/content_generation/jinja2_generate.pyindex 73474136c..995209e75 100644--- a/pipelex/cogt/content_generation/jinja2_generate.py+++ b/pipelex/cogt/content_generation/jinja2_generate.py@@ -5,8 +5,8 @@ from pipelex.tools.templating.jinja2_rendering import render_jinja2from pipelex.tools.templating.template_preprocessor import preprocess_template-# TODO: get rid of this intermediate call which seems useless, or explain why it staysasync def jinja2_gen_text(jinja2_assignment: Jinja2Assignment) -> str:+ # Intermediate call to preprocess the template with our syntax patterns (@, $, @?, etc.)if jinja2_assignment.jinja2:jinja2_assignment.jinja2 = preprocess_template(template=jinja2_assignment.jinja2)check_jinja2_parsing(jinja2_assignment.jinja2)diff --git a/pipelex/cogt/llm/llm_prompt_factory_abstract.py b/pipelex/cogt/llm/llm_prompt_factory_abstract.pyindex 1a30544a7..319ed2657 100644--- a/pipelex/cogt/llm/llm_prompt_factory_abstract.py+++ b/pipelex/cogt/llm/llm_prompt_factory_abstract.py@@ -6,14 +6,6 @@ from pydantic import BaseModelfrom pipelex.cogt.llm.llm_prompt import LLMPrompt-def make_empty_prompt() -> LLMPrompt:- return LLMPrompt(- system_text=None,- user_text=None,- user_images=[],- )--class LLMPromptFactoryAbstract(ABC, BaseModel):@abstractmethodasync def make_llm_prompt_from_args(@@ -21,8 +13,3 @@ class LLMPromptFactoryAbstract(ABC, BaseModel):**prompt_arguments: Any,) -> LLMPrompt:pass-- @property- @abstractmethod- def desc(self) -> str:- passdiff --git a/pipelex/cogt/llm/llm_prompt_spec.py b/pipelex/cogt/llm/llm_prompt_spec.pyindex 32457def0..5b3c17e1d 100644--- a/pipelex/cogt/llm/llm_prompt_spec.py+++ b/pipelex/cogt/llm/llm_prompt_spec.py@@ -173,7 +173,7 @@ class LLMPromptSpec(BaseModel):jinja2_blueprint.prompting_style = prompting_stylelog.verbose(f"Setting prompting style to {prompting_style}")- context: dict[str, Any] = context_provider.generate_context()+ context: dict[str, Any] = context_provider.generate_jinja2_context()if extra_params:context.update(**extra_params)if jinja2_blueprint.extra_context:diff --git a/pipelex/cogt/llm/llm_prompt_template.py b/pipelex/cogt/llm/llm_prompt_template.pyindex 6be640f18..94a67c3eb 100644--- a/pipelex/cogt/llm/llm_prompt_template.py+++ b/pipelex/cogt/llm/llm_prompt_template.py@@ -6,12 +6,19 @@ from pipelex import logfrom pipelex.cogt.exceptions import LLMPromptFactoryError, LLMPromptTemplateInputsErrorfrom pipelex.cogt.image.prompt_image import PromptImagefrom pipelex.cogt.llm.llm_prompt import LLMPrompt-from pipelex.cogt.llm.llm_prompt_factory_abstract import LLMPromptFactoryAbstract, make_empty_prompt+from pipelex.cogt.llm.llm_prompt_factory_abstract import LLMPromptFactoryAbstractfrom pipelex.cogt.llm.llm_prompt_template_inputs import LLMPromptTemplateInputsfrom pipelex.config import get_configfrom pipelex.hub import get_templatefrom pipelex.tools.misc.string_utils import is_none_or_has_text-from pipelex.tools.templating.template_provider_abstract import TemplateProviderAbstract+++def make_empty_prompt() -> LLMPrompt:+ return LLMPrompt(+ system_text=None,+ user_text=None,+ user_images=[],+ )class LLMPromptTemplate(LLMPromptFactoryAbstract):@@ -22,21 +29,10 @@ class LLMPromptTemplate(LLMPromptFactoryAbstract):source_system_template_name: str | None = Nonesource_user_template_name: str | None = None- @property- @override- def desc(self) -> str:- return f"{LLMPromptTemplate.__name__} based on proto prompt: {self.proto_prompt} and base inputs: {self.base_template_inputs}"-@overrideasync def make_llm_prompt_from_args(self,**prompt_arguments: Any,- ) -> LLMPrompt:- return self.synchro_make_llm_prompt_from_args(**prompt_arguments)-- def synchro_make_llm_prompt_from_args(- self,- **prompt_arguments: Any,) -> LLMPrompt:arguments_dict = prompt_arguments.copy()@@ -117,47 +113,6 @@ class LLMPromptTemplate(LLMPromptFactoryAbstract):return llm_prompt- # factories of LLMPromptTemplate (which is a factory of LLMPrompt)- @classmethod- def from_template_name(- cls,- template_provider: TemplateProviderAbstract,- system_text: str | None = None,- user_text: str | None = None,- user_images: list[PromptImage] | None = None,- system_template_name: str | None = None,- user_template_name: str | None = None,- ) -> "LLMPromptTemplate":- proto_prompt: LLMPrompt = LLMPrompt(- system_text=system_text,- user_text=user_text,- user_images=user_images or [],- )- if user_template_name:- proto_prompt.user_text = template_provider.get_template(user_template_name)- if system_template_name:- proto_prompt.system_text = template_provider.get_template(system_template_name)- return cls(- proto_prompt=proto_prompt,- source_system_template_name=system_template_name,- source_user_template_name=user_template_name,- )-- # factories of LLMPromptTemplate (which is a factory of LLMPrompt)- @classmethod- def from_template_contents(- cls,- system_text: str | None = None,- user_text: str | None = None,- user_images: list[PromptImage] | None = None,- ) -> "LLMPromptTemplate":- proto_prompt = LLMPrompt(- user_text=user_text,- system_text=system_text,- user_images=user_images or [],- )- return cls(proto_prompt=proto_prompt)-@classmethoddef for_structure_from_preliminary_text(cls) -> "LLMPromptTemplate":generic_template_names = get_config().pipelex.generic_template_namesdiff --git a/pipelex/core/memory/working_memory.py b/pipelex/core/memory/working_memory.pyindex 349a125cc..347ce733b 100644--- a/pipelex/core/memory/working_memory.py+++ b/pipelex/core/memory/working_memory.py@@ -196,7 +196,7 @@ class WorkingMemory(BaseModel, ContextProviderAbstract):################################################################################################@override- def generate_context(self) -> dict[str, Any]:+ def generate_jinja2_context(self) -> dict[str, Any]:# TODO: Add unit tests for this methodartefact_dict: StuffArtefactDict = {}for name, stuff in self.root.items():diff --git a/pipelex/core/pipes/specific_pipe.py b/pipelex/core/pipes/specific_pipe.pyindex fb749fef5..963c2ea56 100644--- a/pipelex/core/pipes/specific_pipe.py+++ b/pipelex/core/pipes/specific_pipe.py@@ -6,6 +6,10 @@ class SpecificPipeCodesEnum(StrEnum):# TODO: Implement the break pipe: It should enable to leave the current sequence.# BREAK = "break"+ @classmethod+ def value_list(cls) -> list[str]:+ return list(cls)+# TODO: Rethink this class. They are not pipes really.class SpecificPipe:diff --git a/pipelex/libraries/library_config.py b/pipelex/libraries/library_config.pyindex 65e176f9b..e91e624e3 100644--- a/pipelex/libraries/library_config.py+++ b/pipelex/libraries/library_config.py@@ -2,7 +2,7 @@ from pathlib import Pathfrom typing import ClassVarfrom pipelex.tools.config.config_model import ConfigModel-from pipelex.tools.misc.file_utils import copy_file_from_package, copy_folder_from_package, find_files_in_dir+from pipelex.tools.misc.file_utils import copy_file_from_package, copy_folder_from_packagePIPELEX_LIBRARIES_PATH = "libraries"@@ -19,10 +19,6 @@ class LibraryConfig(ConfigModel):def base_pipelines_dir_path(self) -> str:return f"{self.config_dir_path}/pipelines/base_library"- @property- def templates_dir_path(self) -> str:- return f"{self.config_dir_path}/templates"-@propertydef test_pipelines_dir_path(self) -> str:return "tests/test_pipelines"@@ -31,9 +27,6 @@ class LibraryConfig(ConfigModel):def failing_pipelines_file_paths(self) -> set[Path]:return {Path("tests/test_pipelines/failing_pipelines.plx")}- def get_templates_paths(self) -> list[str]:- return [str(path) for path in find_files_in_dir(dir_path=self.templates_dir_path, pattern="*.toml", is_recursive=True)]-def export_libraries(self, overwrite: bool = False) -> None:"""Duplicate pipelex libraries files in the client project, preserving directory structure."""copy_file_from_package(@@ -56,11 +49,3 @@ class LibraryConfig(ConfigModel):target_path=f"{self.pipelines_dir_path}/__init__.py",overwrite=overwrite,)-- # templates- copy_folder_from_package(- package_name=self.package_name,- folder_path_in_package=f"{PIPELEX_LIBRARIES_PATH}/templates",- target_dir=self.templates_dir_path,- overwrite=overwrite,- )diff --git a/pipelex/libraries/library_manager.py b/pipelex/libraries/library_manager.pyindex 4c43a1706..e2e38d3e9 100644--- a/pipelex/libraries/library_manager.py+++ b/pipelex/libraries/library_manager.py@@ -1,4 +1,3 @@-import osfrom pathlib import Pathfrom typing import ClassVar@@ -37,7 +36,6 @@ from pipelex.libraries.library_manager_abstract import LibraryManagerAbstractfrom pipelex.tools.class_registry_utils import ClassRegistryUtilsfrom pipelex.tools.func_registry_utils import FuncRegistryUtilsfrom pipelex.tools.misc.file_utils import find_files_in_dir-from pipelex.tools.misc.toml_utils import TOMLValidationError, validate_toml_filefrom pipelex.tools.runtime_manager import runtime_managerfrom pipelex.tools.typing.pydantic_utils import format_pydantic_validation_errorfrom pipelex.types import StrEnum@@ -85,21 +83,6 @@ class LibraryManager(LibraryManagerAbstract):self.pipe_library.validate_with_libraries()self.domain_library.validate_with_libraries()- def _validate_toml_files(self):- """Validate all TOML files used by the library manager for formatting issues."""- log.debug("LibraryManager validating TOML file formatting")-- # Validation of template paths- template_paths = self.library_config.get_templates_paths()- for template_path in template_paths:- if os.path.exists(template_path):- try:- validate_toml_file(template_path)- except TOMLValidationError as exc:- msg = f"PLX validation failed for template file '{template_path}': {exc}"- log.error(msg)- raise LibraryError(msg) from exc-@overridedef setup(self) -> None:self.concept_library.setup()diff --git a/pipelex/libraries/pipelines/builder/pipe/pipe_design.plx b/pipelex/libraries/pipelines/builder/pipe/pipe_design.plxindex 39f1a1c0d..308dad3d0 100644--- a/pipelex/libraries/pipelines/builder/pipe/pipe_design.plx+++ b/pipelex/libraries/pipelines/builder/pipe/pipe_design.plx@@ -25,6 +25,7 @@ definition = "Route by signature.type to the correct spec emitter."inputs = { plan_draft = "PlanDraft", pipe_signature = "PipeSignature", concept_specs = "ConceptSpec" }output = "Dynamic"expression = "pipe_signature.type"+default_pipe_code = "continue"[pipe.detail_pipe_spec.pipe_map]PipeSequence = "detail_pipe_sequence"@@ -33,7 +34,6 @@ PipeCondition = "detail_pipe_condition"PipeLLM = "detail_pipe_llm"PipeOcr = "detail_pipe_ocr"PipeImgGen = "detail_pipe_img_gen"-# PipeCompose = "detail_pipe_compose"# ────────────────────────────────────────────────────────────────────────────────# PIPE CONTROLLERSdiff --git a/pipelex/libraries/templates/base_templates.toml b/pipelex/libraries/templates/base_templates.tomldeleted file mode 100644index 484fecacf..000000000--- a/pipelex/libraries/templates/base_templates.toml+++ /dev/null@@ -1,17 +0,0 @@-[generic_prompts]--structure_from_preliminary_text_system = "You are a data modeling expert specialized in extracting structure from text."-# TODO: Should be a proper jinja2-structure_from_preliminary_text_user = """-Your job is to extract and structure information from a text.-Here is the text:-{preliminary_text}--Now generate the JSON in the required format.-Do not create information that is not in the text.-"""--[test_prompts]--jinja2_test_template = "I want a {{ place_holder }} t-shirt."-diff --git a/pipelex/pipe_controllers/condition/pipe_condition.py b/pipelex/pipe_controllers/condition/pipe_condition.pyindex 7905e8c35..1a38a5675 100644--- a/pipelex/pipe_controllers/condition/pipe_condition.py+++ b/pipelex/pipe_controllers/condition/pipe_condition.py@@ -9,12 +9,13 @@ from pipelex.config import StaticValidationReaction, get_configfrom pipelex.core.concepts.concept_factory import ConceptFactoryfrom pipelex.core.concepts.concept_native import NATIVE_CONCEPTS_DATA, NativeConceptEnum, NativeConceptManagerfrom pipelex.core.memory.working_memory import WorkingMemory+from pipelex.core.pipes.pipe_abstract import PipeAbstractfrom pipelex.core.pipes.pipe_input import PipeInputSpecfrom pipelex.core.pipes.pipe_input_blueprint import InputRequirementBlueprintfrom pipelex.core.pipes.pipe_input_factory import PipeInputSpecFactoryfrom pipelex.core.pipes.pipe_output import PipeOutputfrom pipelex.core.pipes.pipe_run_params import PipeRunParams-from pipelex.core.pipes.specific_pipe import SpecificPipe+from pipelex.core.pipes.specific_pipe import SpecificPipe, SpecificPipeCodesEnumfrom pipelex.exceptions import (DryRunError,PipeConditionError,@@ -25,13 +26,13 @@ from pipelex.exceptions import (StaticValidationErrorType,WorkingMemoryStuffNotFoundError,)-from pipelex.hub import get_pipe_router, get_pipeline_tracker, get_required_pipe+from pipelex.hub import get_content_generator, get_pipe_router, get_pipeline_tracker, get_required_pipe, get_template_providerfrom pipelex.pipe_controllers.condition.pipe_condition_details import PipeConditionDetails, PipeConditionPipeMapfrom pipelex.pipe_controllers.pipe_controller import PipeController-from pipelex.pipe_operators.compose.pipe_compose_blueprint import PipeComposeBlueprint-from pipelex.pipe_operators.compose.pipe_compose_factory import PipeComposeFactoryfrom pipelex.pipe_works.pipe_job_factory import PipeJobFactory-from pipelex.pipeline.job_metadata import JobCategory, JobMetadata+from pipelex.pipeline.job_metadata import JobMetadata+from pipelex.tools.templating.jinja2_required_variables import detect_jinja2_required_variables+from pipelex.tools.templating.jinja2_template_category import Jinja2TemplateCategoryfrom pipelex.tools.typing.validation_utils import has_exactly_one_among_attributes_from_listfrom pipelex.types import Self@@ -67,7 +68,7 @@ class PipeCondition(PipeController):f"not matching the output concept code '{pipe.output.concept_string}' of the pipe '{pipe_condition_pipe_map.pipe_code}'")raise PipeConditionError(msg)- if self.default_pipe_code:+ if self.default_pipe_code and self.default_pipe_code not in SpecificPipeCodesEnum.value_list():default_pipe = get_required_pipe(pipe_code=self.default_pipe_code)if self.output.concept_string not in (default_pipe.output.concept_string,@@ -128,12 +129,13 @@ class PipeCondition(PipeController):required_variables: set[str] = set()# TODO: use jinja2 directly without going though a pipe# Variables from the expression/expression_template- pipe_compose = PipeComposeFactory.make_pipe_compose_from_template_str(- domain=self.domain,- template_str=self.applied_expression_template,- inputs=self.inputs,+ expression_required_variables = detect_jinja2_required_variables(+ template_category=Jinja2TemplateCategory.LLM_PROMPT,+ template_provider=get_template_provider(),+ jinja2_name=None,+ jinja2=self.applied_expression_template,)- required_variables.update(pipe_compose.required_variables())+ required_variables.update(expression_required_variables)# Variables from the pipe_mapfor pipe_code in self.pipe_dependencies():@@ -162,13 +164,14 @@ class PipeCondition(PipeController):needed_inputs = PipeInputSpecFactory.make_empty()# 1. Add the variables from the expression/expression_template- pipe_compose = PipeComposeFactory.make_pipe_compose_from_template_str(- domain=self.domain,- template_str=self.applied_expression_template,- inputs=self.inputs,+ required_variables = detect_jinja2_required_variables(+ template_category=Jinja2TemplateCategory.LLM_PROMPT,+ template_provider=get_template_provider(),+ jinja2_name=None,+ jinja2=self.applied_expression_template,)- for var_name in pipe_compose.required_variables():+ for var_name in required_variables:if not var_name.startswith("_"): # exclude internal variables starting with `_`# We don't know the concept code from just the variable name,# so we'll use a generic placeholder that will be validated later@@ -257,64 +260,42 @@ class PipeCondition(PipeController):for pipe_condition_pipe_map in self.pipe_mapif not SpecificPipe.is_continue(pipe_condition_pipe_map.pipe_code)]- if self.default_pipe_code:+ if self.default_pipe_code and self.default_pipe_code not in SpecificPipeCodesEnum.value_list():pipe_codes.append(self.default_pipe_code)return set(pipe_codes)- @override- async def _run_controller_pipe(+ async def _evaluate_expression_and_select_pipe(self,- job_metadata: JobMetadata,working_memory: WorkingMemory,- pipe_run_params: PipeRunParams,- output_name: str | None = None,- ) -> PipeOutput:- log.dev(f"{self.class_name} generating a '{self.output.code}'")+ ) -> PipeAbstract:+ """Evaluate the conditional expression and select the appropriate pipe.- # TODO: restore pipe_layer feature- # pipe_run_params.push_pipe_code(pipe_code=pipe_code)+ Args:+ working_memory: The working memory context for evaluation- # Convert PipeInputSpec to blueprint format- inputs_blueprint: dict[str, str | InputRequirementBlueprint] = {}- for var_name, requirement in self.inputs.root.items():- inputs_blueprint[var_name] = InputRequirementBlueprint(- concept=requirement.concept.concept_string,- multiplicity=requirement.multiplicity,- )+ Returns:+ A tuple of (evaluated_expression, chosen_pipe_code)++ Raises:+ PipeConditionError: If expression evaluation fails or no matching pipe is found+ """+ content_generator = get_content_generator()+ # Evaluate the expression using Jinja2 templating- pipe_compose_blueprint = PipeComposeBlueprint(- definition="Jinja2 template for pipe condition evaluation",+ evaluated_expression = await content_generator.make_jinja2_text(+ context=working_memory.generate_jinja2_context(),jinja2=self.applied_expression_template,- inputs=inputs_blueprint,- output=self.output.code,+ template_category=Jinja2TemplateCategory.LLM_PROMPT,)- # TODO: use jinja2 directly without going though a pipe- pipe_compose = PipeComposeFactory.make_from_blueprint(- domain=self.domain,- pipe_code="evaluation_for_pipe_condition",- blueprint=pipe_compose_blueprint,- )- jinja2_job_metadata = job_metadata.copy_with_update(- updated_metadata=JobMetadata(- job_category=JobCategory.JINJA2_JOB,- ),- )- log.debug(f"Jinja2 expression: {self.applied_expression_template}")- evaluated_expression = (- await pipe_compose.run_pipe(- job_metadata=jinja2_job_metadata,- working_memory=working_memory,- pipe_run_params=pipe_run_params,- )- ).main_stuff_as_str.strip()-+ # Validate the evaluated expressionif not evaluated_expression or evaluated_expression == "None":error_msg = f"Conditional expression returned an empty string in pipe {self.code}:"error_msg += f"\n\nExpression: {self.applied_expression_template}"raise PipeConditionError(error_msg)log.debug(f"evaluated_expression: '{evaluated_expression}'")+ # Add alias if configuredlog.debug(f"add_alias: {evaluated_expression} -> {self.add_alias_from_expression_to}")if self.add_alias_from_expression_to:working_memory.add_alias(@@ -322,6 +303,7 @@ class PipeCondition(PipeController):target=self.add_alias_from_expression_to,)+ # Select the pipe based on the evaluated expressionchosen_pipe_code = next((pipe_condition_pipe_map.pipe_code@@ -330,20 +312,50 @@ class PipeCondition(PipeController):),self.default_pipe_code,)+ # Validate that a pipe was foundif not chosen_pipe_code:error_msg = f"No pipe code found for evaluated expression '{evaluated_expression}' in pipe {self.code}:"error_msg += f"\n\nExpression: {self.applied_expression_template}"error_msg += f"\n\nPipe map: {self.pipe_map}"raise PipeConditionError(error_msg)- if SpecificPipe.is_continue(chosen_pipe_code):+ return get_required_pipe(pipe_code=chosen_pipe_code)++ @override+ async def _run_controller_pipe(+ self,+ job_metadata: JobMetadata,+ working_memory: WorkingMemory,+ pipe_run_params: PipeRunParams,+ output_name: str | None = None,+ ) -> PipeOutput:+ log.dev(f"{self.class_name} generating a '{self.output.code}'")++ # TODO: restore pipe_layer feature+ # pipe_run_params.push_pipe_code(pipe_code=pipe_code)++ # Convert PipeInputSpec to blueprint format+ inputs_blueprint: dict[str, str | InputRequirementBlueprint] = {}+ for var_name, requirement in self.inputs.root.items():+ inputs_blueprint[var_name] = InputRequirementBlueprint(+ concept=requirement.concept.concept_string,+ multiplicity=requirement.multiplicity,+ )++ chosen_pipe = await self._evaluate_expression_and_select_pipe(working_memory=working_memory)++ # Handle continue case+ if SpecificPipe.is_continue(chosen_pipe.code):return PipeOutput(working_memory=working_memory)+ # Create condition details for trackingcondition_details = self._make_pipe_condition_details(- evaluated_expression=evaluated_expression,- chosen_pipe_code=chosen_pipe_code,+ evaluated_expression=self.applied_expression_template,+ chosen_pipe_code=chosen_pipe.code,)- required_variables = pipe_compose.required_variables()++ # Get required variables and validate they exist in working memory+ required_variables = chosen_pipe.required_variables()log.debug(required_variables, title=f"Required variables for PipeCondition '{self.code}'")required_stuff_names = {required_variable for required_variable in required_variables if not required_variable.startswith("_")}try:@@ -355,6 +367,7 @@ class PipeCondition(PipeController):msg = f"Some required stuff(s) not found: {error_details}"raise PipeInputError(msg) from exc+ # Track condition stepsfor required_stuff in required_stuffs:get_pipeline_tracker().add_condition_step(from_stuff=required_stuff,@@ -364,16 +377,19 @@ class PipeCondition(PipeController):comment="PipeCondition required for condition",)- log.debug(f"Chosen pipe: {chosen_pipe_code}")+ # Execute the chosen pipe+ log.debug(f"Chosen pipe: {chosen_pipe.code}")pipe_output = await get_pipe_router().run(pipe_job=PipeJobFactory.make_pipe_job(- pipe=get_required_pipe(pipe_code=chosen_pipe_code),+ pipe=chosen_pipe,job_metadata=job_metadata,working_memory=working_memory,pipe_run_params=pipe_run_params,output_name=output_name,),)++ # Track choice stepget_pipeline_tracker().add_choice_step(from_condition=condition_details,to_stuff=pipe_output.main_stuff,@@ -413,13 +429,12 @@ class PipeCondition(PipeController):# 2. Validate that the expression template is validtry:- pipe_compose = PipeComposeFactory.make_pipe_compose_from_template_str(- domain=self.domain,- template_str=self.applied_expression_template,- inputs=self.inputs,+ required_variables = detect_jinja2_required_variables(+ template_category=Jinja2TemplateCategory.LLM_PROMPT,+ template_provider=get_template_provider(),+ jinja2_name=None,+ jinja2=self.applied_expression_template,)- # Get required variables to validate the template syntax- required_variables = pipe_compose.required_variables()log.debug(f"Expression template is valid, requires variables: {required_variables}")except Exception as exc:log.error(f"Dry run failed: invalid expression template: {exc}")@@ -439,6 +454,8 @@ class PipeCondition(PipeController):missing_pipes: list[str] = []for pipe_code in all_pipe_codes:+ if pipe_code in SpecificPipeCodesEnum.value_list():+ continuetry:get_required_pipe(pipe_code=pipe_code)log.debug(f"Pipe '{pipe_code}' exists and is accessible")diff --git a/pipelex/pipe_operators/compose/pipe_compose.py b/pipelex/pipe_operators/compose/pipe_compose.pyindex 7d5399fe6..5d41bd1b9 100644--- a/pipelex/pipe_operators/compose/pipe_compose.py+++ b/pipelex/pipe_operators/compose/pipe_compose.py@@ -132,7 +132,7 @@ class PipeCompose(PipeOperator[PipeComposeOutput]):msg = f"PipeCompose does not suppport multiple outputs, got output_multiplicity = {pipe_run_params.output_multiplicity}"raise PipeRunParamsError(msg)- context: dict[str, Any] = working_memory.generate_context()+ context: dict[str, Any] = working_memory.generate_jinja2_context()if pipe_run_params:context.update(**pipe_run_params.params)if self.extra_context:diff --git a/pipelex/pipe_operators/llm/pipe_llm.py b/pipelex/pipe_operators/llm/pipe_llm.pyindex 10cfab882..b6773d2f9 100644--- a/pipelex/pipe_operators/llm/pipe_llm.py+++ b/pipelex/pipe_operators/llm/pipe_llm.py@@ -49,6 +49,7 @@ from pipelex.hub import (from pipelex.pipe_operators.llm.pipe_llm_blueprint import StructuringMethodfrom pipelex.pipe_operators.pipe_operator import PipeOperatorfrom pipelex.pipeline.job_metadata import JobMetadata+from pipelex.tools.templating.jinja2_template_category import Jinja2TemplateCategoryfrom pipelex.tools.typing.structure_printer import StructurePrinterfrom pipelex.types import Self@@ -390,7 +391,7 @@ class PipeLLM(PipeOperator[PipeLLMOutput]):output_structure_prompt: str | None = Noneif get_config().cogt.llm_config.is_structure_prompt_enabled:- output_structure_prompt = PipeLLM.get_output_structure_prompt(+ output_structure_prompt = await PipeLLM.get_output_structure_prompt(concept_string=pipe_run_params.dynamic_output_concept_code or output_concept.concept_string,is_with_preliminary_text=is_with_preliminary_text,)@@ -529,10 +530,9 @@ class PipeLLM(PipeOperator[PipeLLMOutput]):)@staticmethod- def get_output_structure_prompt(concept_string: str, is_with_preliminary_text: bool) -> str | None:+ async def get_output_structure_prompt(concept_string: str, is_with_preliminary_text: bool) -> str | None:concept = get_required_concept(concept_string=concept_string)- class_name = concept.structure_class_name- output_class = get_class_registry().get_class(class_name)+ output_class = get_class_registry().get_class(concept.structure_class_name)log.debug(f"get_output_structure_prompt for {concept_string} with {is_with_preliminary_text=}")log.debug(f"output_class: {output_class}")if not output_class:@@ -544,27 +544,10 @@ class PipeLLM(PipeOperator[PipeLLMOutput]):return Noneclass_structure_str = "\n".join(class_structure)- # TODO: use proper prompt templating for this- if is_with_preliminary_text:- output_structure_prompt = f"""------The instance we want to generate will be for the following class:-{class_structure_str}--Don't bother with JSON formatting, we'll do that as a second step.-For now, just output markdown with the details of the instance.-DO NOT create information.-If some information is not present for an attribute, output the default value or None according to the attribute definition.-"""- else:- output_structure_prompt = f"""------The instance we want to generate will be for the following class:-{class_structure_str}--DO NOT create information.-If some information is not present for an attribute, output the default value or None according to the attribute definition.-"""- return output_structure_prompt+ return await get_content_generator().make_jinja2_text(+ context={+ "class_structure_str": class_structure_str,+ },+ template_category=Jinja2TemplateCategory.LLM_PROMPT,+ jinja2_name="output_structure_prompt" if is_with_preliminary_text else "output_structure_prompt_no_preliminary_text",+ )diff --git a/pipelex/pipelex.toml b/pipelex/pipelex.tomlindex 3df6c9fb3..9b3852e0d 100644--- a/pipelex/pipelex.toml+++ b/pipelex/pipelex.toml@@ -6,8 +6,8 @@ observer_dir = "results/observer"[pipelex.feature_config]# WIP/Experimental feature flags-is_pipeline_tracking_enabled = true-is_activity_tracking_enabled = true+is_pipeline_tracking_enabled = false+is_activity_tracking_enabled = falseis_reporting_enabled = true[pipelex.reporting_config]@@ -87,11 +87,11 @@ is_auto_setup_preset_ocr = true[cogt.llm_config]default_max_images = 100-is_structure_prompt_enabled = false+is_structure_prompt_enabled = true[cogt.llm_config.instructor_config]is_openai_structured_output_enabled = false-is_dump_kwargs_enabled = true+is_dump_kwargs_enabled = falseis_dump_response_enabled = falseis_dump_error_enabled = falsediff --git a/pipelex/tools/misc/context_provider_abstract.py b/pipelex/tools/misc/context_provider_abstract.pyindex 1cff3ac83..23ba2c56e 100644--- a/pipelex/tools/misc/context_provider_abstract.py+++ b/pipelex/tools/misc/context_provider_abstract.py@@ -20,5 +20,5 @@ class ContextProviderAbstract(ABC):pass@abstractmethod- def generate_context(self) -> dict[str, Any]:+ def generate_jinja2_context(self) -> dict[str, Any]:passdiff --git a/pipelex/tools/templating/template_library.py b/pipelex/tools/templating/template_library.pyindex 82132d484..92a3946a3 100644--- a/pipelex/tools/templating/template_library.py+++ b/pipelex/tools/templating/template_library.py@@ -7,6 +7,7 @@ from typing_extensions import overridefrom pipelex import logfrom pipelex.libraries.library_config import LibraryConfigfrom pipelex.tools.exceptions import ToolException+from pipelex.tools.misc.file_utils import find_files_in_dirfrom pipelex.tools.misc.toml_utils import load_toml_from_pathfrom pipelex.tools.templating.jinja2_parsing import check_jinja2_parsingfrom pipelex.tools.templating.jinja2_template_category import Jinja2TemplateCategory@@ -32,9 +33,9 @@ class TemplateLibrary(TemplateProviderAbstract, RootModel[TemplateLibraryRoot]):@overridedef setup(self) -> None:- template_toml_paths = self.library_config.get_templates_paths()+ template_toml_paths = find_files_in_dir(dir_path="pipelex/tools/templating/templates", pattern="*.toml", is_recursive=True)for template_toml_path in template_toml_paths:- self._load_from_toml(toml_path=template_toml_path)+ self._load_from_toml(toml_path=str(template_toml_path))self.validate_templates(template_category=Jinja2TemplateCategory.LLM_PROMPT)@overridediff --git a/pipelex/tools/templating/templates/__init__.py b/pipelex/tools/templating/templates/__init__.pynew file mode 100644index 000000000..e69de29bbdiff --git a/pipelex/tools/templating/templates/base_templates.toml b/pipelex/tools/templating/templates/base_templates.tomlnew file mode 100644index 000000000..fb1a8d026--- /dev/null+++ b/pipelex/tools/templating/templates/base_templates.toml@@ -0,0 +1,34 @@+[generic_prompts]+structure_from_preliminary_text_system = "You are a data modeling expert specialized in extracting structure from text."+# TODO: Should be a proper jinja2+structure_from_preliminary_text_user = """+Your job is to extract and structure information from a text.+Here is the text:+{preliminary_text}++Now generate the JSON in the required format.+Do not create information that is not in the text.+"""++output_structure_prompt = """+---+The instance we want to generate will be for the following class:+{{ class_structure_str }}++Don't bother with JSON formatting, we'll do that as a second step.+For now, just output markdown with the details of the instance.+DO NOT create information.+If some information is not present for an attribute, output the default value or None according to the attribute definition.+"""++output_structure_prompt_no_preliminary_text = """+---+The instance we want to generate will be for the following class:+{{ class_structure_str }}++DO NOT create information.+If some information is not present for an attribute, output the default value or None according to the attribute definition.+"""++[test_prompts]+jinja2_test_template = "I want a {{ place_holder }} t-shirt."diff --git a/tests/integration/pipelex/cogt/test_llm_job_async_factory.py b/tests/integration/pipelex/cogt/test_llm_job_async_factory.pydeleted file mode 100644index 0b1ed2920..000000000--- a/tests/integration/pipelex/cogt/test_llm_job_async_factory.py+++ /dev/null@@ -1,50 +0,0 @@-import random--import pytest--from pipelex import pretty_print-from pipelex.cogt.llm.llm_job import LLMJobParams-from pipelex.cogt.llm.llm_job_factory import LLMJobFactory-from pipelex.cogt.llm.llm_prompt_template import LLMPromptTemplate-from pipelex.hub import get_llm_worker-from tests.integration.pipelex.cogt.test_data import LLMTestConstants--[email protected](loop_scope="class")-class TestAsyncLLMJobFactory:- async def test_llm_job_using_generic_prompt_factory(self):- llm_prompt_template = LLMPromptTemplate.from_template_contents(user_text=LLMTestConstants.PROMPT_TEMPLATE_TEXT)- random_color = random.choice(LLMTestConstants.PROMPT_COLOR_EXAMPLES)- llm_job = await LLMJobFactory.make_llm_job_from_prompt_factory(- llm_prompt_factory=llm_prompt_template,- llm_prompt_arguments={"color": random_color},- llm_job_params=LLMJobParams(temperature=0.5, max_tokens=50, seed=None),- )- assert llm_job- pretty_print(llm_job)-- async def test_llm_job_using_prompt_template(self):- llm_prompt_template = LLMPromptTemplate.from_template_contents(user_text=LLMTestConstants.PROMPT_TEMPLATE_TEXT)- random_color = random.choice(LLMTestConstants.PROMPT_COLOR_EXAMPLES)- llm_job = await LLMJobFactory.make_llm_job_from_prompt_template(- llm_prompt_template=llm_prompt_template,- llm_prompt_arguments={"color": random_color},- llm_job_params=LLMJobParams(temperature=0.5, max_tokens=50, seed=None),- )- assert llm_job- pretty_print(llm_job)-- @pytest.mark.llm- @pytest.mark.inference- async def test_llm_job_and_gen_text(self, llm_handle: str):- llm_worker = get_llm_worker(llm_handle=llm_handle)- llm_prompt_template = LLMPromptTemplate.from_template_contents(user_text=LLMTestConstants.PROMPT_TEMPLATE_TEXT)- random_color = random.choice(LLMTestConstants.PROMPT_COLOR_EXAMPLES)- llm_job = await LLMJobFactory.make_llm_job_from_prompt_template(- llm_prompt_template=llm_prompt_template,- llm_prompt_arguments={"color": random_color},- llm_job_params=LLMJobParams(temperature=0.5, max_tokens=50, seed=None),- )- generated_text = await llm_worker.gen_text(llm_job=llm_job)- assert generated_text- pretty_print(generated_text)