Skip to content

Build AI methods with your coding agent, then run them anywhere: as an MCP for chatbots, as a webapp for people, or via API for your software.

Refactor/remove pipe compose in pipe condition (#303)

Thomas Hebrard33bf9b78cb1a
diff --git a/.badges/tests.json b/.badges/tests.json
index 056566a85..a2aa3e5b9 100644
--- a/.badges/tests.json
+++ b/.badges/tests.json
@@ -1,7 +1,7 @@
{
"schemaVersion": 1,
"label": "tests",
- "message": "1045",
+ "message": "1052",
"color": "blue",
"cacheSeconds": 300
}
\ No newline at end of file
diff --git a/.github/workflows/check-test-count-badge.yml b/.github/workflows/check-test-count-badge.yml
index ee6e243c1..bc4ddb7c7 100644
--- a/.github/workflows/check-test-count-badge.yml
+++ b/.github/workflows/check-test-count-badge.yml
@@ -2,10 +2,6 @@ name: Check test count matches badge
on:
pull_request:
- branches:
- - main
- - dev
- - "release/v[0-9]+.[0-9]+.[0-9]+"
jobs:
check:
diff --git a/.github/workflows/doc-check.yml b/.github/workflows/doc-check.yml
index 54a70d29a..362c2d833 100644
--- a/.github/workflows/doc-check.yml
+++ b/.github/workflows/doc-check.yml
@@ -2,10 +2,6 @@ name: Documentation Check
on:
pull_request:
- branches:
- - main
- - dev
- - "release/v[0-9]+.[0-9]+.[0-9]+"
paths:
- 'docs/**'
- 'mkdocs.yml'
diff --git a/.github/workflows/lint-check.yml b/.github/workflows/lint-check.yml
index 2eaed36dd..c4e82aa3d 100644
--- a/.github/workflows/lint-check.yml
+++ b/.github/workflows/lint-check.yml
@@ -2,10 +2,6 @@ name: Lint check
on:
pull_request:
- branches:
- - main
- - dev
- - "release/v[0-9]+.[0-9]+.[0-9]+"
jobs:
# --------------------------------------------------------------------------
diff --git a/.github/workflows/tests-check.yml b/.github/workflows/tests-check.yml
index 8e7229e63..532b4c43f 100644
--- a/.github/workflows/tests-check.yml
+++ b/.github/workflows/tests-check.yml
@@ -2,10 +2,6 @@ name: Tests check
on:
pull_request:
- branches:
- - main
- - dev
- - "release/v[0-9]+.[0-9]+.[0-9]+"
jobs:
matrix-test:
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 95b03a0cf..bea021d52 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -26,6 +26,7 @@
- Modified `any_env_var_is_placeholder()` to use new placeholder detection logic
- Updated test environment setup to use dynamic placeholder generation instead of hardcoded values
- Restructured test classes and methods in environment tests
+ - Lint and tests GHA should run on every PR
### Fixed
- Fixed logic error in `any_env_var_is_placeholder()` function - now correctly returns False when no placeholders are found
@@ -35,6 +36,8 @@
- Removed `get_rooted_path()` and `get_env_rooted_path()` utility functions
- Removed hardcoded placeholder dictionary in test setup
- Removed function `run_pipe_code` in pipe router because it was not relevant (used mostly in tests)
+ - Remove the use of `PipeCompose` in `PipeCondition`, to only use jinja2 with the `ContentGenerator`
+ - Remove the template libraries from the pipelex libraries.
## [v0.10.2] - 2025-09-18
diff --git a/pipelex/cogt/content_generation/jinja2_generate.py b/pipelex/cogt/content_generation/jinja2_generate.py
index 73474136c..995209e75 100644
--- a/pipelex/cogt/content_generation/jinja2_generate.py
+++ b/pipelex/cogt/content_generation/jinja2_generate.py
@@ -5,8 +5,8 @@ from pipelex.tools.templating.jinja2_rendering import render_jinja2
from pipelex.tools.templating.template_preprocessor import preprocess_template
-# TODO: get rid of this intermediate call which seems useless, or explain why it stays
async def jinja2_gen_text(jinja2_assignment: Jinja2Assignment) -> str:
+ # Intermediate call to preprocess the template with our syntax patterns (@, $, @?, etc.)
if jinja2_assignment.jinja2:
jinja2_assignment.jinja2 = preprocess_template(template=jinja2_assignment.jinja2)
check_jinja2_parsing(jinja2_assignment.jinja2)
diff --git a/pipelex/cogt/llm/llm_prompt_factory_abstract.py b/pipelex/cogt/llm/llm_prompt_factory_abstract.py
index 1a30544a7..319ed2657 100644
--- a/pipelex/cogt/llm/llm_prompt_factory_abstract.py
+++ b/pipelex/cogt/llm/llm_prompt_factory_abstract.py
@@ -6,14 +6,6 @@ from pydantic import BaseModel
from pipelex.cogt.llm.llm_prompt import LLMPrompt
-def make_empty_prompt() -> LLMPrompt:
- return LLMPrompt(
- system_text=None,
- user_text=None,
- user_images=[],
- )
-
-
class LLMPromptFactoryAbstract(ABC, BaseModel):
@abstractmethod
async def make_llm_prompt_from_args(
@@ -21,8 +13,3 @@ class LLMPromptFactoryAbstract(ABC, BaseModel):
**prompt_arguments: Any,
) -> LLMPrompt:
pass
-
- @property
- @abstractmethod
- def desc(self) -> str:
- pass
diff --git a/pipelex/cogt/llm/llm_prompt_spec.py b/pipelex/cogt/llm/llm_prompt_spec.py
index 32457def0..5b3c17e1d 100644
--- a/pipelex/cogt/llm/llm_prompt_spec.py
+++ b/pipelex/cogt/llm/llm_prompt_spec.py
@@ -173,7 +173,7 @@ class LLMPromptSpec(BaseModel):
jinja2_blueprint.prompting_style = prompting_style
log.verbose(f"Setting prompting style to {prompting_style}")
- context: dict[str, Any] = context_provider.generate_context()
+ context: dict[str, Any] = context_provider.generate_jinja2_context()
if extra_params:
context.update(**extra_params)
if jinja2_blueprint.extra_context:
diff --git a/pipelex/cogt/llm/llm_prompt_template.py b/pipelex/cogt/llm/llm_prompt_template.py
index 6be640f18..94a67c3eb 100644
--- a/pipelex/cogt/llm/llm_prompt_template.py
+++ b/pipelex/cogt/llm/llm_prompt_template.py
@@ -6,12 +6,19 @@ from pipelex import log
from pipelex.cogt.exceptions import LLMPromptFactoryError, LLMPromptTemplateInputsError
from pipelex.cogt.image.prompt_image import PromptImage
from pipelex.cogt.llm.llm_prompt import LLMPrompt
-from pipelex.cogt.llm.llm_prompt_factory_abstract import LLMPromptFactoryAbstract, make_empty_prompt
+from pipelex.cogt.llm.llm_prompt_factory_abstract import LLMPromptFactoryAbstract
from pipelex.cogt.llm.llm_prompt_template_inputs import LLMPromptTemplateInputs
from pipelex.config import get_config
from pipelex.hub import get_template
from pipelex.tools.misc.string_utils import is_none_or_has_text
-from pipelex.tools.templating.template_provider_abstract import TemplateProviderAbstract
+
+
+def make_empty_prompt() -> LLMPrompt:
+ return LLMPrompt(
+ system_text=None,
+ user_text=None,
+ user_images=[],
+ )
class LLMPromptTemplate(LLMPromptFactoryAbstract):
@@ -22,21 +29,10 @@ class LLMPromptTemplate(LLMPromptFactoryAbstract):
source_system_template_name: str | None = None
source_user_template_name: str | None = None
- @property
- @override
- def desc(self) -> str:
- return f"{LLMPromptTemplate.__name__} based on proto prompt: {self.proto_prompt} and base inputs: {self.base_template_inputs}"
-
@override
async def make_llm_prompt_from_args(
self,
**prompt_arguments: Any,
- ) -> LLMPrompt:
- return self.synchro_make_llm_prompt_from_args(**prompt_arguments)
-
- def synchro_make_llm_prompt_from_args(
- self,
- **prompt_arguments: Any,
) -> LLMPrompt:
arguments_dict = prompt_arguments.copy()
@@ -117,47 +113,6 @@ class LLMPromptTemplate(LLMPromptFactoryAbstract):
return llm_prompt
- # factories of LLMPromptTemplate (which is a factory of LLMPrompt)
- @classmethod
- def from_template_name(
- cls,
- template_provider: TemplateProviderAbstract,
- system_text: str | None = None,
- user_text: str | None = None,
- user_images: list[PromptImage] | None = None,
- system_template_name: str | None = None,
- user_template_name: str | None = None,
- ) -> "LLMPromptTemplate":
- proto_prompt: LLMPrompt = LLMPrompt(
- system_text=system_text,
- user_text=user_text,
- user_images=user_images or [],
- )
- if user_template_name:
- proto_prompt.user_text = template_provider.get_template(user_template_name)
- if system_template_name:
- proto_prompt.system_text = template_provider.get_template(system_template_name)
- return cls(
- proto_prompt=proto_prompt,
- source_system_template_name=system_template_name,
- source_user_template_name=user_template_name,
- )
-
- # factories of LLMPromptTemplate (which is a factory of LLMPrompt)
- @classmethod
- def from_template_contents(
- cls,
- system_text: str | None = None,
- user_text: str | None = None,
- user_images: list[PromptImage] | None = None,
- ) -> "LLMPromptTemplate":
- proto_prompt = LLMPrompt(
- user_text=user_text,
- system_text=system_text,
- user_images=user_images or [],
- )
- return cls(proto_prompt=proto_prompt)
-
@classmethod
def for_structure_from_preliminary_text(cls) -> "LLMPromptTemplate":
generic_template_names = get_config().pipelex.generic_template_names
diff --git a/pipelex/core/memory/working_memory.py b/pipelex/core/memory/working_memory.py
index 349a125cc..347ce733b 100644
--- a/pipelex/core/memory/working_memory.py
+++ b/pipelex/core/memory/working_memory.py
@@ -196,7 +196,7 @@ class WorkingMemory(BaseModel, ContextProviderAbstract):
################################################################################################
@override
- def generate_context(self) -> dict[str, Any]:
+ def generate_jinja2_context(self) -> dict[str, Any]:
# TODO: Add unit tests for this method
artefact_dict: StuffArtefactDict = {}
for name, stuff in self.root.items():
diff --git a/pipelex/core/pipes/specific_pipe.py b/pipelex/core/pipes/specific_pipe.py
index fb749fef5..963c2ea56 100644
--- a/pipelex/core/pipes/specific_pipe.py
+++ b/pipelex/core/pipes/specific_pipe.py
@@ -6,6 +6,10 @@ class SpecificPipeCodesEnum(StrEnum):
# TODO: Implement the break pipe: It should enable to leave the current sequence.
# BREAK = "break"
+ @classmethod
+ def value_list(cls) -> list[str]:
+ return list(cls)
+
# TODO: Rethink this class. They are not pipes really.
class SpecificPipe:
diff --git a/pipelex/libraries/library_config.py b/pipelex/libraries/library_config.py
index 65e176f9b..e91e624e3 100644
--- a/pipelex/libraries/library_config.py
+++ b/pipelex/libraries/library_config.py
@@ -2,7 +2,7 @@ from pathlib import Path
from typing import ClassVar
from pipelex.tools.config.config_model import ConfigModel
-from pipelex.tools.misc.file_utils import copy_file_from_package, copy_folder_from_package, find_files_in_dir
+from pipelex.tools.misc.file_utils import copy_file_from_package, copy_folder_from_package
PIPELEX_LIBRARIES_PATH = "libraries"
@@ -19,10 +19,6 @@ class LibraryConfig(ConfigModel):
def base_pipelines_dir_path(self) -> str:
return f"{self.config_dir_path}/pipelines/base_library"
- @property
- def templates_dir_path(self) -> str:
- return f"{self.config_dir_path}/templates"
-
@property
def test_pipelines_dir_path(self) -> str:
return "tests/test_pipelines"
@@ -31,9 +27,6 @@ class LibraryConfig(ConfigModel):
def failing_pipelines_file_paths(self) -> set[Path]:
return {Path("tests/test_pipelines/failing_pipelines.plx")}
- def get_templates_paths(self) -> list[str]:
- return [str(path) for path in find_files_in_dir(dir_path=self.templates_dir_path, pattern="*.toml", is_recursive=True)]
-
def export_libraries(self, overwrite: bool = False) -> None:
"""Duplicate pipelex libraries files in the client project, preserving directory structure."""
copy_file_from_package(
@@ -56,11 +49,3 @@ class LibraryConfig(ConfigModel):
target_path=f"{self.pipelines_dir_path}/__init__.py",
overwrite=overwrite,
)
-
- # templates
- copy_folder_from_package(
- package_name=self.package_name,
- folder_path_in_package=f"{PIPELEX_LIBRARIES_PATH}/templates",
- target_dir=self.templates_dir_path,
- overwrite=overwrite,
- )
diff --git a/pipelex/libraries/library_manager.py b/pipelex/libraries/library_manager.py
index 4c43a1706..e2e38d3e9 100644
--- a/pipelex/libraries/library_manager.py
+++ b/pipelex/libraries/library_manager.py
@@ -1,4 +1,3 @@
-import os
from pathlib import Path
from typing import ClassVar
@@ -37,7 +36,6 @@ from pipelex.libraries.library_manager_abstract import LibraryManagerAbstract
from pipelex.tools.class_registry_utils import ClassRegistryUtils
from pipelex.tools.func_registry_utils import FuncRegistryUtils
from pipelex.tools.misc.file_utils import find_files_in_dir
-from pipelex.tools.misc.toml_utils import TOMLValidationError, validate_toml_file
from pipelex.tools.runtime_manager import runtime_manager
from pipelex.tools.typing.pydantic_utils import format_pydantic_validation_error
from pipelex.types import StrEnum
@@ -85,21 +83,6 @@ class LibraryManager(LibraryManagerAbstract):
self.pipe_library.validate_with_libraries()
self.domain_library.validate_with_libraries()
- def _validate_toml_files(self):
- """Validate all TOML files used by the library manager for formatting issues."""
- log.debug("LibraryManager validating TOML file formatting")
-
- # Validation of template paths
- template_paths = self.library_config.get_templates_paths()
- for template_path in template_paths:
- if os.path.exists(template_path):
- try:
- validate_toml_file(template_path)
- except TOMLValidationError as exc:
- msg = f"PLX validation failed for template file '{template_path}': {exc}"
- log.error(msg)
- raise LibraryError(msg) from exc
-
@override
def setup(self) -> None:
self.concept_library.setup()
diff --git a/pipelex/libraries/pipelines/builder/pipe/pipe_design.plx b/pipelex/libraries/pipelines/builder/pipe/pipe_design.plx
index 39f1a1c0d..308dad3d0 100644
--- a/pipelex/libraries/pipelines/builder/pipe/pipe_design.plx
+++ b/pipelex/libraries/pipelines/builder/pipe/pipe_design.plx
@@ -25,6 +25,7 @@ definition = "Route by signature.type to the correct spec emitter."
inputs = { plan_draft = "PlanDraft", pipe_signature = "PipeSignature", concept_specs = "ConceptSpec" }
output = "Dynamic"
expression = "pipe_signature.type"
+default_pipe_code = "continue"
[pipe.detail_pipe_spec.pipe_map]
PipeSequence = "detail_pipe_sequence"
@@ -33,7 +34,6 @@ PipeCondition = "detail_pipe_condition"
PipeLLM = "detail_pipe_llm"
PipeOcr = "detail_pipe_ocr"
PipeImgGen = "detail_pipe_img_gen"
-# PipeCompose = "detail_pipe_compose"
# ────────────────────────────────────────────────────────────────────────────────
# PIPE CONTROLLERS
diff --git a/pipelex/libraries/templates/base_templates.toml b/pipelex/libraries/templates/base_templates.toml
deleted file mode 100644
index 484fecacf..000000000
--- a/pipelex/libraries/templates/base_templates.toml
+++ /dev/null
@@ -1,17 +0,0 @@
-[generic_prompts]
-
-structure_from_preliminary_text_system = "You are a data modeling expert specialized in extracting structure from text."
-# TODO: Should be a proper jinja2
-structure_from_preliminary_text_user = """
-Your job is to extract and structure information from a text.
-Here is the text:
-{preliminary_text}
-
-Now generate the JSON in the required format.
-Do not create information that is not in the text.
-"""
-
-[test_prompts]
-
-jinja2_test_template = "I want a {{ place_holder }} t-shirt."
-
diff --git a/pipelex/pipe_controllers/condition/pipe_condition.py b/pipelex/pipe_controllers/condition/pipe_condition.py
index 7905e8c35..1a38a5675 100644
--- a/pipelex/pipe_controllers/condition/pipe_condition.py
+++ b/pipelex/pipe_controllers/condition/pipe_condition.py
@@ -9,12 +9,13 @@ from pipelex.config import StaticValidationReaction, get_config
from pipelex.core.concepts.concept_factory import ConceptFactory
from pipelex.core.concepts.concept_native import NATIVE_CONCEPTS_DATA, NativeConceptEnum, NativeConceptManager
from pipelex.core.memory.working_memory import WorkingMemory
+from pipelex.core.pipes.pipe_abstract import PipeAbstract
from pipelex.core.pipes.pipe_input import PipeInputSpec
from pipelex.core.pipes.pipe_input_blueprint import InputRequirementBlueprint
from pipelex.core.pipes.pipe_input_factory import PipeInputSpecFactory
from pipelex.core.pipes.pipe_output import PipeOutput
from pipelex.core.pipes.pipe_run_params import PipeRunParams
-from pipelex.core.pipes.specific_pipe import SpecificPipe
+from pipelex.core.pipes.specific_pipe import SpecificPipe, SpecificPipeCodesEnum
from pipelex.exceptions import (
DryRunError,
PipeConditionError,
@@ -25,13 +26,13 @@ from pipelex.exceptions import (
StaticValidationErrorType,
WorkingMemoryStuffNotFoundError,
)
-from pipelex.hub import get_pipe_router, get_pipeline_tracker, get_required_pipe
+from pipelex.hub import get_content_generator, get_pipe_router, get_pipeline_tracker, get_required_pipe, get_template_provider
from pipelex.pipe_controllers.condition.pipe_condition_details import PipeConditionDetails, PipeConditionPipeMap
from pipelex.pipe_controllers.pipe_controller import PipeController
-from pipelex.pipe_operators.compose.pipe_compose_blueprint import PipeComposeBlueprint
-from pipelex.pipe_operators.compose.pipe_compose_factory import PipeComposeFactory
from pipelex.pipe_works.pipe_job_factory import PipeJobFactory
-from pipelex.pipeline.job_metadata import JobCategory, JobMetadata
+from pipelex.pipeline.job_metadata import JobMetadata
+from pipelex.tools.templating.jinja2_required_variables import detect_jinja2_required_variables
+from pipelex.tools.templating.jinja2_template_category import Jinja2TemplateCategory
from pipelex.tools.typing.validation_utils import has_exactly_one_among_attributes_from_list
from pipelex.types import Self
@@ -67,7 +68,7 @@ class PipeCondition(PipeController):
f"not matching the output concept code '{pipe.output.concept_string}' of the pipe '{pipe_condition_pipe_map.pipe_code}'"
)
raise PipeConditionError(msg)
- if self.default_pipe_code:
+ if self.default_pipe_code and self.default_pipe_code not in SpecificPipeCodesEnum.value_list():
default_pipe = get_required_pipe(pipe_code=self.default_pipe_code)
if self.output.concept_string not in (
default_pipe.output.concept_string,
@@ -128,12 +129,13 @@ class PipeCondition(PipeController):
required_variables: set[str] = set()
# TODO: use jinja2 directly without going though a pipe
# Variables from the expression/expression_template
- pipe_compose = PipeComposeFactory.make_pipe_compose_from_template_str(
- domain=self.domain,
- template_str=self.applied_expression_template,
- inputs=self.inputs,
+ expression_required_variables = detect_jinja2_required_variables(
+ template_category=Jinja2TemplateCategory.LLM_PROMPT,
+ template_provider=get_template_provider(),
+ jinja2_name=None,
+ jinja2=self.applied_expression_template,
)
- required_variables.update(pipe_compose.required_variables())
+ required_variables.update(expression_required_variables)
# Variables from the pipe_map
for pipe_code in self.pipe_dependencies():
@@ -162,13 +164,14 @@ class PipeCondition(PipeController):
needed_inputs = PipeInputSpecFactory.make_empty()
# 1. Add the variables from the expression/expression_template
- pipe_compose = PipeComposeFactory.make_pipe_compose_from_template_str(
- domain=self.domain,
- template_str=self.applied_expression_template,
- inputs=self.inputs,
+ required_variables = detect_jinja2_required_variables(
+ template_category=Jinja2TemplateCategory.LLM_PROMPT,
+ template_provider=get_template_provider(),
+ jinja2_name=None,
+ jinja2=self.applied_expression_template,
)
- for var_name in pipe_compose.required_variables():
+ for var_name in required_variables:
if not var_name.startswith("_"): # exclude internal variables starting with `_`
# We don't know the concept code from just the variable name,
# so we'll use a generic placeholder that will be validated later
@@ -257,64 +260,42 @@ class PipeCondition(PipeController):
for pipe_condition_pipe_map in self.pipe_map
if not SpecificPipe.is_continue(pipe_condition_pipe_map.pipe_code)
]
- if self.default_pipe_code:
+ if self.default_pipe_code and self.default_pipe_code not in SpecificPipeCodesEnum.value_list():
pipe_codes.append(self.default_pipe_code)
return set(pipe_codes)
- @override
- async def _run_controller_pipe(
+ async def _evaluate_expression_and_select_pipe(
self,
- job_metadata: JobMetadata,
working_memory: WorkingMemory,
- pipe_run_params: PipeRunParams,
- output_name: str | None = None,
- ) -> PipeOutput:
- log.dev(f"{self.class_name} generating a '{self.output.code}'")
+ ) -> PipeAbstract:
+ """Evaluate the conditional expression and select the appropriate pipe.
- # TODO: restore pipe_layer feature
- # pipe_run_params.push_pipe_code(pipe_code=pipe_code)
+ Args:
+ working_memory: The working memory context for evaluation
- # Convert PipeInputSpec to blueprint format
- inputs_blueprint: dict[str, str | InputRequirementBlueprint] = {}
- for var_name, requirement in self.inputs.root.items():
- inputs_blueprint[var_name] = InputRequirementBlueprint(
- concept=requirement.concept.concept_string,
- multiplicity=requirement.multiplicity,
- )
+ Returns:
+ A tuple of (evaluated_expression, chosen_pipe_code)
+
+ Raises:
+ PipeConditionError: If expression evaluation fails or no matching pipe is found
+ """
+ content_generator = get_content_generator()
+ # Evaluate the expression using Jinja2 templating
- pipe_compose_blueprint = PipeComposeBlueprint(
- definition="Jinja2 template for pipe condition evaluation",
+ evaluated_expression = await content_generator.make_jinja2_text(
+ context=working_memory.generate_jinja2_context(),
jinja2=self.applied_expression_template,
- inputs=inputs_blueprint,
- output=self.output.code,
+ template_category=Jinja2TemplateCategory.LLM_PROMPT,
)
- # TODO: use jinja2 directly without going though a pipe
- pipe_compose = PipeComposeFactory.make_from_blueprint(
- domain=self.domain,
- pipe_code="evaluation_for_pipe_condition",
- blueprint=pipe_compose_blueprint,
- )
- jinja2_job_metadata = job_metadata.copy_with_update(
- updated_metadata=JobMetadata(
- job_category=JobCategory.JINJA2_JOB,
- ),
- )
- log.debug(f"Jinja2 expression: {self.applied_expression_template}")
- evaluated_expression = (
- await pipe_compose.run_pipe(
- job_metadata=jinja2_job_metadata,
- working_memory=working_memory,
- pipe_run_params=pipe_run_params,
- )
- ).main_stuff_as_str.strip()
-
+ # Validate the evaluated expression
if not evaluated_expression or evaluated_expression == "None":
error_msg = f"Conditional expression returned an empty string in pipe {self.code}:"
error_msg += f"\n\nExpression: {self.applied_expression_template}"
raise PipeConditionError(error_msg)
log.debug(f"evaluated_expression: '{evaluated_expression}'")
+ # Add alias if configured
log.debug(f"add_alias: {evaluated_expression} -> {self.add_alias_from_expression_to}")
if self.add_alias_from_expression_to:
working_memory.add_alias(
@@ -322,6 +303,7 @@ class PipeCondition(PipeController):
target=self.add_alias_from_expression_to,
)
+ # Select the pipe based on the evaluated expression
chosen_pipe_code = next(
(
pipe_condition_pipe_map.pipe_code
@@ -330,20 +312,50 @@ class PipeCondition(PipeController):
),
self.default_pipe_code,
)
+ # Validate that a pipe was found
if not chosen_pipe_code:
error_msg = f"No pipe code found for evaluated expression '{evaluated_expression}' in pipe {self.code}:"
error_msg += f"\n\nExpression: {self.applied_expression_template}"
error_msg += f"\n\nPipe map: {self.pipe_map}"
raise PipeConditionError(error_msg)
- if SpecificPipe.is_continue(chosen_pipe_code):
+ return get_required_pipe(pipe_code=chosen_pipe_code)
+
+ @override
+ async def _run_controller_pipe(
+ self,
+ job_metadata: JobMetadata,
+ working_memory: WorkingMemory,
+ pipe_run_params: PipeRunParams,
+ output_name: str | None = None,
+ ) -> PipeOutput:
+ log.dev(f"{self.class_name} generating a '{self.output.code}'")
+
+ # TODO: restore pipe_layer feature
+ # pipe_run_params.push_pipe_code(pipe_code=pipe_code)
+
+ # Convert PipeInputSpec to blueprint format
+ inputs_blueprint: dict[str, str | InputRequirementBlueprint] = {}
+ for var_name, requirement in self.inputs.root.items():
+ inputs_blueprint[var_name] = InputRequirementBlueprint(
+ concept=requirement.concept.concept_string,
+ multiplicity=requirement.multiplicity,
+ )
+
+ chosen_pipe = await self._evaluate_expression_and_select_pipe(working_memory=working_memory)
+
+ # Handle continue case
+ if SpecificPipe.is_continue(chosen_pipe.code):
return PipeOutput(working_memory=working_memory)
+ # Create condition details for tracking
condition_details = self._make_pipe_condition_details(
- evaluated_expression=evaluated_expression,
- chosen_pipe_code=chosen_pipe_code,
+ evaluated_expression=self.applied_expression_template,
+ chosen_pipe_code=chosen_pipe.code,
)
- required_variables = pipe_compose.required_variables()
+
+ # Get required variables and validate they exist in working memory
+ required_variables = chosen_pipe.required_variables()
log.debug(required_variables, title=f"Required variables for PipeCondition '{self.code}'")
required_stuff_names = {required_variable for required_variable in required_variables if not required_variable.startswith("_")}
try:
@@ -355,6 +367,7 @@ class PipeCondition(PipeController):
msg = f"Some required stuff(s) not found: {error_details}"
raise PipeInputError(msg) from exc
+ # Track condition steps
for required_stuff in required_stuffs:
get_pipeline_tracker().add_condition_step(
from_stuff=required_stuff,
@@ -364,16 +377,19 @@ class PipeCondition(PipeController):
comment="PipeCondition required for condition",
)
- log.debug(f"Chosen pipe: {chosen_pipe_code}")
+ # Execute the chosen pipe
+ log.debug(f"Chosen pipe: {chosen_pipe.code}")
pipe_output = await get_pipe_router().run(
pipe_job=PipeJobFactory.make_pipe_job(
- pipe=get_required_pipe(pipe_code=chosen_pipe_code),
+ pipe=chosen_pipe,
job_metadata=job_metadata,
working_memory=working_memory,
pipe_run_params=pipe_run_params,
output_name=output_name,
),
)
+
+ # Track choice step
get_pipeline_tracker().add_choice_step(
from_condition=condition_details,
to_stuff=pipe_output.main_stuff,
@@ -413,13 +429,12 @@ class PipeCondition(PipeController):
# 2. Validate that the expression template is valid
try:
- pipe_compose = PipeComposeFactory.make_pipe_compose_from_template_str(
- domain=self.domain,
- template_str=self.applied_expression_template,
- inputs=self.inputs,
+ required_variables = detect_jinja2_required_variables(
+ template_category=Jinja2TemplateCategory.LLM_PROMPT,
+ template_provider=get_template_provider(),
+ jinja2_name=None,
+ jinja2=self.applied_expression_template,
)
- # Get required variables to validate the template syntax
- required_variables = pipe_compose.required_variables()
log.debug(f"Expression template is valid, requires variables: {required_variables}")
except Exception as exc:
log.error(f"Dry run failed: invalid expression template: {exc}")
@@ -439,6 +454,8 @@ class PipeCondition(PipeController):
missing_pipes: list[str] = []
for pipe_code in all_pipe_codes:
+ if pipe_code in SpecificPipeCodesEnum.value_list():
+ continue
try:
get_required_pipe(pipe_code=pipe_code)
log.debug(f"Pipe '{pipe_code}' exists and is accessible")
diff --git a/pipelex/pipe_operators/compose/pipe_compose.py b/pipelex/pipe_operators/compose/pipe_compose.py
index 7d5399fe6..5d41bd1b9 100644
--- a/pipelex/pipe_operators/compose/pipe_compose.py
+++ b/pipelex/pipe_operators/compose/pipe_compose.py
@@ -132,7 +132,7 @@ class PipeCompose(PipeOperator[PipeComposeOutput]):
msg = f"PipeCompose does not suppport multiple outputs, got output_multiplicity = {pipe_run_params.output_multiplicity}"
raise PipeRunParamsError(msg)
- context: dict[str, Any] = working_memory.generate_context()
+ context: dict[str, Any] = working_memory.generate_jinja2_context()
if pipe_run_params:
context.update(**pipe_run_params.params)
if self.extra_context:
diff --git a/pipelex/pipe_operators/llm/pipe_llm.py b/pipelex/pipe_operators/llm/pipe_llm.py
index 10cfab882..b6773d2f9 100644
--- a/pipelex/pipe_operators/llm/pipe_llm.py
+++ b/pipelex/pipe_operators/llm/pipe_llm.py
@@ -49,6 +49,7 @@ from pipelex.hub import (
from pipelex.pipe_operators.llm.pipe_llm_blueprint import StructuringMethod
from pipelex.pipe_operators.pipe_operator import PipeOperator
from pipelex.pipeline.job_metadata import JobMetadata
+from pipelex.tools.templating.jinja2_template_category import Jinja2TemplateCategory
from pipelex.tools.typing.structure_printer import StructurePrinter
from pipelex.types import Self
@@ -390,7 +391,7 @@ class PipeLLM(PipeOperator[PipeLLMOutput]):
output_structure_prompt: str | None = None
if get_config().cogt.llm_config.is_structure_prompt_enabled:
- output_structure_prompt = PipeLLM.get_output_structure_prompt(
+ output_structure_prompt = await PipeLLM.get_output_structure_prompt(
concept_string=pipe_run_params.dynamic_output_concept_code or output_concept.concept_string,
is_with_preliminary_text=is_with_preliminary_text,
)
@@ -529,10 +530,9 @@ class PipeLLM(PipeOperator[PipeLLMOutput]):
)
@staticmethod
- def get_output_structure_prompt(concept_string: str, is_with_preliminary_text: bool) -> str | None:
+ async def get_output_structure_prompt(concept_string: str, is_with_preliminary_text: bool) -> str | None:
concept = get_required_concept(concept_string=concept_string)
- class_name = concept.structure_class_name
- output_class = get_class_registry().get_class(class_name)
+ output_class = get_class_registry().get_class(concept.structure_class_name)
log.debug(f"get_output_structure_prompt for {concept_string} with {is_with_preliminary_text=}")
log.debug(f"output_class: {output_class}")
if not output_class:
@@ -544,27 +544,10 @@ class PipeLLM(PipeOperator[PipeLLMOutput]):
return None
class_structure_str = "\n".join(class_structure)
- # TODO: use proper prompt templating for this
- if is_with_preliminary_text:
- output_structure_prompt = f"""
-
----
-The instance we want to generate will be for the following class:
-{class_structure_str}
-
-Don't bother with JSON formatting, we'll do that as a second step.
-For now, just output markdown with the details of the instance.
-DO NOT create information.
-If some information is not present for an attribute, output the default value or None according to the attribute definition.
-"""
- else:
- output_structure_prompt = f"""
-
----
-The instance we want to generate will be for the following class:
-{class_structure_str}
-
-DO NOT create information.
-If some information is not present for an attribute, output the default value or None according to the attribute definition.
-"""
- return output_structure_prompt
+ return await get_content_generator().make_jinja2_text(
+ context={
+ "class_structure_str": class_structure_str,
+ },
+ template_category=Jinja2TemplateCategory.LLM_PROMPT,
+ jinja2_name="output_structure_prompt" if is_with_preliminary_text else "output_structure_prompt_no_preliminary_text",
+ )
diff --git a/pipelex/pipelex.toml b/pipelex/pipelex.toml
index 3df6c9fb3..9b3852e0d 100644
--- a/pipelex/pipelex.toml
+++ b/pipelex/pipelex.toml
@@ -6,8 +6,8 @@ observer_dir = "results/observer"
[pipelex.feature_config]
# WIP/Experimental feature flags
-is_pipeline_tracking_enabled = true
-is_activity_tracking_enabled = true
+is_pipeline_tracking_enabled = false
+is_activity_tracking_enabled = false
is_reporting_enabled = true
[pipelex.reporting_config]
@@ -87,11 +87,11 @@ is_auto_setup_preset_ocr = true
[cogt.llm_config]
default_max_images = 100
-is_structure_prompt_enabled = false
+is_structure_prompt_enabled = true
[cogt.llm_config.instructor_config]
is_openai_structured_output_enabled = false
-is_dump_kwargs_enabled = true
+is_dump_kwargs_enabled = false
is_dump_response_enabled = false
is_dump_error_enabled = false
diff --git a/pipelex/tools/misc/context_provider_abstract.py b/pipelex/tools/misc/context_provider_abstract.py
index 1cff3ac83..23ba2c56e 100644
--- a/pipelex/tools/misc/context_provider_abstract.py
+++ b/pipelex/tools/misc/context_provider_abstract.py
@@ -20,5 +20,5 @@ class ContextProviderAbstract(ABC):
pass
@abstractmethod
- def generate_context(self) -> dict[str, Any]:
+ def generate_jinja2_context(self) -> dict[str, Any]:
pass
diff --git a/pipelex/tools/templating/template_library.py b/pipelex/tools/templating/template_library.py
index 82132d484..92a3946a3 100644
--- a/pipelex/tools/templating/template_library.py
+++ b/pipelex/tools/templating/template_library.py
@@ -7,6 +7,7 @@ from typing_extensions import override
from pipelex import log
from pipelex.libraries.library_config import LibraryConfig
from pipelex.tools.exceptions import ToolException
+from pipelex.tools.misc.file_utils import find_files_in_dir
from pipelex.tools.misc.toml_utils import load_toml_from_path
from pipelex.tools.templating.jinja2_parsing import check_jinja2_parsing
from pipelex.tools.templating.jinja2_template_category import Jinja2TemplateCategory
@@ -32,9 +33,9 @@ class TemplateLibrary(TemplateProviderAbstract, RootModel[TemplateLibraryRoot]):
@override
def setup(self) -> None:
- template_toml_paths = self.library_config.get_templates_paths()
+ template_toml_paths = find_files_in_dir(dir_path="pipelex/tools/templating/templates", pattern="*.toml", is_recursive=True)
for template_toml_path in template_toml_paths:
- self._load_from_toml(toml_path=template_toml_path)
+ self._load_from_toml(toml_path=str(template_toml_path))
self.validate_templates(template_category=Jinja2TemplateCategory.LLM_PROMPT)
@override
diff --git a/pipelex/tools/templating/templates/__init__.py b/pipelex/tools/templating/templates/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/pipelex/tools/templating/templates/base_templates.toml b/pipelex/tools/templating/templates/base_templates.toml
new file mode 100644
index 000000000..fb1a8d026
--- /dev/null
+++ b/pipelex/tools/templating/templates/base_templates.toml
@@ -0,0 +1,34 @@
+[generic_prompts]
+structure_from_preliminary_text_system = "You are a data modeling expert specialized in extracting structure from text."
+# TODO: Should be a proper jinja2
+structure_from_preliminary_text_user = """
+Your job is to extract and structure information from a text.
+Here is the text:
+{preliminary_text}
+
+Now generate the JSON in the required format.
+Do not create information that is not in the text.
+"""
+
+output_structure_prompt = """
+---
+The instance we want to generate will be for the following class:
+{{ class_structure_str }}
+
+Don't bother with JSON formatting, we'll do that as a second step.
+For now, just output markdown with the details of the instance.
+DO NOT create information.
+If some information is not present for an attribute, output the default value or None according to the attribute definition.
+"""
+
+output_structure_prompt_no_preliminary_text = """
+---
+The instance we want to generate will be for the following class:
+{{ class_structure_str }}
+
+DO NOT create information.
+If some information is not present for an attribute, output the default value or None according to the attribute definition.
+"""
+
+[test_prompts]
+jinja2_test_template = "I want a {{ place_holder }} t-shirt."
diff --git a/tests/integration/pipelex/cogt/test_llm_job_async_factory.py b/tests/integration/pipelex/cogt/test_llm_job_async_factory.py
deleted file mode 100644
index 0b1ed2920..000000000
--- a/tests/integration/pipelex/cogt/test_llm_job_async_factory.py
+++ /dev/null
@@ -1,50 +0,0 @@
-import random
-
-import pytest
-
-from pipelex import pretty_print
-from pipelex.cogt.llm.llm_job import LLMJobParams
-from pipelex.cogt.llm.llm_job_factory import LLMJobFactory
-from pipelex.cogt.llm.llm_prompt_template import LLMPromptTemplate
-from pipelex.hub import get_llm_worker
-from tests.integration.pipelex.cogt.test_data import LLMTestConstants
-
-
[email protected](loop_scope="class")
-class TestAsyncLLMJobFactory:
- async def test_llm_job_using_generic_prompt_factory(self):
- llm_prompt_template = LLMPromptTemplate.from_template_contents(user_text=LLMTestConstants.PROMPT_TEMPLATE_TEXT)
- random_color = random.choice(LLMTestConstants.PROMPT_COLOR_EXAMPLES)
- llm_job = await LLMJobFactory.make_llm_job_from_prompt_factory(
- llm_prompt_factory=llm_prompt_template,
- llm_prompt_arguments={"color": random_color},
- llm_job_params=LLMJobParams(temperature=0.5, max_tokens=50, seed=None),
- )
- assert llm_job
- pretty_print(llm_job)
-
- async def test_llm_job_using_prompt_template(self):
- llm_prompt_template = LLMPromptTemplate.from_template_contents(user_text=LLMTestConstants.PROMPT_TEMPLATE_TEXT)
- random_color = random.choice(LLMTestConstants.PROMPT_COLOR_EXAMPLES)
- llm_job = await LLMJobFactory.make_llm_job_from_prompt_template(
- llm_prompt_template=llm_prompt_template,
- llm_prompt_arguments={"color": random_color},
- llm_job_params=LLMJobParams(temperature=0.5, max_tokens=50, seed=None),
- )
- assert llm_job
- pretty_print(llm_job)
-
- @pytest.mark.llm
- @pytest.mark.inference
- async def test_llm_job_and_gen_text(self, llm_handle: str):
- llm_worker = get_llm_worker(llm_handle=llm_handle)
- llm_prompt_template = LLMPromptTemplate.from_template_contents(user_text=LLMTestConstants.PROMPT_TEMPLATE_TEXT)
- random_color = random.choice(LLMTestConstants.PROMPT_COLOR_EXAMPLES)
- llm_job = await LLMJobFactory.make_llm_job_from_prompt_template(
- llm_prompt_template=llm_prompt_template,
- llm_prompt_arguments={"color": random_color},
- llm_job_params=LLMJobParams(temperature=0.5, max_tokens=50, seed=None),
- )
- generated_text = await llm_worker.gen_text(llm_job=llm_job)
- assert generated_text
- pretty_print(generated_text)