Build AI methods with your coding agent, then run them anywhere: as an MCP for chatbots, as a webapp for people, or via API for your software.
Refactor/model and template choices (#348)
Louis Choquel68af1f5db7bd
diff --git a/.cursor/rules/pipelex.mdc b/.cursor/rules/pipelex.mdcindex fcb06a02a..ca99c0a6d 100644--- a/.cursor/rules/pipelex.mdc+++ b/.cursor/rules/pipelex.mdc@@ -428,7 +428,7 @@ description = "Compose a personalized follow-up email for CRM"inputs = { customer = "Customer", deal = "Deal", sales_rep = "SalesRep" }output = "Text"template_category = "html"-prompting_style = { tag_style = "square_brackets", text_format = "html" }+templating_style = { tag_style = "square_brackets", text_format = "html" }jinja2 = """Subject: Following up on our $deal.product_name discussion@@ -464,7 +464,7 @@ $sales_rep.phone | $sales_rep.email- `jinja2`: Inline Jinja2 template (mutually exclusive with jinja2_name)- `jinja2_name`: Name of a predefined template (mutually exclusive with jinja2)- `template_category`: Template type ("llm_prompt", "html", "markdown", "mermaid", etc.)-- `prompting_style`: Styling options for template rendering+- `templating_style`: Styling options for template rendering- `extra_context`: Additional context variables for template## Template Variablesdiff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.mdindex 4678b098f..f54bd0b20 100644--- a/.github/copilot-instructions.md+++ b/.github/copilot-instructions.md@@ -566,7 +566,7 @@ description = "Compose a personalized follow-up email for CRM"inputs = { customer = "Customer", deal = "Deal", sales_rep = "SalesRep" }output = "Text"template_category = "html"-prompting_style = { tag_style = "square_brackets", text_format = "html" }+templating_style = { tag_style = "square_brackets", text_format = "html" }jinja2 = """Subject: Following up on our $deal.product_name discussion@@ -602,7 +602,7 @@ $sales_rep.phone | $sales_rep.email- `jinja2`: Inline Jinja2 template (mutually exclusive with jinja2_name)- `jinja2_name`: Name of a predefined template (mutually exclusive with jinja2)- `template_category`: Template type ("llm_prompt", "html", "markdown", "mermaid", etc.)-- `prompting_style`: Styling options for template rendering+- `templating_style`: Styling options for template rendering- `extra_context`: Additional context variables for template### Template Variablesdiff --git a/AGENTS.md b/AGENTS.mdindex e5221fd6f..6cc4503be 100644--- a/AGENTS.md+++ b/AGENTS.md@@ -566,7 +566,7 @@ description = "Compose a personalized follow-up email for CRM"inputs = { customer = "Customer", deal = "Deal", sales_rep = "SalesRep" }output = "Text"template_category = "html"-prompting_style = { tag_style = "square_brackets", text_format = "html" }+templating_style = { tag_style = "square_brackets", text_format = "html" }jinja2 = """Subject: Following up on our $deal.product_name discussion@@ -602,7 +602,7 @@ $sales_rep.phone | $sales_rep.email- `jinja2`: Inline Jinja2 template (mutually exclusive with jinja2_name)- `jinja2_name`: Name of a predefined template (mutually exclusive with jinja2)- `template_category`: Template type ("llm_prompt", "html", "markdown", "mermaid", etc.)-- `prompting_style`: Styling options for template rendering+- `templating_style`: Styling options for template rendering- `extra_context`: Additional context variables for template### Template Variablesdiff --git a/CLAUDE.md b/CLAUDE.mdindex 7842b12c9..7339ec32a 100644--- a/CLAUDE.md+++ b/CLAUDE.md@@ -566,7 +566,7 @@ description = "Compose a personalized follow-up email for CRM"inputs = { customer = "Customer", deal = "Deal", sales_rep = "SalesRep" }output = "Text"template_category = "html"-prompting_style = { tag_style = "square_brackets", text_format = "html" }+templating_style = { tag_style = "square_brackets", text_format = "html" }jinja2 = """Subject: Following up on our $deal.product_name discussion@@ -602,7 +602,7 @@ $sales_rep.phone | $sales_rep.email- `jinja2`: Inline Jinja2 template (mutually exclusive with jinja2_name)- `jinja2_name`: Name of a predefined template (mutually exclusive with jinja2)- `template_category`: Template type ("llm_prompt", "html", "markdown", "mermaid", etc.)-- `prompting_style`: Styling options for template rendering+- `templating_style`: Styling options for template rendering- `extra_context`: Additional context variables for template### Template Variablesdiff --git a/Makefile b/Makefileindex 1ae3487f4..e578a9da8 100644--- a/Makefile+++ b/Makefile@@ -75,6 +75,7 @@ make test-quiet - Run unit tests without prints (no inference)make tq - Shorthand -> test-quietmake test-with-prints - Run tests with prints (no inference)make tp - Shorthand -> test-with-prints+make tb - Shorthand -> `make test-with-prints TEST=test_boot`make test-inference - Run unit tests only for inference (with prints)make ti - Shorthand -> test-inferencemake tip - Shorthand -> test-inference-with-prints (parallelized inference tests)@@ -285,6 +286,11 @@ test-with-prints: envtp: test-with-prints@echo "> done: tp = test-with-prints"+tb: env+ $(call PRINT_TITLE,"Unit testing a simple boot")+ @echo "• Running unit test test_boot"+ $(VENV_PYTEST) -s -m $(USUAL_PYTEST_MARKERS) -k "test_boot" $(if $(filter 1,$(VERBOSE)),-v,$(if $(filter 2,$(VERBOSE)),-vv,$(if $(filter 3,$(VERBOSE)),-vvv,)));+test-inference-with-prints: env$(call PRINT_TITLE,"Unit testing")@if [ -n "$(TEST)" ]; then \diff --git a/pipelex/cli/commands/build_cmd.py b/pipelex/cli/commands/build_cmd.pyindex f6e4db3e7..403252540 100644--- a/pipelex/cli/commands/build_cmd.py+++ b/pipelex/cli/commands/build_cmd.py@@ -20,6 +20,7 @@ build_app = typer.Typer(help="Build artifacts like pipelines", no_args_is_help=T"""Today's example:pipelex build pipe "Given an expense report, apply company rules"+pipelex build pipe "Take a CV in a PDF file, a Job offer text, and analyze if they match"pipelex build partial "Given an expense report, apply company rules" -o results/generated.jsonpipelex build flow "Given an expense report, apply company rules" -o results/flow.jsondiff --git a/pipelex/cogt/config_cogt.py b/pipelex/cogt/config_cogt.pyindex b673644a2..eb6a2e8c9 100644--- a/pipelex/cogt/config_cogt.py+++ b/pipelex/cogt/config_cogt.py@@ -1,3 +1,4 @@+from pipelex.cogt.exceptions import LLMConfigErrorfrom pipelex.cogt.img_gen.img_gen_job_components import ImgGenJobConfig, ImgGenJobParams, ImgGenJobParamsDefaultsfrom pipelex.cogt.llm.llm_job_components import LLMJobConfigfrom pipelex.plugins.fal.fal_config import FalConfig@@ -31,6 +32,16 @@ class LLMConfig(ConfigModel):llm_job_config: LLMJobConfigis_structure_prompt_enabled: booldefault_max_images: int+ is_dump_text_prompts_enabled: bool+ is_dump_response_text_enabled: bool+ generic_templates: dict[str, str]++ def get_template(self, template_name: str) -> str:+ template = self.generic_templates.get(template_name)+ if not template:+ msg = f"Template '{template_name}' not found in generic_templates"+ raise LLMConfigError(msg)+ return templateclass InferenceManagerConfig(ConfigModel):diff --git a/pipelex/cogt/content_generation/assignment_models.py b/pipelex/cogt/content_generation/assignment_models.pyindex af0274019..7f6471c52 100644--- a/pipelex/cogt/content_generation/assignment_models.py+++ b/pipelex/cogt/content_generation/assignment_models.py@@ -12,10 +12,10 @@ from pipelex.cogt.llm.llm_job_components import LLMJobParamsfrom pipelex.cogt.llm.llm_prompt import LLMPromptfrom pipelex.cogt.llm.llm_prompt_factory_abstract import LLMPromptFactoryAbstractfrom pipelex.cogt.llm.llm_setting import LLMSetting+from pipelex.cogt.templating.template_category import TemplateCategory+from pipelex.cogt.templating.templating_style import TemplatingStylefrom pipelex.hub import get_class_registryfrom pipelex.pipeline.job_metadata import JobMetadata-from pipelex.tools.templating.jinja2_template_category import Jinja2TemplateCategory-from pipelex.tools.templating.templating_models import PromptingStyleclass LLMAssignmentFactory(BaseModel):@@ -126,12 +126,11 @@ class ImgGenAssignment(BaseModel):nb_images: int-class Jinja2Assignment(BaseModel):+class TemplatingAssignment(BaseModel):context: dict[str, Any]- jinja2_name: str | None = None- jinja2: str | None = None- prompting_style: PromptingStyle | None = None- template_category: Jinja2TemplateCategory = Jinja2TemplateCategory.LLM_PROMPT+ template: str+ templating_style: TemplatingStyle | None = None+ category: TemplateCategoryclass ExtractAssignment(BaseModel):diff --git a/pipelex/cogt/content_generation/content_generator.py b/pipelex/cogt/content_generation/content_generator.pyindex 6d8509e5a..1d5b87c8d 100644--- a/pipelex/cogt/content_generation/content_generator.py+++ b/pipelex/cogt/content_generation/content_generator.py@@ -6,17 +6,17 @@ from pipelex import logfrom pipelex.cogt.content_generation.assignment_models import (ExtractAssignment,ImgGenAssignment,- Jinja2Assignment,LLMAssignment,LLMAssignmentFactory,ObjectAssignment,+ TemplatingAssignment,TextThenObjectAssignment,)from pipelex.cogt.content_generation.content_generator_protocol import ContentGeneratorProtocol, update_job_metadatafrom pipelex.cogt.content_generation.extract_generate import extract_gen_pagesfrom pipelex.cogt.content_generation.img_gen_generate import img_gen_image_list, img_gen_single_image-from pipelex.cogt.content_generation.jinja2_generate import jinja2_gen_textfrom pipelex.cogt.content_generation.llm_generate import llm_gen_object, llm_gen_object_list, llm_gen_text+from pipelex.cogt.content_generation.templating_generate import templating_gen_textfrom pipelex.cogt.extract.extract_input import ExtractInputfrom pipelex.cogt.extract.extract_job_components import ExtractJobConfig, ExtractJobParamsfrom pipelex.cogt.extract.extract_output import ExtractOutput@@ -27,10 +27,10 @@ from pipelex.cogt.llm.llm_prompt import LLMPromptfrom pipelex.cogt.llm.llm_prompt_factory_abstract import LLMPromptFactoryAbstractfrom pipelex.cogt.llm.llm_prompt_template import LLMPromptTemplatefrom pipelex.cogt.llm.llm_setting import LLMSetting+from pipelex.cogt.templating.template_category import TemplateCategory+from pipelex.cogt.templating.templating_style import TemplatingStylefrom pipelex.config import get_configfrom pipelex.pipeline.job_metadata import JobMetadata-from pipelex.tools.templating.jinja2_template_category import Jinja2TemplateCategory-from pipelex.tools.templating.templating_models import PromptingStylefrom pipelex.tools.typing.pydantic_utils import BaseModelTypeVar@@ -242,23 +242,21 @@ class ContentGenerator(ContentGeneratorProtocol):return generated_image_list@override- async def make_jinja2_text(+ async def make_templated_text(self,context: dict[str, Any],- jinja2_name: str | None = None,- jinja2: str | None = None,- prompting_style: PromptingStyle | None = None,- template_category: Jinja2TemplateCategory = Jinja2TemplateCategory.LLM_PROMPT,+ template: str,+ templating_style: TemplatingStyle | None = None,+ template_category: TemplateCategory | None = None,) -> str:log.debug(f"context: {context}")- jinja2_assignment = Jinja2Assignment(+ templating_assignment = TemplatingAssignment(context=context,- jinja2_name=jinja2_name,- jinja2=jinja2,- prompting_style=prompting_style,- template_category=template_category,+ template=template,+ templating_style=templating_style,+ category=template_category or TemplateCategory.BASIC,)- return await jinja2_gen_text(jinja2_assignment=jinja2_assignment)+ return await templating_gen_text(templating_assignment=templating_assignment)@overrideasync def make_extract_pages(diff --git a/pipelex/cogt/content_generation/content_generator_dry.py b/pipelex/cogt/content_generation/content_generator_dry.pyindex 2a66b9154..578fd30b3 100644--- a/pipelex/cogt/content_generation/content_generator_dry.py+++ b/pipelex/cogt/content_generation/content_generator_dry.py@@ -14,10 +14,11 @@ from pipelex.cogt.img_gen.img_gen_prompt import ImgGenPromptfrom pipelex.cogt.llm.llm_prompt import LLMPromptfrom pipelex.cogt.llm.llm_prompt_factory_abstract import LLMPromptFactoryAbstractfrom pipelex.cogt.llm.llm_setting import LLMSetting+from pipelex.cogt.templating.template_category import TemplateCategory+from pipelex.cogt.templating.templating_style import TemplatingStylefrom pipelex.config import get_configfrom pipelex.pipeline.job_metadata import JobMetadata-from pipelex.tools.templating.jinja2_template_category import Jinja2TemplateCategory-from pipelex.tools.templating.templating_models import PromptingStyle+from pipelex.tools.jinja2.jinja2_parsing import check_jinja2_parsingfrom pipelex.tools.typing.pydantic_utils import BaseModelTypeVar@@ -177,21 +178,20 @@ class ContentGeneratorDry(ContentGeneratorProtocol):]@override- async def make_jinja2_text(+ async def make_templated_text(self,context: dict[str, Any],- jinja2_name: str | None = None,- jinja2: str | None = None,- prompting_style: PromptingStyle | None = None,- template_category: Jinja2TemplateCategory = Jinja2TemplateCategory.LLM_PROMPT,+ template: str,+ templating_style: TemplatingStyle | None = None,+ template_category: TemplateCategory | None = None,) -> str:- # TODO: Use the code that checks if the jinja2 is a valid template- func_name = "make_jinja2_text"+ check_jinja2_parsing(template_source=template, template_category=template_category or TemplateCategory.BASIC)+ func_name = "make_templated_text"log.dev(f"🤡 DRY RUN: {self.__class__.__name__}.{func_name}")- jinja2_truncated = jinja2[: self._text_gen_truncate_length] if jinja2 else None+ jinja2_truncated = template[: self._text_gen_truncate_length]return (- f"DRY RUN: {func_name} • context={context} • jinja2_name={jinja2_name} • "- f"jinja2={jinja2_truncated} • prompting_style={prompting_style} • template_category={template_category}"+ f"DRY RUN: {func_name} • context={context} • "+ f"jinja2={jinja2_truncated} • templating_style={templating_style} • template_category={template_category}")@overridediff --git a/pipelex/cogt/content_generation/content_generator_protocol.py b/pipelex/cogt/content_generation/content_generator_protocol.pyindex 5112cbc8e..4de591978 100644--- a/pipelex/cogt/content_generation/content_generator_protocol.py+++ b/pipelex/cogt/content_generation/content_generator_protocol.py@@ -11,9 +11,9 @@ from pipelex.cogt.img_gen.img_gen_prompt import ImgGenPromptfrom pipelex.cogt.llm.llm_prompt import LLMPromptfrom pipelex.cogt.llm.llm_prompt_factory_abstract import LLMPromptFactoryAbstractfrom pipelex.cogt.llm.llm_setting import LLMSetting+from pipelex.cogt.templating.template_category import TemplateCategory+from pipelex.cogt.templating.templating_style import TemplatingStylefrom pipelex.pipeline.job_metadata import JobMetadata-from pipelex.tools.templating.jinja2_template_category import Jinja2TemplateCategory-from pipelex.tools.templating.templating_models import PromptingStylefrom pipelex.tools.typing.pydantic_utils import BaseModelTypeVarF = TypeVar("F", bound=Callable[..., Awaitable[Any]])@@ -109,13 +109,12 @@ class ContentGeneratorProtocol(Protocol):img_gen_job_config: ImgGenJobConfig | None = None,) -> list[GeneratedImage]: ...- async def make_jinja2_text(+ async def make_templated_text(self,context: dict[str, Any],- jinja2_name: str | None = None,- jinja2: str | None = None,- prompting_style: PromptingStyle | None = None,- template_category: Jinja2TemplateCategory = Jinja2TemplateCategory.LLM_PROMPT,+ template: str,+ templating_style: TemplatingStyle | None = None,+ template_category: TemplateCategory | None = None,) -> str: ...async def make_extract_pages(diff --git a/pipelex/cogt/content_generation/jinja2_generate.py b/pipelex/cogt/content_generation/jinja2_generate.pydeleted file mode 100644index 995209e75..000000000--- a/pipelex/cogt/content_generation/jinja2_generate.py+++ /dev/null@@ -1,23 +0,0 @@-from pipelex.cogt.content_generation.assignment_models import Jinja2Assignment-from pipelex.hub import get_template_provider-from pipelex.tools.templating.jinja2_parsing import check_jinja2_parsing-from pipelex.tools.templating.jinja2_rendering import render_jinja2-from pipelex.tools.templating.template_preprocessor import preprocess_template---async def jinja2_gen_text(jinja2_assignment: Jinja2Assignment) -> str:- # Intermediate call to preprocess the template with our syntax patterns (@, $, @?, etc.)- if jinja2_assignment.jinja2:- jinja2_assignment.jinja2 = preprocess_template(template=jinja2_assignment.jinja2)- check_jinja2_parsing(jinja2_assignment.jinja2)-- jinja2_text: str = await render_jinja2(- template_category=jinja2_assignment.template_category,- template_provider=get_template_provider(),- temlating_context=jinja2_assignment.context,- jinja2_name=jinja2_assignment.jinja2_name,- jinja2=jinja2_assignment.jinja2,- prompting_style=jinja2_assignment.prompting_style,- )-- return jinja2_textdiff --git a/pipelex/cogt/content_generation/templating_generate.py b/pipelex/cogt/content_generation/templating_generate.pynew file mode 100644index 000000000..8feaaa46b--- /dev/null+++ b/pipelex/cogt/content_generation/templating_generate.py@@ -0,0 +1,20 @@+from pipelex.cogt.content_generation.assignment_models import TemplatingAssignment+from pipelex.cogt.templating.template_preprocessor import preprocess_template+from pipelex.tools.jinja2.jinja2_parsing import check_jinja2_parsing+from pipelex.tools.jinja2.jinja2_rendering import render_jinja2+++async def templating_gen_text(templating_assignment: TemplatingAssignment) -> str:+ # Intermediate call to preprocess the template with our syntax patterns (@, $, @?, etc.)+ if templating_assignment.template:+ templating_assignment.template = preprocess_template(template=templating_assignment.template)+ check_jinja2_parsing(templating_assignment.template)++ templated_text: str = await render_jinja2(+ template_category=templating_assignment.category,+ temlating_context=templating_assignment.context,+ template_source=templating_assignment.template,+ templating_style=templating_assignment.templating_style,+ )++ return templated_textdiff --git a/pipelex/cogt/exceptions.py b/pipelex/cogt/exceptions.pyindex 22d99b0f4..22c7c4dfb 100644--- a/pipelex/cogt/exceptions.py+++ b/pipelex/cogt/exceptions.py@@ -6,6 +6,10 @@ class CogtError(RootException):pass+class LLMConfigError(CogtError):+ pass++class InferenceManagerWorkerSetupError(CogtError, FatalError):passdiff --git a/pipelex/cogt/llm/llm_prompt_spec.py b/pipelex/cogt/llm/llm_prompt_spec.pydeleted file mode 100644index a4198d7ef..000000000--- a/pipelex/cogt/llm/llm_prompt_spec.py+++ /dev/null@@ -1,233 +0,0 @@-from typing import TYPE_CHECKING, Any--from pydantic import BaseModel, model_validator--from pipelex import log-from pipelex.cogt.exceptions import LLMPromptSpecError-from pipelex.cogt.image.prompt_image_factory import PromptImageFactory-from pipelex.cogt.llm.llm_prompt import LLMPrompt-from pipelex.core.stuffs.image_content import ImageContent-from pipelex.hub import get_content_generator, get_template, get_template_provider-from pipelex.tools.misc.context_provider_abstract import ContextProviderAbstract, ContextProviderException-from pipelex.tools.templating.jinja2_blueprint import Jinja2Blueprint-from pipelex.tools.templating.jinja2_required_variables import detect_jinja2_required_variables-from pipelex.tools.templating.template_preprocessor import preprocess_template-from pipelex.tools.templating.templating_models import PromptingStyle-from pipelex.tools.typing.validation_utils import has_exactly_one_among_attributes_from_list, has_more_than_one_among_attributes_from_list-from pipelex.types import Self--if TYPE_CHECKING:- from pipelex.cogt.image.prompt_image import PromptImage---# TODO: move this to pipe operators-class LLMPromptSpec(BaseModel):- prompting_style: PromptingStyle | None = None-- system_prompt_jinja2_blueprint: Jinja2Blueprint | None = None- system_prompt_verbatim_name: str | None = None- system_prompt: str | None = None-- user_text_jinja2_blueprint: Jinja2Blueprint | None = None- user_prompt_verbatim_name: str | None = None- user_text: str | None = None-- user_images: list[str] | None = None-- @model_validator(mode="after")- def validate_user_text(self) -> Self:- if not has_exactly_one_among_attributes_from_list(- obj=self,- attributes_list=[- "user_text_jinja2_blueprint",- "user_prompt_verbatim_name",- "user_text",- ],- ):- msg = f"LLMPromptSpec user text must have exactly one of user_text, user_text_jinja2_blueprint or user_prompt_verbatim_name: {self}"- raise LLMPromptSpecError(msg)- if has_more_than_one_among_attributes_from_list(- obj=self,- attributes_list=[- "system_prompt_jinja2_blueprint",- "system_prompt_verbatim_name",- "system_prompt",- ],- ):- msg = f"LLMPromptSpec system got more than one of system_prompt, system_prompt_jinja2_blueprint, system_prompt_verbatim_name: {self}"- raise LLMPromptSpecError(msg)- return self-- def validate_with_libraries(self):- if self.user_prompt_verbatim_name:- get_template(template_name=self.user_prompt_verbatim_name)- if self.system_prompt_verbatim_name:- get_template(template_name=self.system_prompt_verbatim_name)-- if self.user_text_jinja2_blueprint and self.user_text_jinja2_blueprint.jinja2_name:- the_template = get_template(template_name=self.user_text_jinja2_blueprint.jinja2_name)- log.debug(f"Validated jinja2 template '{self.user_text_jinja2_blueprint.jinja2_name}':\n{the_template}")- if self.system_prompt_jinja2_blueprint and self.system_prompt_jinja2_blueprint.jinja2_name:- the_template = get_template(template_name=self.system_prompt_jinja2_blueprint.jinja2_name)- log.debug(f"Validated jinja2 template '{self.system_prompt_jinja2_blueprint.jinja2_name}':\n{the_template}")-- def required_variables(self) -> set[str]:- required_variables: set[str] = set()- if self.user_images:- user_images_top_object_name = [user_image.split(".", 1)[0] for user_image in self.user_images]- required_variables.update(user_images_top_object_name)-- if self.user_text_jinja2_blueprint and self.user_text_jinja2_blueprint.jinja2:- template_source = preprocess_template(self.user_text_jinja2_blueprint.jinja2)- required_variables = detect_jinja2_required_variables(- template_category=self.user_text_jinja2_blueprint.template_category,- template_provider=get_template_provider(),- jinja2=template_source,- )- return {- variable_name- for variable_name in required_variables- if not variable_name.startswith("_") and variable_name not in ("preliminary_text", "place_holder")- }-- # TODO: make this consistent with `LLMPromptFactoryAbstract` or `LLMPromptTemplate`,- # let's get back to it when we have a better solution for structuring_method- async def make_llm_prompt(- self,- output_concept_string: str,- context_provider: ContextProviderAbstract,- output_structure_prompt: str | None = None,- extra_params: dict[str, Any] | None = None,- ) -> LLMPrompt:- ############################################################- # User images- ############################################################- prompt_user_images: dict[str, PromptImage] = {}- if self.user_images:- for user_image_name in self.user_images:- log.debug(f"Getting user image '{user_image_name}' from context")- # Try to get as a single ImageContent first- try:- prompt_image_content = context_provider.get_typed_object_or_attribute(name=user_image_name, wanted_type=ImageContent)- if prompt_image_content is not None: # An ImageContent can be optional- if base_64 := prompt_image_content.base_64:- user_image = PromptImageFactory.make_prompt_image(base_64=base_64)- else:- image_uri = prompt_image_content.url- user_image = PromptImageFactory.make_prompt_image_from_uri(uri=image_uri)- prompt_user_images[user_image_name] = user_image- except ContextProviderException:- # If single image failed, try to get as a collection (list or tuple)- try:- image_collection = context_provider.get_typed_object_or_attribute(name=user_image_name, wanted_type=None)- # Check if it's a list or tuple- if isinstance(image_collection, (list, tuple)):- for image_item in image_collection: # type: ignore[assignment]- if isinstance(image_item, ImageContent):- item_base_64 = image_item.base_64- if item_base_64:- user_image = PromptImageFactory.make_prompt_image(base_64=item_base_64) # type: ignore[arg-type]- else:- image_uri = image_item.url- user_image = PromptImageFactory.make_prompt_image_from_uri(uri=image_uri)- prompt_user_images[user_image_name] = user_image- else:- msg = (- f"Could not find a valid user image or image collection named '{user_image_name}' from the provided context_provider"- )- raise LLMPromptSpecError(msg)- except ContextProviderException as exc:- msg = f"Could not find a valid user image named '{user_image_name}' from the provided context_provider: {exc}"- raise LLMPromptSpecError(msg) from exc-- ############################################################- # User text- ############################################################- # replace the image variables with numbered tags- if prompt_user_images:- if not extra_params:- extra_params = {}- for image_index, image_name in enumerate(prompt_user_images.keys()):- extra_params[image_name] = f"[Image {image_index + 1}]"- log.warning(f"Replacing image variable '{image_name}' with numbered tag '[Image {image_index + 1}]'")- user_text = await self._unravel_text(- context_provider=context_provider,- jinja2_blueprint=self.user_text_jinja2_blueprint,- text_verbatim_name=self.user_prompt_verbatim_name,- fixed_text=self.user_text,- extra_params=extra_params,- )- if not user_text:- # TODO: link to docs- msg = "Could not unravel user_text, we need either a jinja2_blueprint, a text_verbatim_name or a fixed user_text"- raise LLMPromptSpecError(msg)-- if output_structure_prompt:- user_text += output_structure_prompt-- log.verbose(f"User text with {output_concept_string=}:\n {user_text}")-- ############################################################- # System text- ############################################################- system_text = await self._unravel_text(- context_provider=context_provider,- jinja2_blueprint=self.system_prompt_jinja2_blueprint,- text_verbatim_name=self.system_prompt_verbatim_name,- fixed_text=self.system_prompt,- extra_params=extra_params,- )-- ############################################################- # Full LLMPrompt- ############################################################- return LLMPrompt(- system_text=system_text,- user_text=user_text,- user_images=list(prompt_user_images.values()),- )-- async def _unravel_text(- self,- context_provider: ContextProviderAbstract,- jinja2_blueprint: Jinja2Blueprint | None,- text_verbatim_name: str | None,- fixed_text: str | None,- extra_params: dict[str, Any] | None = None,- ) -> str | None:- the_text: str | None- if jinja2_blueprint:- log.verbose(f"Working with Jinja2 pipe '{jinja2_blueprint.jinja2_name}'")- if (prompting_style := self.prompting_style) and not jinja2_blueprint.prompting_style:- jinja2_blueprint.prompting_style = prompting_style- log.verbose(f"Setting prompting style to {prompting_style}")-- log.info(f"extra_params: {extra_params}")- log.info(f"jinja2_blueprint.extra_context: {jinja2_blueprint.extra_context}")-- context: dict[str, Any] = context_provider.generate_jinja2_context()- if extra_params:- context.update(**extra_params)- if jinja2_blueprint.extra_context:- context.update(**jinja2_blueprint.extra_context)-- the_text = await get_content_generator().make_jinja2_text(- context=context,- jinja2_name=jinja2_blueprint.jinja2_name,- jinja2=jinja2_blueprint.jinja2,- prompting_style=self.prompting_style,- template_category=jinja2_blueprint.template_category,- )- elif text_verbatim_name:- user_text_verbatim = get_template(- template_name=text_verbatim_name,- )- if not user_text_verbatim:- msg = f"Could not find text_verbatim template '{text_verbatim_name}'"- raise LLMPromptSpecError(msg)- the_text = user_text_verbatim- elif fixed_text:- the_text = fixed_text- else:- the_text = None- return the_textdiff --git a/pipelex/cogt/llm/llm_prompt_template.py b/pipelex/cogt/llm/llm_prompt_template.pyindex 9f3f4ea3c..c46c6bdc2 100644--- a/pipelex/cogt/llm/llm_prompt_template.py+++ b/pipelex/cogt/llm/llm_prompt_template.py@@ -9,7 +9,6 @@ from pipelex.cogt.llm.llm_prompt import LLMPromptfrom pipelex.cogt.llm.llm_prompt_factory_abstract import LLMPromptFactoryAbstractfrom pipelex.cogt.llm.llm_prompt_template_inputs import LLMPromptTemplateInputsfrom pipelex.config import get_config-from pipelex.hub import get_templatefrom pipelex.tools.misc.string_utils import is_none_or_has_text@@ -93,6 +92,7 @@ class LLMPromptTemplate(LLMPromptFactoryAbstract):# input variables can be applied to prompt texts used as templatesif llm_prompt.system_text:try:+ # TODO: use jinja2 templating herellm_prompt.system_text = llm_prompt.system_text.format(**all_template_inputs.root)except KeyError as exc:error_msg = f"Could not apply inputs to system_text. KeyError = {exc}. system_text = '{llm_prompt.system_text}'"@@ -102,6 +102,7 @@ class LLMPromptTemplate(LLMPromptFactoryAbstract):raise LLMPromptFactoryError(message=error_msg) from excif llm_prompt.user_text:try:+ # TODO: use jinja2 templating herellm_prompt.user_text = llm_prompt.user_text.format(**all_template_inputs.root)except KeyError as exc:error_msg = f"Could not apply inputs to user_text. KeyError = {exc}. user_text = '{llm_prompt.user_text}'"@@ -114,9 +115,9 @@ class LLMPromptTemplate(LLMPromptFactoryAbstract):@classmethoddef for_structure_from_preliminary_text(cls) -> "LLMPromptTemplate":- generic_template_names = get_config().pipelex.generic_template_names+ llm_config = get_config().cogt.llm_configproto_prompt = LLMPrompt(- system_text=get_template(generic_template_names.structure_from_preliminary_text_system),- user_text=get_template(generic_template_names.structure_from_preliminary_text_user),+ system_text=llm_config.get_template("structure_from_preliminary_text_system"),+ user_text=llm_config.get_template("structure_from_preliminary_text_user"),)return cls(proto_prompt=proto_prompt)diff --git a/pipelex/cogt/llm/llm_utils.py b/pipelex/cogt/llm/llm_utils.pynew file mode 100644index 000000000..7437b3794--- /dev/null+++ b/pipelex/cogt/llm/llm_utils.py@@ -0,0 +1,32 @@+from typing import Any++from pipelex import pretty_print+from pipelex.cogt.llm.llm_prompt import LLMPrompt+++def dump_prompt(llm_prompt: LLMPrompt) -> None:+ prompt_dump = ""+ if user_text := llm_prompt.user_text:+ prompt_dump += f"\n# User text:\n{user_text}\n"+ if system_text := llm_prompt.system_text:+ prompt_dump = f"\n# System text:\n{system_text}\n" + prompt_dump+ if llm_prompt.user_images:+ images_desc = "\n".join([f"- {image}" for image in llm_prompt.user_images])+ prompt_dump += f"\n# User images:\n{images_desc}\n"+ pretty_print(prompt_dump, title="Prompt sent to LLM provider")+++def dump_response_from_text_gen(response: Any) -> None:+ pretty_print(response, title="Response from LLM provider")+++def dump_kwargs(*_: Any, **kwargs: Any) -> None:+ pretty_print(kwargs, title="Instructor about to send to LLM provider")+++def dump_response_from_structured_gen(response: Any) -> None:+ pretty_print(response, title="Instructor response from LLM provider")+++def dump_error(error: Exception) -> None:+ pretty_print(error, title="Instructor error from LLM provider")diff --git a/pipelex/cogt/llm/llm_worker_abstract.py b/pipelex/cogt/llm/llm_worker_abstract.pyindex 12851364e..4b77ad942 100644--- a/pipelex/cogt/llm/llm_worker_abstract.py+++ b/pipelex/cogt/llm/llm_worker_abstract.py@@ -1,7 +1,7 @@from __future__ import annotationsfrom abc import ABC, abstractmethod-from typing import TYPE_CHECKING+from typing import TYPE_CHECKING, Anyfrom typing_extensions import override@@ -55,6 +55,7 @@ class LLMWorkerAbstract(InferenceWorkerAbstract, ABC):async def _after_job(self,llm_job: LLMJob,+ result: Any, # noqa: ARG002):# Report jobllm_job.llm_job_after_complete()@@ -79,7 +80,7 @@ class LLMWorkerAbstract(InferenceWorkerAbstract, ABC):result = await self._gen_text(llm_job=llm_job)- await self._after_job(llm_job=llm_job)+ await self._after_job(llm_job=llm_job, result=result)return result@@ -110,7 +111,7 @@ class LLMWorkerAbstract(InferenceWorkerAbstract, ABC):if hasattr(result, "_raw_response"):delattr(result, "_raw_response")- await self._after_job(llm_job=llm_job)+ await self._after_job(llm_job=llm_job, result=result)return resultdiff --git a/pipelex/cogt/llm/llm_worker_internal_abstract.py b/pipelex/cogt/llm/llm_worker_internal_abstract.pyindex dd422171f..81c26e25a 100644--- a/pipelex/cogt/llm/llm_worker_internal_abstract.py+++ b/pipelex/cogt/llm/llm_worker_internal_abstract.py@@ -3,9 +3,11 @@ from typing_extensions import overridefrom pipelex import logfrom pipelex.cogt.exceptions import LLMCapabilityErrorfrom pipelex.cogt.llm.llm_job import LLMJob+from pipelex.cogt.llm.llm_utils import dump_prompt, dump_response_from_text_genfrom pipelex.cogt.llm.llm_worker_abstract import LLMWorkerAbstractfrom pipelex.cogt.llm.structured_output import StructureMethodfrom pipelex.cogt.model_backends.model_spec import InferenceModelSpec+from pipelex.config import get_configfrom pipelex.reporting.reporting_protocol import ReportingProtocol@@ -50,6 +52,18 @@ class LLMWorkerInternalAbstract(LLMWorkerAbstract):log.info(f"LLM Worker: {self.desc} --------------------------------")await super()._before_job(llm_job=llm_job)llm_job.llm_job_before_start(inference_model=self.inference_model)+ if get_config().cogt.llm_config.is_dump_text_prompts_enabled:+ dump_prompt(llm_prompt=llm_job.llm_prompt)++ @override+ async def _after_job(+ self,+ llm_job: LLMJob,+ result: str,+ ):+ if get_config().cogt.llm_config.is_dump_response_text_enabled:+ dump_response_from_text_gen(response=result)+ await super()._after_job(llm_job=llm_job, result=result)@overridedef _check_can_perform_job(self, llm_job: LLMJob):diff --git a/pipelex/tools/templating/__init__.py b/pipelex/cogt/templating/__init__.pysimilarity index 100%rename from pipelex/tools/templating/__init__.pyrename to pipelex/cogt/templating/__init__.pydiff --git a/pipelex/cogt/templating/template_blueprint.py b/pipelex/cogt/templating/template_blueprint.pynew file mode 100644index 000000000..6c9c682aa--- /dev/null+++ b/pipelex/cogt/templating/template_blueprint.py@@ -0,0 +1,21 @@+from typing import Any++from pydantic import BaseModel, Field, model_validator++from pipelex.cogt.templating.template_category import TemplateCategory+from pipelex.cogt.templating.templating_style import TemplatingStyle+from pipelex.tools.jinja2.jinja2_parsing import check_jinja2_parsing+++class TemplateBlueprint(BaseModel):+ source: str = Field(description="Raw template source")+ templating_style: TemplatingStyle | None = Field(default=None, description="Style of prompting to use (typically for different LLMs)")+ category: TemplateCategory = Field(+ description="Category of the template (could also be HTML, MARKDOWN, MERMAID, etc.), influences template rendering rules",+ )+ extra_context: dict[str, Any] | None = Field(default=None, description="Additional context variables for template rendering")++ @model_validator(mode="after")+ def validate_template(self) -> "TemplateBlueprint":+ check_jinja2_parsing(template_source=self.source, template_category=self.category)+ return selfdiff --git a/pipelex/cogt/templating/template_category.py b/pipelex/cogt/templating/template_category.pynew file mode 100644index 000000000..2529efa58--- /dev/null+++ b/pipelex/cogt/templating/template_category.py@@ -0,0 +1,41 @@+from collections.abc import Callable+from typing import Any++from jinja2.runtime import Context++from pipelex.cogt.templating.templating_style import TextFormat+from pipelex.tools.jinja2.jinja2_filters import tag, text_format+from pipelex.tools.jinja2.jinja2_models import Jinja2FilterName+from pipelex.types import StrEnum+++class TemplateCategory(StrEnum):+ BASIC = "basic"+ EXPRESSION = "expression"+ HTML = "html"+ MARKDOWN = "markdown"+ MERMAID = "mermaid"+ LLM_PROMPT = "llm_prompt"++ @property+ def filters(self) -> dict[Jinja2FilterName, Callable[[Context, Any, TextFormat | None], Any]]:+ match self:+ case TemplateCategory.BASIC:+ return {+ Jinja2FilterName.FORMAT: text_format,+ Jinja2FilterName.TAG: tag,+ }+ case TemplateCategory.EXPRESSION:+ return {}+ case TemplateCategory.HTML | TemplateCategory.MARKDOWN:+ return {+ Jinja2FilterName.FORMAT: text_format,+ Jinja2FilterName.TAG: tag,+ }+ case TemplateCategory.LLM_PROMPT:+ return {+ Jinja2FilterName.FORMAT: text_format,+ Jinja2FilterName.TAG: tag,+ }+ case TemplateCategory.MERMAID:+ return {}diff --git a/pipelex/tools/templating/template_preprocessor.py b/pipelex/cogt/templating/template_preprocessor.pysimilarity index 97%rename from pipelex/tools/templating/template_preprocessor.pyrename to pipelex/cogt/templating/template_preprocessor.pyindex 1db627b24..e2a1c2c3e 100644--- a/pipelex/tools/templating/template_preprocessor.py+++ b/pipelex/cogt/templating/template_preprocessor.py@@ -8,7 +8,7 @@ from re import Match# template_str: The template string to check# Raises:-# Jinja2TemplateError: If any non-allowed filters are found+# TemplateSyntaxError: If any non-allowed filters are found# """# # Pattern to match Jinja2 filter syntax: {{ variable|filter() }} or {{ variable|filter(param) }}@@ -28,7 +28,7 @@ from re import Match# if invalid_filters:# msg = f"Invalid Jinja2 filters found: {invalid_filters}. Only the following filters are allowed: {ALLOWED_FILTERS}"-# raise Jinja2TemplateError(msg)+# raise TemplateSyntaxError(msg)# Handle @variable patternsdiff --git a/pipelex/tools/templating/templating_models.py b/pipelex/cogt/templating/templating_style.pysimilarity index 96%rename from pipelex/tools/templating/templating_models.pyrename to pipelex/cogt/templating/templating_style.pyindex 098128a61..5023a4073 100644--- a/pipelex/tools/templating/templating_models.py+++ b/pipelex/cogt/templating/templating_style.py@@ -23,7 +23,7 @@ class TagStyle(StrEnum):SQUARE_BRACKETS = "square_brackets"-class PromptingStyle(BaseModel):+class TemplatingStyle(BaseModel):tag_style: TagStyle = Field(strict=False)text_format: TextFormat = Field(TextFormat.PLAIN, strict=False)diff --git a/pipelex/config.py b/pipelex/config.pyindex 9a2402519..fb6a63757 100644--- a/pipelex/config.py+++ b/pipelex/config.py@@ -5,6 +5,7 @@ from pydantic import Field, field_validatorfrom pipelex.cogt.config_cogt import Cogtfrom pipelex.cogt.model_backends.prompting_target import PromptingTarget+from pipelex.cogt.templating.templating_style import TemplatingStylefrom pipelex.exceptions import PipelexConfigError, StaticValidationErrorTypefrom pipelex.hub import get_required_configfrom pipelex.language.plx_config import PlxConfig@@ -14,7 +15,6 @@ from pipelex.tools.aws.aws_config import AwsConfigfrom pipelex.tools.config.config_model import ConfigModelfrom pipelex.tools.config.config_root import ConfigRootfrom pipelex.tools.log.log_config import LogConfig-from pipelex.tools.templating.templating_models import PromptingStylefrom pipelex.types import StrEnum@@ -29,8 +29,8 @@ class StaticValidationConfig(ConfigModel):reactions: dict[StaticValidationErrorType, StaticValidationReaction]@field_validator("reactions", mode="before")- @staticmethod- def validate_reactions(value: dict[str, str]) -> dict[StaticValidationErrorType, StaticValidationReaction]:+ @classmethod+ def validate_reactions(cls, value: dict[str, str]) -> dict[StaticValidationErrorType, StaticValidationReaction]:return cast("dict[StaticValidationErrorType, StaticValidationReaction]",ConfigModel.transform_dict_str_to_enum(@@ -54,8 +54,8 @@ class DryRunConfig(ConfigModel):allowed_to_fail_pipes: list[str] = Field(default_factory=list)@field_validator("image_urls", mode="before")- @staticmethod- def validate_image_urls(value: list[str]) -> list[str]:+ @classmethod+ def validate_image_urls(cls, value: list[str]) -> list[str]:if not value:msg = "dry_run_config.image_urls must be a non-empty list"raise PipelexConfigError(msg)@@ -72,10 +72,10 @@ class StructureConfig(ConfigModel):class PromptingConfig(ConfigModel):- default_prompting_style: PromptingStyle- prompting_styles: dict[str, PromptingStyle]+ default_prompting_style: TemplatingStyle+ prompting_styles: dict[str, TemplatingStyle]- def get_prompting_style(self, prompting_target: PromptingTarget | None = None) -> PromptingStyle | None:+ def get_prompting_style(self, prompting_target: PromptingTarget | None = None) -> TemplatingStyle | None:if prompting_target:return self.prompting_styles.get(prompting_target, self.default_prompting_style)return Nonediff --git a/pipelex/core/stuffs/list_content.py b/pipelex/core/stuffs/list_content.pyindex a067a8522..766f6963b 100644--- a/pipelex/core/stuffs/list_content.py+++ b/pipelex/core/stuffs/list_content.py@@ -3,8 +3,8 @@ from typing import Any, Genericfrom json2html import json2htmlfrom typing_extensions import override+from pipelex.cogt.templating.templating_style import TextFormatfrom pipelex.core.stuffs.stuff_content import StuffContent, StuffContentType-from pipelex.tools.templating.templating_models import TextFormatclass ListContent(StuffContent, Generic[StuffContentType]):diff --git a/pipelex/core/stuffs/stuff_artefact.py b/pipelex/core/stuffs/stuff_artefact.pyindex cf494b212..9b9c9bf23 100644--- a/pipelex/core/stuffs/stuff_artefact.py+++ b/pipelex/core/stuffs/stuff_artefact.py@@ -4,10 +4,10 @@ from jinja2.runtime import Contextfrom pydantic import RootModelfrom typing_extensions import override+from pipelex.cogt.templating.templating_style import TextFormatfrom pipelex.core.stuffs.stuff_content import StuffContentfrom pipelex.exceptions import StuffArtefactError-from pipelex.tools.templating.jinja2_models import Jinja2ContextKey, Jinja2TaggableAbstract-from pipelex.tools.templating.templating_models import TextFormat+from pipelex.tools.jinja2.jinja2_models import Jinja2ContextKey, Jinja2TaggableAbstractclass StuffArtefact(RootModel[dict[str, Any]], Jinja2TaggableAbstract):diff --git a/pipelex/core/stuffs/stuff_content.py b/pipelex/core/stuffs/stuff_content.pyindex fabdbdf75..b0a2fc8b7 100644--- a/pipelex/core/stuffs/stuff_content.py+++ b/pipelex/core/stuffs/stuff_content.py@@ -4,7 +4,7 @@ from typing import Any, TypeVarfrom kajson import kajsonfrom typing_extensions import override-from pipelex.tools.templating.templating_models import TextFormat+from pipelex.cogt.templating.templating_style import TextFormatfrom pipelex.tools.typing.pydantic_utils import CustomBaseModelStuffContentType = TypeVar("StuffContentType", bound="StuffContent")diff --git a/pipelex/hub.py b/pipelex/hub.pyindex ca260fd39..70b32ec16 100644--- a/pipelex/hub.py+++ b/pipelex/hub.py@@ -32,7 +32,6 @@ from pipelex.tools.config.config_root import ConfigRootfrom pipelex.tools.config.manager import config_managerfrom pipelex.tools.secrets.secrets_provider_abstract import SecretsProviderAbstractfrom pipelex.tools.storage.storage_provider_abstract import StorageProviderAbstract-from pipelex.tools.templating.template_provider_abstract import TemplateProviderAbstractclass PipelexHub:@@ -47,7 +46,6 @@ class PipelexHub:# toolsself._config: ConfigRoot | None = Noneself._secrets_provider: SecretsProviderAbstract | None = None- self._template_provider: TemplateProviderAbstract | None = Noneself._class_registry: ClassRegistryAbstract | None = Noneself._storage_provider: StorageProviderAbstract | None = None# cogt@@ -118,9 +116,6 @@ class PipelexHub:def set_storage_provider(self, storage_provider: StorageProviderAbstract | None):self._storage_provider = storage_provider- def set_template_provider(self, template_provider: TemplateProviderAbstract):- self._template_provider = template_provider-def set_class_registry(self, class_registry: ClassRegistryAbstract):self._class_registry = class_registry@@ -199,12 +194,6 @@ class PipelexHub:raise RuntimeError(msg)return self._secrets_provider- def get_required_template_provider(self) -> TemplateProviderAbstract:- if self._template_provider is None:- msg = "Template provider is not set. You must initialize Pipelex first."- raise RuntimeError(msg)- return self._template_provider-def get_required_class_registry(self) -> ClassRegistryAbstract:if self._class_registry is None:msg = "ClassRegistry is not initialized"@@ -328,14 +317,6 @@ def get_storage_provider() -> StorageProviderAbstract:return get_pipelex_hub().get_storage_provider()-def get_template_provider() -> TemplateProviderAbstract:- return get_pipelex_hub().get_required_template_provider()---def get_template(template_name: str) -> str:- return get_template_provider().get_template(template_name=template_name)--def get_class_registry() -> ClassRegistryAbstract:return get_pipelex_hub().get_required_class_registry()diff --git a/pipelex/language/plx_factory.py b/pipelex/language/plx_factory.pyindex 294985259..2d9fd666e 100644--- a/pipelex/language/plx_factory.py+++ b/pipelex/language/plx_factory.py@@ -21,6 +21,7 @@ class SectionKey(StrEnum):CONCEPT_STRUCTURE_FIELD_KEY = "structure"+PIPE_TEMPLATE_FIELD_KEY = "template"class PlxFactory:@@ -93,17 +94,24 @@ class PlxFactory:This creates a second-level standard table, and only uses inline tables forthird level and deeper structures.++ Special case: template field is converted to a nested table section instead of inline."""tbl = table()# If field ordering is provided, add fields in the specified order firstif field_ordering:for field_key in field_ordering:- if field_key in mapping and field_key != "category": # Skip category field+ if field_key in mapping and field_key != "category": # Skip category field (pipe metadata)field_value = mapping[field_key]if isinstance(field_value, Mapping):- # Third-level mapping -> inline table- tbl.add(field_key, cls.convert_dicts_to_inline_tables(field_value))+ # Special handling for template field - create nested table instead of inline+ if field_key == PIPE_TEMPLATE_FIELD_KEY:+ field_value = cast("Mapping[str, Any]", field_value)+ tbl.add(field_key, cls.make_template_table(template_value=field_value))+ else:+ # Third-level mapping -> inline table+ tbl.add(field_key, cls.convert_dicts_to_inline_tables(field_value))else:tbl.add(field_key, cls.convert_dicts_to_inline_tables(field_value))@@ -111,20 +119,30 @@ class PlxFactory:for field_key, field_value in mapping.items():if field_key not in field_ordering and field_key != "category":if isinstance(field_value, Mapping):- # Third-level mapping -> inline table- tbl.add(field_key, cls.convert_dicts_to_inline_tables(field_value))+ # Special handling for template field - create nested table instead of inline+ if field_key == PIPE_TEMPLATE_FIELD_KEY:+ field_value = cast("Mapping[str, Any]", field_value)+ tbl.add(field_key, cls.make_template_table(template_value=field_value))+ else:+ # Third-level mapping -> inline table+ tbl.add(field_key, cls.convert_dicts_to_inline_tables(field_value))else:tbl.add(field_key, cls.convert_dicts_to_inline_tables(field_value))else:# No field ordering provided, use original logicfor field_key, field_value in mapping.items():- # Skip the category field as it's not needed in PLX output+ # Skip the category field as it's not needed in PLX output (pipe metadata)if field_key == "category":continueif isinstance(field_value, Mapping):- # Third-level mapping -> inline table- tbl.add(field_key, cls.convert_dicts_to_inline_tables(field_value))+ # Special handling for template field - create nested table instead of inline+ if field_key == PIPE_TEMPLATE_FIELD_KEY:+ field_value = cast("Mapping[str, Any]", field_value)+ tbl.add(field_key, cls.make_template_table(template_value=field_value))+ else:+ # Third-level mapping -> inline table+ tbl.add(field_key, cls.convert_dicts_to_inline_tables(field_value))else:tbl.add(field_key, cls.convert_dicts_to_inline_tables(field_value))return tbl@@ -195,6 +213,16 @@ class PlxFactory:return find_and_replace_inline_tables(toml_string)+ @classmethod+ def make_template_table(cls, template_value: Mapping[str, Any]) -> Any:+ """Create a table for template fields, preserving all fields including category."""+ tbl = table()+ # For template, we want to keep all fields including 'category'+ # which has a different meaning than the pipe's category field+ for template_field_key, template_field_value in template_value.items():+ tbl.add(template_field_key, cls.convert_dicts_to_inline_tables(template_field_value))+ return tbl+@classmethoddef make_table_obj_for_pipe(cls, section_value: Mapping[str, Any]) -> Any:"""Make a table object for a pipe section."""@@ -208,6 +236,7 @@ class PlxFactory:continuelog.debug(f"Field is a mapping: key = {field_key}, value = {field_value}")field_value = cast("Mapping[str, Any]", field_value)+ # Convert pipe configuration to table (handles template field specially)table_obj.add(field_key, cls.convert_mapping_to_table(field_value, field_ordering=cls._plx_config().pipes.field_ordering))return table_objdiff --git a/pipelex/libraries/library_manager.py b/pipelex/libraries/library_manager.pyindex e8358e174..174ac4e53 100644--- a/pipelex/libraries/library_manager.py+++ b/pipelex/libraries/library_manager.py@@ -243,16 +243,16 @@ class LibraryManager(LibraryManagerAbstract):for plx_file_path in valid_plx_paths:try:blueprint = PipelexInterpreter(file_path=plx_file_path).make_pipelex_bundle_blueprint()- except FileNotFoundError as exc:+ except FileNotFoundError as domain_def_error:msg = f"Could not find PLX blueprint at '{plx_file_path}'"- raise LibraryLoadingError(msg) from exc- except ValidationError as exc:- formatted_error_msg = format_pydantic_validation_error(exc)+ raise LibraryLoadingError(msg) from domain_def_error+ except PipeDefinitionError as domain_def_error:+ msg = f"Could not load PLX blueprint from '{plx_file_path}': {domain_def_error}"+ raise LibraryLoadingError(msg) from domain_def_error+ except ValidationError as domain_def_error:+ formatted_error_msg = format_pydantic_validation_error(domain_def_error)msg = f"Could not load PLX blueprint from '{plx_file_path}' because of: {formatted_error_msg}"- raise LibraryLoadingError(msg) from exc- except PipeDefinitionError as exc:- msg = f"Could not load PLX blueprint from '{plx_file_path}': {exc}"- raise LibraryLoadingError(msg) from exc+ raise LibraryLoadingError(msg) from domain_def_errorblueprint.source = str(plx_file_path)blueprints.append(blueprint)@@ -261,9 +261,13 @@ class LibraryManager(LibraryManagerAbstract):for blueprint in blueprints:try:domain = self._load_domain_from_blueprint(blueprint)- except DomainDefinitionError as exc:- msg = f"Could not load domain from PLX blueprint at '{blueprint.source}', domain code: '{blueprint.domain}': {exc}"- raise LibraryLoadingError(msg) from exc+ except DomainDefinitionError as domain_def_error:+ msg = f"Could not load domain from PLX blueprint at '{blueprint.source}', domain code: '{blueprint.domain}': {domain_def_error}"+ raise LibraryLoadingError(msg) from domain_def_error+ except ValidationError as validation_error:+ formatted_error_msg = format_pydantic_validation_error(validation_error)+ msg = f"Could not load domain from PLX blueprint at '{blueprint.source}', domain code: '{blueprint.domain}': {formatted_error_msg}"+ raise LibraryLoadingError(msg) from validation_errorall_domains.append(domain)self.domain_library.add_domains(domains=all_domains)@@ -272,9 +276,13 @@ class LibraryManager(LibraryManagerAbstract):for blueprint in blueprints:try:concepts = self._load_concepts_from_blueprint(blueprint)- except ConceptDefinitionError as exc:- msg = f"Could not load concepts from PLX blueprint at '{blueprint.source}', domain code: '{blueprint.domain}': {exc}"- raise LibraryLoadingError(msg) from exc+ except ConceptDefinitionError as concept_def_error:+ msg = f"Could not load concepts from PLX blueprint at '{blueprint.source}', domain code: '{blueprint.domain}': {concept_def_error}"+ raise LibraryLoadingError(msg) from concept_def_error+ except ValidationError as validation_error:+ formatted_error_msg = format_pydantic_validation_error(validation_error)+ msg = f"Could not load concepts from PLX blueprint at '{blueprint.source}', domain code: '{blueprint.domain}': {formatted_error_msg}"+ raise LibraryLoadingError(msg) from validation_errorall_concepts.extend(concepts)self.concept_library.add_concepts(concepts=all_concepts)@@ -283,8 +291,12 @@ class LibraryManager(LibraryManagerAbstract):for blueprint in blueprints:try:pipes = self._load_pipes_from_blueprint(blueprint)- except PipeDefinitionError as exc:- msg = f"Could not load pipes from PLX blueprint at '{blueprint.source}', domain code: '{blueprint.domain}': {exc}"- raise LibraryLoadingError(msg) from exc+ except PipeDefinitionError as pipe_def_error:+ msg = f"Could not load pipes from PLX blueprint at '{blueprint.source}', domain code: '{blueprint.domain}': {pipe_def_error}"+ raise LibraryLoadingError(msg) from pipe_def_error+ except ValidationError as validation_error:+ formatted_error_msg = format_pydantic_validation_error(validation_error)+ msg = f"Could not load pipes from PLX blueprint at '{blueprint.source}', domain code: '{blueprint.domain}': {formatted_error_msg}"+ raise LibraryLoadingError(msg) from validation_errorall_pipes.extend(pipes)self.pipe_library.add_pipes(pipes=all_pipes)diff --git a/pipelex/libraries/pipelines/builder/builder.plx b/pipelex/libraries/pipelines/builder/builder.plxindex 7e31ba871..5f425db42 100644--- a/pipelex/libraries/pipelines/builder/builder.plx+++ b/pipelex/libraries/pipelines/builder/builder.plx@@ -36,7 +36,7 @@ description = "Turn the brief into a DomainInformation object."inputs = { brief = "UserBrief" }output = "DomainInformation"model = "llm_to_engineer"-prompt_template = """+prompt = """Name and define the domain of this process:@brief@@ -52,7 +52,7 @@ description = "Turn the brief into a pseudo-code plan describing controllers, piinputs = { brief = "UserBrief" }output = "PlanDraft"model = "llm_to_engineer"-prompt_template = """+prompt = """Return a draft of a plan that narrates the pipeline as pseudo-steps (no code):- Explicitly indicate when you are running things in sequence,or in parallel (several independant steps in parallel),@@ -100,7 +100,7 @@ description = "Interpret the draft of a plan to create an AI pipeline, and defininputs = { plan_draft = "PlanDraft", brief = "UserBrief" }output = "ConceptDrafts"model = "llm_to_engineer"-prompt_template = """+prompt = """We are working on writing an AI pipeline to fulfill this brief:@brief@@ -149,7 +149,7 @@ model = "llm_to_engineer"system_prompt = """You are an expert at data extraction and json formatting."""-prompt_template = """+prompt = """You are on a big journey to construct a pipeline, and this is one of the steps.Here is the overalle mission of the user:@brief@@ -169,7 +169,7 @@ system_prompt = """You are a Senior engineer, very well versed in creating pipelines.You are very thorough about naming stuff, structured and rigorous in your planning."""-prompt_template = """+prompt = """Your job is to structure the required PipeSignatures for defining a pipeline which has already been drafted.@briefdiff --git a/pipelex/libraries/pipelines/builder/concept/concept.plx b/pipelex/libraries/pipelines/builder/concept/concept.plxindex aa4862e6b..e9b6f96ef 100644--- a/pipelex/libraries/pipelines/builder/concept/concept.plx+++ b/pipelex/libraries/pipelines/builder/concept/concept.plx@@ -23,7 +23,7 @@ inputs = { concept_spec_draft = "ConceptSpecDraft" }output = "ConceptStructureSpec"multiple_output = truemodel = "llm_to_engineer"-prompt_template = """+prompt = """Create a ConceptStructureSpec from the ConceptSpecDraft.Please focus only on the structure.diff --git a/pipelex/libraries/pipelines/builder/pipe/pipe_compose_spec.py b/pipelex/libraries/pipelines/builder/pipe/pipe_compose_spec.pyindex 647d0bd42..d578720bd 100644--- a/pipelex/libraries/pipelines/builder/pipe/pipe_compose_spec.py+++ b/pipelex/libraries/pipelines/builder/pipe/pipe_compose_spec.py@@ -4,10 +4,11 @@ from pydantic import Field, field_validatorfrom pydantic.json_schema import SkipJsonSchemafrom typing_extensions import override+from pipelex.cogt.templating.template_blueprint import TemplateBlueprint+from pipelex.cogt.templating.template_category import TemplateCategory+from pipelex.cogt.templating.templating_style import TagStyle, TemplatingStyle, TextFormatfrom pipelex.libraries.pipelines.builder.pipe.pipe_signature import PipeSpecfrom pipelex.pipe_operators.compose.pipe_compose_blueprint import PipeComposeBlueprint-from pipelex.tools.templating.jinja2_template_category import Jinja2TemplateCategory-from pipelex.tools.templating.templating_models import PromptingStyle, TagStyle, TextFormatfrom pipelex.types import StrEnum@@ -52,24 +53,24 @@ class TargetFormat(StrEnum):return TextFormat.PLAIN@property- def prompting_style(self) -> PromptingStyle:- return PromptingStyle(tag_style=self.tag_style, text_format=self.text_format)+ def templating_style(self) -> TemplatingStyle:+ return TemplatingStyle(tag_style=self.tag_style, text_format=self.text_format)@property- def template_category(self) -> Jinja2TemplateCategory:+ def category(self) -> TemplateCategory:match self:case TargetFormat.PLAIN:- return Jinja2TemplateCategory.MARKDOWN+ return TemplateCategory.MARKDOWNcase TargetFormat.MARKDOWN:- return Jinja2TemplateCategory.MARKDOWN+ return TemplateCategory.MARKDOWNcase TargetFormat.HTML:- return Jinja2TemplateCategory.HTML+ return TemplateCategory.HTMLcase TargetFormat.JSON:- return Jinja2TemplateCategory.HTML+ return TemplateCategory.HTMLcase TargetFormat.SPREADSHEET:- return Jinja2TemplateCategory.HTML+ return TemplateCategory.HTMLcase TargetFormat.MERMAID:- return Jinja2TemplateCategory.MERMAID+ return TemplateCategory.MERMAIDclass PipeComposeSpec(PipeSpec):@@ -77,7 +78,7 @@ class PipeComposeSpec(PipeSpec):type: SkipJsonSchema[Literal["PipeCompose"]] = "PipeCompose"category: SkipJsonSchema[Literal["PipeOperator"]] = "PipeOperator"- jinja2: str | None = Field(default=None, description="Jinja2 template string")+ template: str = Field(description="Jinja2 template string")target_format: TargetFormat | str = Field(description="Target format for the output")@field_validator("target_format", mode="before")@@ -90,8 +91,15 @@ class PipeComposeSpec(PipeSpec):base_blueprint = super().to_blueprint()target_format = TargetFormat(self.target_format)- prompting_style = target_format.prompting_style- template_category = target_format.template_category+ templating_style = target_format.templating_style+ category = target_format.category++ template_blueprint = TemplateBlueprint(+ source=self.template,+ templating_style=templating_style,+ category=category,+ extra_context=None,+ )return PipeComposeBlueprint(description=base_blueprint.description,@@ -99,9 +107,5 @@ class PipeComposeSpec(PipeSpec):output=base_blueprint.output,type=self.type,category=self.category,- jinja2_name=None,- jinja2=self.jinja2,- prompting_style=prompting_style,- template_category=template_category,- extra_context=None,+ template=template_blueprint,)diff --git a/pipelex/libraries/pipelines/builder/pipe/pipe_design.plx b/pipelex/libraries/pipelines/builder/pipe/pipe_design.plxindex 0beb526b8..a0c0c601f 100644--- a/pipelex/libraries/pipelines/builder/pipe/pipe_design.plx+++ b/pipelex/libraries/pipelines/builder/pipe/pipe_design.plx@@ -45,7 +45,7 @@ description = "Build a PipeSequenceSpec from the signature (children referencedinputs = { plan_draft = "PlanDraft", pipe_signature = "PipeSignature", concept_specs = "concept.ConceptSpec" }output = "PipeSequenceSpec"model = "llm_to_engineer"-prompt_template = """+prompt = """Your job is to design a PipeSequenceSpec to orchestrate a sequence of pipe steps that will run one after the other.This PipeSequence is part of a larger pipeline:@@ -64,7 +64,7 @@ description = "Build a PipeParallelSpec from the signature."inputs = { plan_draft = "PlanDraft", pipe_signature = "PipeSignature", concept_specs = "concept.ConceptSpec" }output = "PipeParallelSpec"model = "llm_to_engineer"-prompt_template = """+prompt = """Your job is to design a PipeParallelSpec to orchestrate a bunch of pipe steps that will run in parallel.This PipeParallel is part of a larger pipeline:@@ -83,7 +83,7 @@ description = "Build a PipeConditionSpec from the signature (provide expression/inputs = { plan_draft = "PlanDraft", pipe_signature = "PipeSignature", concept_specs = "concept.ConceptSpec" }output = "PipeConditionSpec"model = "llm_to_engineer"-prompt_template = """+prompt = """Your job is to design a PipeConditionSpec to route to the correct pipe step based on a conditional expression.This PipeCondition is part of a larger pipeline:@@ -106,7 +106,7 @@ description = "Build a PipeLLMSpec from the signature."inputs = { plan_draft = "PlanDraft", pipe_signature = "PipeSignature", concept_specs = "concept.ConceptSpec" }output = "PipeLLMSpec"model = "llm_to_engineer"-prompt_template = """+prompt = """Your job is to design a PipeLLMSpec to use an LLM to generate a text, or a structured object using different kinds of inputs.Whatever it's really going to do has already been decided, as you can see:@@ -130,7 +130,7 @@ description = "Build a PipeExtractSpec from the signature."inputs = { plan_draft = "PlanDraft", pipe_signature = "PipeSignature", concept_specs = "concept.ConceptSpec" }output = "PipeExtractSpec"model = "llm_to_engineer"-prompt_template = """+prompt = """Your job is to design a PipeExtractSpec to extract text from an image or a pdf.This PipeExtract is part of a larger pipeline:@@ -149,7 +149,7 @@ description = "Build a PipeImgGenSpec from the signature."inputs = { plan_draft = "PlanDraft", pipe_signature = "PipeSignature", concept_specs = "concept.ConceptSpec" }output = "PipeImgGenSpec"model = "llm_to_engineer"-prompt_template = """+prompt = """Your job is to design a PipeImgGenSpec to generate an image from a text prompt.This PipeImgGen is part of a larger pipeline:@@ -170,7 +170,7 @@ description = "Build a PipeComposeSpec from the signature."inputs = { plan_draft = "PlanDraft", pipe_signature = "PipeSignature", concept_specs = "concept.ConceptSpec" }output = "PipeComposeSpec"model = "llm_to_engineer"-prompt_template = """+prompt = """Your job is to design a PipeComposeSpec to render a jinja2 template.This PipeCompose is part of a larger pipeline:diff --git a/pipelex/libraries/pipelines/builder/pipe/pipe_llm_spec.py b/pipelex/libraries/pipelines/builder/pipe/pipe_llm_spec.pyindex 6806bee6c..c9aa5b442 100644--- a/pipelex/libraries/pipelines/builder/pipe/pipe_llm_spec.py+++ b/pipelex/libraries/pipelines/builder/pipe/pipe_llm_spec.py@@ -85,8 +85,8 @@ class PipeLLMSpec(PipeSpec):category: SkipJsonSchema[Literal["PipeOperator"]] = "PipeOperator"llm: LLMSkill | str = Field(description="Select the most adequate LLM model skill according to the task to be performed.")temperature: float | None = Field(default=None, ge=0, le=1)- system_prompt: str | None = Field(default=None, description="A system-level prompt to guide the LLM's behavior, style and skills.")- prompt_template: str | None = Field(+ system_prompt: str | None = Field(default=None, description="A system prompt to guide the LLM's behavior, style and skills. Can be a template.")+ prompt: str | None = Field(description=("A template for the user prompt. Use `$` prefix for inline variables (e.g., `$topic`) and `@` prefix ""to insert content as a block with delimiters (e.g., `@extracted_text` --> extracted_text: ```\n[the extracted_text goes here]\n```). "@@ -148,7 +148,7 @@ class PipeLLMSpec(PipeSpec):inputs=base_blueprint.inputs,output=base_blueprint.output,system_prompt=self.system_prompt,- prompt_template=self.prompt_template,+ prompt=self.prompt,model=llm_choice,nb_output=self.nb_output,multiple_output=self.multiple_output,diff --git a/pipelex/pipe_controllers/condition/pipe_condition.py b/pipelex/pipe_controllers/condition/pipe_condition.pyindex 3f68f4a78..7d03c4bf1 100644--- a/pipelex/pipe_controllers/condition/pipe_condition.py+++ b/pipelex/pipe_controllers/condition/pipe_condition.py@@ -5,6 +5,7 @@ from pydantic import model_validatorfrom typing_extensions import overridefrom pipelex import log+from pipelex.cogt.templating.template_category import TemplateCategoryfrom pipelex.config import StaticValidationReaction, get_configfrom pipelex.core.concepts.concept_factory import ConceptFactoryfrom pipelex.core.concepts.concept_native import NativeConceptCode@@ -23,15 +24,14 @@ from pipelex.exceptions import (StaticValidationErrorType,WorkingMemoryStuffNotFoundError,)-from pipelex.hub import get_content_generator, get_optional_pipe, get_pipe_router, get_pipeline_tracker, get_required_pipe, get_template_provider+from pipelex.hub import get_content_generator, get_optional_pipe, get_pipe_router, get_pipeline_tracker, get_required_pipefrom pipelex.pipe_controllers.condition.pipe_condition_details import PipeConditionDetailsfrom pipelex.pipe_controllers.condition.special_outcome import SpecialOutcomefrom pipelex.pipe_controllers.pipe_controller import PipeControllerfrom pipelex.pipe_run.pipe_job_factory import PipeJobFactoryfrom pipelex.pipe_run.pipe_run_params import PipeRunParamsfrom pipelex.pipeline.job_metadata import JobMetadata-from pipelex.tools.templating.jinja2_required_variables import detect_jinja2_required_variables-from pipelex.tools.templating.jinja2_template_category import Jinja2TemplateCategory+from pipelex.tools.jinja2.jinja2_required_variables import detect_jinja2_required_variablesfrom pipelex.tools.typing.validation_utils import has_exactly_one_among_attributes_from_listfrom pipelex.types import Self@@ -106,13 +106,10 @@ class PipeCondition(PipeController):@overridedef required_variables(self) -> set[str]:required_variables: set[str] = set()- # TODO: use jinja2 directly without going though a pipe# Variables from the expression/expression_templateexpression_required_variables = detect_jinja2_required_variables(- template_category=Jinja2TemplateCategory.LLM_PROMPT,- template_provider=get_template_provider(),- jinja2_name=None,- jinja2=self.applied_expression_template,+ template_category=TemplateCategory.EXPRESSION,+ template_source=self.applied_expression_template,)required_variables.update(expression_required_variables)@@ -144,10 +141,8 @@ class PipeCondition(PipeController):# 1. Add the variables from the expression/expression_templaterequired_variables = detect_jinja2_required_variables(- template_category=Jinja2TemplateCategory.LLM_PROMPT,- template_provider=get_template_provider(),- jinja2_name=None,- jinja2=self.applied_expression_template,+ template_category=TemplateCategory.EXPRESSION,+ template_source=self.applied_expression_template,)for var_name in required_variables:@@ -251,13 +246,12 @@ class PipeCondition(PipeController):PipeConditionError: If expression evaluation fails or no matching pipe is found"""content_generator = get_content_generator()- # Evaluate the expression using Jinja2 templating- # TODO: create a proper category instead of using Jinja2TemplateCategory.LLM_PROMPT- evaluated_expression = await content_generator.make_jinja2_text(+ # Evaluate the expression using templating+ evaluated_expression = await content_generator.make_templated_text(context=working_memory.generate_jinja2_context(),- jinja2=self.applied_expression_template,- template_category=Jinja2TemplateCategory.LLM_PROMPT,+ template=self.applied_expression_template,+ template_category=TemplateCategory.EXPRESSION,)# Validate the evaluated expression@@ -391,10 +385,8 @@ class PipeCondition(PipeController):# 2. Validate that the expression template is validtry:required_variables = detect_jinja2_required_variables(- template_category=Jinja2TemplateCategory.LLM_PROMPT,- template_provider=get_template_provider(),- jinja2_name=None,- jinja2=self.applied_expression_template,+ template_category=TemplateCategory.EXPRESSION,+ template_source=self.applied_expression_template,)log.debug(f"Expression template is valid, requires variables: {required_variables}")except Exception as exc:diff --git a/pipelex/pipe_operators/compose/pipe_compose.py b/pipelex/pipe_operators/compose/pipe_compose.pyindex 9c30d3013..99def447e 100644--- a/pipelex/pipe_operators/compose/pipe_compose.py+++ b/pipelex/pipe_operators/compose/pipe_compose.py@@ -1,12 +1,13 @@-from typing import Any, ClassVar, Literal+from typing import Any, Literal-from jinja2 import TemplateSyntaxErrorfrom pydantic import ConfigDict, model_validatorfrom typing_extensions import overridefrom pipelex import logfrom pipelex.cogt.content_generation.content_generator_dry import ContentGeneratorDryfrom pipelex.cogt.content_generation.content_generator_protocol import ContentGeneratorProtocol+from pipelex.cogt.templating.template_category import TemplateCategory+from pipelex.cogt.templating.templating_style import TemplatingStylefrom pipelex.config import get_configfrom pipelex.core.concepts.concept import Conceptfrom pipelex.core.concepts.concept_factory import ConceptFactory@@ -18,17 +19,14 @@ from pipelex.core.pipes.pipe_output import PipeOutputfrom pipelex.core.stuffs.stuff_factory import StuffFactoryfrom pipelex.core.stuffs.text_content import TextContentfrom pipelex.exceptions import PipeDefinitionError, PipeRunParamsError-from pipelex.hub import get_content_generator, get_template, get_template_provider+from pipelex.hub import get_content_generatorfrom pipelex.pipe_operators.pipe_operator import PipeOperatorfrom pipelex.pipe_run.pipe_run_params import PipeRunMode, PipeRunParamsfrom pipelex.pipe_run.pipe_run_params_factory import PipeRunParamsFactoryfrom pipelex.pipeline.job_metadata import JobMetadata-from pipelex.tools.templating.jinja2_errors import Jinja2TemplateError-from pipelex.tools.templating.jinja2_parsing import check_jinja2_parsing-from pipelex.tools.templating.jinja2_required_variables import detect_jinja2_required_variables-from pipelex.tools.templating.jinja2_template_category import Jinja2TemplateCategory-from pipelex.tools.templating.templating_models import PromptingStyle-from pipelex.tools.typing.validation_utils import has_exactly_one_among_attributes_from_list+from pipelex.tools.jinja2.jinja2_errors import Jinja2TemplateSyntaxError+from pipelex.tools.jinja2.jinja2_parsing import check_jinja2_parsing+from pipelex.tools.jinja2.jinja2_required_variables import detect_jinja2_required_variablesfrom pipelex.types import Self@@ -40,28 +38,22 @@ class PipeCompose(PipeOperator[PipeComposeOutput]):type: Literal["PipeCompose"] = "PipeCompose"model_config = ConfigDict(extra="forbid", strict=False)- adhoc_pipe_code: ClassVar[str] = "jinja2_render"output: Concept = ConceptFactory.make_native_concept(native_concept_code=NativeConceptCode.TEXT,)- jinja2_name: str | None = None- jinja2: str | None = None- prompting_style: PromptingStyle | None = None- template_category: Jinja2TemplateCategory = Jinja2TemplateCategory.LLM_PROMPT+ template: str+ templating_style: TemplatingStyle | None = None+ template_category: TemplateCategory = TemplateCategory.BASICextra_context: dict[str, Any] | None = None@model_validator(mode="after")- def validate_jinja2(self) -> Self:- if not has_exactly_one_among_attributes_from_list(self, attributes_list=["jinja2_name", "jinja2"]):- msg = "PipeCompose should have exactly one of jinja2_name or jinja2"- raise PipeDefinitionError(msg)- if self.jinja2:- try:- check_jinja2_parsing(jinja2_template_source=self.jinja2, template_category=self.template_category)- except TemplateSyntaxError as exc:- msg = f"Could not parse Jinja2 template included in PipeCompose: {exc}"- raise Jinja2TemplateError(msg) from exc+ def validate_template(self) -> Self:+ try:+ check_jinja2_parsing(template_source=self.template, template_category=self.template_category)+ except Jinja2TemplateSyntaxError as exc:+ msg = f"Could not parse template for PipeCompose '{self.code}: {exc}"+ raise PipeDefinitionError(msg) from excreturn self@model_validator(mode="after")@@ -84,9 +76,7 @@ class PipeCompose(PipeOperator[PipeComposeOutput]):@overridedef validate_with_libraries(self):- if self.jinja2_name:- the_template = get_template(template_name=self.jinja2_name)- log.debug(f"Validated jinja2 template '{self.jinja2_name}':\n{the_template}")+ pass@overridedef needed_inputs(self, visited_pipes: set[str] | None = None) -> InputRequirements:@@ -97,20 +87,13 @@ class PipeCompose(PipeOperator[PipeComposeOutput]):@propertydef desc(self) -> str:- if self.jinja2:- return f"Jinja2 included template, prompting style {self.prompting_style}"- elif jinja2_name := self.jinja2_name:- return f"Jinja2 template '{jinja2_name}', prompting style {self.prompting_style}"- else:- return "Jinja2 template not defined"+ return f"Jinja2 included template, prompting style {self.templating_style}"@overridedef required_variables(self) -> set[str]:required_variables = detect_jinja2_required_variables(template_category=self.template_category,- template_provider=get_template_provider(),- jinja2_name=self.jinja2_name,- jinja2=self.jinja2,+ template_source=self.template,)return {variable_name@@ -138,11 +121,10 @@ class PipeCompose(PipeOperator[PipeComposeOutput]):if self.extra_context:context.update(**self.extra_context)- jinja2_text = await content_generator.make_jinja2_text(+ jinja2_text = await content_generator.make_templated_text(context=context,- jinja2_name=self.jinja2_name,- jinja2=self.jinja2,- prompting_style=self.prompting_style,+ template=self.template,+ templating_style=self.templating_style,template_category=self.template_category,)log.verbose(f"Jinja2 rendered text:\n{jinja2_text}")diff --git a/pipelex/pipe_operators/compose/pipe_compose_blueprint.py b/pipelex/pipe_operators/compose/pipe_compose_blueprint.pyindex ac9422040..22af36c1b 100644--- a/pipelex/pipe_operators/compose/pipe_compose_blueprint.py+++ b/pipelex/pipe_operators/compose/pipe_compose_blueprint.py@@ -1,9 +1,39 @@-from typing import Literal+from typing import Any, Literal+from pipelex.cogt.templating.template_blueprint import TemplateBlueprint+from pipelex.cogt.templating.template_category import TemplateCategory+from pipelex.cogt.templating.templating_style import TemplatingStylefrom pipelex.core.pipes.pipe_blueprint import PipeBlueprint-from pipelex.tools.templating.jinja2_blueprint import Jinja2Blueprint-class PipeComposeBlueprint(PipeBlueprint, Jinja2Blueprint):+class PipeComposeBlueprint(PipeBlueprint):type: Literal["PipeCompose"] = "PipeCompose"category: Literal["PipeOperator"] = "PipeOperator"+ template: str | TemplateBlueprint++ @property+ def template_source(self) -> str:+ if isinstance(self.template, TemplateBlueprint):+ return self.template.source+ return self.template++ @property+ def template_category(self) -> TemplateCategory:+ if isinstance(self.template, TemplateBlueprint):+ return self.template.category+ else:+ return TemplateCategory.BASIC++ @property+ def templating_style(self) -> TemplatingStyle | None:+ if isinstance(self.template, TemplateBlueprint):+ return self.template.templating_style+ else:+ return None++ @property+ def extra_context(self) -> dict[str, Any] | None:+ if isinstance(self.template, TemplateBlueprint):+ return self.template.extra_context+ else:+ return Nonediff --git a/pipelex/pipe_operators/compose/pipe_compose_factory.py b/pipelex/pipe_operators/compose/pipe_compose_factory.pyindex ee5b7b30e..c63e6f558 100644--- a/pipelex/pipe_operators/compose/pipe_compose_factory.py+++ b/pipelex/pipe_operators/compose/pipe_compose_factory.py@@ -1,16 +1,15 @@from typing_extensions import override+from pipelex.cogt.templating.template_category import TemplateCategory+from pipelex.cogt.templating.template_preprocessor import preprocess_templatefrom pipelex.core.concepts.concept_factory import ConceptFactoryfrom pipelex.core.pipes.input_requirements import InputRequirementsfrom pipelex.core.pipes.input_requirements_factory import InputRequirementsFactoryfrom pipelex.core.pipes.pipe_factory import PipeFactoryProtocol-from pipelex.exceptions import PipeDefinitionErrorfrom pipelex.hub import get_required_conceptfrom pipelex.pipe_operators.compose.pipe_compose import PipeComposefrom pipelex.pipe_operators.compose.pipe_compose_blueprint import PipeComposeBlueprint-from pipelex.tools.templating.jinja2_parsing import check_jinja2_parsing-from pipelex.tools.templating.jinja2_template_category import Jinja2TemplateCategory-from pipelex.tools.templating.template_preprocessor import preprocess_template+from pipelex.tools.jinja2.jinja2_parsing import check_jinja2_parsingclass PipeComposeFactory(PipeFactoryProtocol[PipeComposeBlueprint, PipeCompose]):@@ -23,15 +22,11 @@ class PipeComposeFactory(PipeFactoryProtocol[PipeComposeBlueprint, PipeCompose])blueprint: PipeComposeBlueprint,concept_codes_from_the_same_domain: list[str] | None = None,) -> PipeCompose:- preprocessed_template: str | None = None- if blueprint.jinja2:- preprocessed_template = preprocess_template(blueprint.jinja2)- check_jinja2_parsing(- jinja2_template_source=preprocessed_template,- template_category=blueprint.template_category,- )- else:- preprocessed_template = None+ preprocessed_template = preprocess_template(blueprint.template_source)+ check_jinja2_parsing(+ template_source=preprocessed_template,+ template_category=blueprint.template_category,+ )output_domain_and_code = ConceptFactory.make_domain_and_concept_code_from_concept_string_or_code(domain=domain,@@ -53,9 +48,8 @@ class PipeComposeFactory(PipeFactoryProtocol[PipeComposeBlueprint, PipeCompose])concept_code=output_domain_and_code.concept_code,),),- jinja2_name=blueprint.jinja2_name,- jinja2=preprocessed_template,- prompting_style=blueprint.prompting_style,+ template=preprocessed_template,+ templating_style=blueprint.templating_style,template_category=blueprint.template_category,extra_context=blueprint.extra_context,)@@ -64,29 +58,17 @@ class PipeComposeFactory(PipeFactoryProtocol[PipeComposeBlueprint, PipeCompose])def make_pipe_compose_from_template_str(cls,domain: str,+ template_str: str,inputs: InputRequirements | None = None,- template_str: str | None = None,- template_name: str | None = None,) -> PipeCompose:- if template_str:- preprocessed_template = preprocess_template(template_str)- check_jinja2_parsing(- jinja2_template_source=preprocessed_template,- template_category=Jinja2TemplateCategory.LLM_PROMPT,- )- return PipeCompose(- domain=domain,- code="adhoc_pipe_compose_from_template_str",- jinja2=preprocessed_template,- inputs=inputs or InputRequirementsFactory.make_empty(),- )- elif template_name:- return PipeCompose(- domain=domain,- code="adhoc_pipe_compose_from_template_name",- jinja2_name=template_name,- inputs=inputs or InputRequirementsFactory.make_empty(),- )- else:- msg = "Could not make a PipeCompose because neither template_str nor template_name were provided"- raise PipeDefinitionError(msg)+ preprocessed_template = preprocess_template(template_str)+ check_jinja2_parsing(+ template_source=preprocessed_template,+ template_category=TemplateCategory.LLM_PROMPT,+ )+ return PipeCompose(+ domain=domain,+ code="adhoc_pipe_compose_from_template_str",+ template=preprocessed_template,+ inputs=inputs or InputRequirementsFactory.make_empty(),+ )diff --git a/pipelex/pipe_operators/llm/llm_prompt_blueprint.py b/pipelex/pipe_operators/llm/llm_prompt_blueprint.pynew file mode 100644index 000000000..2df0dc2da--- /dev/null+++ b/pipelex/pipe_operators/llm/llm_prompt_blueprint.py@@ -0,0 +1,180 @@+from typing import TYPE_CHECKING, Any++from pydantic import BaseModel++from pipelex import log+from pipelex.cogt.exceptions import LLMPromptSpecError+from pipelex.cogt.image.prompt_image_factory import PromptImageFactory+from pipelex.cogt.llm.llm_prompt import LLMPrompt+from pipelex.cogt.templating.template_blueprint import TemplateBlueprint+from pipelex.cogt.templating.template_preprocessor import preprocess_template+from pipelex.cogt.templating.templating_style import TemplatingStyle+from pipelex.core.stuffs.image_content import ImageContent+from pipelex.hub import get_content_generator+from pipelex.tools.jinja2.jinja2_required_variables import detect_jinja2_required_variables+from pipelex.tools.misc.context_provider_abstract import ContextProviderAbstract, ContextProviderException++if TYPE_CHECKING:+ from pipelex.cogt.image.prompt_image import PromptImage+++class LLMPromptBlueprint(BaseModel):+ templating_style: TemplatingStyle | None = None+ system_prompt_blueprint: TemplateBlueprint | None = None+ prompt_blueprint: TemplateBlueprint | None = None+ user_images: list[str] | None = None++ def validate_with_libraries(self):+ pass++ def required_variables(self) -> set[str]:+ required_variables: set[str] = set()+ if self.user_images:+ user_images_top_object_name = [user_image.split(".", 1)[0] for user_image in self.user_images]+ required_variables.update(user_images_top_object_name)++ if self.prompt_blueprint:+ template_source = preprocess_template(self.prompt_blueprint.source)+ required_variables.update(+ detect_jinja2_required_variables(+ template_category=self.prompt_blueprint.category,+ template_source=template_source,+ )+ )++ if self.system_prompt_blueprint:+ system_prompt_template_source = preprocess_template(self.system_prompt_blueprint.source)+ required_variables.update(+ detect_jinja2_required_variables(+ template_category=self.system_prompt_blueprint.category,+ template_source=system_prompt_template_source,+ )+ )++ return {+ variable_name+ for variable_name in required_variables+ if not variable_name.startswith("_") and variable_name not in ("preliminary_text", "place_holder")+ }++ # TODO: make this consistent with `LLMPromptFactoryAbstract` or `LLMPromptTemplate`,+ # let's get back to it when we have a better solution for structuring_method+ async def make_llm_prompt(+ self,+ output_concept_string: str,+ context_provider: ContextProviderAbstract,+ output_structure_prompt: str | None = None,+ extra_params: dict[str, Any] | None = None,+ ) -> LLMPrompt:+ ############################################################+ # User images+ ############################################################+ prompt_user_images: dict[str, PromptImage] = {}+ if self.user_images:+ for user_image_name in self.user_images:+ log.debug(f"Getting user image '{user_image_name}' from context")+ # Try to get as a single ImageContent first+ try:+ prompt_image_content = context_provider.get_typed_object_or_attribute(name=user_image_name, wanted_type=ImageContent)+ if prompt_image_content is not None: # An ImageContent can be optional+ if base_64 := prompt_image_content.base_64:+ user_image = PromptImageFactory.make_prompt_image(base_64=base_64)+ else:+ image_uri = prompt_image_content.url+ user_image = PromptImageFactory.make_prompt_image_from_uri(uri=image_uri)+ prompt_user_images[user_image_name] = user_image+ except ContextProviderException:+ # If single image failed, try to get as a collection (list or tuple)+ try:+ image_collection = context_provider.get_typed_object_or_attribute(name=user_image_name, wanted_type=None)+ # Check if it's a list or tuple+ if isinstance(image_collection, (list, tuple)):+ for image_item in image_collection: # type: ignore[assignment]+ if isinstance(image_item, ImageContent):+ item_base_64 = image_item.base_64+ if item_base_64:+ user_image = PromptImageFactory.make_prompt_image(base_64=item_base_64) # type: ignore[arg-type]+ else:+ image_uri = image_item.url+ user_image = PromptImageFactory.make_prompt_image_from_uri(uri=image_uri)+ prompt_user_images[user_image_name] = user_image+ else:+ msg = (+ f"Could not find a valid user image or image collection named '{user_image_name}' from the provided context_provider"+ )+ raise LLMPromptSpecError(msg)+ except ContextProviderException as exc:+ msg = f"Could not find a valid user image named '{user_image_name}' from the provided context_provider: {exc}"+ raise LLMPromptSpecError(msg) from exc++ ############################################################+ # User text+ ############################################################+ # replace the image variables with numbered tags+ if prompt_user_images:+ if not extra_params:+ extra_params = {}+ for image_index, image_name in enumerate(prompt_user_images.keys()):+ extra_params[image_name] = f"[Image {image_index + 1}]"+ log.warning(f"Replacing image variable '{image_name}' with numbered tag '[Image {image_index + 1}]'")+ user_text: str | None = None+ if self.prompt_blueprint:+ user_text = await self._unravel_text(+ context_provider=context_provider,+ jinja2_blueprint=self.prompt_blueprint,+ extra_params=extra_params,+ )+ if output_structure_prompt:+ user_text += output_structure_prompt+ else:+ user_text = output_structure_prompt+ # Note that output_structure_prompt can be None+ # it's OK to have a null user_text++ log.verbose(f"User text with {output_concept_string=}:\n {user_text}")++ ############################################################+ # System text+ ############################################################+ system_text: str | None = None+ if self.system_prompt_blueprint:+ system_text = await self._unravel_text(+ context_provider=context_provider,+ jinja2_blueprint=self.system_prompt_blueprint,+ extra_params=extra_params,+ )++ ############################################################+ # Full LLMPrompt+ ############################################################+ return LLMPrompt(+ system_text=system_text,+ user_text=user_text,+ user_images=list(prompt_user_images.values()),+ )++ async def _unravel_text(+ self,+ context_provider: ContextProviderAbstract,+ jinja2_blueprint: TemplateBlueprint,+ extra_params: dict[str, Any] | None = None,+ ) -> str:+ if (templating_style := self.templating_style) and not jinja2_blueprint.templating_style:+ jinja2_blueprint.templating_style = templating_style+ log.verbose(f"Setting prompting style to {templating_style}")++ log.info(f"extra_params: {extra_params}")+ log.info(f"jinja2_blueprint.extra_context: {jinja2_blueprint.extra_context}")++ context: dict[str, Any] = context_provider.generate_jinja2_context()+ if extra_params:+ context.update(**extra_params)+ if jinja2_blueprint.extra_context:+ context.update(**jinja2_blueprint.extra_context)++ return await get_content_generator().make_templated_text(+ context=context,+ template=jinja2_blueprint.source,+ templating_style=self.templating_style,+ template_category=jinja2_blueprint.category,+ )diff --git a/pipelex/pipe_operators/llm/pipe_llm.py b/pipelex/pipe_operators/llm/pipe_llm.pyindex 302398574..b1393fb57 100644--- a/pipelex/pipe_operators/llm/pipe_llm.py+++ b/pipelex/pipe_operators/llm/pipe_llm.py@@ -8,10 +8,10 @@ from pipelex.cogt.content_generation.content_generator_dry import ContentGeneratfrom pipelex.cogt.content_generation.content_generator_protocol import ContentGeneratorProtocolfrom pipelex.cogt.llm.llm_prompt import LLMPromptfrom pipelex.cogt.llm.llm_prompt_factory_abstract import LLMPromptFactoryAbstract-from pipelex.cogt.llm.llm_prompt_spec import LLMPromptSpecfrom pipelex.cogt.llm.llm_prompt_template import LLMPromptTemplatefrom pipelex.cogt.llm.llm_setting import LLMModelChoice, LLMSetting, LLMSettingChoicesfrom pipelex.cogt.models.model_deck_check import check_llm_choice_with_deck+from pipelex.cogt.templating.template_category import TemplateCategoryfrom pipelex.config import StaticValidationReaction, get_configfrom pipelex.core.concepts.concept_factory import ConceptFactoryfrom pipelex.core.concepts.concept_native import NativeConceptCode@@ -39,8 +39,8 @@ from pipelex.hub import (get_required_concept,get_required_domain,get_required_pipe,- get_template,)+from pipelex.pipe_operators.llm.llm_prompt_blueprint import LLMPromptBlueprintfrom pipelex.pipe_operators.llm.pipe_llm_blueprint import StructuringMethodfrom pipelex.pipe_operators.pipe_operator import PipeOperatorfrom pipelex.pipe_run.pipe_run_params import (@@ -50,7 +50,6 @@ from pipelex.pipe_run.pipe_run_params import (output_multiplicity_to_apply,)from pipelex.pipeline.job_metadata import JobMetadata-from pipelex.tools.templating.jinja2_template_category import Jinja2TemplateCategoryfrom pipelex.tools.typing.structure_printer import StructurePrinterfrom pipelex.types import Self@@ -61,7 +60,7 @@ class PipeLLMOutput(PipeOutput):class PipeLLM(PipeOperator[PipeLLMOutput]):type: Literal["PipeLLM"] = "PipeLLM"- llm_prompt_spec: LLMPromptSpec+ llm_prompt_spec: LLMPromptBlueprintllm_choices: LLMSettingChoices | None = Nonestructuring_method: StructuringMethod | None = Noneprompt_template_to_structure: str | None = None@@ -85,12 +84,13 @@ class PipeLLM(PipeOperator[PipeLLMOutput]):@overridedef validate_with_libraries(self):+ llm_config = get_config().cogt.llm_configself.validate_inputs()self.llm_prompt_spec.validate_with_libraries()if self.prompt_template_to_structure:- get_template(template_name=self.prompt_template_to_structure)+ llm_config.get_template(template_name=self.prompt_template_to_structure)if self.system_prompt_to_structure:- get_template(template_name=self.system_prompt_to_structure)+ llm_config.get_template(template_name=self.system_prompt_to_structure)if self.llm_choices:for llm_choice in self.llm_choices.list_choices():check_llm_choice_with_deck(llm_choice=llm_choice)@@ -180,6 +180,7 @@ class PipeLLM(PipeOperator[PipeLLMOutput]):output_name: str | None = None,content_generator: ContentGeneratorProtocol | None = None,) -> PipeLLMOutput:+ llm_config = get_config().cogt.llm_configcontent_generator = content_generator or get_content_generator()# interpret / unwrap the argumentslog.debug(f"PipeLLM pipe_code = {self.code}")@@ -228,13 +229,13 @@ class PipeLLM(PipeOperator[PipeLLMOutput]):)llm_setting_for_object: LLMSetting = model_deck.get_llm_setting(llm_choice=llm_setting_or_preset_id_for_object)- if (not self.llm_prompt_spec.prompting_style) and (+ if (not self.llm_prompt_spec.templating_style) and (inference_model := model_deck.get_optional_inference_model(model_handle=llm_setting_main.model)):# Note: the case where we don't get an inference model corresponds to the use of an external LLM Plugin# TODO: improve this by making it possible to get the inference model for external LLM Pluginsprompting_target = llm_setting_main.prompting_target or inference_model.prompting_target- self.llm_prompt_spec.prompting_style = get_config().pipelex.prompting_config.get_prompting_style(+ self.llm_prompt_spec.templating_style = get_config().pipelex.prompting_config.get_prompting_style(prompting_target=prompting_target,)@@ -298,7 +299,7 @@ class PipeLLM(PipeOperator[PipeLLMOutput]):prompt_template_to_structure = (self.prompt_template_to_structureor domain.prompt_template_to_structure- or get_template(template_name="structure_from_preliminary_text_user")+ or llm_config.get_template(template_name="structure_from_preliminary_text_user"))system_prompt = self.system_prompt_to_structure or domain.system_promptllm_prompt_2_proto = LLMPrompt(@@ -318,7 +319,7 @@ class PipeLLM(PipeOperator[PipeLLMOutput]):prompt_template_to_structure = (self.prompt_template_to_structureor domain.prompt_template_to_structure- or get_template(template_name="structure_from_preliminary_text_user")+ or llm_config.get_template(template_name="structure_from_preliminary_text_user"))system_prompt = self.system_prompt_to_structure or domain.system_promptllm_prompt_2_proto = LLMPrompt(@@ -333,7 +334,7 @@ class PipeLLM(PipeOperator[PipeLLMOutput]):llm_prompt_2_factory = Noneoutput_structure_prompt: str | None = None- if get_config().cogt.llm_config.is_structure_prompt_enabled:+ if llm_config.is_structure_prompt_enabled:output_structure_prompt = await PipeLLM.get_output_structure_prompt(concept_string=pipe_run_params.dynamic_output_concept_code or output_concept.concept_string,is_with_preliminary_text=is_with_preliminary_text,@@ -486,11 +487,16 @@ class PipeLLM(PipeOperator[PipeLLMOutput]):if not class_structure:return Noneclass_structure_str = "\n".join(class_structure)+ llm_config = get_config().cogt.llm_config+ if is_with_preliminary_text:+ template_source = llm_config.get_template(template_name="output_structure_prompt")+ else:+ template_source = llm_config.get_template(template_name="output_structure_prompt_no_preliminary_text")- return await get_content_generator().make_jinja2_text(+ return await get_content_generator().make_templated_text(context={"class_structure_str": class_structure_str,},- template_category=Jinja2TemplateCategory.LLM_PROMPT,- jinja2_name="output_structure_prompt" if is_with_preliminary_text else "output_structure_prompt_no_preliminary_text",+ template=template_source,+ template_category=TemplateCategory.LLM_PROMPT,)diff --git a/pipelex/pipe_operators/llm/pipe_llm_blueprint.py b/pipelex/pipe_operators/llm/pipe_llm_blueprint.pyindex 08ec46d03..19ce6aa13 100644--- a/pipelex/pipe_operators/llm/pipe_llm_blueprint.py+++ b/pipelex/pipe_operators/llm/pipe_llm_blueprint.py@@ -5,7 +5,7 @@ from pydantic import field_validator, model_validatorfrom pipelex.cogt.llm.llm_setting import LLMModelChoicefrom pipelex.core.pipes.pipe_blueprint import PipeBlueprintfrom pipelex.exceptions import PipeDefinitionError-from pipelex.tools.typing.validation_utils import has_more_than_one_among_attributes_from_lists+from pipelex.tools.typing.validation_utils import has_more_than_one_among_attributes_from_listfrom pipelex.types import Self, StrEnum@@ -21,14 +21,7 @@ class PipeLLMBlueprint(PipeBlueprint):model: LLMModelChoice | None = Nonemodel_to_structure: LLMModelChoice | None = None- system_prompt_template: str | None = None- system_prompt_template_name: str | None = None- system_prompt_name: str | None = Nonesystem_prompt: str | None = None-- prompt_template: str | None = None- template_name: str | None = None- prompt_name: str | None = Noneprompt: str | None = Nonestructuring_method: StructuringMethod | None = None@@ -48,13 +41,9 @@ class PipeLLMBlueprint(PipeBlueprint):@model_validator(mode="after")def validate_multiple_output(self) -> Self:- if excess_attributes_list := has_more_than_one_among_attributes_from_lists(+ if excess_attributes_list := has_more_than_one_among_attributes_from_list(self,- attributes_lists=[- ["nb_output", "multiple_output"],- ["system_prompt", "system_prompt_name", "system_prompt_template", "system_prompt_template_name"],- ["prompt", "prompt_name", "prompt_template", "template_name"],- ],+ attributes_list=["nb_output", "multiple_output"],):msg = f"PipeLLMBlueprint should have no more than one of {excess_attributes_list} among them"raise PipeDefinitionError(msg)diff --git a/pipelex/pipe_operators/llm/pipe_llm_factory.py b/pipelex/pipe_operators/llm/pipe_llm_factory.pyindex baf055f48..e9c73e751 100644--- a/pipelex/pipe_operators/llm/pipe_llm_factory.py+++ b/pipelex/pipe_operators/llm/pipe_llm_factory.py@@ -1,7 +1,8 @@from typing_extensions import override-from pipelex.cogt.llm.llm_prompt_spec import LLMPromptSpecfrom pipelex.cogt.llm.llm_setting import LLMSettingChoices+from pipelex.cogt.templating.template_blueprint import TemplateBlueprint+from pipelex.cogt.templating.template_category import TemplateCategoryfrom pipelex.core.concepts.concept import Conceptfrom pipelex.core.concepts.concept_factory import ConceptFactoryfrom pipelex.core.concepts.concept_native import NativeConceptCode@@ -10,11 +11,11 @@ from pipelex.core.pipes.input_requirements_factory import InputRequirementsFactofrom pipelex.core.pipes.pipe_factory import PipeFactoryProtocolfrom pipelex.exceptions import PipeDefinitionErrorfrom pipelex.hub import get_native_concept, get_optional_domain, get_required_concept+from pipelex.pipe_operators.llm.llm_prompt_blueprint import LLMPromptBlueprintfrom pipelex.pipe_operators.llm.pipe_llm import PipeLLMfrom pipelex.pipe_operators.llm.pipe_llm_blueprint import PipeLLMBlueprintfrom pipelex.pipe_run.pipe_run_params import make_output_multiplicity-from pipelex.tools.templating.jinja2_blueprint import Jinja2Blueprint-from pipelex.tools.templating.jinja2_errors import Jinja2TemplateError+from pipelex.tools.jinja2.jinja2_errors import Jinja2TemplateSyntaxErrorclass PipeLLMFactory(PipeFactoryProtocol[PipeLLMBlueprint, PipeLLM]):@@ -27,39 +28,35 @@ class PipeLLMFactory(PipeFactoryProtocol[PipeLLMBlueprint, PipeLLM]):blueprint: PipeLLMBlueprint,concept_codes_from_the_same_domain: list[str] | None = None,) -> PipeLLM:- system_prompt_jinja2_blueprint: Jinja2Blueprint | None = None- system_prompt: str | None = None- if blueprint.system_prompt_template or blueprint.system_prompt_template_name:+ system_prompt = blueprint.system_prompt+ if not system_prompt and (domain_obj := get_optional_domain(domain=domain)):+ system_prompt = domain_obj.system_prompt++ system_prompt_jinja2_blueprint: TemplateBlueprint | None = None+ if system_prompt:try:- system_prompt_jinja2_blueprint = Jinja2Blueprint(- jinja2=blueprint.system_prompt_template,- jinja2_name=blueprint.system_prompt_template_name,+ system_prompt_jinja2_blueprint = TemplateBlueprint(+ source=system_prompt,+ category=TemplateCategory.LLM_PROMPT,+ )+ except Jinja2TemplateSyntaxError as exc:+ error_msg = (+ f"Template syntax error in system prompt for pipe '{pipe_code}' "+ f"in domain '{domain}': {exc}. Template source:\n{blueprint.system_prompt}")- except Jinja2TemplateError as exc:- error_msg = f"Jinja2 template error in system prompt for pipe '{pipe_code}' in domain '{domain}': {exc}."- if blueprint.system_prompt_template:- error_msg += f"\nThe system prompt template is:\n{blueprint.system_prompt_template}"- else:- error_msg += "The system prompt template is not provided."raise PipeDefinitionError(error_msg) from exc- elif not blueprint.system_prompt and not blueprint.system_prompt_name:- # really no system prompt provided, let's use the domain's default system prompt- if domain_obj := get_optional_domain(domain=domain):- system_prompt = domain_obj.system_prompt- user_text_jinja2_blueprint: Jinja2Blueprint | None = None- if blueprint.prompt_template or blueprint.template_name:+ user_text_jinja2_blueprint: TemplateBlueprint | None = None+ if blueprint.prompt:try:- user_text_jinja2_blueprint = Jinja2Blueprint(- jinja2=blueprint.prompt_template,- jinja2_name=blueprint.template_name,+ user_text_jinja2_blueprint = TemplateBlueprint(+ source=blueprint.prompt,+ category=TemplateCategory.LLM_PROMPT,+ )+ except Jinja2TemplateSyntaxError as exc:+ error_msg = (+ f"Template syntax error in user prompt for pipe '{pipe_code}' in domain '{domain}': {exc}. Template source:\n{blueprint.prompt}")- except Jinja2TemplateError as exc:- error_msg = f"Jinja2 syntax error in user prompt for pipe '{pipe_code}' in domain '{domain}': {exc}."- if blueprint.prompt_template:- error_msg += f"\nThe prompt template is:\n{blueprint.prompt_template}"- else:- error_msg += "The prompt template is not provided."raise PipeDefinitionError(error_msg) from excuser_images: list[str] = []@@ -93,13 +90,9 @@ class PipeLLMFactory(PipeFactoryProtocol[PipeLLMBlueprint, PipeLLM]):for field_path in image_field_paths:user_images.append(f"{stuff_name}.{field_path}")- llm_prompt_spec = LLMPromptSpec(- system_prompt_jinja2_blueprint=system_prompt_jinja2_blueprint,- system_prompt_verbatim_name=blueprint.system_prompt_name,- system_prompt=blueprint.system_prompt or system_prompt,- user_text_jinja2_blueprint=user_text_jinja2_blueprint,- user_prompt_verbatim_name=blueprint.prompt_name,- user_text=blueprint.prompt,+ llm_prompt_spec = LLMPromptBlueprint(+ system_prompt_blueprint=system_prompt_jinja2_blueprint,+ prompt_blueprint=user_text_jinja2_blueprint,user_images=user_images or None,)diff --git a/pipelex/pipelex.py b/pipelex/pipelex.pyindex f4fecc9c1..f6aa4d883 100644--- a/pipelex/pipelex.py+++ b/pipelex/pipelex.py@@ -51,7 +51,6 @@ from pipelex.tools.runtime_manager import runtime_managerfrom pipelex.tools.secrets.env_secrets_provider import EnvSecretsProviderfrom pipelex.tools.secrets.secrets_provider_abstract import SecretsProviderAbstractfrom pipelex.tools.storage.storage_provider_abstract import StorageProviderAbstract-from pipelex.tools.templating.template_library import TemplateLibraryfrom pipelex.tools.typing.pydantic_utils import format_pydantic_validation_errorfrom pipelex.types import Self@@ -67,7 +66,6 @@ class Pipelex(metaclass=MetaSingleton):pipelex_hub: PipelexHub | None = None,config_cls: type[ConfigRoot] | None = None,class_registry: ClassRegistryAbstract | None = None,- template_provider: TemplateLibrary | None = None,models_manager: ModelManagerAbstract | None = None,inference_manager: InferenceManager | None = None,pipeline_manager: PipelineManager | None = None,@@ -94,9 +92,6 @@ class Pipelex(metaclass=MetaSingleton):log.debug("Logs are configured")# tools- self.template_provider = template_provider or TemplateLibrary.make_empty(config_dir_path=config_dir_path)- self.pipelex_hub.set_template_provider(self.template_provider)-self.class_registry = class_registry or ClassRegistry()self.pipelex_hub.set_class_registry(self.class_registry)self.kajson_manager = KajsonManager(class_registry=self.class_registry)@@ -217,14 +212,8 @@ class Pipelex(metaclass=MetaSingleton):log.debug(f"{PACKAGE_NAME} version {PACKAGE_VERSION} setup done for {get_config().project_name}")def setup_libraries(self):- try:- self.template_provider.setup()- self.library_manager.setup()- self.library_manager.load_libraries()- except ValidationError as exc:- formatted_error_msg = format_pydantic_validation_error(exc)- msg = f"Could not setup libraries because of: {formatted_error_msg}"- raise PipelexSetupError(msg) from exc+ self.library_manager.setup()+ self.library_manager.load_libraries()log.debug(f"{PACKAGE_NAME} version {PACKAGE_VERSION} setup libraries done for {get_config().project_name}")def validate_libraries(self):@@ -241,7 +230,6 @@ class Pipelex(metaclass=MetaSingleton):self.pipeline_manager.teardown()self.pipeline_tracker.teardown()self.library_manager.teardown()- self.template_provider.teardown()self.activity_manager.teardown()# cogtdiff --git a/pipelex/pipelex.toml b/pipelex/pipelex.tomlindex 19cbc73da..8d86c747b 100644--- a/pipelex/pipelex.toml+++ b/pipelex/pipelex.toml@@ -88,6 +88,8 @@ is_auto_setup_preset_extract = true[cogt.llm_config]default_max_images = 100is_structure_prompt_enabled = true+is_dump_text_prompts_enabled = false+is_dump_response_text_enabled = false[cogt.llm_config.instructor_config]is_openai_structured_output_enabled = false@@ -99,6 +101,42 @@ is_dump_error_enabled = falsemax_retries = 3is_streaming_enabled = false+[cogt.llm_config.generic_templates]+structure_from_preliminary_text_system = """+You are a data modeling expert specialized in extracting structure from text.+"""++structure_from_preliminary_text_user = """+Your job is to extract and structure information from a text.+Here is the text:++{preliminary_text}++Now generate the JSON in the required format.+Do not create information that is not in the text.+"""++output_structure_prompt = """++---+The instance we want to generate will be for the following class:+{{ class_structure_str }}++Don't bother with JSON formatting, we'll do that as a second step.+For now, just output markdown with the details of the instance.+DO NOT create information.+If some information is not present for an attribute, output the default value or None according to the attribute definition.+"""++output_structure_prompt_no_preliminary_text = """+---+The instance we want to generate will be for the following class:+{{ class_structure_str }}++DO NOT create information.+If some information is not present for an attribute, output the default value or None according to the attribute definition.+"""+##################################################################################################### Image generation config####################################################################################################diff --git a/pipelex/plugins/anthropic/anthropic_llm_worker.py b/pipelex/plugins/anthropic/anthropic_llm_worker.pyindex bc533de71..88beb29b0 100644--- a/pipelex/plugins/anthropic/anthropic_llm_worker.py+++ b/pipelex/plugins/anthropic/anthropic_llm_worker.py@@ -7,6 +7,7 @@ from typing_extensions import overridefrom pipelex import logfrom pipelex.cogt.exceptions import LLMCompletionError, SdkTypeErrorfrom pipelex.cogt.llm.llm_job import LLMJob+from pipelex.cogt.llm.llm_utils import dump_error, dump_kwargs, dump_response_from_structured_genfrom pipelex.cogt.llm.llm_worker_internal_abstract import LLMWorkerInternalAbstractfrom pipelex.cogt.llm.structured_output import StructureMethodfrom pipelex.cogt.model_backends.model_spec import InferenceModelSpec@@ -14,7 +15,6 @@ from pipelex.config import get_configfrom pipelex.plugins.anthropic.anthropic_exceptions import AnthropicWorkerConfigurationErrorfrom pipelex.plugins.anthropic.anthropic_factory import AnthropicFactory, AnthropicSdkVariantfrom pipelex.reporting.reporting_protocol import ReportingProtocol-from pipelex.tools.misc.instructor_utils import dump_error, dump_kwargs, dump_responsefrom pipelex.tools.typing.pydantic_utils import BaseModelTypeVarfrom pipelex.types import StrEnum@@ -81,7 +81,7 @@ class AnthropicLLMWorker(LLMWorkerInternalAbstract):if instructor_config.is_dump_kwargs_enabled:self.instructor_for_objects.on(hook_name="completion:kwargs", handler=dump_kwargs)if instructor_config.is_dump_response_enabled:- self.instructor_for_objects.on(hook_name="completion:response", handler=dump_response)+ self.instructor_for_objects.on(hook_name="completion:response", handler=dump_response_from_structured_gen)if instructor_config.is_dump_error_enabled:self.instructor_for_objects.on(hook_name="completion:error", handler=dump_error)diff --git a/pipelex/plugins/google/google_llm_worker.py b/pipelex/plugins/google/google_llm_worker.pyindex 9ccdd02bf..30adf36b0 100644--- a/pipelex/plugins/google/google_llm_worker.py+++ b/pipelex/plugins/google/google_llm_worker.py@@ -11,6 +11,7 @@ from typing_extensions import overridefrom pipelex import logfrom pipelex.cogt.exceptions import LLMCompletionErrorfrom pipelex.cogt.llm.llm_job import LLMJob+from pipelex.cogt.llm.llm_utils import dump_error, dump_kwargs, dump_response_from_structured_genfrom pipelex.cogt.llm.llm_worker_internal_abstract import LLMWorkerInternalAbstractfrom pipelex.cogt.llm.structured_output import StructureMethodfrom pipelex.cogt.model_backends.model_spec import InferenceModelSpec@@ -18,7 +19,6 @@ from pipelex.cogt.usage.token_category import NbTokensByCategoryDict, TokenCategfrom pipelex.config import get_configfrom pipelex.plugins.google.google_factory import GoogleFactoryfrom pipelex.reporting.reporting_protocol import ReportingProtocol-from pipelex.tools.misc.instructor_utils import dump_error, dump_kwargs, dump_responsefrom pipelex.tools.typing.pydantic_utils import BaseModelTypeVar@@ -52,7 +52,7 @@ class GoogleLLMWorker(LLMWorkerInternalAbstract):if instructor_config.is_dump_kwargs_enabled:self.instructor_for_objects.on(hook_name="completion:kwargs", handler=dump_kwargs)if instructor_config.is_dump_response_enabled:- self.instructor_for_objects.on(hook_name="completion:response", handler=dump_response)+ self.instructor_for_objects.on(hook_name="completion:response", handler=dump_response_from_structured_gen)if instructor_config.is_dump_error_enabled:self.instructor_for_objects.on(hook_name="completion:error", handler=dump_error)diff --git a/pipelex/plugins/openai/openai_llm_worker.py b/pipelex/plugins/openai/openai_llm_worker.pyindex 6fd8b2b04..706919790 100644--- a/pipelex/plugins/openai/openai_llm_worker.py+++ b/pipelex/plugins/openai/openai_llm_worker.py@@ -12,6 +12,7 @@ from typing_extensions import overridefrom pipelex import logfrom pipelex.cogt.exceptions import LLMCompletionError, LLMModelNotFoundError, SdkTypeErrorfrom pipelex.cogt.llm.llm_job import LLMJob+from pipelex.cogt.llm.llm_utils import dump_error, dump_kwargs, dump_response_from_structured_genfrom pipelex.cogt.llm.llm_worker_internal_abstract import LLMWorkerInternalAbstractfrom pipelex.cogt.llm.structured_output import StructureMethodfrom pipelex.cogt.model_backends.model_constraints import ModelConstraints@@ -19,7 +20,6 @@ from pipelex.cogt.model_backends.model_spec import InferenceModelSpecfrom pipelex.config import get_configfrom pipelex.plugins.openai.openai_factory import OpenAIFactoryfrom pipelex.reporting.reporting_protocol import ReportingProtocol-from pipelex.tools.misc.instructor_utils import dump_error, dump_kwargs, dump_responsefrom pipelex.tools.typing.pydantic_utils import BaseModelTypeVar@@ -54,7 +54,7 @@ class OpenAILLMWorker(LLMWorkerInternalAbstract):if instructor_config.is_dump_kwargs_enabled:self.instructor_for_objects.on(hook_name="completion:kwargs", handler=dump_kwargs)if instructor_config.is_dump_response_enabled:- self.instructor_for_objects.on(hook_name="completion:response", handler=dump_response)+ self.instructor_for_objects.on(hook_name="completion:response", handler=dump_response_from_structured_gen)if instructor_config.is_dump_error_enabled:self.instructor_for_objects.on(hook_name="completion:error", handler=dump_error)diff --git a/pipelex/tools/jinja2/__init__.py b/pipelex/tools/jinja2/__init__.pynew file mode 100644index 000000000..8b1378917--- /dev/null+++ b/pipelex/tools/jinja2/__init__.py@@ -0,0 +1 @@+diff --git a/pipelex/tools/jinja2/jinja2_environment.py b/pipelex/tools/jinja2/jinja2_environment.pynew file mode 100644index 000000000..391450d08--- /dev/null+++ b/pipelex/tools/jinja2/jinja2_environment.py@@ -0,0 +1,71 @@+from jinja2 import BaseLoader, Environment, PackageLoader++from pipelex.cogt.templating.template_category import TemplateCategory+++def make_jinja2_env_from_loader(+ template_category: TemplateCategory,+ loader: BaseLoader,+) -> Environment:+ autoescape: bool+ trim_blocks: bool+ lstrip_blocks: bool+ match template_category:+ case TemplateCategory.BASIC:+ autoescape = False+ trim_blocks = False+ lstrip_blocks = False+ case TemplateCategory.EXPRESSION:+ autoescape = False+ trim_blocks = False+ lstrip_blocks = False+ case TemplateCategory.HTML:+ autoescape = False+ trim_blocks = True+ lstrip_blocks = True+ case TemplateCategory.MARKDOWN:+ autoescape = False+ trim_blocks = True+ lstrip_blocks = True+ case TemplateCategory.MERMAID:+ autoescape = False+ trim_blocks = False+ lstrip_blocks = False+ case TemplateCategory.LLM_PROMPT:+ autoescape = False+ trim_blocks = False+ lstrip_blocks = False++ return Environment(+ loader=loader,+ enable_async=True,+ autoescape=autoescape,+ trim_blocks=trim_blocks,+ lstrip_blocks=lstrip_blocks,+ )+++def make_jinja2_env_from_package(+ template_category: TemplateCategory,+ package_name: str,+ package_path: str,+) -> tuple[Environment, BaseLoader]:+ full_package_path = f"{package_path}/jinja2_{template_category}"+ loader = PackageLoader(+ package_name=package_name,+ package_path=full_package_path,+ )+ jinja2_env = make_jinja2_env_from_loader(template_category=template_category, loader=loader)+ return jinja2_env, loader+++def make_jinja2_env_without_loader(+ template_category: TemplateCategory,+) -> Environment:+ loader = BaseLoader()+ jinja2_env = make_jinja2_env_from_loader(template_category=template_category, loader=loader)++ filters = template_category.filters+ for filter_name, filter_function in filters.items():+ jinja2_env.filters[filter_name] = filter_function # pyright: ignore[reportArgumentType]+ return jinja2_envdiff --git a/pipelex/tools/jinja2/jinja2_errors.py b/pipelex/tools/jinja2/jinja2_errors.pynew file mode 100644index 000000000..27f918f7d--- /dev/null+++ b/pipelex/tools/jinja2/jinja2_errors.py@@ -0,0 +1,21 @@+from pipelex.tools.exceptions import ToolException+++class Jinja2TemplateSyntaxError(ToolException):+ pass+++class Jinja2TemplateRenderError(ToolException):+ pass+++class Jinja2StuffError(ToolException):+ pass+++class Jinja2ContextError(ToolException):+ pass+++class Jinja2DetectVariablesError(ToolException):+ passdiff --git a/pipelex/tools/templating/jinja2_filters.py b/pipelex/tools/jinja2/jinja2_filters.pysimilarity index 93%rename from pipelex/tools/templating/jinja2_filters.pyrename to pipelex/tools/jinja2/jinja2_filters.pyindex da0341de1..c8690456c 100644--- a/pipelex/tools/templating/jinja2_filters.py+++ b/pipelex/tools/jinja2/jinja2_filters.py@@ -3,9 +3,9 @@ from typing import Anyfrom jinja2 import pass_contextfrom jinja2.runtime import Context, Undefined-from pipelex.tools.templating.jinja2_errors import Jinja2ContextError-from pipelex.tools.templating.jinja2_models import Jinja2ContextKey, Jinja2TaggableAbstract-from pipelex.tools.templating.templating_models import TagStyle, TextFormat+from pipelex.cogt.templating.templating_style import TagStyle, TextFormat+from pipelex.tools.jinja2.jinja2_errors import Jinja2ContextError+from pipelex.tools.jinja2.jinja2_models import Jinja2ContextKey, Jinja2TaggableAbstractfrom pipelex.types import StrEnum########################################################################################diff --git a/pipelex/tools/templating/jinja2_models.py b/pipelex/tools/jinja2/jinja2_models.pysimilarity index 100%rename from pipelex/tools/templating/jinja2_models.pyrename to pipelex/tools/jinja2/jinja2_models.pydiff --git a/pipelex/tools/jinja2/jinja2_parsing.py b/pipelex/tools/jinja2/jinja2_parsing.pynew file mode 100644index 000000000..5bd638792--- /dev/null+++ b/pipelex/tools/jinja2/jinja2_parsing.py@@ -0,0 +1,17 @@+import jinja2++from pipelex.cogt.templating.template_category import TemplateCategory+from pipelex.tools.jinja2.jinja2_environment import make_jinja2_env_without_loader+from pipelex.tools.jinja2.jinja2_errors import Jinja2TemplateSyntaxError+++def check_jinja2_parsing(+ template_source: str,+ template_category: TemplateCategory = TemplateCategory.LLM_PROMPT,+):+ jinja2_env = make_jinja2_env_without_loader(template_category=template_category)+ try:+ jinja2_env.parse(template_source)+ except jinja2.exceptions.TemplateSyntaxError as exc:+ msg = f"Could not parse Jinja2 template because of: {exc}. Template source:\n{template_source}"+ raise Jinja2TemplateSyntaxError(msg) from excdiff --git a/pipelex/tools/jinja2/jinja2_rendering.py b/pipelex/tools/jinja2/jinja2_rendering.pynew file mode 100644index 000000000..ad6fbb9a2--- /dev/null+++ b/pipelex/tools/jinja2/jinja2_rendering.py@@ -0,0 +1,85 @@+from typing import Any++from jinja2 import Template, meta+from jinja2.exceptions import (+ TemplateAssertionError,+ TemplateSyntaxError,+ UndefinedError,+)++from pipelex import log+from pipelex.cogt.templating.template_category import TemplateCategory+from pipelex.cogt.templating.template_preprocessor import preprocess_template+from pipelex.cogt.templating.templating_style import TemplatingStyle+from pipelex.tools.jinja2.jinja2_environment import make_jinja2_env_without_loader+from pipelex.tools.jinja2.jinja2_errors import (+ Jinja2ContextError,+ Jinja2StuffError,+ Jinja2TemplateRenderError,+)+from pipelex.tools.jinja2.jinja2_models import Jinja2ContextKey+from pipelex.tools.jinja2.jinja2_parsing import check_jinja2_parsing+++def _add_to_templating_context(temlating_context: dict[str, Any], jinja2_context_key: Jinja2ContextKey, value: Any) -> None:+ if jinja2_context_key in temlating_context:+ msg = f"Jinja2 context key '{jinja2_context_key}' already in temlating_context"+ raise Jinja2StuffError(msg)+ temlating_context[jinja2_context_key] = value+++async def render_jinja2(+ template_source: str,+ template_category: TemplateCategory,+ temlating_context: dict[str, Any],+ templating_style: TemplatingStyle | None = None,+) -> str:+ jinja2_env = make_jinja2_env_without_loader(+ template_category=template_category,+ )++ template: Template+ try:+ template = jinja2_env.from_string(template_source)+ except TemplateAssertionError as exc:+ msg = f"Jinja2 render error: '{exc}', template_source:\n{template_source}"+ raise Jinja2TemplateRenderError(msg) from exc+ template_source = preprocess_template(template_source)+ check_jinja2_parsing(+ template_source=template_source,+ template_category=template_category,+ )++ parsed_ast = jinja2_env.parse(template_source)+ if undeclared_variables := meta.find_undeclared_variables(parsed_ast):+ undeclared_variables.discard("preliminary_text")+ if undeclared_variables:+ log.verbose(undeclared_variables, "Jinja2 undeclared_variables")+ temlating_context = temlating_context.copy()+ if templating_style:+ _add_to_templating_context(+ temlating_context=temlating_context,+ jinja2_context_key=Jinja2ContextKey.TAG_STYLE,+ value=templating_style.tag_style,+ )+ _add_to_templating_context(+ temlating_context=temlating_context,+ jinja2_context_key=Jinja2ContextKey.TEXT_FORMAT,+ value=templating_style.text_format,+ )++ try:+ generated_text: str = await template.render_async(**temlating_context)+ except Jinja2StuffError as stuff_error:+ msg = f"Jinja2 render — stuff error: '{stuff_error}', template_source:\n{template_source}"+ raise Jinja2TemplateRenderError(msg) from stuff_error+ except TemplateSyntaxError as syntax_error:+ msg = f"Jinja2 render — syntax error: '{syntax_error}', template_source:\n{template_source}"+ raise Jinja2TemplateRenderError(msg) from syntax_error+ except UndefinedError as undef_error:+ msg = f"Jinja2 render — undefined error: '{undef_error}', template_source:\n{template_source}"+ raise Jinja2TemplateRenderError(msg) from undef_error+ except Jinja2ContextError as context_error:+ msg = f"Jinja2 render — context error: '{context_error}', template_source:\n{template_source}"+ raise Jinja2TemplateRenderError(msg) from context_error+ return generated_textdiff --git a/pipelex/tools/jinja2/jinja2_required_variables.py b/pipelex/tools/jinja2/jinja2_required_variables.pynew file mode 100644index 000000000..faca570d9--- /dev/null+++ b/pipelex/tools/jinja2/jinja2_required_variables.py@@ -0,0 +1,48 @@+from jinja2 import meta+from jinja2.exceptions import (+ TemplateSyntaxError,+ UndefinedError,+)++from pipelex.cogt.templating.template_category import TemplateCategory+from pipelex.tools.jinja2.jinja2_environment import make_jinja2_env_without_loader+from pipelex.tools.jinja2.jinja2_errors import Jinja2DetectVariablesError, Jinja2StuffError+++def detect_jinja2_required_variables(+ template_category: TemplateCategory,+ template_source: str,+) -> set[str]:+ """Returns a list of variables required by the Jinja2 template.++ Args:+ template_category: Category of the template (HTML, MARKDOWN, etc.), used to set the appropriate jinja2 environment settings+ template_source: Jinja2 template string++ Returns:+ List of variable names required by the template++ Raises:+ Jinja2DetectVariablesError: If there is an error parsing the template++ """+ jinja2_env = make_jinja2_env_without_loader(+ template_category=template_category,+ )++ try:+ parsed_ast = jinja2_env.parse(template_source)+ undeclared_variables = meta.find_undeclared_variables(parsed_ast)+ except Jinja2StuffError as stuff_error:+ msg = f"Jinja2 detect variables — stuff error: '{stuff_error}', template_category: {template_category}, template_source:\n{template_source}"+ raise Jinja2DetectVariablesError(msg) from stuff_error+ except TemplateSyntaxError as syntax_error:+ msg = f"Jinja2 detect variables — syntax error: '{syntax_error}', template_category: {template_category}, template_source:\n{template_source}"+ raise Jinja2DetectVariablesError(msg) from syntax_error+ except UndefinedError as undef_error:+ msg = (+ f"Jinja2 detect variables — undefined error: '{undef_error}', template_category: {template_category}, template_source:\n{template_source}"+ )+ raise Jinja2DetectVariablesError(msg) from undef_error++ return undeclared_variablesdiff --git a/pipelex/tools/misc/instructor_utils.py b/pipelex/tools/misc/instructor_utils.pydeleted file mode 100644index b64340270..000000000--- a/pipelex/tools/misc/instructor_utils.py+++ /dev/null@@ -1,15 +0,0 @@-from typing import Any--from pipelex import pretty_print---def dump_kwargs(*_: Any, **kwargs: Any) -> None:- pretty_print(kwargs, title="Instructor about to send to LLM provider")---def dump_response(response: Any) -> None:- pretty_print(response, title="Instructor response from LLM provider")---def dump_error(error: Exception) -> None:- pretty_print(error, title="Instructor error from LLM provider")diff --git a/pipelex/tools/templating/jinja2_blueprint.py b/pipelex/tools/templating/jinja2_blueprint.pydeleted file mode 100644index 8210ba811..000000000--- a/pipelex/tools/templating/jinja2_blueprint.py+++ /dev/null@@ -1,17 +0,0 @@-from typing import Any--from pydantic import BaseModel, Field--from pipelex.tools.templating.jinja2_template_category import Jinja2TemplateCategory-from pipelex.tools.templating.templating_models import PromptingStyle---class Jinja2Blueprint(BaseModel):- jinja2_name: str | None = Field(default=None, description="Name of the Jinja2 template to use")- jinja2: str | None = Field(default=None, description="Raw Jinja2 template string")- prompting_style: PromptingStyle | None = Field(default=None, description="Style of prompting to use (typically for different LLMs)")- template_category: Jinja2TemplateCategory = Field(- default=Jinja2TemplateCategory.LLM_PROMPT,- description="Category of the template (could also be HTML, MARKDOWN, MERMAID, etc.), influences Jinja2 rendering environment config",- )- extra_context: dict[str, Any] | None = Field(default=None, description="Additional context variables for template rendering")diff --git a/pipelex/tools/templating/jinja2_environment.py b/pipelex/tools/templating/jinja2_environment.pydeleted file mode 100644index ba8dd6262..000000000--- a/pipelex/tools/templating/jinja2_environment.py+++ /dev/null@@ -1,79 +0,0 @@-from jinja2 import BaseLoader, Environment, PackageLoader--from pipelex.tools.templating.jinja2_template_category import Jinja2TemplateCategory-from pipelex.tools.templating.jinja2_template_loader import Jinja2TemplateLoader-from pipelex.tools.templating.template_provider_abstract import TemplateProviderAbstract---def make_jinja2_env_from_loader(- template_category: Jinja2TemplateCategory,- loader: BaseLoader,-) -> Environment:- match template_category:- case Jinja2TemplateCategory.HTML:- jinja2_env = Environment(- loader=loader,- enable_async=True,- autoescape=False,- trim_blocks=True,- lstrip_blocks=True,- )- case Jinja2TemplateCategory.MARKDOWN:- jinja2_env = Environment(- loader=loader,- enable_async=True,- autoescape=False,- trim_blocks=True,- lstrip_blocks=True,- )- case Jinja2TemplateCategory.MERMAID:- jinja2_env = Environment(- loader=loader,- enable_async=True,- autoescape=False,- trim_blocks=False,- lstrip_blocks=False,- )- case Jinja2TemplateCategory.LLM_PROMPT:- jinja2_env = Environment(- loader=loader,- enable_async=True,- autoescape=False,- trim_blocks=False,- lstrip_blocks=False,- )- return jinja2_env---def make_jinja2_env_from_package(- template_category: Jinja2TemplateCategory,- package_name: str,- package_path: str,-) -> tuple[Environment, BaseLoader]:- full_package_path = f"{package_path}/jinja2_{template_category}"- loader = PackageLoader(- package_name=package_name,- package_path=full_package_path,- )- jinja2_env = make_jinja2_env_from_loader(template_category=template_category, loader=loader)- return jinja2_env, loader---def make_jinja2_env_without_loader(- template_category: Jinja2TemplateCategory,-) -> Environment:- loader = BaseLoader()- return make_jinja2_env_from_loader(template_category=template_category, loader=loader)---def make_jinja2_env_from_template_provider(- template_category: Jinja2TemplateCategory,- template_provider: TemplateProviderAbstract,-) -> tuple[Environment, BaseLoader]:- loader = Jinja2TemplateLoader(template_provider=template_provider)- jinja2_env = make_jinja2_env_from_loader(template_category=template_category, loader=loader)-- filters = template_category.filters- for filter_name, filter_function in filters.items():- jinja2_env.filters[filter_name] = filter_function # pyright: ignore[reportArgumentType]- return jinja2_env, loaderdiff --git a/pipelex/tools/templating/jinja2_errors.py b/pipelex/tools/templating/jinja2_errors.pydeleted file mode 100644index 97ac0c3cb..000000000--- a/pipelex/tools/templating/jinja2_errors.py+++ /dev/null@@ -1,32 +0,0 @@-from pipelex.tools.exceptions import ToolException---class Jinja2TemplateError(ToolException):- pass---class Jinja2StuffError(ToolException):- pass---class Jinja2ContextError(ToolException):- pass---class Jinja2RenderError(ToolException):- pass---class Jinja2DetectVariablesError(ToolException):- pass---def make_jinja2_error_explanation(jinja2_name: str | None, template_text: str | None) -> str:- explanation = ""- if jinja2_name:- explanation += f"\nJinja2 name: '{jinja2_name}'\n"- if template_text:- explanation += f"\ntemplate:\n\n{template_text}'\n"- if not explanation:- explanation = "No template text or Jinja2 name"- return explanationdiff --git a/pipelex/tools/templating/jinja2_parsing.py b/pipelex/tools/templating/jinja2_parsing.pydeleted file mode 100644index eb6a773b7..000000000--- a/pipelex/tools/templating/jinja2_parsing.py+++ /dev/null@@ -1,20 +0,0 @@-from jinja2.exceptions import TemplateSyntaxError--from pipelex.tools.templating.jinja2_environment import make_jinja2_env_without_loader-from pipelex.tools.templating.jinja2_template_category import Jinja2TemplateCategory---class Jinja2ParsingError(Exception):- pass---def check_jinja2_parsing(- jinja2_template_source: str,- template_category: Jinja2TemplateCategory = Jinja2TemplateCategory.LLM_PROMPT,-):- jinja2_env = make_jinja2_env_without_loader(template_category=template_category)- try:- jinja2_env.parse(jinja2_template_source)- except TemplateSyntaxError as exc:- msg = f"Could not parse Jinja2 template because of: {exc}. The template is:\n{jinja2_template_source}"- raise Jinja2ParsingError(msg) from excdiff --git a/pipelex/tools/templating/jinja2_rendering.py b/pipelex/tools/templating/jinja2_rendering.pydeleted file mode 100644index be00123aa..000000000--- a/pipelex/tools/templating/jinja2_rendering.py+++ /dev/null@@ -1,116 +0,0 @@-from typing import Any--from jinja2 import Template, meta-from jinja2.exceptions import (- TemplateAssertionError,- TemplateSyntaxError,- UndefinedError,-)--from pipelex import log-from pipelex.tools.templating.jinja2_environment import make_jinja2_env_from_template_provider-from pipelex.tools.templating.jinja2_errors import (- Jinja2ContextError,- Jinja2RenderError,- Jinja2StuffError,- make_jinja2_error_explanation,-)-from pipelex.tools.templating.jinja2_models import Jinja2ContextKey-from pipelex.tools.templating.jinja2_parsing import check_jinja2_parsing-from pipelex.tools.templating.jinja2_template_category import Jinja2TemplateCategory-from pipelex.tools.templating.template_preprocessor import preprocess_template-from pipelex.tools.templating.template_provider_abstract import TemplateProviderAbstract-from pipelex.tools.templating.templating_models import PromptingStyle--########################################################################################-# Jinja2 rendering-########################################################################################---def _add_to_templating_context(temlating_context: dict[str, Any], jinja2_context_key: Jinja2ContextKey, value: Any) -> None:- if jinja2_context_key in temlating_context:- msg = f"Jinja2 context key '{jinja2_context_key}' already in temlating_context"- raise Jinja2StuffError(msg)- temlating_context[jinja2_context_key] = value---async def render_jinja2(- template_category: Jinja2TemplateCategory,- template_provider: TemplateProviderAbstract,- temlating_context: dict[str, Any],- jinja2_name: str | None = None,- jinja2: str | None = None,- prompting_style: PromptingStyle | None = None,-) -> str:- jinja2_env, loader = make_jinja2_env_from_template_provider(- template_category=template_category,- template_provider=template_provider,- )-- template: Template- template_source: str- try:- if jinja2:- template = jinja2_env.from_string(jinja2)- template_source = jinja2- elif jinja2_name:- template = jinja2_env.get_template(jinja2_name)- template_source = loader.get_source(jinja2_env, jinja2_name)[0]- else:- msg = "No jinja2 or jinja2_name in Jinja2Assignment"- raise Jinja2StuffError(msg)- except TemplateAssertionError as exc:- explanation = make_jinja2_error_explanation(jinja2_name=jinja2_name, template_text=jinja2)- msg = f"Jinja2 render error: '{exc}' {explanation}"- raise Jinja2RenderError(msg) from exc- template_source = preprocess_template(template_source)- check_jinja2_parsing(- jinja2_template_source=template_source,- template_category=template_category,- )-- parsed_ast = jinja2_env.parse(template_source)- if undeclared_variables := meta.find_undeclared_variables(parsed_ast):- undeclared_variables.discard("preliminary_text")- if undeclared_variables:- log.verbose(undeclared_variables, "Jinja2 undeclared_variables")- temlating_context = temlating_context.copy()- if prompting_style:- _add_to_templating_context(- temlating_context=temlating_context,- jinja2_context_key=Jinja2ContextKey.TAG_STYLE,- value=prompting_style.tag_style,- )- _add_to_templating_context(- temlating_context=temlating_context,- jinja2_context_key=Jinja2ContextKey.TEXT_FORMAT,- value=prompting_style.text_format,- )-- # TODO: restore preferences using templating manager- # if get_config().pipelex.preferences.is_include_prefs_in_jinja2_context:- # _add_to_templating_context(- # temlating_context=temlating_context,- # jinja2_context_key=Jinja2ContextKey.PREFERENCES,- # value=get_config().pipelex.preferences,- # )-- try:- generated_text: str = await template.render_async(**temlating_context)- except Jinja2StuffError as stuff_error:- explanation = make_jinja2_error_explanation(jinja2_name=jinja2_name, template_text=template_source)- msg = f"Jinja2 render — stuff error: '{stuff_error}' {explanation}"- raise Jinja2RenderError(msg) from stuff_error- except TemplateSyntaxError as syntax_error:- explanation = make_jinja2_error_explanation(jinja2_name=jinja2_name, template_text=template_source)- msg = f"Jinja2 render — syntax error: '{syntax_error}' {explanation}"- raise Jinja2RenderError(msg) from syntax_error- except UndefinedError as undef_error:- explanation = make_jinja2_error_explanation(jinja2_name=jinja2_name, template_text=template_source)- msg = f"Jinja2 render — undefined error: '{undef_error}' {explanation}"- raise Jinja2RenderError(msg) from undef_error- except Jinja2ContextError as context_error:- explanation = make_jinja2_error_explanation(jinja2_name=jinja2_name, template_text=template_source)- msg = f"Jinja2 render — context error: '{context_error}' {explanation}"- raise Jinja2RenderError(msg) from context_error- return generated_textdiff --git a/pipelex/tools/templating/jinja2_required_variables.py b/pipelex/tools/templating/jinja2_required_variables.pydeleted file mode 100644index 6de93c39c..000000000--- a/pipelex/tools/templating/jinja2_required_variables.py+++ /dev/null@@ -1,63 +0,0 @@-from jinja2 import meta-from jinja2.exceptions import (- TemplateSyntaxError,- UndefinedError,-)--from pipelex.tools.templating.jinja2_environment import make_jinja2_env_from_template_provider-from pipelex.tools.templating.jinja2_errors import Jinja2DetectVariablesError, Jinja2StuffError, make_jinja2_error_explanation-from pipelex.tools.templating.jinja2_template_category import Jinja2TemplateCategory-from pipelex.tools.templating.template_provider_abstract import TemplateProviderAbstract---def detect_jinja2_required_variables(- template_category: Jinja2TemplateCategory,- template_provider: TemplateProviderAbstract,- jinja2_name: str | None = None,- jinja2: str | None = None,-) -> set[str]:- """Returns a list of variables required by the Jinja2 template.-- Args:- template_category: Category of the template (HTML, MARKDOWN, etc.), used to set the appropriate jinja2 environment settings- template_provider: Library containing templates- jinja2_name: Name of template in library (optional)- jinja2: Direct Jinja2 template string (optional)-- Returns:- List of variable names required by the template-- Raises:- Jinja2StuffError: If neither jinja2 nor jinja2_name is provided-- """- jinja2_env, loader = make_jinja2_env_from_template_provider(- template_category=template_category,- template_provider=template_provider,- )-- template_source: str- if jinja2:- template_source = jinja2- elif jinja2_name:- template_source = loader.get_source(jinja2_env, jinja2_name)[0]- else:- msg = "No jinja2 or jinja2_name provided"- raise Jinja2StuffError(msg)- try:- parsed_ast = jinja2_env.parse(template_source)- undeclared_variables = meta.find_undeclared_variables(parsed_ast)- except Jinja2StuffError as stuff_error:- explanation = make_jinja2_error_explanation(jinja2_name=jinja2_name, template_text=template_source)- msg = f"Jinja2 detect variables — stuff error: '{stuff_error}' {explanation}"- raise Jinja2DetectVariablesError(msg) from stuff_error- except TemplateSyntaxError as syntax_error:- explanation = make_jinja2_error_explanation(jinja2_name=jinja2_name, template_text=template_source)- msg = f"Jinja2 detect variables — syntax error: '{syntax_error}' {explanation}"- raise Jinja2DetectVariablesError(msg) from syntax_error- except UndefinedError as undef_error:- explanation = make_jinja2_error_explanation(jinja2_name=jinja2_name, template_text=template_source)- msg = f"Jinja2 detect variables — undefined error: '{undef_error}' {explanation}"- raise Jinja2DetectVariablesError(msg) from undef_error-- return undeclared_variablesdiff --git a/pipelex/tools/templating/jinja2_template_category.py b/pipelex/tools/templating/jinja2_template_category.pydeleted file mode 100644index 3c8c09780..000000000--- a/pipelex/tools/templating/jinja2_template_category.py+++ /dev/null@@ -1,33 +0,0 @@-from collections.abc import Callable-from typing import Any--from jinja2.runtime import Context--from pipelex.tools.templating.jinja2_filters import tag, text_format-from pipelex.tools.templating.jinja2_models import Jinja2FilterName-from pipelex.tools.templating.templating_models import TextFormat-from pipelex.types import StrEnum---# TODO: add more categories-class Jinja2TemplateCategory(StrEnum):- HTML = "html"- MARKDOWN = "markdown"- MERMAID = "mermaid"- LLM_PROMPT = "llm_prompt"-- @property- def filters(self) -> dict[Jinja2FilterName, Callable[[Context, Any, TextFormat | None], Any]]:- match self:- case Jinja2TemplateCategory.MERMAID:- return {}- case Jinja2TemplateCategory.HTML | Jinja2TemplateCategory.MARKDOWN:- return {- Jinja2FilterName.FORMAT: text_format,- Jinja2FilterName.TAG: tag,- }- case Jinja2TemplateCategory.LLM_PROMPT:- return {- Jinja2FilterName.FORMAT: text_format,- Jinja2FilterName.TAG: tag,- }diff --git a/pipelex/tools/templating/jinja2_template_loader.py b/pipelex/tools/templating/jinja2_template_loader.pydeleted file mode 100644index 417d36254..000000000--- a/pipelex/tools/templating/jinja2_template_loader.py+++ /dev/null@@ -1,19 +0,0 @@-from collections.abc import Callable--from jinja2 import BaseLoader, Environment-from typing_extensions import override--from pipelex import log-from pipelex.tools.templating.template_provider_abstract import TemplateProviderAbstract---class Jinja2TemplateLoader(BaseLoader):- def __init__(self, template_provider: TemplateProviderAbstract) -> None:- super().__init__()- self.template_provider = template_provider-- @override- def get_source(self, environment: Environment, template: str) -> tuple[str, str | None, Callable[[], bool] | None]:- the_template = self.template_provider.get_template(template_name=template)- log.debug(f"TemplateLoader.get_source: template='{template}'")- return the_template, None, Nonediff --git a/pipelex/tools/templating/template_library.py b/pipelex/tools/templating/template_library.pydeleted file mode 100644index 990d38063..000000000--- a/pipelex/tools/templating/template_library.py+++ /dev/null@@ -1,107 +0,0 @@-from pathlib import Path-from typing import Any, ClassVar, cast--from jinja2 import TemplateSyntaxError-from pydantic import Field, RootModel, ValidationError-from typing_extensions import override--from pipelex import log-from pipelex.libraries.library_config import LibraryConfig-from pipelex.tools.exceptions import ToolException-from pipelex.tools.misc.file_utils import find_files_in_dir-from pipelex.tools.misc.toml_utils import load_toml_from_path-from pipelex.tools.templating.jinja2_parsing import check_jinja2_parsing-from pipelex.tools.templating.jinja2_template_category import Jinja2TemplateCategory-from pipelex.tools.templating.template_preprocessor import preprocess_template-from pipelex.tools.templating.template_provider_abstract import TemplateNotFoundError, TemplateProviderAbstract-from pipelex.tools.typing.pydantic_utils import format_pydantic_validation_error--TemplateLibraryRoot = dict[str, str]---class TemplateLibraryError(ToolException):- pass---class TemplateLibrary(TemplateProviderAbstract, RootModel[TemplateLibraryRoot]):- root: TemplateLibraryRoot = Field(default_factory=dict)- library_config: ClassVar[LibraryConfig]-- @classmethod- def make_empty(cls, config_dir_path: str) -> "TemplateLibrary":- cls.library_config = LibraryConfig(config_dir_path=config_dir_path)- return cls()-- @override- def setup(self) -> None:- templates_dir = Path(__file__).parent / "templates"- template_toml_paths = find_files_in_dir(dir_path=str(templates_dir), pattern="*.toml", is_recursive=True)- for template_toml_path in template_toml_paths:- self._load_from_toml(toml_path=str(template_toml_path))- self.validate_templates(template_category=Jinja2TemplateCategory.LLM_PROMPT)-- @override- def teardown(self) -> None:- self.root = {}-- @override- def get_template(self, template_name: str) -> str:- try:- return self.root[template_name]- except KeyError as exc:- msg = f"Template '{template_name}' not found in template library"- raise TemplateNotFoundError(msg) from exc-- def _set_template(self, template: str, name: str):- preprocessed_template = preprocess_template(template)- self.root[name] = preprocessed_template-- def _add_new_template(self, template: str, name: str):- if name in self.root:- msg = f"Template '{name}' already exists in the library"- raise TemplateLibraryError(msg)- self._set_template(template=template, name=name)-- def _load_from_toml(self, toml_path: str):- nb_concepts_before = len(self.root)- library_dict = load_toml_from_path(path=toml_path)- for start_domain, templates in library_dict.items():- self._load_from_recursive_dict(domain=start_domain, recursive_dict=templates)- toml_name = toml_path.split("/")[-1]- log.debug(f"Loaded {len(self.root) - nb_concepts_before} templates from '{toml_name}'")-- def _load_from_recursive_dict(self, domain: str, recursive_dict: dict[str, Any]):- for name, obj in recursive_dict.items():- try:- if isinstance(obj, str):- # it's a template- template = obj- self._add_new_template(template=template, name=name)- elif isinstance(obj, dict):- # this is not a templae but a subdomain- sub_recursive_dict = cast("dict[str, str]", obj)- domain = f"{domain}/{name}"- self._load_from_recursive_dict(domain=domain, recursive_dict=sub_recursive_dict)- else:- msg = f"Unexpected type for key '{name}' in recursive_dict: {type(obj)}"- raise TemplateLibraryError(msg)- except ValidationError as exc:- error_msg = format_pydantic_validation_error(exc)- msg = f"Error loading concept '{name}' of domain '{domain}' because of: {error_msg}"- raise TemplateLibraryError(msg) from exc-- def validate_templates(self, template_category: Jinja2TemplateCategory):- for template_name, template in self.root.items():- try:- check_jinja2_parsing(- jinja2_template_source=template,- template_category=template_category,- )- except TemplateSyntaxError as exc:- error_msg = f"Jinja2 syntax error in template '{template_name}': {exc}."- if template:- error_msg += f"\nThe template is:\n{template}"- else:- error_msg += "The template is empty."- msg = error_msg- raise TemplateLibraryError(msg) from excdiff --git a/pipelex/tools/templating/template_provider_abstract.py b/pipelex/tools/templating/template_provider_abstract.pydeleted file mode 100644index ea9a23d9d..000000000--- a/pipelex/tools/templating/template_provider_abstract.py+++ /dev/null@@ -1,21 +0,0 @@-from abc import ABC, abstractmethod--from pipelex.tools.exceptions import ToolException---class TemplateNotFoundError(ToolException):- pass---class TemplateProviderAbstract(ABC):- @abstractmethod- def setup(self) -> None:- pass-- @abstractmethod- def teardown(self) -> None:- pass-- @abstractmethod- def get_template(self, template_name: str) -> str:- passdiff --git a/pipelex/tools/templating/templates/__init__.py b/pipelex/tools/templating/templates/__init__.pydeleted file mode 100644index e69de29bb..000000000diff --git a/pipelex/tools/templating/templates/base_templates.toml b/pipelex/tools/templating/templates/base_templates.tomldeleted file mode 100644index fb1a8d026..000000000--- a/pipelex/tools/templating/templates/base_templates.toml+++ /dev/null@@ -1,34 +0,0 @@-[generic_prompts]-structure_from_preliminary_text_system = "You are a data modeling expert specialized in extracting structure from text."-# TODO: Should be a proper jinja2-structure_from_preliminary_text_user = """-Your job is to extract and structure information from a text.-Here is the text:-{preliminary_text}--Now generate the JSON in the required format.-Do not create information that is not in the text.-"""--output_structure_prompt = """-----The instance we want to generate will be for the following class:-{{ class_structure_str }}--Don't bother with JSON formatting, we'll do that as a second step.-For now, just output markdown with the details of the instance.-DO NOT create information.-If some information is not present for an attribute, output the default value or None according to the attribute definition.-"""--output_structure_prompt_no_preliminary_text = """-----The instance we want to generate will be for the following class:-{{ class_structure_str }}--DO NOT create information.-If some information is not present for an attribute, output the default value or None according to the attribute definition.-"""--[test_prompts]-jinja2_test_template = "I want a {{ place_holder }} t-shirt."diff --git a/tests/cases/templates.py b/tests/cases/templates.pyindex 9f1d3e719..fd70714b6 100644--- a/tests/cases/templates.py+++ b/tests/cases/templates.py@@ -2,7 +2,7 @@from typing import Any, ClassVar-from pipelex.tools.templating.templating_models import PromptingStyle, TagStyle, TextFormat+from pipelex.cogt.templating.templating_style import TagStyle, TemplatingStyle, TextFormatfrom .registry import Fruit@@ -90,20 +90,20 @@ Format filter html:]# Style configurations- STYLE: ClassVar[list[PromptingStyle]] = [- PromptingStyle(+ STYLE: ClassVar[list[TemplatingStyle]] = [+ TemplatingStyle(tag_style=TagStyle.NO_TAG,text_format=TextFormat.PLAIN,),- PromptingStyle(+ TemplatingStyle(tag_style=TagStyle.TICKS,text_format=TextFormat.MARKDOWN,),- PromptingStyle(+ TemplatingStyle(tag_style=TagStyle.XML,text_format=TextFormat.HTML,),- PromptingStyle(+ TemplatingStyle(tag_style=TagStyle.SQUARE_BRACKETS,text_format=TextFormat.JSON,),diff --git a/tests/integration/pipelex/cogt/test_content_generator.py b/tests/integration/pipelex/cogt/test_content_generator.pyindex 91c43952d..e0f866614 100644--- a/tests/integration/pipelex/cogt/test_content_generator.py+++ b/tests/integration/pipelex/cogt/test_content_generator.py@@ -115,14 +115,14 @@ class TestContentGenerator:assert isinstance(image, GeneratedImage)@pytest.mark.usefixtures("request")- async def test_make_jinja2_text(self):+ async def test_make_templated_text(self):context = {"the_answer": "elementary, my dear Watson",}- jinja2_text: str = await get_content_generator().make_jinja2_text(+ jinja2_text: str = await get_content_generator().make_templated_text(context=context,- jinja2="The answer is: {{ the_answer }}",+ template="The answer is: {{ the_answer }}",)assert isinstance(jinja2_text, str)assert jinja2_text == "The answer is: elementary, my dear Watson"diff --git a/tests/integration/pipelex/pipes/controller/pipe_parallel/test_pipe_parallel_validation.py b/tests/integration/pipelex/pipes/controller/pipe_parallel/test_pipe_parallel_validation.pyindex 3940b1899..e7c07751d 100644--- a/tests/integration/pipelex/pipes/controller/pipe_parallel/test_pipe_parallel_validation.py+++ b/tests/integration/pipelex/pipes/controller/pipe_parallel/test_pipe_parallel_validation.py@@ -46,7 +46,7 @@ class TestPipeParallelValidation:},description="Analysis pipe for document processing",output=ConceptFactory.make_concept_string_with_domain(domain=domain, concept_code=concept_3.code),- prompt_template="Analyze this document: \n@context\n@document",+ prompt="Analyze this document: \n@context\n@document",)real_pipe = PipeLLMFactory.make_from_blueprint(diff --git a/tests/integration/pipelex/pipes/operator/pipe_llm/test_pipe_llm.py b/tests/integration/pipelex/pipes/operator/pipe_llm/test_pipe_llm.pyindex 0e92c94cf..ee6d5019f 100644--- a/tests/integration/pipelex/pipes/operator/pipe_llm/test_pipe_llm.py+++ b/tests/integration/pipelex/pipes/operator/pipe_llm/test_pipe_llm.py@@ -11,7 +11,7 @@ from pipelex.pipe_operators.llm.pipe_llm_factory import PipeLLMFactoryfrom pipelex.pipe_run.pipe_job_factory import PipeJobFactoryfrom pipelex.pipe_run.pipe_run_params import PipeRunModefrom pipelex.pipe_run.pipe_run_params_factory import PipeRunParamsFactory-from tests.integration.pipelex.test_data import BasicStructuredDataTestCases, ComplexStructuredDataTestCases, PipeTestCases+from tests.integration.pipelex.test_data import BasicStructuredDataTestCases, PipeTestCases@pytest.mark.dry_runnable@@ -51,21 +51,22 @@ class TestPipeLLM:@pytest.mark.parametrize(("topic", "data", "concept"),- BasicStructuredDataTestCases.STRUCTURE_TEST_CASES + ComplexStructuredDataTestCases.STRUCTURE_TEST_CASES,+ BasicStructuredDataTestCases.STRUCTURE_TEST_CASES,+ # + ComplexStructuredDataTestCases.STRUCTURE_TEST_CASES,)@pytest.mark.parametrize("structuring_method",[- StructuringMethod.DIRECT,+ # StructuringMethod.DIRECT,StructuringMethod.PRELIMINARY_TEXT,],)@pytest.mark.parametrize(("llm", "llm_to_structure"),[- # ("gpt-4o-mini", "gpt-4o-mini"),+ ("gpt-4o-mini", "gpt-4o-mini"),# ("gemini-2.5-flash-lite", "gemini-2.5-flash-lite"),- ("gemini-2.5-flash-lite", "gpt-4o-mini"),+ # ("gemini-2.5-flash-lite", "gpt-4o-mini"),],)async def test_pipe_llm_structured(@@ -87,7 +88,7 @@ class TestPipeLLM:description=f"Extract {concept} from text",inputs={"data": "Text"},output=f"test_structured_generations.{concept}",- prompt_template=BasicStructuredDataTestCases.EXTRACTION_PROMPT,+ prompt=BasicStructuredDataTestCases.EXTRACTION_PROMPT,model=llm,model_to_structure=llm_to_structure,structuring_method=structuring_method,diff --git a/tests/integration/pipelex/pipes/test_image_inputs.py b/tests/integration/pipelex/pipes/test_image_inputs.pyindex 226d0cc16..569e3eeb9 100644--- a/tests/integration/pipelex/pipes/test_image_inputs.py+++ b/tests/integration/pipelex/pipes/test_image_inputs.py@@ -108,7 +108,7 @@ class TestImageInputs:description="Test that a pipe can accept a PageContent input, give to the LLM the image via subattributes",inputs={"page": "Page"},output="Text",- prompt_template="Describe the page: @page",+ prompt="Describe the page: @page",)pipe_llm = PipeLLMFactory.make_from_blueprint(diff --git a/tests/integration/pipelex/pipes/test_pipe_compose.py b/tests/integration/pipelex/pipes/test_pipe_compose.pyindex 4e870e147..91b90621b 100644--- a/tests/integration/pipelex/pipes/test_pipe_compose.py+++ b/tests/integration/pipelex/pipes/test_pipe_compose.py@@ -3,6 +3,9 @@ from typing import TYPE_CHECKING, castimport pytestfrom pipelex import pretty_print+from pipelex.cogt.templating.template_blueprint import TemplateBlueprint+from pipelex.cogt.templating.template_category import TemplateCategory+from pipelex.cogt.templating.templating_style import TagStyle, TemplatingStyle, TextFormatfrom pipelex.core.concepts.concept_native import NativeConceptCodefrom pipelex.core.memory.working_memory_factory import WorkingMemoryFactoryfrom pipelex.hub import get_pipe_router@@ -11,7 +14,6 @@ from pipelex.pipe_operators.compose.pipe_compose_factory import PipeComposeFactofrom pipelex.pipe_run.pipe_job_factory import PipeJobFactoryfrom pipelex.pipe_run.pipe_run_params import PipeRunModefrom pipelex.pipe_run.pipe_run_params_factory import PipeRunParamsFactory-from pipelex.tools.templating.templating_models import PromptingStyle, TagStyle, TextFormatfrom tests.cases import JINJA2TestCasesif TYPE_CHECKING:@@ -21,17 +23,21 @@ if TYPE_CHECKING:@pytest.mark.dry_runnable@pytest.mark.asyncio(loop_scope="class")class TestPipeCompose:- @pytest.mark.parametrize("jinja2", JINJA2TestCases.JINJA2_FOR_ANY)+ @pytest.mark.parametrize("template_source", JINJA2TestCases.JINJA2_FOR_ANY)async def test_pipe_compose_for_any(self,pipe_run_mode: PipeRunMode,- jinja2: str,+ template_source: str,):pipe_compose_blueprint = PipeComposeBlueprint(description="Jinja2 test for any context",- jinja2=jinja2,+ template=TemplateBlueprint(+ source=template_source,+ templating_style=TemplatingStyle(tag_style=TagStyle.TICKS, text_format=TextFormat.MARKDOWN),+ category=TemplateCategory.MARKDOWN,+ extra_context={"place_holder": "[some text from test_pipe_compose_for_any]"},+ ),output=NativeConceptCode.TEXT,- extra_context={"place_holder": "[some text from test_pipe_compose_for_any]"},)pipe_job = PipeJobFactory.make_pipe_job(@@ -46,18 +52,22 @@ class TestPipeCompose:rendered_text = pipe_compose_output.main_stuff_as_strpretty_print(rendered_text)- @pytest.mark.parametrize("jinja2", JINJA2TestCases.JINJA2_FOR_STUFF)+ @pytest.mark.parametrize("template_source", JINJA2TestCases.JINJA2_FOR_STUFF)async def test_pipe_compose_for_stuff(self,pipe_run_mode: PipeRunMode,- jinja2: str,+ template_source: str,):working_memory = WorkingMemoryFactory.make_from_text(text="[some text from test_pipe_compose_for_stuff]", name="place_holder")pipe_compose_blueprint = PipeComposeBlueprint(description="Jinja2 test for stuff context",- jinja2=jinja2,- prompting_style=PromptingStyle(tag_style=TagStyle.TICKS, text_format=TextFormat.MARKDOWN),+ template=TemplateBlueprint(+ source=template_source,+ templating_style=TemplatingStyle(tag_style=TagStyle.TICKS, text_format=TextFormat.MARKDOWN),+ category=TemplateCategory.MARKDOWN,+ extra_context={"place_holder": "[some text from test_pipe_compose_for_stuff]"},+ ),output=NativeConceptCode.TEXT,)diff --git a/tests/integration/pipelex/test_data.py b/tests/integration/pipelex/test_data.pyindex a115b250d..4af15eb79 100644--- a/tests/integration/pipelex/test_data.py+++ b/tests/integration/pipelex/test_data.py@@ -303,17 +303,17 @@ Extract information from the following text:# Combined test cases for parametrized testsSTRUCTURE_TEST_CASES: ClassVar[list[tuple[str, str, str]]] = [ # topic, data, concept- ("Simple structure basic", SIMPLE_STRUCTURE_1, "ConceptWithSimpleStructure"),+ # ("Simple structure basic", SIMPLE_STRUCTURE_1, "ConceptWithSimpleStructure"),("Simple structure narrative", SIMPLE_STRUCTURE_2, "ConceptWithSimpleStructure"),- ("Optionals all present", OPTIONAL_ALL_PRESENT, "ConceptWithOptionals"),- ("Optionals some missing", OPTIONAL_SOME_MISSING, "ConceptWithOptionals"),- ("Optionals all missing", OPTIONAL_ALL_MISSING, "ConceptWithOptionals"),- ("Lists with data", LISTS_WITH_DATA, "ConceptWithLists"),- ("Lists empty", LISTS_EMPTY, "ConceptWithLists"),- ("Lists mixed", LISTS_MIXED, "ConceptWithLists"),- ("Nested full", NESTED_FULL, "ConceptWithNestedStructures"),- ("Nested partial", NESTED_PARTIAL, "ConceptWithNestedStructures"),- ("Nested complex", NESTED_COMPLEX, "ConceptWithNestedStructures"),+ # ("Optionals all present", OPTIONAL_ALL_PRESENT, "ConceptWithOptionals"),+ # ("Optionals some missing", OPTIONAL_SOME_MISSING, "ConceptWithOptionals"),+ # ("Optionals all missing", OPTIONAL_ALL_MISSING, "ConceptWithOptionals"),+ # ("Lists with data", LISTS_WITH_DATA, "ConceptWithLists"),+ # ("Lists empty", LISTS_EMPTY, "ConceptWithLists"),+ # ("Lists mixed", LISTS_MIXED, "ConceptWithLists"),+ # ("Nested full", NESTED_FULL, "ConceptWithNestedStructures"),+ # ("Nested partial", NESTED_PARTIAL, "ConceptWithNestedStructures"),+ # ("Nested complex", NESTED_COMPLEX, "ConceptWithNestedStructures"),]diff --git a/tests/test_pipelines/discord_newsletter.plx b/tests/test_pipelines/discord_newsletter.plxindex 2a1fa937b..8081a23cf 100644--- a/tests/test_pipelines/discord_newsletter.plx+++ b/tests/test_pipelines/discord_newsletter.plx@@ -33,7 +33,7 @@ description = "Summarize the new member announcements"inputs = { discord_channel_update = "DiscordChannelUpdate" }output = "ChannelSummary"system_prompt = "You are a newsletter editor who creates engaging summaries of Discord channel content. You extract key information, preserve important links, and write in a clear, concise style suitable for newsletter readers."-prompt_template = """+prompt = """Analyze this Discord channel update and create a newsletter-friendly summary.Channel Information:@@ -48,7 +48,7 @@ description = "Summarize a Discord channel's messages into newsletter-friendly cinputs = { discord_channel_update = "DiscordChannelUpdate" }output = "ChannelSummary"system_prompt = "You are a newsletter editor who creates engaging summaries of Discord channel content. You extract key information, preserve important links, and write in a clear, concise style suitable for newsletter readers."-prompt_template = """+prompt = """Analyze this Discord channel update and create a newsletter-friendly summary.Channel Information:@@ -66,7 +66,7 @@ type = "PipeLLM"description = "Combine channel summaries into a short summary of the week's Share channel content (200 characters)"inputs = { channel_summaries = "ChannelSummary" }output = "Text"-prompt_template = """+prompt = """Write a single overall summary of the week's content based on the following Share channel summaries:{% for channel in channel_summaries.content.items if channel.category == "Share" %}@@ -81,8 +81,10 @@ type = "PipeCompose"description = "Combine weekly and channel summaries into a complete newsletter following specific formatting requirements"inputs = { weekly_summary = "Text", channel_summaries = "ChannelSummary" }output = "HtmlNewsletter"-template_category = "html"-jinja2 = """++[pipe.format_html_newsletter.template]+category = "html"+source = """<!-- Weekly Summary --><h2>☀️ Weekly Summary</h2><p>diff --git a/tests/test_pipelines/failing_pipelines.plx b/tests/test_pipelines/failing_pipelines.plxindex 99981e222..b66b7e3ac 100644--- a/tests/test_pipelines/failing_pipelines.plx+++ b/tests/test_pipelines/failing_pipelines.plx@@ -8,7 +8,7 @@ type = "PipeLLM"description = "This pipe is a dummy pipe"output = "Text"model = { model = "gpt-4o-mini", temperature = 1, max_tokens = 50 }-prompt_template = """+prompt = """Output only "Hello"."""diff --git a/tests/test_pipelines/misc_tests/flows.plx b/tests/test_pipelines/misc_tests/flows.plxindex 3d2357030..7571feb1c 100644--- a/tests/test_pipelines/misc_tests/flows.plx+++ b/tests/test_pipelines/misc_tests/flows.plx@@ -11,7 +11,7 @@ Color = "A color"type = "PipeLLM"description = "Choose Colors"output = "Color"-prompt_template = """+prompt = """Choose {{ _nb_output }} colors."""model = "llm_for_creative_writing"@@ -31,7 +31,7 @@ type = "PipeLLM"description = "Capitalize Colors"inputs = { color = "Color" }output = "Color"-prompt_template = """+prompt = """Put the first letter of a word that represents a color as a capital letter.Here is the word:@@ -45,7 +45,7 @@ type = "PipeLLM"description = "Capitalize Last Letter"inputs = { capitalized_color = "Color" }output = "Color"-prompt_template = """+prompt = """Put the last letter of a word that represents a color as a capital letter.Here is the word:diff --git a/tests/test_pipelines/misc_tests/multiplicity.plx b/tests/test_pipelines/misc_tests/multiplicity.plxindex e8c9a2a3f..f3e842522 100644--- a/tests/test_pipelines/misc_tests/multiplicity.plx+++ b/tests/test_pipelines/misc_tests/multiplicity.plx@@ -12,7 +12,7 @@ FantasyScene = "A fantasy scene"type = "PipeLLM"description = "Recall the colors of the original power rangers"output = "Color"-prompt_template = """+prompt = """Remind me of the colors of the {{ _nb_output }} original power rangers."""model = "llm_for_creative_writing"@@ -22,7 +22,7 @@ nb_output = 5type = "PipeLLM"description = "Recall the colors of all the power rangers"output = "Color"-prompt_template = """+prompt = """Remind me of the colors of all the power rangers over all series.Don't list the names of the characters or the name of the series: just list the colors."""@@ -33,7 +33,7 @@ type = "PipeLLM"description = "Imagine a product of nature"inputs = { color = "Color" }output = "ProductOfNature"-prompt_template = """+prompt = """Propose a product of nature of this color: $color.Just state what it is in a single sentence."""@@ -43,7 +43,7 @@ type = "PipeLLM"description = "Imagine a fantasy scene including products of nature"inputs = { product_of_nature = "ProductOfNature" }output = "images.ImgGenPrompt"-prompt_template = """+prompt = """Imagine a fantasy scene including the following products of nature:@product_of_naturediff --git a/tests/test_pipelines/misc_tests/subfolder_1/cars.plx b/tests/test_pipelines/misc_tests/subfolder_1/cars.plxindex 99db4493a..d8ce8851c 100644--- a/tests/test_pipelines/misc_tests/subfolder_1/cars.plx+++ b/tests/test_pipelines/misc_tests/subfolder_1/cars.plx@@ -14,7 +14,7 @@ type = "PipeLLM"description = "Generate a description of a car"inputs = { car = "Car" }output = "CarDescription"-prompt_template = """+prompt = """Given the reference to a car, generate a description of the car.@cardiff --git a/tests/test_pipelines/misc_tests/subfolder_2/animals.plx b/tests/test_pipelines/misc_tests/subfolder_2/animals.plxindex dc1696b30..adfbf094c 100644--- a/tests/test_pipelines/misc_tests/subfolder_2/animals.plx+++ b/tests/test_pipelines/misc_tests/subfolder_2/animals.plx@@ -14,7 +14,7 @@ type = "PipeLLM"description = "Generate a description of an animal"inputs = { animal = "Animal" }output = "AnimalDescription"-prompt_template = """+prompt = """Given the reference to an animal, generate a description of the animal.@animaldiff --git a/tests/test_pipelines/misc_tests/subfolder_2/subfolder/flowers.plx b/tests/test_pipelines/misc_tests/subfolder_2/subfolder/flowers.plxindex ef4d310fc..77e85bc8f 100644--- a/tests/test_pipelines/misc_tests/subfolder_2/subfolder/flowers.plx+++ b/tests/test_pipelines/misc_tests/subfolder_2/subfolder/flowers.plx@@ -12,7 +12,7 @@ type = "PipeLLM"description = "Generate a description of a flower"inputs = { flower = "Flower" }output = "FlowerDescription"-prompt_template = """+prompt = """Given the reference to a flower, generate a description of the flower.@flowerdiff --git a/tests/test_pipelines/misc_tests/test_image_inputs.plx b/tests/test_pipelines/misc_tests/test_image_inputs.plxindex 7e2a53e04..78f454bf2 100644--- a/tests/test_pipelines/misc_tests/test_image_inputs.plx+++ b/tests/test_pipelines/misc_tests/test_image_inputs.plx@@ -13,7 +13,7 @@ output = "Article"system_prompt = """You are an expert at describing page contents."""-prompt_template = """+prompt = """Extract the full text (all the text that represents a "title") and the date of the article in the image.@image"""@@ -26,7 +26,7 @@ output = "Article"system_prompt = """You are an expert at describing page contents."""-prompt_template = """+prompt = """Extract the date and title of the article.@page.page_viewdiff --git a/tests/test_pipelines/misc_tests/test_jinja2.plx b/tests/test_pipelines/misc_tests/test_jinja2.plxindex 7fb93f334..ca9ef194e 100644--- a/tests/test_pipelines/misc_tests/test_jinja2.plx+++ b/tests/test_pipelines/misc_tests/test_jinja2.plx@@ -6,10 +6,12 @@ type = "PipeCompose"description = "Jinja2 test 1"inputs = { text = "Text" }output = "Text"-jinja2 = """++[pipe.jinja2_test_1.template]+source = """This is a simple test prompt:@text"""-prompting_style = { tag_style = "square_brackets", text_format = "markdown" }-template_category = "llm_prompt"+templating_style = { tag_style = "square_brackets", text_format = "markdown" }+category = "llm_prompt"diff --git a/tests/test_pipelines/misc_tests/tests.plx b/tests/test_pipelines/misc_tests/tests.plxindex c22fb87af..58d6cacc7 100644--- a/tests/test_pipelines/misc_tests/tests.plx+++ b/tests/test_pipelines/misc_tests/tests.plx@@ -13,7 +13,7 @@ type = "PipeLLM"description = "Simple LLM test from image"inputs = { image = "Image" }output = "Text"-prompt_template = """+prompt = """Describe the using rap lyrics, including puns and references to the image.@image"""@@ -23,7 +23,7 @@ type = "PipeLLM"description = "No Input"output = "Text"model = "llm_for_testing_gen_text"-prompt_template = """+prompt = """Explain that this is a test prompt which took no input from the user."""@@ -32,7 +32,7 @@ type = "PipeLLM"description = "No Input but generates a text that could be long"output = "Text"model = { model = "gpt-4o-mini", temperature = 0.5, max_tokens = 1000 }-prompt_template = """+prompt = """Tell me a short story about a red baloon."""diff --git a/tests/test_pipelines/pipe_controllers/pipe_batch/uppercase_transformer.plx b/tests/test_pipelines/pipe_controllers/pipe_batch/uppercase_transformer.plxindex 88b6ad088..c3e82f1fb 100644--- a/tests/test_pipelines/pipe_controllers/pipe_batch/uppercase_transformer.plx+++ b/tests/test_pipelines/pipe_controllers/pipe_batch/uppercase_transformer.plx@@ -20,7 +20,7 @@ type = "PipeLLM"description = "Transform text to uppercase"inputs = { text_item = "Text" }output = "UppercaseText"-prompt_template = """+prompt = """Transform the following text to uppercase and add the prefix "UPPER: ":@text_itemdiff --git a/tests/test_pipelines/pipe_controllers/pipe_condition/pipe_condition_1.plx b/tests/test_pipelines/pipe_controllers/pipe_condition/pipe_condition_1.plxindex 749b655cd..b85e3d6f8 100644--- a/tests/test_pipelines/pipe_controllers/pipe_condition/pipe_condition_1.plx+++ b/tests/test_pipelines/pipe_controllers/pipe_condition/pipe_condition_1.plx@@ -22,7 +22,7 @@ large = "process_large"type = "PipeLLM"description = "Generate random text for small items"output = "native.Text"-prompt_template = """+prompt = """Output this only: "small""""@@ -30,7 +30,7 @@ Output this only: "small"type = "PipeLLM"description = "Generate random text for medium items"output = "native.Text"-prompt_template = """+prompt = """Output this only: "medium""""@@ -38,7 +38,7 @@ Output this only: "medium"type = "PipeLLM"description = "Generate random text for large items"output = "native.Text"-prompt_template = """+prompt = """Output this only: "large""""diff --git a/tests/test_pipelines/pipe_controllers/pipe_condition/pipe_condition_2.plx b/tests/test_pipelines/pipe_controllers/pipe_condition/pipe_condition_2.plxindex b11255aa3..c476e7874 100644--- a/tests/test_pipelines/pipe_controllers/pipe_condition/pipe_condition_2.plx+++ b/tests/test_pipelines/pipe_controllers/pipe_condition/pipe_condition_2.plx@@ -22,7 +22,7 @@ large = "process_large_2"type = "PipeLLM"description = "Generate random text for small items"output = "native.Text"-prompt_template = """+prompt = """Output this only: "small""""@@ -30,7 +30,7 @@ Output this only: "small"type = "PipeLLM"description = "Generate random text for medium items"output = "native.Text"-prompt_template = """+prompt = """Output this only: "medium""""@@ -38,7 +38,7 @@ Output this only: "medium"type = "PipeLLM"description = "Generate random text for large items"output = "native.Text"-prompt_template = """+prompt = """Output this only: "large""""diff --git a/tests/test_pipelines/pipe_controllers/pipe_condition/pipe_condition_complex.plx b/tests/test_pipelines/pipe_controllers/pipe_condition/pipe_condition_complex.plxindex 3a9893cc5..cd5c0df13 100644--- a/tests/test_pipelines/pipe_controllers/pipe_condition/pipe_condition_complex.plx+++ b/tests/test_pipelines/pipe_controllers/pipe_condition/pipe_condition_complex.plx@@ -70,7 +70,7 @@ type = "PipeLLM"description = "Process urgent technical documents with high priority"inputs = { doc_request = "DocumentRequest", user_profile = "UserProfile" }output = "native.Text"-prompt_template = """+prompt = """Process this urgent technical document with immediate attention.@doc_request.document_type@doc_request.priority@@ -86,7 +86,7 @@ type = "PipeLLM"description = "Process complex technical documents for expert users"inputs = { doc_request = "DocumentRequest", user_profile = "UserProfile" }output = "native.Text"-prompt_template = """+prompt = """Process this complex technical document for expert user.@doc_request.document_type@doc_request.complexity@@ -100,7 +100,7 @@ type = "PipeLLM"description = "Process standard technical documents"inputs = { doc_request = "DocumentRequest", user_profile = "UserProfile" }output = "native.Text"-prompt_template = """+prompt = """Process this standard technical document.@doc_request.document_type@user_profile.user_level@@ -114,7 +114,7 @@ type = "PipeLLM"description = "Process urgent business documents"inputs = { doc_request = "DocumentRequest", user_profile = "UserProfile" }output = "native.Text"-prompt_template = """+prompt = """Process this urgent business document.@doc_request.priority@user_profile.department@@ -127,7 +127,7 @@ type = "PipeLLM"description = "Process finance business documents"inputs = { doc_request = "DocumentRequest", user_profile = "UserProfile" }output = "native.Text"-prompt_template = """+prompt = """Process this finance business document.@user_profile.department@doc_request.document_type@@ -140,7 +140,7 @@ type = "PipeLLM"description = "Process marketing business documents"inputs = { doc_request = "DocumentRequest", user_profile = "UserProfile" }output = "native.Text"-prompt_template = """+prompt = """Process this marketing business document.@user_profile.department@doc_request.language@@ -153,7 +153,7 @@ type = "PipeLLM"description = "Process general business documents"inputs = { doc_request = "DocumentRequest", user_profile = "UserProfile" }output = "native.Text"-prompt_template = """+prompt = """Process this general business document.@doc_request.document_type@user_profile.department@@ -167,7 +167,7 @@ type = "PipeLLM"description = "Process complex legal documents"inputs = { doc_request = "DocumentRequest", user_profile = "UserProfile" }output = "native.Text"-prompt_template = """+prompt = """Process this complex legal document.@doc_request.complexity@user_profile.user_level@@ -181,7 +181,7 @@ type = "PipeLLM"description = "Process international legal documents"inputs = { doc_request = "DocumentRequest" }output = "native.Text"-prompt_template = """+prompt = """Process this international legal document.@doc_request.language@doc_request.document_type@@ -194,7 +194,7 @@ type = "PipeLLM"description = "Process standard legal documents"inputs = { doc_request = "DocumentRequest", user_profile = "UserProfile" }output = "native.Text"-prompt_template = """+prompt = """Process this standard legal document.@doc_request.document_type@user_profile.user_leveldiff --git a/tests/test_pipelines/pipe_controllers/pipe_condition/text_length_condition.plx b/tests/test_pipelines/pipe_controllers/pipe_condition/text_length_condition.plxindex 755973778..160053049 100644--- a/tests/test_pipelines/pipe_controllers/pipe_condition/text_length_condition.plx+++ b/tests/test_pipelines/pipe_controllers/pipe_condition/text_length_condition.plx@@ -8,7 +8,7 @@ type = "PipeLLM"description = "Capitalize text for long inputs"inputs = { input_text = "Text" }output = "Text"-prompt_template = """Transform the following LONG text to uppercase and add "LONG: " prefix:+prompt = """Transform the following LONG text to uppercase and add "LONG: " prefix:@input_text.text@@ -19,7 +19,7 @@ type = "PipeLLM"description = "Add prefix for short text inputs"inputs = { input_text = "Text" }output = "Text"-prompt_template = """Add the prefix "SHORT: " to the beginning of the following text:+prompt = """Add the prefix "SHORT: " to the beginning of the following text:@input_text.textdiff --git a/tests/test_pipelines/pipe_controllers/pipe_parallel/parallel_text_analysis.plx b/tests/test_pipelines/pipe_controllers/pipe_parallel/parallel_text_analysis.plxindex ad60c02d9..1c3ad3e6e 100644--- a/tests/test_pipelines/pipe_controllers/pipe_parallel/parallel_text_analysis.plx+++ b/tests/test_pipelines/pipe_controllers/pipe_parallel/parallel_text_analysis.plx@@ -9,7 +9,7 @@ description = "Analyze sentiment of text"inputs = { input_text = "Text" }output = "Text"structuring_method = "preliminary_text"-prompt_template = """PLEASE OUTPUT ONLY ONE WORD: 'positive', 'negative', or 'neutral'. Ignore the Hello world+prompt = """PLEASE OUTPUT ONLY ONE WORD: 'positive', 'negative', or 'neutral'. Ignore the Hello world@input_text.text"""@@ -20,7 +20,7 @@ description = "Count words in text"inputs = { input_text = "Text" }output = "Text"structuring_method = "preliminary_text"-prompt_template = """+prompt = """Count the number of words in the following text and return only the number:@input_text.text@@ -31,7 +31,7 @@ type = "PipeLLM"description = "Extract keywords from text"inputs = { input_text = "Text" }output = "Text"-prompt_template = """+prompt = """structuring_method = "preliminary_text"Extract the top 3 keywords from the following text. Return them as a comma-separated list:@input_text.textdiff --git a/tests/test_pipelines/pipe_controllers/pipe_parallel/pipe_parallel_1.plx b/tests/test_pipelines/pipe_controllers/pipe_parallel/pipe_parallel_1.plxindex 2f7077580..2f82d936f 100644--- a/tests/test_pipelines/pipe_controllers/pipe_parallel/pipe_parallel_1.plx+++ b/tests/test_pipelines/pipe_controllers/pipe_parallel/pipe_parallel_1.plx@@ -26,7 +26,7 @@ description = "Analyze document length and structure"inputs = { document = "DocumentInput" }output = "LengthAnalysis"model = "llm_for_testing_gen_text"-prompt_template = """+prompt = """Analyze the length and structure of this document:@document.text@@ -43,7 +43,7 @@ description = "Analyze document content and themes"inputs = { document = "DocumentInput" }output = "ContentAnalysis"model = "llm_for_testing_gen_text"-prompt_template = """+prompt = """Analyze the content and themes of this document:@document.textdiff --git a/tests/test_pipelines/pipe_controllers/pipe_sequence/capitalize_text.plx b/tests/test_pipelines/pipe_controllers/pipe_sequence/capitalize_text.plxindex e1de8685b..f5e5fdd6b 100644--- a/tests/test_pipelines/pipe_controllers/pipe_sequence/capitalize_text.plx+++ b/tests/test_pipelines/pipe_controllers/pipe_sequence/capitalize_text.plx@@ -7,7 +7,7 @@ type = "PipeLLM"description = "Transform text to uppercase"inputs = { input_text = "Text" }output = "Text"-prompt_template = """Transform the following text to uppercase:+prompt = """Transform the following text to uppercase:@input_text.text@@ -18,7 +18,7 @@ type = "PipeLLM"description = "Add PROCESSED prefix to text"inputs = { capitalized_text = "Text" }output = "Text"-prompt_template = """Add the prefix "PROCESSED: " to the beginning of the following text:+prompt = """Add the prefix "PROCESSED: " to the beginning of the following text:@capitalized_text.textdiff --git a/tests/test_pipelines/pipe_controllers/pipe_sequence/pipe_sequence_1.plx b/tests/test_pipelines/pipe_controllers/pipe_sequence/pipe_sequence_1.plxindex 37a1bb3a0..2ce422ba3 100644--- a/tests/test_pipelines/pipe_controllers/pipe_sequence/pipe_sequence_1.plx+++ b/tests/test_pipelines/pipe_controllers/pipe_sequence/pipe_sequence_1.plx@@ -24,7 +24,7 @@ description = "Clean and normalize the input text"inputs = { raw_text = "RawText" }output = "CleanedText"model = "llm_for_testing_gen_text"-prompt_template = """+prompt = """Clean and normalize the following text by:- Removing extra whitespace- Fixing basic grammar and punctuation@@ -39,7 +39,7 @@ description = "Generate a summary of the cleaned text"inputs = { cleaned_text = "CleanedText" }output = "SummaryText"model = "llm_for_testing_gen_text"-prompt_template = """+prompt = """Generate a concise summary of the following text in 2-3 sentences:@cleaned_textdiff --git a/tests/test_pipelines/pipe_controllers/pipe_sequence/pipe_sequence_2.plx b/tests/test_pipelines/pipe_controllers/pipe_sequence/pipe_sequence_2.plxindex 5792f3844..37ed5a102 100644--- a/tests/test_pipelines/pipe_controllers/pipe_sequence/pipe_sequence_2.plx+++ b/tests/test_pipelines/pipe_controllers/pipe_sequence/pipe_sequence_2.plx@@ -27,7 +27,7 @@ inputs = { document = "Document" }output = "CustomerReview"multiple_output = truemodel = "llm_for_testing_gen_text"-prompt_template = """+prompt = """Extract each individual customer review from this document as separate items:@document@@ -39,7 +39,7 @@ description = "Analyze sentiment of a single review"inputs = { single_review = "CustomerReview" }output = "SentimentAnalysis"model = "llm_for_testing_gen_text"-prompt_template = """+prompt = """Analyze the sentiment of this customer review:@single_review@@ -53,7 +53,7 @@ description = "Aggregate all review analyses into final rating"inputs = { sentiment_analyses = "SentimentAnalysis" }output = "ProductRating"model = "llm_for_testing_gen_text"-prompt_template = """+prompt = """Based on these sentiment analyses, provide an overall product rating:@sentiment_analysesdiff --git a/tests/test_pipelines/pipe_controllers/pipe_sequence/pipe_sequence_3.plx b/tests/test_pipelines/pipe_controllers/pipe_sequence/pipe_sequence_3.plxindex 3d2c40808..5d9dfb63f 100644--- a/tests/test_pipelines/pipe_controllers/pipe_sequence/pipe_sequence_3.plx+++ b/tests/test_pipelines/pipe_controllers/pipe_sequence/pipe_sequence_3.plx@@ -29,7 +29,7 @@ inputs = { topic = "CreativeTopic" }output = "CreativeIdea"nb_output = 5model = "llm_for_creative_writing"-prompt_template = """+prompt = """Generate $_nb_output creative and innovative ideas for the following topic:@topic@@ -49,7 +49,7 @@ inputs = { topic = "CreativeTopic" }output = "CreativeIdea"multiple_output = truemodel = "llm_for_creative_writing"-prompt_template = """+prompt = """Brainstorm creative concepts related to this topic. Generate as many good ideas as come to mind:@topic@@ -63,7 +63,7 @@ description = "Analyze a single idea in detail"inputs = { single_idea = "CreativeIdea" }output = "IdeaAnalysis"model = "llm_for_testing_gen_text"-prompt_template = """+prompt = """Perform a detailed analysis of this creative idea:@single_idea@@ -84,7 +84,7 @@ description = "Evaluate and rank all analyzed ideas"inputs = { detailed_analyses = "IdeaAnalysis" }output = "IdeaEvaluation"model = "llm_for_testing_gen_text"-prompt_template = """+prompt = """Based on these detailed analyses, rank all ideas from best to worst:@detailed_analyses@@ -101,7 +101,7 @@ description = "Select the best idea based on evaluation"inputs = { evaluation = "IdeaEvaluation" }output = "BestIdea"model = "llm_for_testing_gen_text"-prompt_template = """+prompt = """Based on this evaluation, select the single best idea and explain why it stands out:@evaluationdiff --git a/tests/test_pipelines/pipe_operators/pipe_llm_vision.plx b/tests/test_pipelines/pipe_operators/pipe_llm_vision.plxindex acf219e53..140ab6ff8 100644--- a/tests/test_pipelines/pipe_operators/pipe_llm_vision.plx+++ b/tests/test_pipelines/pipe_operators/pipe_llm_vision.plx@@ -11,7 +11,7 @@ description = "Describe what is in the image"inputs = { image = "Image" }output = "Text"model = "llm_for_testing_vision"-prompt_template = """+prompt = """Describe what you see in this image in 1-2 sentences, be concise.$image"""@@ -22,7 +22,7 @@ description = "Describe what is in the image"inputs = { imageA = "Image", imageB = "Image" }output = "BasicDescription"model = "llm_to_extract_diagram"-prompt_template = """+prompt = """Describe what you see in $imageA only.Completely ignore $imageB."""@@ -33,7 +33,7 @@ description = "Describe what is in the image"inputs = { imageA = "Image", imageB = "Image" }output = "BasicDescription"model = "llm_to_extract_diagram"-prompt_template = """+prompt = """Describe what you see in $imageB only.Completely ignore $imageA."""@@ -46,7 +46,7 @@ inputs = { image = "images.Photo" }output = "VisionAnalysis"model = "llm_to_extract_diagram"system_prompt = "You are an expert image analyst. Provide detailed, accurate descriptions."-prompt_template = """+prompt = """Analyze this image and describe what's the main topic etc.$image--------------------------------diff --git a/tests/test_pipelines/test_tweet.plx b/tests/test_pipelines/test_tweet.plxindex b8851dd68..25fe6e43b 100644--- a/tests/test_pipelines/test_tweet.plx+++ b/tests/test_pipelines/test_tweet.plx@@ -17,7 +17,7 @@ system_prompt = """You are an expert in social media optimization, particularly for tech content on Twitter/X.Your role is to analyze tech tweets and check if they display typical startup communication pitfalls."""-prompt_template = """+prompt = """Evaluate the tweet for these key issues:**Fluffiness** - Overuse of buzzwords without concrete meaning (e.g., "synergizing disruptive paradigms")@@ -45,7 +45,7 @@ system_prompt = """You are an expert in writing engaging tech tweets that drive meaningful discussions and engagement.Your goal is to rewrite tweets to be impactful and avoid the pitfalls identified in the analysis."""-prompt_template = """+prompt = """Rewrite this tech tweet to be more engaging and effective, based on the analysis:Original tweet:diff --git a/tests/test_pipelines/tricky_questions.plx b/tests/test_pipelines/tricky_questions.plxindex 09ac5234f..393f118ae 100644--- a/tests/test_pipelines/tricky_questions.plx+++ b/tests/test_pipelines/tricky_questions.plx@@ -17,7 +17,7 @@ description = "Analyze a question to determine whether it's straightforward or tinputs = { question = "Question" }output = "QuestionAnalysis"model = "llm_to_reason"-prompt_template = """+prompt = """Here is a question for an LLM:@questionDo you think it's tricky, or maybe even a deceptive trap?@@ -33,7 +33,7 @@ description = "Answer knowingly after analyzing a question"inputs = { question = "Question", question_analysis = "QuestionAnalysis" }output = "ThoughtfulAnswer"model = "llm_to_reason"-prompt_template = """+prompt = """A question was asked:@questionA thoughtful analysis was given:@@ -51,7 +51,7 @@ type = "PipeCompose"description = "Conclude a thoughtful answer"inputs = { thoughtful_answer = "ThoughtfulAnswer" }output = "ThoughtfulAnswerConclusion"-jinja2 = "After analyzing the question, here is my answer: $thoughtful_answer.the_answer"+template = "After analyzing the question, here is my answer: $thoughtful_answer.the_answer"[pipe.conclude_tricky_question_by_steps]diff --git a/tests/unit/pipelex/core/pipes/pipe_operator/pipe_llm/data.py b/tests/unit/pipelex/core/pipes/pipe_operator/pipe_llm/data.pyindex 6ee8592d8..1b7e5d706 100644--- a/tests/unit/pipelex/core/pipes/pipe_operator/pipe_llm/data.py+++ b/tests/unit/pipelex/core/pipes/pipe_operator/pipe_llm/data.py@@ -13,7 +13,7 @@ class PipeLLMInputTestCases:description="Test case: valid_text_input",inputs={"user_text": "native.Text"},output="native.Text",- prompt_template="Process this text: @user_text",+ prompt="Process this text: @user_text",),)@@ -23,7 +23,7 @@ class PipeLLMInputTestCases:description="Test case: valid_image_input",inputs={"user_image": "native.Image"},output="native.Text",- prompt_template="Describe this image: @user_image",+ prompt="Describe this image: @user_image",),)@@ -33,7 +33,7 @@ class PipeLLMInputTestCases:description="Test case: valid_two_inputs",inputs={"text_input": "native.Text", "another_text": "native.Text"},output="native.Text",- prompt_template="Compare @text_input with @another_text",+ prompt="Compare @text_input with @another_text",),)@@ -43,7 +43,7 @@ class PipeLLMInputTestCases:description="Test case: valid_text_and_image",inputs={"user_text": "native.Text", "user_image": "native.Image"},output="native.Text",- prompt_template="Analyze this text: @user_text with the image: @user_image",+ prompt="Analyze this text: @user_text with the image: @user_image",),)@@ -53,7 +53,7 @@ class PipeLLMInputTestCases:description="Test case: image_mentioned_in_prompt",inputs={"user_image": "native.Image"},output="native.Text",- prompt_template="Describe @user_image",+ prompt="Describe @user_image",),)@@ -63,7 +63,7 @@ class PipeLLMInputTestCases:description="Test case: text_and_image_both_in_prompt",inputs={"user_text": "native.Text", "user_image": "native.Image"},output="native.Text",- prompt_template="Analyze @user_text and describe @user_image",+ prompt="Analyze @user_text and describe @user_image",),)@@ -83,7 +83,7 @@ class PipeLLMInputTestCases:description="Test case: missing_input_in_prompt",inputs={},output="native.Text",- prompt_template="Process this text: @user_text",+ prompt="Process this text: @user_text",),"missing_input_variable",)@@ -94,7 +94,7 @@ class PipeLLMInputTestCases:description="Test case: extraneous_input",inputs={"user_text": "native.Text", "extra_input": "native.Text"},output="native.Text",- prompt_template="Process this: @user_text",+ prompt="Process this: @user_text",),"extraneous_input",)@@ -105,7 +105,7 @@ class PipeLLMInputTestCases:description="Test case: two_inputs_three_variables",inputs={"input_one": "native.Text", "input_two": "native.Text"},output="native.Text",- prompt_template="Process @input_one, @input_two, and @input_three",+ prompt="Process @input_one, @input_two, and @input_three",),"missing_input_variable",)diff --git a/tests/unit/pipelex/core/test_data/complex/multi_feature.py b/tests/unit/pipelex/core/test_data/complex/multi_feature.pyindex 51dace512..8930658ae 100644--- a/tests/unit/pipelex/core/test_data/complex/multi_feature.py+++ b/tests/unit/pipelex/core/test_data/complex/multi_feature.py@@ -17,7 +17,7 @@ type = "PipeLLM"description = "Generate content"inputs = { data = "InputData" }output = "ProcessedData"-prompt_template = "Process this data: @data"+prompt = "Process this data: @data"[pipe.sequence_pipe]type = "PipeSequence"@@ -42,7 +42,7 @@ steps = [description="Generate content",inputs={"data": "InputData"},output="ProcessedData",- prompt_template="Process this data: @data",+ prompt="Process this data: @data",),"sequence_pipe": PipeSequenceBlueprint(type="PipeSequence",diff --git a/tests/unit/pipelex/core/test_data/pipes/operators/compose/pipe_compose.py b/tests/unit/pipelex/core/test_data/pipes/operators/compose/pipe_compose.pyindex 981429745..aa7bc6dc6 100644--- a/tests/unit/pipelex/core/test_data/pipes/operators/compose/pipe_compose.py+++ b/tests/unit/pipelex/core/test_data/pipes/operators/compose/pipe_compose.py@@ -1,10 +1,11 @@+from pipelex.cogt.templating.template_blueprint import TemplateBlueprint+from pipelex.cogt.templating.template_category import TemplateCategoryfrom pipelex.core.bundles.pipelex_bundle_blueprint import PipelexBundleBlueprintfrom pipelex.core.concepts.concept_native import NativeConceptCodefrom pipelex.pipe_operators.compose.pipe_compose_blueprint import PipeComposeBlueprint-from pipelex.tools.templating.jinja2_template_category import Jinja2TemplateCategory-PIPE_COMPOSE = (- "pipe_compose",+PIPE_COMPOSE_WITHOUT_CATEGORY = (+ "pipe_compose_without_category","""domain = "test_pipes"description = "Domain with template processing pipe"@@ -12,8 +13,7 @@ description = "Domain with template processing pipe"type = "PipeCompose"description = "Process a Jinja2 template"output = "Text"-jinja2 = "Hello {{ name }}!"-template_category = "markdown"+template = "Hello {{ name }}!"""",PipelexBundleBlueprint(domain="test_pipes",@@ -23,8 +23,38 @@ template_category = "markdown"type="PipeCompose",description="Process a Jinja2 template",output=NativeConceptCode.TEXT,- jinja2="Hello {{ name }}!",- template_category=Jinja2TemplateCategory.MARKDOWN,+ template="Hello {{ name }}!",+ ),+ },+ ),+)++PIPE_COMPOSE_WITH_CATEGORY = (+ "pipe_compose",+ """domain = "test_pipes"+description = "Domain with template processing pipe"++[pipe.compose_output]+type = "PipeCompose"+description = "Process a Jinja2 template"+output = "Text"++[pipe.compose_output.template]+source = "Hello {{ name }}!"+category = "markdown"+""",+ PipelexBundleBlueprint(+ domain="test_pipes",+ description="Domain with template processing pipe",+ pipe={+ "compose_output": PipeComposeBlueprint(+ type="PipeCompose",+ description="Process a Jinja2 template",+ output=NativeConceptCode.TEXT,+ template=TemplateBlueprint(+ source="Hello {{ name }}!",+ category=TemplateCategory.MARKDOWN,+ ),),},),@@ -32,5 +62,6 @@ template_category = "markdown"# Export all PipeCompose test casesPIPE_COMPOSE_TEST_CASES = [- PIPE_COMPOSE,+ PIPE_COMPOSE_WITHOUT_CATEGORY,+ PIPE_COMPOSE_WITH_CATEGORY,]diff --git a/tests/unit/pipelex/core/test_data/pipes/operators/llm/pipe_llm.py b/tests/unit/pipelex/core/test_data/pipes/operators/llm/pipe_llm.pyindex acc9ded15..d1e28ff1d 100644--- a/tests/unit/pipelex/core/test_data/pipes/operators/llm/pipe_llm.py+++ b/tests/unit/pipelex/core/test_data/pipes/operators/llm/pipe_llm.py@@ -14,7 +14,7 @@ description = "Domain with pipe definitions"type = "PipeLLM"description = "Generate text using LLM"output = "Text"-prompt_template = "Generate a story about a programmer"+prompt = "Generate a story about a programmer"""",PipelexBundleBlueprint(domain="test_pipes",@@ -24,7 +24,7 @@ prompt_template = "Generate a story about a programmer"type="PipeLLM",description="Generate text using LLM",output=NativeConceptCode.TEXT,- prompt_template="Generate a story about a programmer",+ prompt="Generate a story about a programmer",),},),@@ -41,7 +41,7 @@ type = "PipeLLM"description = "Extract information from text"inputs = { text = "Text", topic = "Text" }output = "Text"-prompt_template = """+prompt = """Extract information about $topic from this text:@text@@ -56,7 +56,7 @@ Extract information about $topic from this text:description="Extract information from text",inputs={"text": "Text", "topic": "Text"},output=NativeConceptCode.TEXT,- prompt_template="Extract information about $topic from this text:\n\n@text\n",+ prompt="Extract information about $topic from this text:\n\n@text\n",),},),@@ -73,7 +73,7 @@ type = "PipeLLM"description = "Expert analysis with system prompt"output = "Text"system_prompt = "You are a data analysis expert with 20 years of experience"-prompt_template = "Analyze the following data and provide insights"+prompt = "Analyze the following data and provide insights"""",PipelexBundleBlueprint(domain="test_pipes",@@ -84,7 +84,7 @@ prompt_template = "Analyze the following data and provide insights"description="Expert analysis with system prompt",output=NativeConceptCode.TEXT,system_prompt="You are a data analysis expert with 20 years of experience",- prompt_template="Analyze the following data and provide insights",+ prompt="Analyze the following data and provide insights",),},),@@ -100,7 +100,7 @@ description = "Domain with pipe definitions"type = "PipeLLM"description = "Expert analysis with system prompt"output = "Text"-prompt_template = """+prompt = """Extract all articles/items from this invoice text: $extracted_text. For each item find: item name, quantity, unit price, total price, description, and product code ifavailable. Return each article as separate structured data."""@@ -113,7 +113,7 @@ Extract all articles/items from this invoice text: $extracted_text. For each itetype="PipeLLM",description="Expert analysis with system prompt",output=NativeConceptCode.TEXT,- prompt_template="""Extract all articles/items from this invoice text: $extracted_text. For each item find: item name, quantity, unit price, total price, description, and product code if+ prompt="""Extract all articles/items from this invoice text: $extracted_text. For each item find: item name, quantity, unit price, total price, description, and product code ifavailable. Return each article as separate structured data.""",),@@ -130,7 +130,7 @@ description = "Domain with pipe definitions"type = "PipeLLM"description = "Generate multiple ideas"output = "Text"-prompt_template = "Generate creative ideas for a mobile app"+prompt = "Generate creative ideas for a mobile app"nb_output = 3""",PipelexBundleBlueprint(@@ -142,7 +142,7 @@ nb_output = 3description="Generate multiple ideas",output=NativeConceptCode.TEXT,nb_output=3,- prompt_template="Generate creative ideas for a mobile app",+ prompt="Generate creative ideas for a mobile app",),},),@@ -159,7 +159,7 @@ type = "PipeLLM"description = "Brainstorm multiple solutions"inputs = { problem = { concept = "Text" } }output = "Text"-prompt_template = "Brainstorm solutions for this problem: $problem"+prompt = "Brainstorm solutions for this problem: $problem"multiple_output = true""",PipelexBundleBlueprint(@@ -172,7 +172,7 @@ multiple_output = trueinputs={"problem": InputRequirementBlueprint(concept="Text")},output=NativeConceptCode.TEXT,multiple_output=True,- prompt_template="Brainstorm solutions for this problem: $problem",+ prompt="Brainstorm solutions for this problem: $problem",),},),@@ -189,7 +189,7 @@ type = "PipeLLM"description = "Analyze image content"inputs = { image = "Image" }output = "Text"-prompt_template = "Describe what you see in this image in detail"+prompt = "Describe what you see in this image in detail"""",PipelexBundleBlueprint(domain="test_pipes",@@ -200,41 +200,13 @@ prompt_template = "Describe what you see in this image in detail"description="Analyze image content",inputs={"image": "Image"},output=NativeConceptCode.TEXT,- prompt_template="Describe what you see in this image in detail",+ prompt="Describe what you see in this image in detail",),},),)-# PipeLLM with template name instead of inline template-PIPE_LLM_TEMPLATE_NAME = (- "pipe_llm_template_name",- """domain = "test_pipes"-description = "Domain with pipe definitions"--[pipe.use_template]-type = "PipeLLM"-description = "Use named template"-inputs = { data = "Text" }-output = "Text"-template_name = "analysis_template"-""",- PipelexBundleBlueprint(- domain="test_pipes",- description="Domain with pipe definitions",- pipe={- "use_template": PipeLLMBlueprint(- type="PipeLLM",- description="Use named template",- inputs={"data": "Text"},- output=NativeConceptCode.TEXT,- template_name="analysis_template",- ),- },- ),-)--# PipeLLM with fixed prompt (no template)+# PipeLLM with fixed prompt (not an actual template)PIPE_LLM_FIXED_PROMPT = ("pipe_llm_fixed_prompt","""domain = "test_pipes"@@ -271,8 +243,8 @@ type = "PipeLLM"description = "Analysis with dynamic system prompt"inputs = { expertise_level = "Text", content = "Text" }output = "Text"-system_prompt_template = "You are an expert with $expertise_level level knowledge"-prompt_template = "Analyze this content: @content"+system_prompt = "You are an expert with $expertise_level level knowledge"+prompt = "Analyze this content: @content"""",PipelexBundleBlueprint(domain="test_pipes",@@ -283,8 +255,8 @@ prompt_template = "Analyze this content: @content"description="Analysis with dynamic system prompt",inputs={"expertise_level": "Text", "content": "Text"},output=NativeConceptCode.TEXT,- system_prompt_template="You are an expert with $expertise_level level knowledge",- prompt_template="Analyze this content: @content",+ system_prompt="You are an expert with $expertise_level level knowledge",+ prompt="Analyze this content: @content",),},),@@ -304,7 +276,7 @@ type = "PipeLLM"description = "Extract structured person information"inputs = { text = { concept = "Text", multiplicity = 1 } }output = "PersonInfo"-prompt_template = "Extract person information from this text: @text"+prompt = "Extract person information from this text: @text"""",PipelexBundleBlueprint(domain="test_pipes",@@ -316,7 +288,7 @@ prompt_template = "Extract person information from this text: @text"description="Extract structured person information",inputs={"text": InputRequirementBlueprint(concept="Text", multiplicity=1)},output="PersonInfo",- prompt_template="Extract person information from this text: @text",+ prompt="Extract person information from this text: @text",),},),@@ -336,7 +308,7 @@ type = "PipeLLM"description = "Analyze multiple documents and single query"inputs = { documents = { concept = "Text", multiplicity = true }, query = { concept = "Text", multiplicity = false } }output = "DocumentSummary"-prompt_template = """+prompt = """Analyze these documents based on the query: $queryDocuments: @documents@@ -355,7 +327,7 @@ Documents: @documents"query": InputRequirementBlueprint(concept="Text", multiplicity=False),},output="DocumentSummary",- prompt_template="""Analyze these documents based on the query: $query+ prompt="""Analyze these documents based on the query: $queryDocuments: @documents""",@@ -373,7 +345,6 @@ PIPE_LLM_TEST_CASES = [PIPE_LLM_MULTIPLE_OUTPUTS,PIPE_LLM_DYNAMIC_MULTIPLE,PIPE_LLM_VISION,- PIPE_LLM_TEMPLATE_NAME,PIPE_LLM_FIXED_PROMPT,PIPE_LLM_SYSTEM_TEMPLATE,PIPE_LLM_STRUCTURED_OUTPUT,diff --git a/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_operator/pipe_compose/test_data.py b/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_operator/pipe_compose/test_data.pyindex 828e21062..d519826f3 100644--- a/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_operator/pipe_compose/test_data.py+++ b/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_operator/pipe_compose/test_data.py@@ -1,10 +1,11 @@from typing import ClassVar+from pipelex.cogt.templating.template_blueprint import TemplateBlueprint+from pipelex.cogt.templating.template_category import TemplateCategory+from pipelex.cogt.templating.templating_style import TagStyle, TemplatingStyle, TextFormatfrom pipelex.core.pipes.input_requirement_blueprint import InputRequirementBlueprintfrom pipelex.libraries.pipelines.builder.pipe.pipe_compose_spec import PipeComposeSpecfrom pipelex.pipe_operators.compose.pipe_compose_blueprint import PipeComposeBlueprint-from pipelex.tools.templating.jinja2_template_category import Jinja2TemplateCategory-from pipelex.tools.templating.templating_models import PromptingStyle, TagStyle, TextFormatclass PipeComposeTestCases:@@ -15,7 +16,7 @@ class PipeComposeTestCases:description="Render a template",inputs={"data": "Data"},output="RenderedText",- jinja2="Hello {{ data.name }}!",+ template="Hello {{ data.name }}!",target_format="markdown",),PipeComposeBlueprint(@@ -24,14 +25,15 @@ class PipeComposeTestCases:output="RenderedText",type="PipeCompose",category="PipeOperator",- jinja2_name=None,- jinja2="Hello {{ data.name }}!",- prompting_style=PromptingStyle(- tag_style=TagStyle.TICKS,- text_format=TextFormat.MARKDOWN,+ template=TemplateBlueprint(+ source="Hello {{ data.name }}!",+ category=TemplateCategory.MARKDOWN,+ templating_style=TemplatingStyle(+ tag_style=TagStyle.TICKS,+ text_format=TextFormat.MARKDOWN,+ ),+ extra_context=None,),- template_category=Jinja2TemplateCategory.MARKDOWN,- extra_context=None,),)diff --git a/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_operator/pipe_llm/test_data.py b/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_operator/pipe_llm/test_data.pyindex c6f70e239..8821adff8 100644--- a/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_operator/pipe_llm/test_data.py+++ b/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_operator/pipe_llm/test_data.py@@ -15,7 +15,7 @@ class PipeLLMTestCases:inputs={"topic": "Text"},output="Text",llm="llm_for_creative_writing",- prompt_template="Write about $topic",+ prompt="Write about $topic",),PipeLLMBlueprint(source=None,@@ -23,7 +23,7 @@ class PipeLLMTestCases:description="Generate text",inputs={"topic": InputRequirementBlueprint(concept="Text")},output="Text",- prompt_template="Write about $topic",+ prompt="Write about $topic",model="claude-4.1-opus",),)@@ -36,14 +36,14 @@ class PipeLLMTestCases:inputs={},output="Text",llm="llm_for_creative_writing",- prompt_template="Generate something interesting",+ prompt="Generate something interesting",),PipeLLMBlueprint(source=None,type="PipeLLM",description="Generate without inputs",output="Text",- prompt_template="Generate something interesting",+ prompt="Generate something interesting",model="claude-4.1-opus",),)@@ -55,7 +55,7 @@ class PipeLLMTestCases:description="Analyze image",inputs={"image": "Image"},output="Text",- prompt_template="Analyze the image: $image",+ prompt="Analyze the image: $image",llm="llm_cheap_for_vision",),PipeLLMBlueprint(@@ -64,7 +64,7 @@ class PipeLLMTestCases:description="Analyze image",inputs={"image": InputRequirementBlueprint(concept="Image")},output="Text",- prompt_template="Analyze the image: $image",+ prompt="Analyze the image: $image",model="gemini-2.5-flash-lite",),)@@ -76,7 +76,7 @@ class PipeLLMTestCases:description="Generate with preset",inputs={},output="Text",- prompt_template="Generate text",+ prompt="Generate text",llm="llm_to_reason",),PipeLLMBlueprint(@@ -85,7 +85,7 @@ class PipeLLMTestCases:description="Generate with preset",model="claude-4.5-sonnet",output="Text",- prompt_template="Generate text",+ prompt="Generate text",),)@@ -96,7 +96,7 @@ class PipeLLMTestCases:description="Generate with settings",inputs={},output="Text",- prompt_template="Generate text",+ prompt="Generate text",llm="llm_cheap_for_easy_questions",temperature=0.7,),@@ -110,7 +110,7 @@ class PipeLLMTestCases:max_tokens=None, # "auto" is handled at conversion to core),output="Text",- prompt_template="Generate text",+ prompt="Generate text",),)@@ -122,7 +122,7 @@ class PipeLLMTestCases:inputs={"data": "Data"},output="Analysis",system_prompt="You are a data analyst",- prompt_template="Analyze: @data",+ prompt="Analyze: @data",llm="llm_to_analyze_data",),PipeLLMBlueprint(@@ -131,7 +131,7 @@ class PipeLLMTestCases:description="Generate with system prompt",inputs={"data": InputRequirementBlueprint(concept="Data")},system_prompt="You are a data analyst",- prompt_template="Analyze: @data",+ prompt="Analyze: @data",output="Analysis",model="claude-4.5-sonnet",),@@ -144,7 +144,7 @@ class PipeLLMTestCases:description="Generate multiple items",inputs={},output="Item",- prompt_template="Generate items",+ prompt="Generate items",multiple_output=True,llm="llm_cheap_for_easy_questions",),@@ -155,7 +155,7 @@ class PipeLLMTestCases:multiple_output=True,nb_output=None,output="Item",- prompt_template="Generate items",+ prompt="Generate items",model="claude-4.5-sonnet",),)@@ -167,7 +167,7 @@ class PipeLLMTestCases:description="Generate exactly 5 items",inputs={},output="Item",- prompt_template="Generate items",+ prompt="Generate items",nb_output=5,llm="llm_cheap_for_easy_questions",),@@ -178,7 +178,7 @@ class PipeLLMTestCases:nb_output=5,multiple_output=None,output="Item",- prompt_template="Generate items",+ prompt="Generate items",model="claude-4.5-sonnet",),)diff --git a/tests/unit/pipelex/tools/misc/test_toml_utils.py b/tests/unit/pipelex/tools/misc/test_toml_utils.pyindex cfb1a7b6a..46f3fab04 100644--- a/tests/unit/pipelex/tools/misc/test_toml_utils.py+++ b/tests/unit/pipelex/tools/misc/test_toml_utils.py@@ -21,7 +21,7 @@ TestConcept = "A test concept"[pipe.test_pipe]type = "PipeLLM"description = "Test pipe definition"-prompt_template = '''+user_prompt = '''This is a test prompt'''"""@@ -51,7 +51,7 @@ description = "Test"[pipe.test_pipe]type = "PipeLLM"description = "Test"-prompt_template = """+user_prompt = """Output this only: "test""""'''@@ -128,7 +128,7 @@ TestConcept = "A test concept"description = "Test"[pipe.test_pipe]-prompt_template = \"\"\"+user_prompt = \"\"\"Output: "test"\"\"\""""@@ -168,7 +168,7 @@ large = "process_large_2"type = "PipeLLM"description = "Generate random text for large items"output = "native.Text"-prompt_template = """+user_prompt = """Output this only: "large"""" '''toml_file = tmp_path / "pipe_condition_real_case.toml"diff --git a/tests/unit/pipelex/tools/templating/test_jinja2.py b/tests/unit/pipelex/tools/templating/test_jinja2.pyindex e5df43dbf..17466259f 100644--- a/tests/unit/pipelex/tools/templating/test_jinja2.py+++ b/tests/unit/pipelex/tools/templating/test_jinja2.py@@ -3,10 +3,9 @@ from typing import Anyimport pytestfrom pipelex import log, pretty_print-from pipelex.hub import get_template_provider-from pipelex.tools.templating.jinja2_rendering import render_jinja2-from pipelex.tools.templating.jinja2_template_category import Jinja2TemplateCategory-from pipelex.tools.templating.templating_models import PromptingStyle, TagStyle, TextFormat+from pipelex.cogt.templating.template_category import TemplateCategory+from pipelex.cogt.templating.templating_style import TagStyle, TemplatingStyle, TextFormat+from pipelex.tools.jinja2.jinja2_rendering import render_jinja2from tests.cases import Fruit, JINJA2TestCasesPLACE_HOLDER = "place_holder"@@ -14,76 +13,51 @@ PLACE_HOLDER = "place_holder"@pytest.mark.asyncio(loop_scope="class")class TestRenderJinja2:- @pytest.mark.parametrize("jinja2_name", JINJA2TestCases.JINJA2_NAME)+ @pytest.mark.parametrize("template", JINJA2TestCases.JINJA2_FOR_ANY)@pytest.mark.parametrize("color", JINJA2TestCases.COLOR)- async def test_render_jinja2_name_from_text(self, jinja2_name: str, color: str):+ async def test_render_jinja2_from_text(self, template: str, color: str):temlating_context = {PLACE_HOLDER: color}- prompting_style = PromptingStyle(+ templating_style = TemplatingStyle(tag_style=TagStyle.NO_TAG,text_format=TextFormat.MARKDOWN,)jinja2_text: str = await render_jinja2(- template_category=Jinja2TemplateCategory.LLM_PROMPT,- template_provider=get_template_provider(),+ template_category=TemplateCategory.LLM_PROMPT,temlating_context=temlating_context,- jinja2_name=jinja2_name,- jinja2=None,- prompting_style=prompting_style,+ template_source=template,+ templating_style=templating_style,)pretty_print(jinja2_text, title="jinja2_text")- @pytest.mark.parametrize("jinja2", JINJA2TestCases.JINJA2_FOR_ANY)- @pytest.mark.parametrize("color", JINJA2TestCases.COLOR)- async def test_render_jinja2_from_text(self, jinja2: str, color: str):- temlating_context = {PLACE_HOLDER: color}- prompting_style = PromptingStyle(- tag_style=TagStyle.NO_TAG,- text_format=TextFormat.MARKDOWN,- )-- jinja2_text: str = await render_jinja2(- template_category=Jinja2TemplateCategory.LLM_PROMPT,- template_provider=get_template_provider(),- temlating_context=temlating_context,- jinja2_name=None,- jinja2=jinja2,- prompting_style=prompting_style,- )- pretty_print(jinja2_text, title="jinja2_text")-- @pytest.mark.parametrize("jinja2", JINJA2TestCases.JINJA2_FOR_ANY)+ @pytest.mark.parametrize("template", JINJA2TestCases.JINJA2_FOR_ANY)@pytest.mark.parametrize("fruit", JINJA2TestCases.FRUIT)- async def test_render_jinja2_from_specific_object(self, jinja2: str, fruit: Fruit):+ async def test_render_jinja2_from_specific_object(self, template: str, fruit: Fruit):temlating_context = {PLACE_HOLDER: fruit}- prompting_style = PromptingStyle(+ templating_style = TemplatingStyle(tag_style=TagStyle.NO_TAG,text_format=TextFormat.MARKDOWN,)jinja2_text: str = await render_jinja2(- template_category=Jinja2TemplateCategory.LLM_PROMPT,- template_provider=get_template_provider(),+ template_category=TemplateCategory.LLM_PROMPT,temlating_context=temlating_context,- jinja2_name=None,- jinja2=jinja2,- prompting_style=prompting_style,+ template_source=template,+ templating_style=templating_style,)pretty_print(jinja2_text, title="jinja2_text")- @pytest.mark.parametrize("jinja2", JINJA2TestCases.JINJA2_FOR_STUFF)- @pytest.mark.parametrize("prompting_style", JINJA2TestCases.STYLE)+ @pytest.mark.parametrize("template", JINJA2TestCases.JINJA2_FOR_STUFF)+ @pytest.mark.parametrize("templating_style", JINJA2TestCases.STYLE)@pytest.mark.parametrize(("topic", "any_object"), JINJA2TestCases.ANY_OBJECT)- async def test_render_jinja2_from_any_object(self, jinja2: str, prompting_style: PromptingStyle, topic: str, any_object: Any):+ async def test_render_jinja2_from_any_object(self, template: str, templating_style: TemplatingStyle, topic: str, any_object: Any):temlating_context = {PLACE_HOLDER: any_object}- log.verbose(f"Rendering Jinja2 for '{topic}' with style '{prompting_style}'")+ log.verbose(f"Rendering Jinja2 for '{topic}' with style '{templating_style}'")jinja2_text: str = await render_jinja2(- template_category=Jinja2TemplateCategory.LLM_PROMPT,- template_provider=get_template_provider(),+ template_category=TemplateCategory.LLM_PROMPT,temlating_context=temlating_context,- jinja2_name=None,- jinja2=jinja2,- prompting_style=prompting_style,+ template_source=template,+ templating_style=templating_style,)- log.verbose(f"Jinja2 rendered Jinja2 for '{topic}' with style '{prompting_style}':\n{jinja2_text}")+ log.verbose(f"Jinja2 rendered Jinja2 for '{topic}' with style '{templating_style}':\n{jinja2_text}")pretty_print(jinja2_text, title="jinja2_text")diff --git a/tests/unit/pipelex/tools/templating/test_template_preprocessor.py b/tests/unit/pipelex/tools/templating/test_template_preprocessor.pyindex d6b414880..fb90125ea 100644--- a/tests/unit/pipelex/tools/templating/test_template_preprocessor.py+++ b/tests/unit/pipelex/tools/templating/test_template_preprocessor.py@@ -1,5 +1,5 @@from pipelex import pretty_print-from pipelex.tools.templating.template_preprocessor import preprocess_template+from pipelex.cogt.templating.template_preprocessor import preprocess_templateclass TestTemplatePreprocessor: