Build AI methods with your coding agent, then run them anywhere: as an MCP for chatbots, as a webapp for people, or via API for your software.
Refactor/remove yattag replace by jinja2 (#515)
Louis Choquel2436b1388049
diff --git a/pipelex/cogt/templating/template_rendering.py b/pipelex/cogt/templating/template_rendering.pyindex 62f428718..0eb8863a0 100644--- a/pipelex/cogt/templating/template_rendering.py+++ b/pipelex/cogt/templating/template_rendering.py@@ -3,7 +3,7 @@ from typing import Anyfrom pipelex.cogt.templating.template_category import TemplateCategoryfrom pipelex.cogt.templating.template_preprocessor import preprocess_templatefrom pipelex.cogt.templating.templating_style import TemplatingStyle-from pipelex.tools.jinja2.jinja2_rendering import render_jinja2+from pipelex.tools.jinja2.jinja2_rendering import render_jinja2_asyncasync def render_template(@@ -14,7 +14,7 @@ async def render_template() -> str:template_source = preprocess_template(template)- return await render_jinja2(+ return await render_jinja2_async(template_source=template_source,template_category=category,temlating_context=context,diff --git a/pipelex/core/stuffs/html_content.py b/pipelex/core/stuffs/html_content.pyindex 1349573eb..0ecf35985 100644--- a/pipelex/core/stuffs/html_content.py+++ b/pipelex/core/stuffs/html_content.py@@ -1,9 +1,10 @@import jsonfrom typing_extensions import override-from yattag import Doc+from pipelex.cogt.templating.template_category import TemplateCategoryfrom pipelex.core.stuffs.stuff_content import StuffContent+from pipelex.tools.jinja2.jinja2_rendering import render_jinja2_syncclass HtmlContent(StuffContent):@@ -25,10 +26,15 @@ class HtmlContent(StuffContent):@overridedef rendered_html(self) -> str:- doc, tag, text = Doc().tagtext()- with tag("div", klass=self.css_class):- text(self.inner_html)- return doc.getvalue()+ template_source = '<div class="{{ css_class|e }}">{{ inner_html | safe }}</div>'+ return render_jinja2_sync(+ template_source=template_source,+ template_category=TemplateCategory.HTML,+ temlating_context={+ "inner_html": self.inner_html,+ "css_class": self.css_class,+ },+ )@overridedef rendered_markdown(self, level: int = 1, is_pretty: bool = False) -> str:diff --git a/pipelex/core/stuffs/image_content.py b/pipelex/core/stuffs/image_content.pyindex 175a35f23..d5fc5d3f5 100644--- a/pipelex/core/stuffs/image_content.py+++ b/pipelex/core/stuffs/image_content.py@@ -4,11 +4,12 @@ from io import BytesIOfrom PIL import Imagefrom typing_extensions import override-from yattag import Docfrom pipelex.cogt.exceptions import ImageContentErrorfrom pipelex.cogt.extract.extract_output import ExtractedImage+from pipelex.cogt.templating.template_category import TemplateCategoryfrom pipelex.core.stuffs.stuff_content import StuffContent+from pipelex.tools.jinja2.jinja2_rendering import render_jinja2_syncfrom pipelex.tools.misc.base_64_utils import prefixed_base64_str_from_base64_str, save_base_64_str_to_binary_filefrom pipelex.tools.misc.file_utils import ensure_directory_exists, get_incremental_file_path, save_text_to_pathfrom pipelex.tools.misc.filetype_utils import detect_file_type_from_base64@@ -34,10 +35,14 @@ class ImageContent(StuffContent):@overridedef rendered_html(self) -> str:- doc = Doc()- doc.stag("img", src=self.url, klass="msg-img")-- return doc.getvalue()+ template_source = '<img src="{{ url|e }}" class="msg-img">'+ return render_jinja2_sync(+ template_source=template_source,+ template_category=TemplateCategory.HTML,+ temlating_context={+ "url": self.url,+ },+ )@overridedef rendered_markdown(self, level: int = 1, is_pretty: bool = False) -> str:diff --git a/pipelex/core/stuffs/mermaid_content.py b/pipelex/core/stuffs/mermaid_content.pyindex 1c1f1a63b..fd392f2f6 100644--- a/pipelex/core/stuffs/mermaid_content.py+++ b/pipelex/core/stuffs/mermaid_content.py@@ -1,9 +1,10 @@import jsonfrom typing_extensions import override-from yattag import Doc+from pipelex.cogt.templating.template_category import TemplateCategoryfrom pipelex.core.stuffs.stuff_content import StuffContent+from pipelex.tools.jinja2.jinja2_rendering import render_jinja2_syncclass MermaidContent(StuffContent):@@ -25,10 +26,14 @@ class MermaidContent(StuffContent):@overridedef rendered_html(self) -> str:- doc, tag, text = Doc().tagtext()- with tag("div", klass="mermaid"):- text(self.mermaid_code)- return doc.getvalue()+ template_source = '<div class="mermaid">{{ mermaid_code|e }}</div>'+ return render_jinja2_sync(+ template_source=template_source,+ template_category=TemplateCategory.HTML,+ temlating_context={+ "mermaid_code": self.mermaid_code,+ },+ )@overridedef rendered_markdown(self, level: int = 1, is_pretty: bool = False) -> str:diff --git a/pipelex/core/stuffs/pdf_content.py b/pipelex/core/stuffs/pdf_content.pyindex 3e7ca4f8a..49981e58a 100644--- a/pipelex/core/stuffs/pdf_content.py+++ b/pipelex/core/stuffs/pdf_content.py@@ -1,7 +1,8 @@from typing_extensions import override-from yattag import Doc+from pipelex.cogt.templating.template_category import TemplateCategoryfrom pipelex.core.stuffs.stuff_content import StuffContent+from pipelex.tools.jinja2.jinja2_rendering import render_jinja2_syncfrom pipelex.tools.misc.path_utils import interpret_path_or_url@@ -20,11 +21,14 @@ class PDFContent(StuffContent):@overridedef rendered_html(self) -> str:- doc = Doc()- doc.stag("a", href=self.url, klass="msg-pdf")- doc.text(self.url)-- return doc.getvalue()+ template_source = '<a href="{{ url|e }}" class="msg-pdf">{{ url|e }}</a>'+ return render_jinja2_sync(+ template_source=template_source,+ template_category=TemplateCategory.HTML,+ temlating_context={+ "url": self.url,+ },+ )@overridedef rendered_markdown(self, level: int = 1, is_pretty: bool = False) -> str:diff --git a/pipelex/tools/jinja2/jinja2_rendering.py b/pipelex/tools/jinja2/jinja2_rendering.pyindex a251a50cf..e781fea99 100644--- a/pipelex/tools/jinja2/jinja2_rendering.py+++ b/pipelex/tools/jinja2/jinja2_rendering.py@@ -1,4 +1,5 @@-from typing import Any+from collections.abc import Awaitable+from typing import Any, Protocolfrom jinja2.exceptions import (TemplateAssertionError,@@ -24,57 +25,141 @@ def _add_to_templating_context(temlating_context: dict[str, Any], jinja2_contexttemlating_context[jinja2_context_key] = value-async def render_jinja2(- template_source: str,- template_category: TemplateCategory,- temlating_context: dict[str, Any],- templating_style: TemplatingStyle | None = None,-) -> str:+class _Jinja2Template(Protocol):+ def render(self, **kwargs: Any) -> str: ...++ def render_async(self, **kwargs: Any) -> Awaitable[str]: ...+++def _compile_jinja2_template(template_source: str, template_category: TemplateCategory) -> _Jinja2Template:jinja2_env = make_jinja2_env_without_loader(template_category=template_category,)try:- template = jinja2_env.from_string(template_source)+ return jinja2_env.from_string(template_source)except TemplateAssertionError as exc:msg = f"Jinja2 render error: '{exc}', template_source:\n{template_source}"raise Jinja2TemplateRenderError(msg) from exc++def _prepare_templating_context(+ temlating_context: dict[str, Any],+ templating_style: TemplatingStyle | None,+) -> dict[str, Any]:# Create a copy to avoid mutating the caller's original dictionary- temlating_context = temlating_context.copy()+ prepared_templating_context = temlating_context.copy()if templating_style:_add_to_templating_context(- temlating_context=temlating_context,+ temlating_context=prepared_templating_context,jinja2_context_key=Jinja2ContextKey.TAG_STYLE,value=templating_style.tag_style,)_add_to_templating_context(- temlating_context=temlating_context,+ temlating_context=prepared_templating_context,jinja2_context_key=Jinja2ContextKey.TEXT_FORMAT,value=templating_style.text_format,)+ return prepared_templating_context+++def _make_type_error_msg(template_source: str, temlating_context: dict[str, Any], type_error: TypeError) -> str:+ context_vars = ", ".join(f"'{key}'" for key in temlating_context)+ return (+ f"Jinja2 render — type error: '{type_error}'. "+ f"This often happens when trying to iterate over a method or accessing an attribute incorrectly. "+ f"Template source:\n{template_source}\n"+ f"Available context variables: {context_vars}"+ )++def _make_non_type_error_msg(+ template_source: str,+ error_label: str,+ error: Jinja2StuffError | TemplateSyntaxError | UndefinedError | Jinja2ContextError,+) -> str:+ return f"Jinja2 render — {error_label}: '{error}', template_source:\n{template_source}"+++def _render_template_sync(template_source: str, template: _Jinja2Template, temlating_context: dict[str, Any]) -> str:+ try:+ generated_text: str = template.render(**temlating_context)+ except Jinja2StuffError as exc:+ msg = _make_non_type_error_msg(template_source=template_source, error_label="stuff error", error=exc)+ raise Jinja2TemplateRenderError(msg) from exc+ except TemplateSyntaxError as exc:+ msg = _make_non_type_error_msg(template_source=template_source, error_label="syntax error", error=exc)+ raise Jinja2TemplateRenderError(msg) from exc+ except UndefinedError as exc:+ msg = _make_non_type_error_msg(template_source=template_source, error_label="undefined error", error=exc)+ raise Jinja2TemplateRenderError(msg) from exc+ except Jinja2ContextError as exc:+ msg = _make_non_type_error_msg(template_source=template_source, error_label="context error", error=exc)+ raise Jinja2TemplateRenderError(msg) from exc+ except TypeError as exc:+ msg = _make_type_error_msg(template_source=template_source, temlating_context=temlating_context, type_error=exc)+ raise Jinja2TemplateRenderError(msg) from exc+ return generated_text+++async def _render_template_async(template_source: str, template: _Jinja2Template, temlating_context: dict[str, Any]) -> str:try:generated_text: str = await template.render_async(**temlating_context)- except Jinja2StuffError as stuff_error:- msg = f"Jinja2 render — stuff error: '{stuff_error}', template_source:\n{template_source}"- raise Jinja2TemplateRenderError(msg) from stuff_error- except TemplateSyntaxError as syntax_error:- msg = f"Jinja2 render — syntax error: '{syntax_error}', template_source:\n{template_source}"- raise Jinja2TemplateRenderError(msg) from syntax_error- except UndefinedError as undef_error:- msg = f"Jinja2 render — undefined error: '{undef_error}', template_source:\n{template_source}"- raise Jinja2TemplateRenderError(msg) from undef_error- except Jinja2ContextError as context_error:- msg = f"Jinja2 render — context error: '{context_error}', template_source:\n{template_source}"- raise Jinja2TemplateRenderError(msg) from context_error- except TypeError as type_error:- context_vars = ", ".join(f"'{key}'" for key in temlating_context)- msg = (- f"Jinja2 render — type error: '{type_error}'. "- f"This often happens when trying to iterate over a method or accessing an attribute incorrectly. "- f"Template source:\n{template_source}\n"- f"Available context variables: {context_vars}"- )- raise Jinja2TemplateRenderError(msg) from type_error+ except Jinja2StuffError as exc:+ msg = _make_non_type_error_msg(template_source=template_source, error_label="stuff error", error=exc)+ raise Jinja2TemplateRenderError(msg) from exc+ except TemplateSyntaxError as exc:+ msg = _make_non_type_error_msg(template_source=template_source, error_label="syntax error", error=exc)+ raise Jinja2TemplateRenderError(msg) from exc+ except UndefinedError as exc:+ msg = _make_non_type_error_msg(template_source=template_source, error_label="undefined error", error=exc)+ raise Jinja2TemplateRenderError(msg) from exc+ except Jinja2ContextError as exc:+ msg = _make_non_type_error_msg(template_source=template_source, error_label="context error", error=exc)+ raise Jinja2TemplateRenderError(msg) from exc+ except TypeError as exc:+ msg = _make_type_error_msg(template_source=template_source, temlating_context=temlating_context, type_error=exc)+ raise Jinja2TemplateRenderError(msg) from excreturn generated_text+++def render_jinja2_sync(+ template_source: str,+ template_category: TemplateCategory,+ temlating_context: dict[str, Any],+ templating_style: TemplatingStyle | None = None,+) -> str:+ template = _compile_jinja2_template(+ template_source=template_source,+ template_category=template_category,+ )+ prepared_templating_context = _prepare_templating_context(+ temlating_context=temlating_context,+ templating_style=templating_style,+ )+ return _render_template_sync(+ template_source=template_source,+ template=template,+ temlating_context=prepared_templating_context,+ )+++async def render_jinja2_async(+ template_source: str,+ template_category: TemplateCategory,+ temlating_context: dict[str, Any],+ templating_style: TemplatingStyle | None = None,+) -> str:+ template = _compile_jinja2_template(+ template_source=template_source,+ template_category=template_category,+ )+ prepared_templating_context = _prepare_templating_context(+ temlating_context=temlating_context,+ templating_style=templating_style,+ )+ return await _render_template_async(+ template_source=template_source,+ template=template,+ temlating_context=prepared_templating_context,+ )diff --git a/pyproject.toml b/pyproject.tomlindex 883d5cdfb..14b15059c 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -47,7 +47,6 @@ dependencies = ["tomlkit>=0.13.2","typer>=0.16.0","typing-extensions>=4.13.2",- "yattag>=1.15.2",][project.urls]diff --git a/tests/unit/pipelex/core/stuffs/data.py b/tests/unit/pipelex/core/stuffs/data.pyindex d9179c911..7b41b56b6 100644--- a/tests/unit/pipelex/core/stuffs/data.py+++ b/tests/unit/pipelex/core/stuffs/data.py@@ -1,11 +1,19 @@+from typing import ClassVar++from markupsafe import escapefrom pydantic import Fieldfrom pipelex.client.protocol import StuffContentOrDatafrom pipelex.core.concepts.concept_factory import ConceptFactoryfrom pipelex.core.concepts.native.concept_native import NativeConceptCode+from pipelex.core.stuffs.html_content import HtmlContent+from pipelex.core.stuffs.image_content import ImageContentfrom pipelex.core.stuffs.list_content import ListContent+from pipelex.core.stuffs.mermaid_content import MermaidContent+from pipelex.core.stuffs.pdf_content import PDFContentfrom pipelex.core.stuffs.structured_content import StructuredContentfrom pipelex.core.stuffs.stuff import DictStuff, Stuff+from pipelex.core.stuffs.stuff_content import StuffContentfrom pipelex.core.stuffs.text_content import TextContent@@ -676,3 +684,43 @@ ERROR_TEST_CASES: list[tuple[str, StuffContentOrData, str | None, str, list[str]"not compatible",),]+++class RenderedHtmlTestData:+ HTML_INNER_HTML = "<h1>Title</h1><p>Hi <strong>there</strong>.</p>"+ HTML_CSS_CLASS = 'report-content x" onclick="alert(1)'+ HTML_EXPECTED = f'<div class="{escape(HTML_CSS_CLASS)!s}">{HTML_INNER_HTML}</div>'++ IMAGE_URL = 'https://example.com/image.png?x=1&y="2"'+ IMAGE_EXPECTED = f'<img src="{escape(IMAGE_URL)!s}" class="msg-img">'++ MERMAID_CODE = 'graph TD; A-- "<b>&</b>" -->B'+ MERMAID_EXPECTED = f'<div class="mermaid">{escape(MERMAID_CODE)!s}</div>'++ PDF_URL = "https://example.com/doc.pdf?x=1&y=2"+ PDF_EXPECTED = f'<a href="{escape(PDF_URL)!s}" class="msg-pdf">{escape(PDF_URL)!s}</a>'++ RENDERED_HTML_TEST_CASES: ClassVar[list[tuple[StuffContent, str]]] = [+ (+ HtmlContent(+ inner_html=HTML_INNER_HTML,+ css_class=HTML_CSS_CLASS,+ ),+ HTML_EXPECTED,+ ),+ (+ ImageContent(url=IMAGE_URL),+ IMAGE_EXPECTED,+ ),+ (+ MermaidContent(+ mermaid_code=MERMAID_CODE,+ mermaid_url="https://example.com/diagram.svg",+ ),+ MERMAID_EXPECTED,+ ),+ (+ PDFContent(url=PDF_URL),+ PDF_EXPECTED,+ ),+ ]diff --git a/tests/unit/pipelex/core/stuffs/test_html_content_rendering.py b/tests/unit/pipelex/core/stuffs/test_html_content_rendering.pynew file mode 100644index 000000000..d1ff639eb--- /dev/null+++ b/tests/unit/pipelex/core/stuffs/test_html_content_rendering.py@@ -0,0 +1,14 @@+import pytest++from pipelex.core.stuffs.stuff_content import StuffContent+from tests.unit.pipelex.core.stuffs.data import RenderedHtmlTestData+++class TestRenderedHtml:+ """Test rendered_html() for HTML-like StuffContent implementations."""++ @pytest.mark.parametrize(("content", "expected"), RenderedHtmlTestData.RENDERED_HTML_TEST_CASES)+ def test_rendered_html(self, content: StuffContent, expected: str):+ rendered = content.rendered_html()++ assert rendered == expecteddiff --git a/tests/unit/pipelex/tools/test_jinja2.py b/tests/unit/pipelex/tools/test_jinja2.pyindex 17466259f..b8074fb41 100644--- a/tests/unit/pipelex/tools/test_jinja2.py+++ b/tests/unit/pipelex/tools/test_jinja2.py@@ -5,7 +5,7 @@ import pytestfrom pipelex import log, pretty_printfrom pipelex.cogt.templating.template_category import TemplateCategoryfrom pipelex.cogt.templating.templating_style import TagStyle, TemplatingStyle, TextFormat-from pipelex.tools.jinja2.jinja2_rendering import render_jinja2+from pipelex.tools.jinja2.jinja2_rendering import render_jinja2_asyncfrom tests.cases import Fruit, JINJA2TestCasesPLACE_HOLDER = "place_holder"@@ -22,7 +22,7 @@ class TestRenderJinja2:text_format=TextFormat.MARKDOWN,)- jinja2_text: str = await render_jinja2(+ jinja2_text: str = await render_jinja2_async(template_category=TemplateCategory.LLM_PROMPT,temlating_context=temlating_context,template_source=template,@@ -39,7 +39,7 @@ class TestRenderJinja2:text_format=TextFormat.MARKDOWN,)- jinja2_text: str = await render_jinja2(+ jinja2_text: str = await render_jinja2_async(template_category=TemplateCategory.LLM_PROMPT,temlating_context=temlating_context,template_source=template,@@ -53,7 +53,7 @@ class TestRenderJinja2:async def test_render_jinja2_from_any_object(self, template: str, templating_style: TemplatingStyle, topic: str, any_object: Any):temlating_context = {PLACE_HOLDER: any_object}log.verbose(f"Rendering Jinja2 for '{topic}' with style '{templating_style}'")- jinja2_text: str = await render_jinja2(+ jinja2_text: str = await render_jinja2_async(template_category=TemplateCategory.LLM_PROMPT,temlating_context=temlating_context,template_source=template,diff --git a/uv.lock b/uv.lockindex 927c5f3b8..ffaf60664 100644--- a/uv.lock+++ b/uv.lock@@ -1860,7 +1860,6 @@ dependencies = [{ name = "tomlkit" },{ name = "typer" },{ name = "typing-extensions" },- { name = "yattag" },][package.optional-dependencies]@@ -1968,7 +1967,6 @@ requires-dist = [{ name = "types-networkx", marker = "extra == 'dev'", specifier = ">=3.3.0.20241020" },{ name = "types-pyyaml", marker = "extra == 'dev'", specifier = ">=6.0.12.20250326" },{ name = "typing-extensions", specifier = ">=4.13.2" },- { name = "yattag", specifier = ">=1.15.2" },]provides-extras = ["anthropic", "bedrock", "fal", "google", "google-genai", "mistralai", "docs", "dev"]@@ -3189,12 +3187,6 @@ wheels = [{ url = "https://files.pythonhosted.org/packages/73/ae/b48f95715333080afb75a4504487cbe142cae1268afc482d06692d605ae6/yarl-1.22.0-py3-none-any.whl", hash = "sha256:1380560bdba02b6b6c90de54133c81c9f2a453dee9912fe58c1dcced1edb7cff", size = 46814, upload-time = "2025-10-06T14:12:53.872Z" },]-[[package]]-name = "yattag"-version = "1.16.1"-source = { registry = "https://pypi.org/simple" }-sdist = { url = "https://files.pythonhosted.org/packages/1c/1a/d3b2a2b8f843f5e7138471c4a5c9172ef62bb41239aa4371784b7448110c/yattag-1.16.1.tar.gz", hash = "sha256:baa8f254e7ea5d3e0618281ad2ff5610e0e5360b3608e695c29bfb3b29d051f4", size = 29069, upload-time = "2024-11-02T22:38:30.443Z" }-[[package]]name = "zipp"version = "3.23.0"