Build AI methods with your coding agent, then run them anywhere: as an MCP for chatbots, as a webapp for people, or via API for your software.
Fix/content should be allowed in stuffs (#484)
Thomas Hebrardc40bed720f89
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex fdfd498a0..e6c243b7f 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -4,6 +4,10 @@**Highlights:** - Previously, in the pipelex config files (`.toml` files in the `.pipelex/` directory, such as `.pipelex/pipelex.toml`, but also the routing profiles files, backends, etc.), when an array was overridden, the new array was concatenated to the old array. Now, the new array overrides the old array.+### Fixed++- **Relaxed concept structure field naming restrictions**: Users can now use field names like `content`, `stuff_code`, `stuff_name`, and `concept` in their concept structures without conflicts. Internal metadata fields in stuff artefacts now use underscore prefixes (`_stuff_name`, `_content_class`, `_concept_code`, `_stuff_code`, `_content`) to avoid collisions with user-defined fields. Reserved field names (Pydantic BaseModel attributes like `model_config`, `model_fields`, etc.) and field names starting with underscore remain forbidden with improved error messages that clearly specify which fields are problematic.+### Changed- Modified the GHA `version-check.yml` so that the check of the version is only applying to release branches.diff --git a/pipelex/builder/pipe/pipe_batch_spec.py b/pipelex/builder/pipe/pipe_batch_spec.pyindex 9cdb7576b..76cfa25c9 100644--- a/pipelex/builder/pipe/pipe_batch_spec.py+++ b/pipelex/builder/pipe/pipe_batch_spec.py@@ -60,8 +60,6 @@ class PipeBatchSpec(PipeSpec):description=base_blueprint.description,inputs=base_blueprint.inputs,output=base_blueprint.output,- type=self.type,- pipe_category=self.pipe_category,branch_pipe_code=self.branch_pipe_code,input_list_name=self.input_list_name,input_item_name=self.input_item_name,diff --git a/pipelex/builder/pipe/pipe_compose_spec.py b/pipelex/builder/pipe/pipe_compose_spec.pyindex 14891b4da..862b1670d 100644--- a/pipelex/builder/pipe/pipe_compose_spec.py+++ b/pipelex/builder/pipe/pipe_compose_spec.py@@ -135,7 +135,5 @@ class PipeComposeSpec(PipeSpec):description=base_blueprint.description,inputs=base_blueprint.inputs,output=base_blueprint.output,- type=self.type,- pipe_category=self.pipe_category,template=template_blueprint,)diff --git a/pipelex/builder/pipe/pipe_extract_spec.py b/pipelex/builder/pipe/pipe_extract_spec.pyindex a048ed333..ee09b1a83 100644--- a/pipelex/builder/pipe/pipe_extract_spec.py+++ b/pipelex/builder/pipe/pipe_extract_spec.py@@ -96,8 +96,6 @@ class PipeExtractSpec(PipeSpec):description=base_blueprint.description,inputs=base_blueprint.inputs,output=base_blueprint.output,- type=self.type,- pipe_category=self.pipe_category,model=extract_model_choice,page_images=self.page_images,page_image_captions=self.page_image_captions,diff --git a/pipelex/builder/pipe/pipe_func_spec.py b/pipelex/builder/pipe/pipe_func_spec.pyindex c5f99fa48..efa474bca 100644--- a/pipelex/builder/pipe/pipe_func_spec.py+++ b/pipelex/builder/pipe/pipe_func_spec.py@@ -40,7 +40,5 @@ class PipeFuncSpec(PipeSpec):description=base_blueprint.description,inputs=base_blueprint.inputs,output=base_blueprint.output,- type=self.type,- pipe_category=self.pipe_category,function_name=self.function_name,)diff --git a/pipelex/builder/pipe/pipe_img_gen_spec.py b/pipelex/builder/pipe/pipe_img_gen_spec.pyindex 788f87e96..2f15b656e 100644--- a/pipelex/builder/pipe/pipe_img_gen_spec.py+++ b/pipelex/builder/pipe/pipe_img_gen_spec.py@@ -71,8 +71,6 @@ class PipeImgGenSpec(PipeSpec):description=base_blueprint.description,inputs=base_blueprint.inputs,output=base_blueprint.output,- type=self.type,- pipe_category=self.pipe_category,img_gen_prompt=None,img_gen_prompt_var_name=base_blueprint.input_names[0] if base_blueprint.input_names else None,model=img_gen_choice,diff --git a/pipelex/builder/pipe/pipe_sequence_spec.py b/pipelex/builder/pipe/pipe_sequence_spec.pyindex 73b3f76f9..a75187be1 100644--- a/pipelex/builder/pipe/pipe_sequence_spec.py+++ b/pipelex/builder/pipe/pipe_sequence_spec.py@@ -65,7 +65,5 @@ class PipeSequenceSpec(PipeSpec):description=base_blueprint.description,inputs=base_blueprint.inputs,output=base_blueprint.output,- type=self.type,- pipe_category=self.pipe_category,steps=core_steps,)diff --git a/pipelex/cogt/templating/template_rendering.py b/pipelex/cogt/templating/template_rendering.pyindex 0fc02f47e..62f428718 100644--- a/pipelex/cogt/templating/template_rendering.py+++ b/pipelex/cogt/templating/template_rendering.py@@ -13,6 +13,7 @@ async def render_template(templating_style: TemplatingStyle | None = None,) -> str:template_source = preprocess_template(template)+return await render_jinja2(template_source=template_source,template_category=category,diff --git a/pipelex/core/concepts/concept_blueprint.py b/pipelex/core/concepts/concept_blueprint.pyindex 911358f3b..38cd4140a 100644--- a/pipelex/core/concepts/concept_blueprint.py+++ b/pipelex/core/concepts/concept_blueprint.py@@ -2,10 +2,12 @@ from typing import Anyfrom pydantic import BaseModel, ConfigDict, field_validator, model_validator-from pipelex.core.concepts.concept_structure_blueprint import ConceptStructureBlueprint+from pipelex.core.concepts.concept_structure_blueprint import RESERVED_FIELD_NAMES, ConceptStructureBlueprintfrom pipelex.core.concepts.native.concept_native import NativeConceptCodefrom pipelex.core.concepts.native.exceptions import NativeConceptDefinitionError+ConceptStructureBlueprintType = str | ConceptStructureBlueprint+class ConceptBlueprint(BaseModel):model_config = ConfigDict(extra="forbid")@@ -13,7 +15,7 @@ class ConceptBlueprint(BaseModel):source: str | None = Nonedescription: str# TODO (non-blockiing): define a type for Union[str, ConceptStructureBlueprint] (ConceptChoice to be consistent with LLMChoice)- structure: str | dict[str, str | ConceptStructureBlueprint] | None = None+ structure: str | dict[str, ConceptStructureBlueprintType] | None = None# TODO: restore possibility of multiple refilesrefines: str | None = None@@ -28,6 +30,38 @@ class ConceptBlueprint(BaseModel):raise ValueError(msg) from excreturn refines+ @field_validator("structure", mode="before")+ @classmethod+ def validate_structure_field_names(+ cls, structure: str | dict[str, ConceptStructureBlueprintType] | None+ ) -> str | dict[str, ConceptStructureBlueprintType] | None:+ if isinstance(structure, dict):+ # Check for reserved field names+ reserved_fields_used = [field_name for field_name in structure if field_name in RESERVED_FIELD_NAMES]+ if reserved_fields_used:+ fields_word = "field" if len(reserved_fields_used) == 1 else "fields"+ reserved_fields_used_list = ", ".join(f"'{name}'" for name in sorted(reserved_fields_used))+ reserved_names = ", ".join(f"'{name}'" for name in sorted(RESERVED_FIELD_NAMES))+ msg = (+ f"Cannot use reserved fields in concept structure. "+ f"Problematic fields: '{reserved_fields_used_list}'. "+ f"All reserved field names are: {reserved_names}"+ )+ raise ValueError(msg)++ # Check for field names starting with underscore (reserved for internal use)+ underscore_fields = [field_name for field_name in structure if field_name.startswith("_")]+ if underscore_fields:+ fields_word = "field" if len(underscore_fields) == 1 else "fields"+ underscore_list = ", ".join(f"'{name}'" for name in sorted(underscore_fields))+ msg = (+ f"Cannot use '{fields_word}' starting with underscore in concept structure. "+ f"Problematic fields: '{underscore_list}'. "+ "Field names starting with '_' are reserved for internal use by Pipelex."+ )+ raise ValueError(msg)+ return structure+@model_validator(mode="before")@classmethoddef validate_blueprint(cls, values: dict[str, Any] | str) -> dict[str, Any] | str:diff --git a/pipelex/core/concepts/concept_structure_blueprint.py b/pipelex/core/concepts/concept_structure_blueprint.pyindex b8f9bfa71..31d884b85 100644--- a/pipelex/core/concepts/concept_structure_blueprint.py+++ b/pipelex/core/concepts/concept_structure_blueprint.py@@ -3,8 +3,35 @@ from typing import Anyfrom pydantic import BaseModel, Field, model_validator+from pipelex.pipe_run.pipe_run_params import PipeRunParamKeyfrom pipelex.types import Self, StrEnum+# Reserved field names that cannot be used in concept structures+# These are either Pydantic BaseModel reserved attributes or internal metadata fields+RESERVED_FIELD_NAMES = frozenset(+ [+ # Pipe run parameter keys (from PipeRunParamKey enum)+ *PipeRunParamKey.value_list(),+ # Pydantic BaseModel reserved attributes+ "model_config",+ "model_fields",+ "model_computed_fields",+ "model_dump",+ "model_dump_json",+ "model_validate",+ "model_validate_json",+ "model_copy",+ "model_fields_set",+ "model_extra",+ # Internal metadata fields (with underscore prefix)+ "_stuff_name",+ "_content_class",+ "_concept_code",+ "_stuff_code",+ "_content",+ ]+)+class ConceptStructureBlueprintFieldType(StrEnum):TEXT = "text"diff --git a/pipelex/core/stuffs/stuff.py b/pipelex/core/stuffs/stuff.pyindex 588c148be..4d9ab5681 100644--- a/pipelex/core/stuffs/stuff.py+++ b/pipelex/core/stuffs/stuff.py@@ -12,7 +12,7 @@ from pipelex.core.stuffs.list_content import ListContentfrom pipelex.core.stuffs.mermaid_content import MermaidContentfrom pipelex.core.stuffs.number_content import NumberContentfrom pipelex.core.stuffs.pdf_content import PDFContent-from pipelex.core.stuffs.stuff_artefact import StuffArtefact+from pipelex.core.stuffs.stuff_artefact import BaseStuffArtefactField, StuffArtefactfrom pipelex.core.stuffs.stuff_content import StuffContent, StuffContentTypefrom pipelex.core.stuffs.text_and_images_content import TextAndImagesContentfrom pipelex.core.stuffs.text_content import TextContent@@ -43,11 +43,11 @@ Forbidden fields are: 'stuff_name', 'content_class', 'concept_code', 'stuff_coderaise StuffArtefactReservedFieldError(message=msg)artefact_dict[key] = value- set_artefact_field("stuff_name", self.stuff_name)- set_artefact_field("content_class", self.content.__class__.__name__)- set_artefact_field("concept_code", self.concept.code)- set_artefact_field("stuff_code", self.stuff_code)- set_artefact_field("content", self.content)+ set_artefact_field(BaseStuffArtefactField.STUFF_NAME, self.stuff_name)+ set_artefact_field(BaseStuffArtefactField.CONTENT_CLASS, self.content.__class__.__name__)+ set_artefact_field(BaseStuffArtefactField.CONCEPT_CODE, self.concept.code)+ set_artefact_field(BaseStuffArtefactField.STUFF_CODE, self.stuff_code)+ set_artefact_field(BaseStuffArtefactField.CONTENT, self.content)return StuffArtefact(artefact_dict)@classmethoddiff --git a/pipelex/core/stuffs/stuff_artefact.py b/pipelex/core/stuffs/stuff_artefact.pyindex 861f1fcdb..e381a6728 100644--- a/pipelex/core/stuffs/stuff_artefact.py+++ b/pipelex/core/stuffs/stuff_artefact.py@@ -8,6 +8,15 @@ from pipelex.cogt.templating.templating_style import TextFormatfrom pipelex.core.stuffs.exceptions import StuffArtefactErrorfrom pipelex.core.stuffs.stuff_content import StuffContentfrom pipelex.tools.jinja2.jinja2_models import Jinja2ContextKey, Jinja2TaggableAbstract+from pipelex.types import StrEnum+++class BaseStuffArtefactField(StrEnum):+ STUFF_NAME = "_stuff_name"+ CONTENT_CLASS = "_content_class"+ CONCEPT_CODE = "_concept_code"+ STUFF_CODE = "_stuff_code"+ CONTENT = "_content"class StuffArtefact(RootModel[dict[str, Any]], Jinja2TaggableAbstract):@@ -21,6 +30,28 @@ class StuffArtefact(RootModel[dict[str, Any]], Jinja2TaggableAbstract):e.g. {{ variable.field_name }} is equivalent to {{ variable['field_name'] }}"""+ @override+ def __getattribute__(self, key: str) -> Any:+ """Prioritize dict keys over methods for attribute access in Jinja2 templates.++ This allows templates to use {{ stuff.items }} to access the 'items' key+ instead of getting the dict.items() method.+ """+ # Allow access to special attributes, 'root', and model-related attributes+ if key.startswith("_") or key in ("root", "model_dump", "model_config"):+ return object.__getattribute__(self, key)++ # Check if it's a key in the root dict first+ try:+ root_dict = object.__getattribute__(self, "root")+ if key in root_dict:+ return root_dict[key]+ except AttributeError:+ pass++ # Fall back to normal attribute/method access+ return object.__getattribute__(self, key)+def __getitem__(self, key: str) -> Any:return self.root[key]@@ -43,7 +74,7 @@ class StuffArtefact(RootModel[dict[str, Any]], Jinja2TaggableAbstract):return self.root.items()def rendered_str(self, text_format: TextFormat) -> str:- content = self.root["content"]+ content = self.root[BaseStuffArtefactField.CONTENT]if not isinstance(content, StuffContent):msg = f"StuffArtefact has no StuffContent, content: {self}"raise StuffArtefactError(msg)@@ -55,6 +86,6 @@ class StuffArtefact(RootModel[dict[str, Any]], Jinja2TaggableAbstract):text_format = context.get(Jinja2ContextKey.TEXT_FORMAT, default=TextFormat.PLAIN)rendered_str = self.rendered_str(text_format=text_format)- tag_name = tag_name or self.get("stuff_name")+ tag_name = tag_name or self.get(BaseStuffArtefactField.STUFF_NAME)return rendered_str, tag_namediff --git a/pipelex/pipe_operators/compose/pipe_compose.py b/pipelex/pipe_operators/compose/pipe_compose.pyindex f3c37ed71..306eb487c 100644--- a/pipelex/pipe_operators/compose/pipe_compose.py+++ b/pipelex/pipe_operators/compose/pipe_compose.py@@ -67,10 +67,11 @@ class PipeCompose(PipeOperator[PipeComposeOutput]):}@override+ # TODO: this needs testing!!!def needed_inputs(self, visited_pipes: set[str] | None = None) -> InputRequirements:needed_inputs = InputRequirementsFactory.make_empty()for input_name, requirement in self.inputs.root.items():- needed_inputs.add_requirement(variable_name=input_name, concept=requirement.concept)+ needed_inputs.add_requirement(variable_name=input_name, concept=requirement.concept, multiplicity=requirement.multiplicity)return needed_inputs@overridediff --git a/pipelex/pipe_operators/llm/pipe_llm.py b/pipelex/pipe_operators/llm/pipe_llm.pyindex 4a1d23e14..fa75e71ab 100644--- a/pipelex/pipe_operators/llm/pipe_llm.py+++ b/pipelex/pipe_operators/llm/pipe_llm.py@@ -116,7 +116,7 @@ class PipeLLM(PipeOperator[PipeLLMOutput]):needed_inputs = InputRequirementsFactory.make_empty()for input_name, requirement in self.inputs.items:- needed_inputs.add_requirement(variable_name=input_name, concept=requirement.concept)+ needed_inputs.add_requirement(variable_name=input_name, concept=requirement.concept, multiplicity=requirement.multiplicity)return needed_inputs@overridediff --git a/pipelex/pipe_run/pipe_run_params.py b/pipelex/pipe_run/pipe_run_params.pyindex f0ce75719..4104b2e79 100644--- a/pipelex/pipe_run/pipe_run_params.py+++ b/pipelex/pipe_run/pipe_run_params.py@@ -15,6 +15,10 @@ class PipeRunParamKey(StrEnum):DYNAMIC_OUTPUT_CONCEPT = "_dynamic_output_concept"NB_OUTPUT = "_nb_output"+ @classmethod+ def value_list(cls) -> list[str]:+ return [member.value for member in cls]+FORCE_DRY_RUN_MODE_ENV_KEY = "PIPELEX_FORCE_DRY_RUN_MODE"diff --git a/pipelex/tools/jinja2/jinja2_rendering.py b/pipelex/tools/jinja2/jinja2_rendering.pyindex 5d984150f..a251a50cf 100644--- a/pipelex/tools/jinja2/jinja2_rendering.py+++ b/pipelex/tools/jinja2/jinja2_rendering.py@@ -68,4 +68,13 @@ async def render_jinja2(except Jinja2ContextError as context_error:msg = f"Jinja2 render — context error: '{context_error}', template_source:\n{template_source}"raise Jinja2TemplateRenderError(msg) from context_error+ except TypeError as type_error:+ context_vars = ", ".join(f"'{key}'" for key in temlating_context)+ msg = (+ f"Jinja2 render — type error: '{type_error}'. "+ f"This often happens when trying to iterate over a method or accessing an attribute incorrectly. "+ f"Template source:\n{template_source}\n"+ f"Available context variables: {context_vars}"+ )+ raise Jinja2TemplateRenderError(msg) from type_errorreturn generated_textdiff --git a/tests/e2e/pipelex/pipes/pipe_controller/pipe_sequence/discord_newsletter.plx b/tests/e2e/pipelex/pipes/pipe_controller/pipe_sequence/discord_newsletter.plxindex e5585be3c..9d6f429d0 100644--- a/tests/e2e/pipelex/pipes/pipe_controller/pipe_sequence/discord_newsletter.plx+++ b/tests/e2e/pipelex/pipes/pipe_controller/pipe_sequence/discord_newsletter.plx@@ -64,12 +64,12 @@ Make sure to preserve the channel name and position from the input for proper or[pipe.write_weekly_summary_e2e]type = "PipeLLM"description = "Combine channel summaries into a short summary of the week's Share channel content (200 characters)"-inputs = { channel_summaries = "ChannelSummaryE2E" }+inputs = { channel_summaries = "ChannelSummaryE2E[]" }output = "Text"prompt = """Write a single overall summary of the week's content based on the following Share channel summaries:-{% for channel in channel_summaries.content.items if channel.category == "Share" %}+{% for channel in channel_summaries.items if channel.category == "Share" %}{{ channel }}{% endfor %}@@ -79,7 +79,7 @@ Keep it short: 200 characters.[pipe.format_html_newsletter_e2e]type = "PipeCompose"description = "Combine weekly and channel summaries into a complete newsletter following specific formatting requirements"-inputs = { weekly_summary = "Text", channel_summaries = "ChannelSummaryE2E" }+inputs = { weekly_summary = "Text", channel_summaries = "ChannelSummaryE2E[]" }output = "HtmlNewsletterE2E"[pipe.format_html_newsletter_e2e.template]@@ -92,7 +92,7 @@ $weekly_summary</p><!-- New Members Section -->-{% set introduce_channels = channel_summaries.content.items | selectattr('category', 'equalto', 'Introduce Yourself') | list %}+{% set introduce_channels = channel_summaries.items | selectattr('category', 'equalto', 'Introduce Yourself') | list %}{% if introduce_channels %}<h2>🙌 New members</h2><ul>@@ -105,7 +105,7 @@ $weekly_summary{% endif %}<!-- Share Channel Section -->-{% set regular_channels = channel_summaries.content.items | selectattr('category', 'equalto', 'Share') | list | sort(attribute='position') %}+{% set regular_channels = channel_summaries.items | selectattr('category', 'equalto', 'Share') | list | sort(attribute='position') %}{% if regular_channels %}{% for channel in regular_channels %}<h2>{{ channel.channel_name }}</h2>@@ -116,7 +116,7 @@ $weekly_summary{% endif %}<!-- Geographic Hubs Section -->-{% set geo_hubs = channel_summaries.content.items | selectattr('category', 'equalto', 'Geographic Hubs') | list | sort(attribute='position') %}+{% set geo_hubs = channel_summaries.items | selectattr('category', 'equalto', 'Geographic Hubs') | list | sort(attribute='position') %}{% if geo_hubs %}<h2>🌎 Geographic hubs</h2>{% for channel in geo_hubs %}diff --git a/tests/e2e/pipelex/pipes/pipe_controller/pipe_sequence/discord_newsletter.py b/tests/e2e/pipelex/pipes/pipe_controller/pipe_sequence/discord_newsletter.pyindex 5bd123297..3cced69e3 100644--- a/tests/e2e/pipelex/pipes/pipe_controller/pipe_sequence/discord_newsletter.py+++ b/tests/e2e/pipelex/pipes/pipe_controller/pipe_sequence/discord_newsletter.py@@ -29,7 +29,7 @@ class DiscordMessage(StructuredContent):link: str = Field(..., description="Link to the message")-class DiscordChannelUpdate(StructuredContent):+class DiscordChannelUpdateE2E(StructuredContent):"""Represents a Discord channel with its messages"""name: str = Field(..., description="Name of the Discord channel")@@ -37,19 +37,21 @@ class DiscordChannelUpdate(StructuredContent):messages: list[DiscordMessage] = Field(default_factory=empty_list_factory_of(DiscordMessage), description="List of messages in the channel")-class ChannelSummary(StructuredContent):+class ChannelSummaryE2E(StructuredContent):"""Represents a summarized Discord channel for newsletter inclusion"""channel_name: str = Field(..., description="Name of the Discord channel")summary_items: list[str] = Field(..., description="Well-written summaries of the channel's activity")-class Newsletter(StructuredContent):+class NewsletterE2E(StructuredContent):"""Represents the final newsletter content"""weekly_summary: str = Field(..., description="200 character summary of weekly Share channel content")new_members: list[str] = Field(default_factory=empty_list_factory_of(str), description="New member introductions in bullet points")- channel_sections: list[ChannelSummary] = Field(default_factory=empty_list_factory_of(ChannelSummary), description="Ordered channel summaries")- geographic_hubs: list[ChannelSummary] = Field(- default_factory=empty_list_factory_of(ChannelSummary), description="Geographic hub channels grouped at end"+ channel_sections: list[ChannelSummaryE2E] = Field(+ default_factory=empty_list_factory_of(ChannelSummaryE2E), description="Ordered channel summaries"+ )+ geographic_hubs: list[ChannelSummaryE2E] = Field(+ default_factory=empty_list_factory_of(ChannelSummaryE2E), description="Geographic hub channels grouped at end")diff --git a/tests/integration/pipelex/pipes/controller/pipe_sequence/discord_newsletter.plx b/tests/integration/pipelex/pipes/controller/pipe_sequence/discord_newsletter.plxindex f912d9308..041302661 100644--- a/tests/integration/pipelex/pipes/controller/pipe_sequence/discord_newsletter.plx+++ b/tests/integration/pipelex/pipes/controller/pipe_sequence/discord_newsletter.plx@@ -64,14 +64,11 @@ Make sure to preserve the channel name and position from the input for proper or[pipe.write_weekly_summary]type = "PipeLLM"description = "Combine channel summaries into a short summary of the week's Share channel content (200 characters)"-inputs = { channel_summaries = "ChannelSummary" }+inputs = { channel_summaries = "ChannelSummary[]" }output = "Text"prompt = """Write a single overall summary of the week's content based on the following Share channel summaries:--{% for channel in channel_summaries.content.items if channel.category == "Share" %}-{{ channel }}-{% endfor %}+@channel_summariesKeep it short: 200 characters."""@@ -79,7 +76,7 @@ Keep it short: 200 characters.[pipe.format_html_newsletter]type = "PipeCompose"description = "Combine weekly and channel summaries into a complete newsletter following specific formatting requirements"-inputs = { weekly_summary = "Text", channel_summaries = "ChannelSummary" }+inputs = { weekly_summary = "Text", channel_summaries = "ChannelSummary[]" }output = "HtmlNewsletter"[pipe.format_html_newsletter.template]@@ -92,12 +89,12 @@ $weekly_summary</p><!-- New Members Section -->-{% set introduce_channels = channel_summaries.content.items | selectattr('category', 'equalto', 'Introduce Yourself') | list %}+{% set introduce_channels = channel_summaries.items | selectattr('channel_name', 'equalto', 'Introduce-Yourself') | list %}{% if introduce_channels %}<h2>🙌 New members</h2><ul>{% for channel in introduce_channels %}- {% for item in channel.summary_items_as_html %}+ {% for item in channel.summary_items %}<li>{{ item }}</li>{% endfor %}{% endfor %}@@ -105,23 +102,23 @@ $weekly_summary{% endif %}<!-- Share Channel Section -->-{% set regular_channels = channel_summaries.content.items | selectattr('category', 'equalto', 'Share') | list | sort(attribute='position') %}+{% set regular_channels = channel_summaries.items | rejectattr('channel_name', 'equalto', 'Introduce-Yourself') | list | sort(attribute='position') %}{% if regular_channels %}{% for channel in regular_channels %}<h2>{{ channel.channel_name }}</h2>- {% for item in channel.summary_items_as_html %}+ {% for item in channel.summary_items %}<p>{{ item }}</p>{% endfor %}{% endfor %}{% endif %}<!-- Geographic Hubs Section -->-{% set geo_hubs = channel_summaries.content.items | selectattr('category', 'equalto', 'Geographic Hubs') | list | sort(attribute='position') %}+{% set geo_hubs = [] %}{% if geo_hubs %}<h2>🌎 Geographic hubs</h2>{% for channel in geo_hubs %}<h3>{{ channel.channel_name }}</h3>- {% for item in channel.summary_items_as_html %}+ {% for item in channel.summary_items %}<p>{{ item }}</p>{% endfor %}{% endfor %}diff --git a/tests/integration/pipelex/pipes/controller/pipe_sequence/test_pipe_sequence_dry_run.py b/tests/integration/pipelex/pipes/controller/pipe_sequence/test_pipe_sequence_dry_run.pyindex d5cfab502..2cadd4715 100644--- a/tests/integration/pipelex/pipes/controller/pipe_sequence/test_pipe_sequence_dry_run.py+++ b/tests/integration/pipelex/pipes/controller/pipe_sequence/test_pipe_sequence_dry_run.py@@ -23,6 +23,8 @@ if TYPE_CHECKING:@pytest.mark.dry_runnable@pytest.mark.asyncio(loop_scope="class")class TestPipeSequenceDryRun:async def test_discord_newsletter_dry_run_working_memory(diff --git a/tests/integration/pipelex/pipes/test_flow_factory/discord_newsletter.plx b/tests/integration/pipelex/pipes/test_flow_factory/discord_newsletter.plxindex 89e5060aa..c4fa3d4d0 100644--- a/tests/integration/pipelex/pipes/test_flow_factory/discord_newsletter.plx+++ b/tests/integration/pipelex/pipes/test_flow_factory/discord_newsletter.plx@@ -64,12 +64,12 @@ Make sure to preserve the channel name and position from the input for proper or[pipe.write_weekly_summary_flow]type = "PipeLLM"description = "Combine channel summaries into a short summary of the week's Share channel content (200 characters)"-inputs = { channel_summaries = "ChannelSummaryFlow" }+inputs = { channel_summaries = "ChannelSummaryFlow[]" }output = "Text"prompt = """Write a single overall summary of the week's content based on the following Share channel summaries:-{% for channel in channel_summaries.content.items if channel.category == "Share" %}+{% for channel in channel_summaries.items if channel.category == "Share" %}{{ channel }}{% endfor %}@@ -79,7 +79,7 @@ Keep it short: 200 characters.[pipe.format_html_newsletter_flow]type = "PipeCompose"description = "Combine weekly and channel summaries into a complete newsletter following specific formatting requirements"-inputs = { weekly_summary = "Text", channel_summaries = "ChannelSummaryFlow" }+inputs = { weekly_summary = "Text", channel_summaries = "ChannelSummaryFlow[]" }output = "HtmlNewsletterFlow"[pipe.format_html_newsletter_flow.template]@@ -92,7 +92,7 @@ $weekly_summary</p><!-- New Members Section -->-{% set introduce_channels = channel_summaries.content.items | selectattr('category', 'equalto', 'Introduce Yourself') | list %}+{% set introduce_channels = channel_summaries.items | selectattr('category', 'equalto', 'Introduce Yourself') | list %}{% if introduce_channels %}<h2>🙌 New members</h2><ul>@@ -105,7 +105,7 @@ $weekly_summary{% endif %}<!-- Share Channel Section -->-{% set regular_channels = channel_summaries.content.items | selectattr('category', 'equalto', 'Share') | list | sort(attribute='position') %}+{% set regular_channels = channel_summaries.items | selectattr('category', 'equalto', 'Share') | list | sort(attribute='position') %}{% if regular_channels %}{% for channel in regular_channels %}<h2>{{ channel.channel_name }}</h2>@@ -116,7 +116,7 @@ $weekly_summary{% endif %}<!-- Geographic Hubs Section -->-{% set geo_hubs = channel_summaries.content.items | selectattr('category', 'equalto', 'Geographic Hubs') | list | sort(attribute='position') %}+{% set geo_hubs = channel_summaries.items | selectattr('category', 'equalto', 'Geographic Hubs') | list | sort(attribute='position') %}{% if geo_hubs %}<h2>🌎 Geographic hubs</h2>{% for channel in geo_hubs %}diff --git a/tests/integration/pipelex/pipes/test_flow_factory/discord_newsletter.py b/tests/integration/pipelex/pipes/test_flow_factory/discord_newsletter.pyindex 5bd123297..c11ac4010 100644--- a/tests/integration/pipelex/pipes/test_flow_factory/discord_newsletter.py+++ b/tests/integration/pipelex/pipes/test_flow_factory/discord_newsletter.py@@ -29,7 +29,7 @@ class DiscordMessage(StructuredContent):link: str = Field(..., description="Link to the message")-class DiscordChannelUpdate(StructuredContent):+class DiscordChannelUpdateFlow(StructuredContent):"""Represents a Discord channel with its messages"""name: str = Field(..., description="Name of the Discord channel")@@ -37,19 +37,21 @@ class DiscordChannelUpdate(StructuredContent):messages: list[DiscordMessage] = Field(default_factory=empty_list_factory_of(DiscordMessage), description="List of messages in the channel")-class ChannelSummary(StructuredContent):+class ChannelSummaryFlow(StructuredContent):"""Represents a summarized Discord channel for newsletter inclusion"""channel_name: str = Field(..., description="Name of the Discord channel")summary_items: list[str] = Field(..., description="Well-written summaries of the channel's activity")-class Newsletter(StructuredContent):+class HtmlNewsletterFlow(StructuredContent):"""Represents the final newsletter content"""weekly_summary: str = Field(..., description="200 character summary of weekly Share channel content")new_members: list[str] = Field(default_factory=empty_list_factory_of(str), description="New member introductions in bullet points")- channel_sections: list[ChannelSummary] = Field(default_factory=empty_list_factory_of(ChannelSummary), description="Ordered channel summaries")- geographic_hubs: list[ChannelSummary] = Field(- default_factory=empty_list_factory_of(ChannelSummary), description="Geographic hub channels grouped at end"+ channel_sections: list[ChannelSummaryFlow] = Field(+ default_factory=empty_list_factory_of(ChannelSummaryFlow), description="Ordered channel summaries"+ )+ geographic_hubs: list[ChannelSummaryFlow] = Field(+ default_factory=empty_list_factory_of(ChannelSummaryFlow), description="Geographic hub channels grouped at end")diff --git a/tests/unit/pipelex/core/concepts/test_special_args_concept.py b/tests/unit/pipelex/core/concepts/test_special_args_concept.pynew file mode 100644index 000000000..1236579e5--- /dev/null+++ b/tests/unit/pipelex/core/concepts/test_special_args_concept.py@@ -0,0 +1,108 @@+from typing import Callable++import pytest++from pipelex import pretty_print+from pipelex.core.concepts.concept_blueprint import ConceptBlueprint+from pipelex.core.concepts.concept_factory import ConceptFactory+from pipelex.core.concepts.concept_structure_blueprint import (+ ConceptStructureBlueprint,+ ConceptStructureBlueprintFieldType,+)+from pipelex.core.memory.working_memory_factory import WorkingMemoryFactory+from pipelex.core.pipes.pipe_factory import PipeFactory+from pipelex.core.stuffs.stuff_factory import StuffContentFactory, StuffFactory+from pipelex.hub import get_concept_library, get_pipe_router+from pipelex.pipe_operators.llm.pipe_llm import PipeLLM+from pipelex.pipe_operators.llm.pipe_llm_blueprint import PipeLLMBlueprint+from pipelex.pipe_run.pipe_job_factory import PipeJobFactory+from pipelex.pipe_run.pipe_run_params import PipeRunMode+from pipelex.pipe_run.pipe_run_params_factory import PipeRunParamsFactory++[email protected]_runnable[email protected](loop_scope="class")+class TestSpecialArgsConcept:+ async def test_create_concept_with_content_field(+ self,+ pipe_run_mode: PipeRunMode,+ load_empty_library: Callable[[], None],+ ):+ """Test creating a concept with a 'content' field and running a pipe that uses it."""+ load_empty_library()+ domain = "test"+ concept_library = get_concept_library()++ # Create a ConceptBlueprint with a structure containing "content"+ concept_blueprint = ConceptBlueprint(+ description="A concept with content field",+ structure={+ "content": ConceptStructureBlueprint(+ description="The main content",+ type=ConceptStructureBlueprintFieldType.TEXT,+ ),+ },+ )++ # Make a Concept using ConceptFactory+ concept = ConceptFactory.make_from_blueprint(+ domain=domain,+ concept_code="ContentConcept",+ blueprint=concept_blueprint,+ )++ concept_library.add_new_concept(concept)+ # Verify the concept was created+ assert concept.code == "ContentConcept"+ assert concept.domain == domain+ assert concept.description == "A concept with content field"++ # Create a PipeLLM that uses the content field+ pipe_llm_blueprint = PipeLLMBlueprint(+ description="A pipe that reads the content",+ inputs={"my_input": "ContentConcept"},+ output="native.Text",+ prompt="read the content, @my_input.content",+ )++ # Make a PipeLLM using PipeFactory+ pipe_llm = PipeFactory[PipeLLM].make_from_blueprint(+ domain_code=domain,+ pipe_code="test_pipe",+ blueprint=pipe_llm_blueprint,+ concept_codes_from_the_same_domain=[concept.code],+ )++ # Verify the pipe was created+ assert pipe_llm.code == "test_pipe"+ assert pipe_llm.domain == domain++ # Create content for the concept+ content = StuffContentFactory.make_stuff_content_from_concept_required(+ concept=concept,+ value={"content": "This is my content to read"},+ )++ # Create a stuff with the content+ stuff = StuffFactory.make_stuff(+ concept=concept,+ content=content,+ name="my_input",+ )++ # Create working memory with the stuff+ working_memory = WorkingMemoryFactory.make_from_single_stuff(stuff=stuff)++ # Create and run the pipe job+ pipe_job = PipeJobFactory.make_pipe_job(+ pipe=pipe_llm,+ pipe_run_params=PipeRunParamsFactory.make_run_params(pipe_run_mode=pipe_run_mode),+ working_memory=working_memory,+ )++ # Run the pipe+ output = await get_pipe_router().run(pipe_job=pipe_job)++ # Verify output was generated+ pretty_print(output, title="Pipe output")+ assert output is not Nonediff --git a/tests/unit/pipelex/core/memory/test_working_memory_generate_context.py b/tests/unit/pipelex/core/memory/test_working_memory_generate_context.pyindex 4a895b892..3f70754c1 100644--- a/tests/unit/pipelex/core/memory/test_working_memory_generate_context.py+++ b/tests/unit/pipelex/core/memory/test_working_memory_generate_context.py@@ -4,7 +4,7 @@ from pipelex.core.stuffs.html_content import HtmlContentfrom pipelex.core.stuffs.image_content import ImageContentfrom pipelex.core.stuffs.list_content import ListContentfrom pipelex.core.stuffs.number_content import NumberContent-from pipelex.core.stuffs.stuff_artefact import StuffArtefact+from pipelex.core.stuffs.stuff_artefact import BaseStuffArtefactField, StuffArtefactfrom pipelex.core.stuffs.text_and_images_content import TextAndImagesContentfrom pipelex.core.stuffs.text_content import TextContentfrom tests.unit.pipelex.core.memory.conftest import TestWorkingMemoryData@@ -33,11 +33,11 @@ class TestWorkingMemoryGenerateContext:# Verify artefact structureartefact = context["sample_text"]assert isinstance(artefact, StuffArtefact)- assert "content" in artefact- assert isinstance(artefact["content"], TextContent)+ assert BaseStuffArtefactField.CONTENT in artefact+ assert isinstance(artefact[BaseStuffArtefactField.CONTENT], TextContent)# Verify actual content value- assert artefact["content"].text == TestWorkingMemoryData.SAMPLE_TEXT+ assert artefact[BaseStuffArtefactField.CONTENT].text == TestWorkingMemoryData.SAMPLE_TEXT# Verify MAIN_STUFF_NAME points to the same artefactassert context[MAIN_STUFF_NAME] is context["sample_text"]@@ -54,10 +54,10 @@ class TestWorkingMemoryGenerateContext:# Verify artefact structureartefact = context["pdf_document"]assert isinstance(artefact, StuffArtefact)- assert "content" in artefact+ assert BaseStuffArtefactField.CONTENT in artefact# Verify actual content value- assert artefact["content"].url == TestWorkingMemoryData.SAMPLE_PDF_URL+ assert artefact[BaseStuffArtefactField.CONTENT].url == TestWorkingMemoryData.SAMPLE_PDF_URL# Verify MAIN_STUFF_NAME points to the same artefactassert context[MAIN_STUFF_NAME] is context["pdf_document"]@@ -74,11 +74,11 @@ class TestWorkingMemoryGenerateContext:# Verify artefact structureartefact = context["sample_image"]assert isinstance(artefact, StuffArtefact)- assert "content" in artefact- assert isinstance(artefact["content"], ImageContent)+ assert BaseStuffArtefactField.CONTENT in artefact+ assert isinstance(artefact[BaseStuffArtefactField.CONTENT], ImageContent)# Verify actual content value- assert artefact["content"].url == TestWorkingMemoryData.SAMPLE_IMAGE_URL+ assert artefact[BaseStuffArtefactField.CONTENT].url == TestWorkingMemoryData.SAMPLE_IMAGE_URL# Verify MAIN_STUFF_NAME points to the same artefactassert context[MAIN_STUFF_NAME] is context["sample_image"]@@ -99,17 +99,17 @@ class TestWorkingMemoryGenerateContext:assert isinstance(context["question"], StuffArtefact)assert isinstance(context["document"], StuffArtefact)assert isinstance(context["diagram"], StuffArtefact)- assert "content" in context["question"]- assert "content" in context["document"]- assert "content" in context["diagram"]+ assert BaseStuffArtefactField.CONTENT in context["question"]+ assert BaseStuffArtefactField.CONTENT in context["document"]+ assert BaseStuffArtefactField.CONTENT in context["diagram"]# Verify actual content values- assert isinstance(context["question"]["content"], TextContent)- assert context["question"]["content"].text == "What are the aerodynamic features?"- assert isinstance(context["document"]["content"], TextContent)- assert context["document"]["content"].text == TestWorkingMemoryData.SAMPLE_TEXT- assert isinstance(context["diagram"]["content"], ImageContent)- assert context["diagram"]["content"].url == TestWorkingMemoryData.SAMPLE_IMAGE_URL+ assert isinstance(context["question"][BaseStuffArtefactField.CONTENT], TextContent)+ assert context["question"][BaseStuffArtefactField.CONTENT].text == "What are the aerodynamic features?"+ assert isinstance(context["document"][BaseStuffArtefactField.CONTENT], TextContent)+ assert context["document"][BaseStuffArtefactField.CONTENT].text == TestWorkingMemoryData.SAMPLE_TEXT+ assert isinstance(context["diagram"][BaseStuffArtefactField.CONTENT], ImageContent)+ assert context["diagram"][BaseStuffArtefactField.CONTENT].url == TestWorkingMemoryData.SAMPLE_IMAGE_URL# Verify MAIN_STUFF_NAME points to document (main_name="document" in fixture)assert context[MAIN_STUFF_NAME] is context["document"]@@ -128,10 +128,10 @@ class TestWorkingMemoryGenerateContext:assert MAIN_STUFF_NAME in context# Verify actual content values- assert isinstance(context["primary_text"]["content"], TextContent)- assert context["primary_text"]["content"].text == "Primary content"- assert isinstance(context["secondary_text"]["content"], TextContent)- assert context["secondary_text"]["content"].text == "Secondary content"+ assert isinstance(context["primary_text"][BaseStuffArtefactField.CONTENT], TextContent)+ assert context["primary_text"][BaseStuffArtefactField.CONTENT].text == "Primary content"+ assert isinstance(context["secondary_text"][BaseStuffArtefactField.CONTENT], TextContent)+ assert context["secondary_text"][BaseStuffArtefactField.CONTENT].text == "Secondary content"# Verify aliases point to the same artefact objects (using 'is')assert context["main_text"] is context["primary_text"]@@ -150,11 +150,11 @@ class TestWorkingMemoryGenerateContext:# Verify artefact structureartefact = context["mixed_list"]assert isinstance(artefact, StuffArtefact)- assert "content" in artefact- assert isinstance(artefact["content"], ListContent)+ assert BaseStuffArtefactField.CONTENT in artefact+ assert isinstance(artefact[BaseStuffArtefactField.CONTENT], ListContent)# Verify actual list content values- list_content = artefact["content"]+ list_content = artefact[BaseStuffArtefactField.CONTENT]assert len(list_content.items) == 3assert isinstance(list_content.items[0], TextContent)assert list_content.items[0].text == "The quick brown fox jumps over the lazy dog"@@ -175,11 +175,11 @@ class TestWorkingMemoryGenerateContext:# Verify artefact structureartefact = context["project_overview"]assert isinstance(artefact, StuffArtefact)- assert "content" in artefact- assert isinstance(artefact["content"], TextAndImagesContent)+ assert BaseStuffArtefactField.CONTENT in artefact+ assert isinstance(artefact[BaseStuffArtefactField.CONTENT], TextAndImagesContent)# Verify actual content values- content = artefact["content"]+ content = artefact[BaseStuffArtefactField.CONTENT]assert content.text is not Noneassert content.text.text == "Project overview with diagrams"assert content.images is not None@@ -199,11 +199,11 @@ class TestWorkingMemoryGenerateContext:# Verify artefact structureartefact = context["test_report"]assert isinstance(artefact, StuffArtefact)- assert "content" in artefact- assert isinstance(artefact["content"], HtmlContent)+ assert BaseStuffArtefactField.CONTENT in artefact+ assert isinstance(artefact[BaseStuffArtefactField.CONTENT], HtmlContent)# Verify actual content values- content = artefact["content"]+ content = artefact[BaseStuffArtefactField.CONTENT]assert content.inner_html == "<h1>Test Report</h1><p>This is a <strong>test</strong> report.</p><ul><li>Item 1</li><li>Item 2</li></ul>"assert content.css_class == "report-content"@@ -219,11 +219,11 @@ class TestWorkingMemoryGenerateContext:# Verify artefact structureartefact = context["pi_value"]assert isinstance(artefact, StuffArtefact)- assert "content" in artefact- assert isinstance(artefact["content"], NumberContent)+ assert BaseStuffArtefactField.CONTENT in artefact+ assert isinstance(artefact[BaseStuffArtefactField.CONTENT], NumberContent)# Verify actual content value- assert artefact["content"].number == 3.14159+ assert artefact[BaseStuffArtefactField.CONTENT].number == 3.14159def test_generate_context_artefact_independence(self, multiple_stuff_memory: WorkingMemory):"""Test that each stuff generates independent artefacts."""diff --git a/tests/unit/pipelex/core/stuffs/test_special_args.py b/tests/unit/pipelex/core/stuffs/test_special_args.pynew file mode 100644index 000000000..bd3b6339d--- /dev/null+++ b/tests/unit/pipelex/core/stuffs/test_special_args.py@@ -0,0 +1,120 @@+"""Test stuff with special args like 'content' that should work without conflicts."""++from typing import Callable, ClassVar++import pytest++from pipelex.core.concepts.concept_blueprint import ConceptBlueprint+from pipelex.core.concepts.concept_factory import ConceptFactory+from pipelex.core.concepts.concept_structure_blueprint import (+ RESERVED_FIELD_NAMES,+ ConceptStructureBlueprint,+ ConceptStructureBlueprintFieldType,+)+from pipelex.core.stuffs.stuff_factory import StuffContentFactory, StuffFactory+from pipelex.hub import get_concept_library+++class TestSpecialArgsStuff:+ """Test stuff with field names that should not conflict with internal fields."""++ # Field names that users should be able to use+ # These look like they might conflict but actually work fine with our underscore-prefix approach+ ALLOWED_FIELD_NAMES: ClassVar[list[str]] = [+ # Stuff class fields - these are OK because artefact uses underscore-prefixed versions+ "stuff_code",+ "stuff_name",+ "concept",+ "content",+ ]++ # Examples of field names that should fail validation+ # Includes the actual RESERVED_FIELD_NAMES plus additional examples for testing+ # Sorted to ensure consistent ordering across pytest-xdist workers+ FAILING_FIELD_NAMES: ClassVar[list[str]] = sorted(+ [+ *RESERVED_FIELD_NAMES,+ # Generic underscore-prefixed names (additional failing examples)+ "_internal",+ "_private",+ "_reserved",+ ]+ )++ @pytest.mark.parametrize("field_name", ALLOWED_FIELD_NAMES)+ def test_make_artefact_with_allowed_field_names(+ self,+ field_name: str,+ load_empty_library: Callable[[], None],+ ):+ """Test that creating an artefact works with field names that look like they might conflict.++ Users should be able to use field names like 'content', 'stuff_name', etc.+ These don't conflict because internal metadata fields use underscore prefixes.+ """+ load_empty_library()+ domain = "test"+ concept_library = get_concept_library()++ # Create a ConceptBlueprint with a structure containing the field name+ concept_blueprint = ConceptBlueprint(+ description=f"A concept with {field_name} field",+ structure={+ field_name: ConceptStructureBlueprint(+ description=f"The {field_name} field",+ type=ConceptStructureBlueprintFieldType.TEXT,+ ),+ },+ )++ # Make a Concept using ConceptFactory+ concept = ConceptFactory.make_from_blueprint(+ domain=domain,+ concept_code=f"Concept{field_name.replace('_', '').title()}",+ blueprint=concept_blueprint,+ )++ concept_library.add_new_concept(concept)++ # Create content for the concept+ content = StuffContentFactory.make_stuff_content_from_concept_required(+ concept=concept,+ value={field_name: f"This is my {field_name}"},+ )++ # Create a stuff with the content+ stuff = StuffFactory.make_stuff(+ concept=concept,+ content=content,+ name="my_stuff",+ )++ # Calling make_artefact should work without errors+ stuff.make_artefact()++ @pytest.mark.parametrize("field_name", FAILING_FIELD_NAMES)+ def test_reserved_field_names_raise_error(+ self,+ field_name: str,+ load_empty_library: Callable[[], None],+ ):+ """Test that reserved field names raise a validation error.++ The following should NOT be allowed as user field names:+ - Pydantic BaseModel reserved attributes (model_config, model_fields, etc.)+ - Internal underscore-prefixed fields (_stuff_name, _content_class, etc.)+ - Any field name starting with underscore (reserved for internal Pipelex use)+ """+ load_empty_library()++ # Creating a ConceptBlueprint with a reserved field name in structure should raise ValueError+ with pytest.raises(ValueError, match=r"(reserved field|underscore)"):+ ConceptBlueprint(+ description=f"A concept with {field_name} field",+ structure={+ field_name: ConceptStructureBlueprint(+ description=f"The {field_name} field",+ type=ConceptStructureBlueprintFieldType.TEXT,+ ),+ },+ )