Build AI methods with your coding agent, then run them anywhere: as an MCP for chatbots, as a webapp for people, or via API for your software.
Fix/output multiplicity pipe sequence (#530)
Thomas Hebrarda770aa141fbe
diff --git a/.vscode/launch.json b/.vscode/launch.jsonindex afb364211..1ad5317d4 100644--- a/.vscode/launch.json+++ b/.vscode/launch.json@@ -65,7 +65,7 @@"justMyCode": false},{- "name": "Debug run pipe",+ "name": "Debug run pipe bundle","type": "debugpy","request": "launch","program": "${workspaceFolder}/.venv/bin/pipelex",@@ -77,6 +77,20 @@],"console": "integratedTerminal","justMyCode": false+ },+ {+ "name": "Debug run pipe code",+ "type": "debugpy",+ "request": "launch",+ "program": "${workspaceFolder}/.venv/bin/pipelex",+ "args": [+ "run",+ "write_discord_newsletter",+ "--inputs",+ "tests/data/discord_newsletter/discord_sample_input.json",+ ],+ "console": "integratedTerminal",+ "justMyCode": false}]}\ No newline at end of filediff --git a/pipelex/core/concepts/concept_factory.py b/pipelex/core/concepts/concept_factory.pyindex 742d83420..521c9e1af 100644--- a/pipelex/core/concepts/concept_factory.py+++ b/pipelex/core/concepts/concept_factory.py@@ -389,7 +389,7 @@ class ConceptFactory:match declaration_type:case ConceptDeclarationType.STRING:- structure_class_name, refines = cls._handle_basic_blueprint(+ structure_class_name, _ = cls._handle_basic_blueprint(concept_code=concept_code,domain=domain,)@@ -398,7 +398,6 @@ class ConceptFactory:code=domain_and_concept_code.concept_code,description=cast("str", blueprint_or_string_description),structure_class_name=structure_class_name,- refines=refines,)case ConceptDeclarationType.BASIC_BLUEPRINT:diff --git a/pipelex/core/pipes/variable_multiplicity.py b/pipelex/core/pipes/variable_multiplicity.pyindex ba466e93a..57b0989ad 100644--- a/pipelex/core/pipes/variable_multiplicity.py+++ b/pipelex/core/pipes/variable_multiplicity.py@@ -113,6 +113,63 @@ def parse_concept_with_multiplicity(concept_spec: str) -> MultiplicityParseResulreturn MultiplicityParseResult(concept=concept, multiplicity=multiplicity)+def is_multiplicity_compatible(source_multiplicity: VariableMultiplicity | None, target_multiplicity: VariableMultiplicity | None) -> bool:+ """Check if a source multiplicity is compatible with a target multiplicity.++ This is used to validate that a pipe's output multiplicity can fulfill a required output multiplicity.+ For example, when validating a PipeSequence, the last step's output multiplicity (source) must be+ compatible with the sequence's declared output multiplicity (target).++ Compatibility rules:+ - If target is None (single item), source must also be None+ - If target is True (variable list), source can be True OR any positive integer+ (a fixed count is compatible with a variable-length expectation)+ - If target is an integer N (fixed count), source must be exactly N++ Args:+ source_multiplicity: The actual multiplicity provided (e.g., from a sub-pipe's output)+ target_multiplicity: The required/expected multiplicity (e.g., declared on a sequence)++ Returns:+ True if source_multiplicity can fulfill target_multiplicity, False otherwise++ Examples:+ >>> is_multiplicity_compatible(None, None)+ True+ >>> is_multiplicity_compatible(True, True)+ True+ >>> is_multiplicity_compatible(3, True) # Fixed count fulfills variable expectation+ True+ >>> is_multiplicity_compatible(True, 3) # Variable cannot fulfill fixed expectation+ False+ >>> is_multiplicity_compatible(3, 3)+ True+ >>> is_multiplicity_compatible(3, 5) # Different fixed counts are incompatible+ False+ >>> is_multiplicity_compatible(None, True) # Single cannot fulfill list expectation+ False+ """+ # Case 1: Target expects single item (None)+ if target_multiplicity is None:+ return source_multiplicity is None++ # Case 2: Target expects variable-length list (True)+ if target_multiplicity is True:+ # Accept True (variable) or any integer (fixed count)+ # Both represent "multiple items", just with different specificity+ # Note: We must explicitly check for bool first because bool is a subclass of int in Python+ # isinstance(False, int) returns True, which would incorrectly match False as a valid multiplicity+ return source_multiplicity is True or (isinstance(source_multiplicity, int) and not isinstance(source_multiplicity, bool))++ # Case 3: Target expects fixed count (integer)+ # Source must match exactly, but must not be a boolean+ # Note: We must explicitly check for bool first because bool is a subclass of int in Python+ # True == 1 evaluates to True, which would incorrectly match True (variable list) as compatible with 1 (fixed count)+ if isinstance(source_multiplicity, bool):+ return False+ return source_multiplicity == target_multiplicity++def format_concept_with_multiplicity(concept_code_or_string: str, multiplicity: VariableMultiplicity | None) -> str:"""Format a concept code or string with multiplicity notation.diff --git a/pipelex/core/stuffs/text_content.py b/pipelex/core/stuffs/text_content.pyindex 42ddf34c7..dfbef2c25 100644--- a/pipelex/core/stuffs/text_content.py+++ b/pipelex/core/stuffs/text_content.py@@ -2,11 +2,12 @@ import jsonfrom typing import Anyimport markdown+from rich.markdown import Markdownfrom typing_extensions import override-from pipelex import pretty_print_mdfrom pipelex.core.stuffs.stuff_content import StuffContentfrom pipelex.tools.misc.file_utils import ensure_directory_exists, save_text_to_path+from pipelex.tools.misc.pretty import PrettyPrintableclass TextContent(StuffContent):@@ -42,11 +43,11 @@ class TextContent(StuffContent):def rendered_json(self) -> str:return json.dumps({"text": self.text})+ @override+ def rendered_pretty(self, title: str | None = None, depth: int = 0) -> PrettyPrintable:+ return Markdown(self.text)+def save_to_directory(self, directory: str):ensure_directory_exists(directory)filename = "text_content.txt"save_text_to_path(text=self.text, path=f"{directory}/{filename}")-- @override- def pretty_print_content(self, title: str | None = None) -> None:- pretty_print_md(self.text, title=title)diff --git a/pipelex/pipe_controllers/batch/pipe_batch.py b/pipelex/pipe_controllers/batch/pipe_batch.pyindex 8e4fe226f..4a3cbd0fb 100644--- a/pipelex/pipe_controllers/batch/pipe_batch.py+++ b/pipelex/pipe_controllers/batch/pipe_batch.py@@ -143,7 +143,17 @@ class PipeBatch(PipeController):required_stuffs = branch_memory.get_existing_stuffs(names=required_variables)required_stuffs = [required_stuff for required_stuff in required_stuffs if required_stuff.stuff_code != input_stuff.stuff_code]required_stuff_lists.append(required_stuffs)- branch_pipe_run_params = pipe_run_params.deep_copy_with_final_stuff_code(final_stuff_code=branch_output_item_code)+ # We create a deep copy of the run params to avoid modifying the original run params,+ # and we set the final stuff code to use the one provided fro the branch pipe.+ # Note: the batching will yield a list by aggregating the outputs of each run of the branch pipe,+ # but each run of the branch pipe will only yield one output, so we set the output multiplicity to False.+ branch_pipe_run_params = pipe_run_params.model_copy(+ deep=True,+ update={+ "final_stuff_code": branch_output_item_code,+ "output_multiplicity": False,+ },+ )task: Coroutine[Any, Any, PipeOutput]if pipe_run_params.run_mode == PipeRunMode.DRY:diff --git a/pipelex/pipe_controllers/sequence/pipe_sequence.py b/pipelex/pipe_controllers/sequence/pipe_sequence.pyindex 6e23ed69f..03119325d 100644--- a/pipelex/pipe_controllers/sequence/pipe_sequence.py+++ b/pipelex/pipe_controllers/sequence/pipe_sequence.py@@ -9,6 +9,7 @@ from pipelex.core.pipes.inputs.exceptions import InputStuffSpecNotFoundErrorfrom pipelex.core.pipes.inputs.input_stuff_specs import InputStuffSpecsfrom pipelex.core.pipes.inputs.input_stuff_specs_factory import InputStuffSpecsFactoryfrom pipelex.core.pipes.pipe_output import PipeOutput+from pipelex.core.pipes.variable_multiplicity import is_multiplicity_compatiblefrom pipelex.hub import get_concept_library, get_required_pipefrom pipelex.pipe_controllers.exceptions import PipeControllerOutputConceptMismatchErrorfrom pipelex.pipe_controllers.parallel.pipe_parallel import PipeParallel@@ -55,15 +56,13 @@ class PipeSequence(PipeController):The output of the pipe sequence should match the output of the last step,both in terms of concept compatibility and multiplicity."""- last_step_pipe_code = self.sequential_sub_pipes[-1].pipe_code- last_step_pipe = get_required_pipe(pipe_code=last_step_pipe_code)- last_step_stuff_spec = last_step_pipe.output+ last_step_pipe = get_required_pipe(pipe_code=self.sequential_sub_pipes[-1].pipe_code)# Check concept compatibility- if not get_concept_library().is_compatible(tested_concept=last_step_stuff_spec.concept, wanted_concept=self.output.concept):+ if not get_concept_library().is_compatible(tested_concept=last_step_pipe.output.concept, wanted_concept=self.output.concept):msg = (- f"PipeSequence concept mismatch: the output concept '{last_step_stuff_spec.concept.concept_string}' "- f"of the last step '{last_step_pipe_code}' of sequence pipe '{self.code}' "+ f"PipeSequence concept mismatch: the output concept '{last_step_pipe.output.concept.concept_string}' "+ f"of the last step '{last_step_pipe.code}' of sequence pipe '{self.code}' "f"is not compatible with the output concept '{self.output.concept.concept_string}' of the sequence.")raise PipeValidationError(@@ -71,23 +70,31 @@ class PipeSequence(PipeController):error_type=PipeValidationErrorType.INADEQUATE_OUTPUT_CONCEPT,domain=self.domain,pipe_code=self.code,- provided_concept_code=last_step_stuff_spec.concept.concept_string,+ provided_concept_code=last_step_pipe.output.concept.concept_string,required_concept_codes=[self.output.concept.concept_string],)- # Check multiplicity match- if self.output.multiplicity != last_step_stuff_spec.multiplicity:+ last_sub_pipe = self.sequential_sub_pipes[-1]+ effective_last_step_output_multiplicity = last_sub_pipe.output_multiplicity+ if not effective_last_step_output_multiplicity:+ effective_last_step_output_multiplicity = get_required_pipe(pipe_code=last_sub_pipe.pipe_code).output.multiplicity++ # Check multiplicity compatibility+ if not is_multiplicity_compatible(+ source_multiplicity=effective_last_step_output_multiplicity,+ target_multiplicity=self.output.multiplicity,+ ):msg = (f"PipeSequence output multiplicity mismatch: the sequence '{self.code}' declares "- f"output multiplicity={self.output.multiplicity}, but the last step '{last_step_pipe_code}' "- f"has output multiplicity={last_step_stuff_spec.multiplicity}. They must be the same."+ f"output multiplicity={self.output.multiplicity}, but the last step '{last_step_pipe.code}' "+ f"has output multiplicity={effective_last_step_output_multiplicity}. They are not compatible.")raise PipeValidationError(message=msg,error_type=PipeValidationErrorType.INADEQUATE_OUTPUT_MULTIPLICITY,domain=self.domain,pipe_code=self.code,- provided_concept_code=last_step_stuff_spec.concept.concept_string,+ provided_concept_code=last_step_pipe.output.concept.concept_string,required_concept_codes=[self.output.concept.concept_string],)diff --git a/pipelex/pipe_controllers/sequence/pipe_sequence_factory.py b/pipelex/pipe_controllers/sequence/pipe_sequence_factory.pyindex d9f2f1c76..006823071 100644--- a/pipelex/pipe_controllers/sequence/pipe_sequence_factory.py+++ b/pipelex/pipe_controllers/sequence/pipe_sequence_factory.py@@ -28,6 +28,7 @@ class PipeSequenceFactory(PipeFactoryProtocol[PipeSequenceBlueprint, PipeSequencblueprint: PipeSequenceBlueprint,) -> PipeSequence:sequential_sub_pipes: list[SubPipe] = []+for step in blueprint.steps:sub_pipe = SubPipeFactory.make_from_blueprint(blueprint=step)sequential_sub_pipes.append(sub_pipe)diff --git a/pipelex/pipe_controllers/sub_pipe.py b/pipelex/pipe_controllers/sub_pipe.pyindex 2f769bf16..5164a7add 100644--- a/pipelex/pipe_controllers/sub_pipe.py+++ b/pipelex/pipe_controllers/sub_pipe.py@@ -32,12 +32,12 @@ class SubPipe(BaseModel):"""Run or dry run a single operation self."""if self.output_multiplicity:sub_pipe_run_params.output_multiplicity = self.output_multiplicity- sub_pipe_run_params.batch_params = self.batch_paramssub_pipe = get_required_pipe(pipe_code=self.pipe_code)# Case 1: Batch processingif batch_params := self.batch_params:+ sub_pipe_run_params.batch_params = batch_paramstry:working_memory.get_typed_object_or_attribute(name=batch_params.input_list_stuff_name, wanted_type=ListContent)except WorkingMemoryStuffNotFoundError as exc:@@ -80,9 +80,10 @@ class SubPipe(BaseModel):},)+ pipe_batch_adhoc_pipe_code = f"{self.pipe_code}_adhoc_batch"pipe_batch = PipeFactory[PipeBatch].make_from_blueprint(domain_code=sub_pipe.domain,- pipe_code=self.pipe_code,+ pipe_code=pipe_batch_adhoc_pipe_code,blueprint=pipe_batch_blueprint,concept_codes_from_the_same_domain=[concept.code for concept in sub_pipe.concept_dependencies],)diff --git a/pipelex/pipe_controllers/sub_pipe_factory.py b/pipelex/pipe_controllers/sub_pipe_factory.pyindex a58f11606..6ec70b66e 100644--- a/pipelex/pipe_controllers/sub_pipe_factory.py+++ b/pipelex/pipe_controllers/sub_pipe_factory.py@@ -1,8 +1,12 @@-from pipelex.core.pipes.variable_multiplicity import make_variable_multiplicity+from typing import TYPE_CHECKING+from pipelex.pipe_controllers.sub_pipe import SubPipefrom pipelex.pipe_controllers.sub_pipe_blueprint import SubPipeBlueprintfrom pipelex.pipe_run.pipe_run_params import BatchParams+if TYPE_CHECKING:+ from pipelex.core.pipes.variable_multiplicity import VariableMultiplicity+class SubPipeFactory:@classmethod@@ -11,18 +15,26 @@ class SubPipeFactory:blueprint: SubPipeBlueprint,) -> SubPipe:"""Create a SubPipe from a SubPipeBlueprint."""- output_multiplicity = make_variable_multiplicity(- nb_items=blueprint.nb_output,- multiple_items=blueprint.multiple_output,- )batch_params: BatchParams | None = None+ output_multiplicity: VariableMultiplicity | None = None+if blueprint.batch_over and blueprint.batch_as:batch_params = BatchParams.make_batch_params(input_list_name=blueprint.batch_over,input_item_name=blueprint.batch_as,)+ output_multiplicity = True+ if blueprint.nb_output:+ output_multiplicity = blueprint.nb_output+ elif blueprint.multiple_output:+ output_multiplicity = True+ elif blueprint.nb_output:+ output_multiplicity = blueprint.nb_output+ elif blueprint.multiple_output:+ output_multiplicity = Trueelse:batch_params = None+return SubPipe(pipe_code=blueprint.pipe,output_name=blueprint.result,diff --git a/pipelex/pipe_run/pipe_run_params.py b/pipelex/pipe_run/pipe_run_params.pyindex 4104b2e79..817db8690 100644--- a/pipelex/pipe_run/pipe_run_params.py+++ b/pipelex/pipe_run/pipe_run_params.py@@ -165,9 +165,6 @@ class PipeRunParams(BaseModel):def make_deep_copy(self) -> Self:return self.model_copy(deep=True)- def deep_copy_with_final_stuff_code(self, final_stuff_code: str) -> Self:- return self.model_copy(deep=True, update={"final_stuff_code": final_stuff_code})-@classmethoddef copy_by_injecting_multiplicity(cls,diff --git a/tests/data/discord_newsletter/discord_sample_input.json b/tests/data/discord_newsletter/discord_sample_input.jsonnew file mode 100644index 000000000..17efb34d0--- /dev/null+++ b/tests/data/discord_newsletter/discord_sample_input.json@@ -0,0 +1,189 @@+{+ "discord_channel_updates": {+ "concept": "DiscordChannelUpdate",+ "content": [+ {+ "id": "1320057977847283834",+ "message_count": 5,+ "messages": [+ {+ "attachments": [+ ],+ "author": "John Smith (TechCorp)",+ "content": "## [**John Smith**](https://www.linkedin.com/in/john-smith/) (mostly on X https://x.com/johnsmith) \n\n*CTO @ [TechCorp](https://techcorp.com) · AI and search optimization.*\n\nI've been building software for 15 years now (love building also for fun), I am a dad, and I spend most of my free time doing fun stuff with the family. \n\nSpecial power: Been playing music for 20 years\n\n📍 Amsterdam, NL (CET)",+ "embeds": [+ ],+ "link": "https://discord.com/channels/1320057304199987230/1320057977847283834/1392231464141193318"+ },+ {+ "attachments": [+ ],+ "author": "Pierre Martin (DataFlow)",+ "content": "## [**Pierre Martin**](https://www.linkedin.com/in/pierre-martin/)\n*Chief Technology Officer [DataFlow](https://dataflow.sh) · Tech consulting with a focus on Data and AI stuff*\nI've been in Tech for more than 25 years. I am software engineer at heart. I am the producer of \"Tech Talk\" podcast, I appear regularly on Tech Business for \"The Tech Reviews\", I've been Director of TechInstitute in Paris, I created a community call tech-meetup (https://tech-meetup.com). I've also been very active in Paris with the AI meetup.\nSpecial power: I am a super networker person. I can sail a boat, fly an airplane, I was also involved in various communities.\n📍 Paris, France (CET)",+ "embeds": [+ ],+ "link": "https://discord.com/channels/1320057304199987230/1320057977847283834/1391681488709488711"+ },+ {+ "attachments": [+ ],+ "author": "Alex Johnson (WorkflowAI)",+ "content": "## [Alex Johnson](https://www.linkedin.com/in/alex-johnson/)\n\n*Co-founder & CTO @ [WorkflowAI](https://workflowai.com/)· WorkflowAI is an open-source language for repeatable AI workflows. Build modular pipes that connect like LEGO blocks, each using its optimal LLM. Turn custom AI code into shareable, versioned pipelines with guaranteed outputs. [Python library available now](https://github.com/WorkflowAI/workflowai), hosted API coming soon.*\nI'm a repeat entrepreneur in tech. Started in digital media distribution (music, VOD, peer-to-peer), then found success in social gaming (free-to-play games on Facebook and mobile). I was born in Sweden and speak Swedish fluently but I'm really a French guy.\n\nWorkflowAI is my first open-source project, so I'm in full learning mode.\n\nSpecial power: Modern rationality. Not Hollywood rationality (like Mr. Spock) but rather the art of fighting biases, seeking truth, and making decisions that work.\n\n📍 Paris, France (CET)",+ "embeds": [+ ],+ "link": "https://discord.com/channels/1320057304199987230/1320057977847283834/1391085336188424363"+ },+ {+ "attachments": [+ ],+ "author": "Mike Schmidt (AutoBot)",+ "content": "## [**Mike Schmidt**](https://www.linkedin.com/in/mike-schmidt-890b96303/)\n*CTO & Co-founder @ [AutoBot](https://autobot.ai) · We enable AI Agents to control mobile apps & phones.*\nI'm a Tech builder passionate about decentralized compute, now building AutoBot's AI automation platform for Android.\nPreviously worked on blockchain projects, especially on various networks.\nSpecial power: Once ran 100m in under 11 seconds as a track & field athlete. Not quite superhero speed, but fast enough to turn heads :fast: \n📍 Berlin, Germany (CET)",+ "embeds": [+ ],+ "link": "https://discord.com/channels/1320057304199987230/1320057977847283834/1390267363785441432"+ },+ {+ "attachments": [+ ],+ "author": "Chris Wilson (AutoBot)",+ "content": "## [**Chris Wilson**](https://www.linkedin.com/in/chris-wilson-22b4b8210/)\n*CEO & Co-founder @ [AutoBot](https://autobot.ai) · We enable AI Agents to control mobile apps & phones - We are open source with 3.2k GitHub stars in 9 weeks post-launch*\n\nOutside the startup grind, I'm passionate about travel and cooking - there's something therapeutic about finding fresh ingredients and turning them into a great meal. 🍄🟫\n\nWe recently incorporated in Delaware and successfully closed our Pre-Seed. Heading to SF/NY soon and excited to connect with some folks there. Please reach out 😄\n\n📍 Berlin, Germany (CET)",+ "embeds": [+ ],+ "link": "https://discord.com/channels/1320057304199987230/1320057977847283834/1390259441852481547"+ }+ ],+ "name": "Introduce-Yourself",+ "parent": "1348223201812938782",+ "position": 4,+ "type": "text"+ },+ {+ "id": "1320059080857096214",+ "message_count": 2,+ "messages": [+ {+ "attachments": [+ ],+ "author": "Robert Davis (AITech)",+ "content": "Nice presentation",+ "embeds": [+ ],+ "link": "https://discord.com/channels/1320057304199987230/1320059080857096214/1388797681568190514"+ },+ {+ "attachments": [+ ],+ "author": "Robert Davis (AITech)",+ "content": "https://youtube.com/watch?v=example123&feature=shared",+ "embeds": [+ {+ "description": "",+ "title": "Multimodal Open Source AI Models, From Online Demos to On-Device - ...",+ "type": "video"+ }+ ],+ "link": "https://discord.com/channels/1320057304199987230/1320059080857096214/1388797614740475906"+ }+ ],+ "name": "🧠-Knowledge",+ "parent": "1320057304199987231",+ "position": 7,+ "type": "text"+ },+ {+ "id": "1324834149965631569",+ "message_count": 5,+ "messages": [+ {+ "attachments": [+ ],+ "author": "Sam Moore (EdgeTech)",+ "content": "https://www.edgetech.cloud/blog/posts/announcing-edge-functions",+ "embeds": [+ {+ "description": "Developers can push edge functions to the EdgeTech Component Registry to generate HTTP responses at the edge.",+ "title": "Announcing the new Edge Functions Components powered by WebAssembly...",+ "type": "article"+ }+ ],+ "link": "https://discord.com/channels/1320057304199987230/1324834149965631569/1392507684573675561"+ },+ {+ "attachments": [+ ],+ "author": "Kevin Thompson (DataAnalytics)",+ "content": "https://www.linkedin.com/posts/example-user_very-cool-to-see-our-joint-analysis-with-activity-7346527049613815808-8P9G?",+ "embeds": [+ {+ "description": "Very cool to see our joint analysis with Cloud Services Provider about GraphRAG being featured in AI Company's blog.\n\nOverall, their article presents the approaches of GraphRAG and Agentic Search side by side. \n\nI think we should go a step further by pointing out that these two approaches are closely intertwined : an agentic system could use G...",+ "title": "Very cool to see our joint analysis with Cloud Services Provider...",+ "type": "article"+ }+ ],+ "link": "https://discord.com/channels/1320057304199987230/1324834149965631569/1390327059057999893"+ },+ {+ "attachments": [+ ],+ "author": "Julian Garcia (BackupTech)",+ "content": "https://backuptech.io/posts/2025-06-27/modern-data-format-introducing-new-archive/",+ "embeds": [+ {+ "description": "",+ "title": "It doesn't make sense to wrap modern data in old formats, introducing new archive format...",+ "type": "article"+ }+ ],+ "link": "https://discord.com/channels/1320057304199987230/1324834149965631569/1388195527321194548"+ },+ {+ "attachments": [+ ],+ "author": "Peter Chen (MLTech)",+ "content": "No I didn't start as one, I did some for a startup which had a product that needed to integrate with CRMs 🙂\n\nIn reply to Kevin Thompson (DataAnalytics)'s message:\nDidn't you start as a CRM integrator 🙈",+ "embeds": [+ ],+ "link": "https://discord.com/channels/1320057304199987230/1324834149965631569/1388135901829267559"+ },+ {+ "attachments": [+ ],+ "author": "Kevin Thompson (DataAnalytics)",+ "content": "Didn't you start as a CRM integrator 🙈",+ "embeds": [+ ],+ "link": "https://discord.com/channels/1320057304199987230/1324834149965631569/1388124441887899648"+ }+ ],+ "name": "🎉-Achievements",+ "parent": "1320057304199987231",+ "position": 10,+ "type": "text"+ },+ {+ "id": "1348214314829615116",+ "message_count": 1,+ "messages": [+ {+ "attachments": [+ ],+ "author": "Ryan Miller (VisionAI)",+ "content": "Also this hasn't been posted enough either but I think https://example-ai.org/ is a really cool model and is open weights and really impressive performance as it lets you do anything essentially",+ "embeds": [+ {+ "description": "A scalable unified multimodal model, revolutionizing how AI interacts with complex systems.",+ "title": "EXAMPLE: Scalable Unified Multimodal Model",+ "type": "article"+ }+ ],+ "link": "https://discord.com/channels/1320057304199987230/1348214314829615116/1389575075795177473"+ }+ ],+ "name": "👁-Computer-Vision",+ "parent": "1320057304199987231",+ "position": 16,+ "type": "text"+ }+ ]+ }+}\ No newline at end of filediff --git a/tests/integration/pipelex/pipes/controller/pipe_sequence/test_pipe_sequence_dry_run.py b/tests/integration/pipelex/pipes/controller/pipe_sequence/test_pipe_sequence_dry_run.pyindex 75af9a84a..7a7990ca3 100644--- a/tests/integration/pipelex/pipes/controller/pipe_sequence/test_pipe_sequence_dry_run.py+++ b/tests/integration/pipelex/pipes/controller/pipe_sequence/test_pipe_sequence_dry_run.py@@ -26,7 +26,7 @@ if TYPE_CHECKING:@pytest.mark.inference@pytest.mark.asyncio(loop_scope="class")class TestPipeSequenceDryRun:- async def test_discord_newsletter_dry_run_working_memory(+ async def test_discord_newsletter(self,pipe_run_mode: PipeRunMode,load_test_library: Callable[[list[Path]], None],@@ -86,44 +86,43 @@ class TestPipeSequenceDryRun:assert discord_updates_stuff_final.concept.code == "DiscordChannelUpdate"assert discord_updates_stuff_final.concept.domain == "discord_newsletter"- if pipe_run_mode == PipeRunMode.DRY:- # The key assertion: verify it's a ListContent with multiple items- discord_updates_list = discord_updates_stuff_final.as_list_of_fixed_content_type(item_type=DiscordChannelUpdate)- assert isinstance(discord_updates_list, ListContent)- assert len(discord_updates_list.items) > 1, "Should have multiple items for batching"-- # Verify each discord update item- for i, update in enumerate(discord_updates_list.items):- assert isinstance(update, DiscordChannelUpdate), f"Update {i} should be DiscordChannelUpdate"- assert len(update.name) > 0, f"Update {i} should have a non-empty channel name"-- # Check that channel_summaries was created as ListContent (result of batched operation)- channel_summaries_stuff: Stuff | None = final_working_memory.get_optional_stuff("channel_summaries")- assert channel_summaries_stuff is not None, "channel_summaries should be in working memory"- assert channel_summaries_stuff.concept.code == "ChannelSummary"- assert channel_summaries_stuff.concept.domain == "discord_newsletter"- assert isinstance(channel_summaries_stuff, Stuff), "ChannelSurrmary Stuff is not a Stuff"-- # Verify channel_summaries is also a ListContent with multiple ChannelSummary items- channel_summaries_list: ListContent[ChannelSummary] = channel_summaries_stuff.as_list_of_fixed_content_type(item_type=ChannelSummary)-- pretty_print(channel_summaries_stuff, "jiodisjdqosj")- pretty_print(channel_summaries_list, title="Channel Summaries List")- assert isinstance(channel_summaries_list, ListContent)- assert len(channel_summaries_list.items) > 1, "Should have multiple ChannelSummary items from batch processing"-- # Verify each summary item (these should be proper ChannelSummary objects from the LLM mock)- for i, summary in enumerate(channel_summaries_list.items):- assert isinstance(summary, ChannelSummary), f"Summary {i} should be ChannelSummary"- assert len(summary.channel_name) > 0, f"Summary {i} should have a non-empty channel name"- assert isinstance(summary.summary_items, list), f"Summary {i} should have a list of summary items"-- # Verify that the number of summaries matches the number of original channel updates- assert len(channel_summaries_list.items) == len(discord_updates_list.items), (- "Number of summaries should match number of original channel updates"- )-- log.info("✅ Successfully verified dry run working memory:")- log.info(f" - discord_channel_updates: ListContent with {len(discord_updates_list.items)} items")- log.info(f" - channel_summaries: ListContent with {len(channel_summaries_list.items)} items")- log.info(f" - Final output: ChannelSummary list with concept code {pipe_output.main_stuff.concept.code}")+ # The key assertion: verify it's a ListContent with multiple items+ discord_updates_list = discord_updates_stuff_final.as_list_of_fixed_content_type(item_type=DiscordChannelUpdate)+ assert isinstance(discord_updates_list, ListContent)+ assert len(discord_updates_list.items) > 1, "Should have multiple items for batching"++ # Verify each discord update item+ for i, update in enumerate(discord_updates_list.items):+ assert isinstance(update, DiscordChannelUpdate), f"Update {i} should be DiscordChannelUpdate"+ assert len(update.name) > 0, f"Update {i} should have a non-empty channel name"++ # Check that channel_summaries was created as ListContent (result of batched operation)+ channel_summaries_stuff: Stuff | None = final_working_memory.get_optional_stuff("channel_summaries")+ assert channel_summaries_stuff is not None, "channel_summaries should be in working memory"+ assert channel_summaries_stuff.concept.code == "ChannelSummary"+ assert channel_summaries_stuff.concept.domain == "discord_newsletter"+ assert isinstance(channel_summaries_stuff, Stuff), "ChannelSurrmary Stuff is not a Stuff"++ # Verify channel_summaries is also a ListContent with multiple ChannelSummary items+ channel_summaries_list = channel_summaries_stuff.as_list_of_fixed_content_type(item_type=ChannelSummary)++ pretty_print(channel_summaries_stuff, title="Channel Summaries Stuff")+ pretty_print(channel_summaries_list, title="Channel Summaries List")+ assert isinstance(channel_summaries_list, ListContent)+ assert len(channel_summaries_list.items) > 1, "Should have multiple ChannelSummary items from batch processing"++ # Verify each summary item (these should be proper ChannelSummary objects from the LLM mock)+ for i, summary in enumerate(channel_summaries_list.items):+ assert isinstance(summary, ChannelSummary), f"Summary {i} should be ChannelSummary"+ assert len(summary.channel_name) > 0, f"Summary {i} should have a non-empty channel name"+ assert isinstance(summary.summary_items, list), f"Summary {i} should have a list of summary items"++ # Verify that the number of summaries matches the number of original channel updates+ assert len(channel_summaries_list.items) == len(discord_updates_list.items), (+ "Number of summaries should match number of original channel updates"+ )++ log.info("✅ Successfully verified dry run working memory:")+ log.info(f" - discord_channel_updates: ListContent with {len(discord_updates_list.items)} items")+ log.info(f" - channel_summaries: ListContent with {len(channel_summaries_list.items)} items")+ log.info(f" - Final output: ChannelSummary list with concept code {pipe_output.main_stuff.concept.code}")diff --git a/tests/integration/pipelex/pipes/controller/pipe_sequence/test_pipe_sequence_output_multiplicity.py b/tests/integration/pipelex/pipes/controller/pipe_sequence/test_pipe_sequence_output_multiplicity.pyindex 8d0ea61d6..92da5e144 100644--- a/tests/integration/pipelex/pipes/controller/pipe_sequence/test_pipe_sequence_output_multiplicity.py+++ b/tests/integration/pipelex/pipes/controller/pipe_sequence/test_pipe_sequence_output_multiplicity.py@@ -221,16 +221,18 @@ class TestPipeSequenceOutputMultiplicity:error_message = str(exc_info.value).lower()assert "multiplicity" in error_message, f"Error should mention multiplicity mismatch, got: {exc_info.value}"- def test_fixed_vs_variable_multiplicity_mismatch(+ def test_fixed_multiplicity_compatible_with_variable_multiplicity(self,load_empty_library: Callable[[], str],):- """Test that validation fails when last step has [3] but sequence has [].+ """Test that validation passes when last step has [3] and sequence has [].Creates:- A PipeLLM with output="Text[3]" (fixed multiplicity)- A PipeSequence with output="Text[]" (variable multiplicity)- The validation should fail because multiplicities don't match (3 vs True).++ A fixed multiplicity is compatible with a variable multiplicity expectation+ because a fixed count of items fulfills the "list of items" requirement."""load_empty_library()@@ -251,18 +253,188 @@ class TestPipeSequenceOutputMultiplicity:)get_pipe_library().add_new_pipe(llm_pipe)- # Create a PipeSequence with variable multiplicity - MISMATCH+ # Create a PipeSequence with variable multiplicity+ # This is COMPATIBLE: fixed count fulfills variable expectationsequence_blueprint = PipeSequenceBlueprint(description="Sequence with variable output",inputs={"input_text": "Text"},- output="Text[]", # Variable multiplicity - MISMATCH!+ output="Text[]", # Variable multiplicity - compatible with fixedsteps=[SubPipeBlueprint(pipe="llm_fixed_3", result="result"),],)sequence_pipe = PipeFactory[PipeSequence].make_from_blueprint(domain_code=domain,- pipe_code="sequence_mismatch_fixed_var",+ pipe_code="sequence_fixed_to_variable",+ blueprint=sequence_blueprint,+ )+ get_pipe_library().add_new_pipe(sequence_pipe)++ # Validation should pass - fixed multiplicity is compatible with variable+ sequence_pipe.validate_with_libraries()++ def test_variable_multiplicity_incompatible_with_fixed_multiplicity(+ self,+ load_empty_library: Callable[[], str],+ ):+ """Test that validation fails when last step has [] but sequence has [3].++ Creates:+ - A PipeLLM with output="Text[]" (variable multiplicity)+ - A PipeSequence with output="Text[3]" (fixed multiplicity)++ A variable multiplicity cannot fulfill a fixed count expectation because+ we cannot guarantee the exact number of items will be produced.+ """+ load_empty_library()++ domain = "test_variable_vs_fixed"++ # Create a PipeLLM with variable multiplicity+ llm_blueprint = PipeLLMBlueprint(+ description="LLM with variable output count",+ inputs={"input_text": "Text"},+ output="Text[]", # Variable multiplicity+ model="llm_for_testing_gen_text",+ prompt="@input_text",+ )+ llm_pipe = PipeFactory[PipeLLM].make_from_blueprint(+ domain_code=domain,+ pipe_code="llm_variable",+ blueprint=llm_blueprint,+ )+ get_pipe_library().add_new_pipe(llm_pipe)++ # Create a PipeSequence with fixed multiplicity - INCOMPATIBLE+ sequence_blueprint = PipeSequenceBlueprint(+ description="Sequence with fixed output count",+ inputs={"input_text": "Text"},+ output="Text[3]", # Fixed multiplicity - cannot be fulfilled by variable+ steps=[+ SubPipeBlueprint(pipe="llm_variable", result="result"),+ ],+ )+ sequence_pipe = PipeFactory[PipeSequence].make_from_blueprint(+ domain_code=domain,+ pipe_code="sequence_variable_to_fixed",+ blueprint=sequence_blueprint,+ )+ get_pipe_library().add_new_pipe(sequence_pipe)++ # Validation should fail - variable cannot fulfill fixed expectation+ with pytest.raises(PipeValidationError) as exc_info:+ sequence_pipe.validate_with_libraries()++ error_message = str(exc_info.value).lower()+ assert "multiplicity" in error_message, f"Error should mention multiplicity mismatch, got: {exc_info.value}"++ def test_batch_over_with_variable_multiplicity_passes(+ self,+ load_empty_library: Callable[[], str],+ ):+ """Test that validation passes when last step uses batch_over/batch_as and sequence has [].++ When a step has batch_over and batch_as, the effective output multiplicity should be+ variable (True), since batching produces multiple outputs.++ Creates:+ - A PipeLLM with output="Text" (single output)+ - A PipeSequence with output="Text[]" where last step has batch_over/batch_as+ The validation should pass because batch_over/batch_as makes the effective multiplicity True.+ """+ load_empty_library()++ domain = "test_batch_over_variable"++ # Create a PipeLLM with single output (process_cv in the example)+ llm_blueprint = PipeLLMBlueprint(+ description="LLM that processes single item",+ inputs={"cv_pdf": "PDF"},+ output="Text", # Single output+ model="llm_for_testing_gen_text",+ prompt="@cv_pdf",+ )+ llm_pipe = PipeFactory[PipeLLM].make_from_blueprint(+ domain_code=domain,+ pipe_code="process_cv",+ blueprint=llm_blueprint,+ )+ get_pipe_library().add_new_pipe(llm_pipe)++ # Create a PipeSequence with variable multiplicity where last step batches+ sequence_blueprint = PipeSequenceBlueprint(+ description="Sequence that batches over CVs",+ inputs={"cvs": "PDF[]"}, # Multiple PDFs input+ output="Text[]", # Variable multiplicity output - should match batch_over effect+ steps=[+ SubPipeBlueprint(+ pipe="process_cv",+ result="match_analyses",+ batch_over="cvs",+ batch_as="cv_pdf",+ ),+ ],+ )+ sequence_pipe = PipeFactory[PipeSequence].make_from_blueprint(+ domain_code=domain,+ pipe_code="sequence_batch_variable",+ blueprint=sequence_blueprint,+ )+ get_pipe_library().add_new_pipe(sequence_pipe)++ # Validation should pass - batch_over/batch_as makes effective multiplicity True+ sequence_pipe.validate_with_libraries()++ def test_batch_over_with_no_multiplicity_fails(+ self,+ load_empty_library: Callable[[], str],+ ):+ """Test that validation fails when last step uses batch_over/batch_as but sequence has no multiplicity.++ When a step has batch_over and batch_as, the effective output multiplicity should be+ variable (True). The sequence must declare output="Concept[]" to match.++ Creates:+ - A PipeLLM with output="Text" (single output)+ - A PipeSequence with output="Text" (no multiplicity) where last step has batch_over/batch_as+ The validation should fail because batch_over/batch_as makes multiplicity True, but sequence says None.+ """+ load_empty_library()++ domain = "test_batch_over_no_mult"++ # Create a PipeLLM with single output+ llm_blueprint = PipeLLMBlueprint(+ description="LLM that processes single item",+ inputs={"cv_pdf": "PDF"},+ output="Text", # Single output+ model="llm_for_testing_gen_text",+ prompt="@cv_pdf",+ )+ llm_pipe = PipeFactory[PipeLLM].make_from_blueprint(+ domain_code=domain,+ pipe_code="process_cv_single",+ blueprint=llm_blueprint,+ )+ get_pipe_library().add_new_pipe(llm_pipe)++ # Create a PipeSequence with NO multiplicity where last step batches - MISMATCH+ sequence_blueprint = PipeSequenceBlueprint(+ description="Sequence that batches but declares single output",+ inputs={"cvs": "PDF[]"}, # Multiple PDFs input+ output="Text", # No multiplicity - MISMATCH with batch effect!+ steps=[+ SubPipeBlueprint(+ pipe="process_cv_single",+ result="match_analyses",+ batch_over="cvs",+ batch_as="cv_pdf",+ ),+ ],+ )+ sequence_pipe = PipeFactory[PipeSequence].make_from_blueprint(+ domain_code=domain,+ pipe_code="sequence_batch_mismatch",blueprint=sequence_blueprint,)get_pipe_library().add_new_pipe(sequence_pipe)@@ -273,3 +445,167 @@ class TestPipeSequenceOutputMultiplicity:error_message = str(exc_info.value).lower()assert "multiplicity" in error_message, f"Error should mention multiplicity mismatch, got: {exc_info.value}"++ def test_batch_over_with_nb_output_fixed_multiplicity(+ self,+ load_empty_library: Callable[[], str],+ ):+ """Test that when last step has batch_over/batch_as AND nb_output, multiplicity uses nb_output.++ When a step has batch_over, batch_as, AND nb_output=3, the effective output multiplicity+ should be 3, not True.++ Creates:+ - A PipeLLM with output="Text" (single output)+ - A PipeSequence with output="Text[3]" where last step has batch_over/batch_as and nb_output=3+ The validation should pass because nb_output=3 overrides to give multiplicity=3.+ """+ load_empty_library()++ domain = "test_batch_over_nb_output"++ # Create a PipeLLM with single output+ llm_blueprint = PipeLLMBlueprint(+ description="LLM that processes single item",+ inputs={"cv_pdf": "PDF"},+ output="Text", # Single output+ model="llm_for_testing_gen_text",+ prompt="@cv_pdf",+ )+ llm_pipe = PipeFactory[PipeLLM].make_from_blueprint(+ domain_code=domain,+ pipe_code="process_cv_fixed",+ blueprint=llm_blueprint,+ )+ get_pipe_library().add_new_pipe(llm_pipe)++ # Create a PipeSequence with fixed multiplicity where last step batches with nb_output+ sequence_blueprint = PipeSequenceBlueprint(+ description="Sequence that batches with fixed output count",+ inputs={"cvs": "PDF[]"}, # Multiple PDFs input+ output="Text[6]", # Fixed multiplicity - should match nb_output+ steps=[+ SubPipeBlueprint(+ pipe="process_cv_fixed",+ result="match_analyses",+ batch_over="cvs",+ batch_as="cv_pdf",+ nb_output=6, # Fixed output count+ ),+ ],+ )+ sequence_pipe = PipeFactory[PipeSequence].make_from_blueprint(+ domain_code=domain,+ pipe_code="sequence_batch_fixed",+ blueprint=sequence_blueprint,+ )+ get_pipe_library().add_new_pipe(sequence_pipe)++ # Validation should pass - nb_output=3 gives multiplicity=3+ sequence_pipe.validate_with_libraries()++ def test_nb_output_only_fixed_multiplicity(+ self,+ load_empty_library: Callable[[], str],+ ):+ """Test that when last step has ONLY nb_output (no batch_over/batch_as), multiplicity uses nb_output.++ Creates:+ - A PipeLLM with output="Text" (single output)+ - A PipeSequence with output="Text[5]" where last step has nb_output=5 only+ The validation should pass because nb_output=5 gives multiplicity=5.+ """+ load_empty_library()++ domain = "test_nb_output_only"++ # Create a PipeLLM with single output+ llm_blueprint = PipeLLMBlueprint(+ description="LLM that processes single item",+ inputs={"input_text": "Text"},+ output="Text", # Single output+ model="llm_for_testing_gen_text",+ prompt="@input_text",+ )+ llm_pipe = PipeFactory[PipeLLM].make_from_blueprint(+ domain_code=domain,+ pipe_code="single_output_llm",+ blueprint=llm_blueprint,+ )+ get_pipe_library().add_new_pipe(llm_pipe)++ # Create a PipeSequence with fixed multiplicity where last step has nb_output only+ sequence_blueprint = PipeSequenceBlueprint(+ description="Sequence with fixed output count via nb_output",+ inputs={"input_text": "Text"},+ output="Text[5]", # Fixed multiplicity - should match nb_output+ steps=[+ SubPipeBlueprint(+ pipe="single_output_llm",+ result="results",+ nb_output=5, # Fixed output count, no batch_over/batch_as+ ),+ ],+ )+ sequence_pipe = PipeFactory[PipeSequence].make_from_blueprint(+ domain_code=domain,+ pipe_code="sequence_nb_output_only",+ blueprint=sequence_blueprint,+ )+ get_pipe_library().add_new_pipe(sequence_pipe)++ # Validation should pass - nb_output=5 gives multiplicity=5+ sequence_pipe.validate_with_libraries()++ def test_multiple_output_only_variable_multiplicity(+ self,+ load_empty_library: Callable[[], str],+ ):+ """Test that when last step has ONLY multiple_output=True (no nb_output, no batch_over/batch_as), multiplicity is True.++ Creates:+ - A PipeLLM with output="Text" (single output)+ - A PipeSequence with output="Text[]" where last step has multiple_output=True only+ The validation should pass because multiple_output=True gives multiplicity=True.+ """+ load_empty_library()++ domain = "test_multiple_output_only"++ # Create a PipeLLM with single output+ llm_blueprint = PipeLLMBlueprint(+ description="LLM that processes single item",+ inputs={"input_text": "Text"},+ output="Text", # Single output+ model="llm_for_testing_gen_text",+ prompt="@input_text",+ )+ llm_pipe = PipeFactory[PipeLLM].make_from_blueprint(+ domain_code=domain,+ pipe_code="single_output_llm_mult",+ blueprint=llm_blueprint,+ )+ get_pipe_library().add_new_pipe(llm_pipe)++ # Create a PipeSequence with variable multiplicity where last step has multiple_output only+ sequence_blueprint = PipeSequenceBlueprint(+ description="Sequence with variable output via multiple_output",+ inputs={"input_text": "Text"},+ output="Text[]", # Variable multiplicity - should match multiple_output=True+ steps=[+ SubPipeBlueprint(+ pipe="single_output_llm_mult",+ result="results",+ multiple_output=True, # Variable output, no nb_output, no batch_over/batch_as+ ),+ ],+ )+ sequence_pipe = PipeFactory[PipeSequence].make_from_blueprint(+ domain_code=domain,+ pipe_code="sequence_multiple_output_only",+ blueprint=sequence_blueprint,+ )+ get_pipe_library().add_new_pipe(sequence_pipe)++ # Validation should pass - multiple_output=True gives multiplicity=True+ sequence_pipe.validate_with_libraries()diff --git a/tests/unit/pipelex/core/pipes/test_is_multiplicity_compatible.py b/tests/unit/pipelex/core/pipes/test_is_multiplicity_compatible.pynew file mode 100644index 000000000..7713574a8--- /dev/null+++ b/tests/unit/pipelex/core/pipes/test_is_multiplicity_compatible.py@@ -0,0 +1,56 @@+import pytest++from pipelex.core.pipes.variable_multiplicity import is_multiplicity_compatible+++class TestIsMultiplicityCompatible:+ """Test multiplicity compatibility checking between source and target multiplicities."""++ @pytest.mark.parametrize(+ ("source_multiplicity", "target_multiplicity", "expected"),+ [+ # Case 1: Target expects single item (None)+ (None, None, True),+ (True, None, False),+ (3, None, False),+ # Case 2: Target expects variable-length list (True)+ (True, True, True),+ (3, True, True), # Fixed count fulfills variable expectation+ (1, True, True), # Even count of 1 is multiple items+ (None, True, False), # Single cannot fulfill list expectation+ # Case 3: Target expects fixed count (integer)+ (3, 3, True),+ (5, 5, True),+ (1, 1, True), # Fixed count of 1 matches itself+ (3, 5, False), # Different fixed counts are incompatible+ (True, 3, False), # Variable cannot fulfill fixed expectation+ (True, 1, False), # Variable (True) cannot fulfill fixed count of 1 (edge case: True == 1 in Python)+ (None, 3, False),+ (None, 1, False), # Single item cannot fulfill fixed count of 1+ ],+ )+ def test_multiplicity_compatibility(+ self,+ source_multiplicity: bool | int | None,+ target_multiplicity: bool | int | None,+ expected: bool,+ ):+ """Test various multiplicity compatibility scenarios."""+ result = is_multiplicity_compatible(source_multiplicity, target_multiplicity)+ assert result == expected++ def test_false_is_not_compatible_with_variable_list(self):+ """Test that False (force single) is not compatible with True (variable list).++ This is a regression test for a bug where isinstance(False, int) returns True+ because bool is a subclass of int in Python.+ """+ # False should be treated as "force single output", not as a valid multiplicity+ assert is_multiplicity_compatible(False, True) is False++ def test_false_is_compatible_with_none(self):+ """Test that False matches None semantically (both mean single item)."""+ # Note: The type system allows False as VariableMultiplicity, and semantically+ # False should mean "force single", which is compatible with None (single item)+ # This test documents the current behavior - False == None as integers+ assert is_multiplicity_compatible(False, None) is False