Build AI methods with your coding agent, then run them anywhere: as an MCP for chatbots, as a webapp for people, or via API for your software.
Fix/multiple images input to llm (#520)
Louis Choquele86867e02132
diff --git a/.vscode/launch.json b/.vscode/launch.jsonindex a03eb1287..afb364211 100644--- a/.vscode/launch.json+++ b/.vscode/launch.json@@ -41,7 +41,7 @@"console": "integratedTerminal"},{- "name": "Debug validate pipe",+ "name": "Debug validate bundle","type": "debugpy","request": "launch","program": "${workspaceFolder}/.venv/bin/pipelex",@@ -52,6 +52,18 @@"console": "integratedTerminal","justMyCode": false},+ {+ "name": "Debug validate pipe",+ "type": "debugpy",+ "request": "launch",+ "program": "${workspaceFolder}/.venv/bin/pipelex",+ "args": [+ "validate",+ "analyze_image_collection",+ ],+ "console": "integratedTerminal",+ "justMyCode": false+ },{"name": "Debug run pipe","type": "debugpy",diff --git a/pipelex/builder/pipe/pipe_img_gen_spec.py b/pipelex/builder/pipe/pipe_img_gen_spec.pyindex b3438495a..898453e85 100644--- a/pipelex/builder/pipe/pipe_img_gen_spec.py+++ b/pipelex/builder/pipe/pipe_img_gen_spec.py@@ -36,7 +36,7 @@ class PipeImgGenSpec(PipeSpec):type: Literal["PipeImgGen"] = "PipeImgGen"pipe_category: Literal["PipeOperator"] = "PipeOperator"img_gen_skill: ImgGenSkill | str = Field(description="Select the most adequate image generation skill according to the task to be performed.")- prompt: str+ prompt: str = Field(description="A finalized image generation prompt or prompt template: use `$` prefix for inline variables (e.g., `$topic`).")@field_validator("img_gen_skill", mode="before")@classmethoddiff --git a/pipelex/core/memory/working_memory.py b/pipelex/core/memory/working_memory.pyindex 621f18c59..1b5e1765b 100644--- a/pipelex/core/memory/working_memory.py+++ b/pipelex/core/memory/working_memory.py@@ -1,5 +1,5 @@from operator import attrgetter-from typing import Any+from typing import Any, castfrom pydantic import BaseModel, Field, model_validatorfrom typing_extensions import override@@ -210,6 +210,37 @@ class WorkingMemory(BaseModel, ContextProviderAbstract):@overridedef get_typed_object_or_attribute(self, name: str, wanted_type: type[Any] | None = None, accept_list: bool = False) -> Any:+ """Retrieve a typed object or nested attribute from working memory.++ This method is primarily used to:++ - Find specific content types that require special handling when passed to LLMs,+ such as Images and Documents (PDFs). These types need to be formatted differently+ in LLM prompts compared to plain text content.+ - Find the list of items to iterate over in a PipeBatch operation.++ Supports dot-notation for accessing nested attributes (e.g., "stuff_name.attribute.sub_attr").+ When the base stuff contains ListContent, extracts the specified attribute from each item.++ Args:+ name: The name of the stuff, optionally followed by dot-separated attribute path+ (e.g., "my_stuff" or "my_stuff.nested_attr.deeper_attr")+ wanted_type: Optional type to validate the retrieved content against. If provided+ and the content doesn't match, raises WorkingMemoryTypeError.+ Commonly used with ImageContent, PDFContent, etc.+ accept_list: If True, allows retrieval from ListContent stuffs. If False and the+ content is ListContent, raises WorkingMemoryTypeError++ Returns:+ The retrieved content or attribute value. For ListContent with dot-notation,+ returns a flattened list of attribute values from each item.++ Raises:+ WorkingMemoryStuffNotFoundError: If the base stuff name doesn't exist+ WorkingMemoryStuffAttributeNotFoundError: If the attribute path is invalid+ WorkingMemoryTypeError: If type validation fails or ListContent is encountered+ when accept_list is False+ """# TODO: Refactor this method. In the python paradigm, we should not have those ".", but arrays with field names.if "." in name:parts = name.split(".", 1) # Split only at the first dot@@ -270,15 +301,75 @@ class WorkingMemory(BaseModel, ContextProviderAbstract):)return stuff_content- content = self.get_stuff(name).content- if wanted_type is not None and not isinstance(content, wanted_type):+ top_level_content = self.get_stuff(name).content+ if isinstance(top_level_content, ListContent):+ top_level_content = cast("ListContent[Any]", top_level_content)+ if wanted_type is ListContent:+ return top_level_content.items+ else:+ if not accept_list:+ raise WorkingMemoryTypeError(+ variable_name=name,+ message=f"Content of '{name}' is ListContent, but accept_list is False",+ )+ top_level_content_items = self._get_typed_items_from_list_content(list_content=top_level_content, wanted_type=wanted_type)+ if top_level_content_items is None and wanted_type is not None:+ raise WorkingMemoryTypeError(+ variable_name=name,+ message=f"Content of '{name}' is ListContent, but some of its items are not of type '{wanted_type.__name__}'",+ )+ if top_level_content_items is not None and len(top_level_content_items) == 0:+ raise WorkingMemoryTypeError(+ variable_name=name,+ message=f"Content of '{name}' is ListContent, but it is empty",+ )+ return top_level_content_items++ if wanted_type is not None and not isinstance(top_level_content, wanted_type):raise WorkingMemoryTypeError(variable_name=name,- message=f"Content of '{name}' is of type '{type(content).__name__}', it should be '{wanted_type.__name__}'",+ message=f"Content of '{name}' is of type '{type(top_level_content).__name__}', it should be '{wanted_type.__name__}'",)- return content+ return top_level_content++ def _get_typed_items_from_list_content(self, list_content: ListContent[Any], wanted_type: type[Any] | None) -> list[Any] | None:+ """Extract items from ListContent with optional type validation.++ Used to collect specific content types (e.g., Images, Documents) from lists+ that require special handling when passed to LLMs.++ Flattens nested lists and filters out None values. If any item fails type+ validation, returns None to indicate failure.++ Args:+ list_content: The ListContent to extract items from+ wanted_type: Optional type to validate each item against (e.g., ImageContent,+ PDFContent). If None, no type checking is performed++ Returns:+ A list of extracted items if all items pass type validation,+ or None if any item fails validation+ """+ the_items: list[Any] = []+ for item in list_content.items:+ item_content = item+ # check type+ if isinstance(item_content, list):+ for item_content_item in item_content: # pyright: ignore[reportUnknownVariableType]+ if item_content_item is None:+ continue+ if wanted_type and not isinstance(item_content_item, wanted_type):+ return None+ the_items.append(item_content_item)+ else:+ if item_content is None:+ continue+ if wanted_type and not isinstance(item_content, wanted_type):+ return None+ the_items.append(item_content)+ return the_items################################################################################################# Stuff accessorsdiff --git a/pipelex/pipe_operators/llm/llm_prompt_blueprint.py b/pipelex/pipe_operators/llm/llm_prompt_blueprint.pyindex f2f9eea6e..035882cfe 100644--- a/pipelex/pipe_operators/llm/llm_prompt_blueprint.py+++ b/pipelex/pipe_operators/llm/llm_prompt_blueprint.py@@ -1,4 +1,4 @@-from typing import TYPE_CHECKING, Any+from typing import TYPE_CHECKING, Any, castfrom pydantic import BaseModel@@ -55,7 +55,7 @@ class LLMPromptBlueprint(BaseModel):if self.user_images:for user_image_name in self.user_images:log.verbose(f"Getting user image '{user_image_name}' from context")- # Try to get as a single ImageContent first+ # First: Try to get as a single ImageContent or ListContent[ImageContent]try:prompt_image_content = context_provider.get_typed_object_or_attribute(name=user_image_name,@@ -66,12 +66,14 @@ class LLMPromptBlueprint(BaseModel):user_image = PromptImageFactory.make_prompt_image(url=prompt_image_content.url, base_64_str=prompt_image_content.base_64)prompt_user_images[user_image_name] = user_imageelif isinstance(prompt_image_content, list):- for image_item in prompt_image_content: # pyright: ignore[reportUnknownVariableType]- if not isinstance(image_item, ImageContent):- msg = f"Item of '{user_image_name}' is of type '{type(image_item).__name__}', it should be ImageContent" # pyright: ignore[reportUnknownArgumentType]+ prompt_image_content = cast("list[ImageContent]", prompt_image_content)+ for image_index, image_item in enumerate(prompt_image_content, start=1):+ if not isinstance(image_item, ImageContent): # pyright: ignore[reportUnnecessaryIsInstance]+ msg = f"Item of '{user_image_name}' is of type '{type(image_item).__name__}', it should be ImageContent"raise LLMPromptBlueprintValueError(msg)user_image = PromptImageFactory.make_prompt_image(url=image_item.url, base_64_str=image_item.base_64)- prompt_user_images[user_image_name] = user_image+ user_image_item_name = f"{user_image_name}[{image_index}]"+ prompt_user_images[user_image_item_name] = user_imageelse:msg = (f"User image '{user_image_name}' is of type '{type(prompt_image_content).__name__}', "@@ -84,10 +86,11 @@ class LLMPromptBlueprint(BaseModel):image_collection = context_provider.get_typed_object_or_attribute(name=user_image_name, wanted_type=None)# Check if it's a list or tupleif isinstance(image_collection, (list, tuple)):- for image_item in image_collection: # type: ignore[assignment]- if isinstance(image_item, ImageContent):- user_image = PromptImageFactory.make_prompt_image(url=image_item.url, base_64_str=image_item.base_64)- prompt_user_images[user_image_name] = user_image+ image_collection = cast("list[ImageContent] | tuple[ImageContent]", image_collection)+ for image_index, image_item in enumerate(image_collection, start=1):+ user_image = PromptImageFactory.make_prompt_image(url=image_item.url, base_64_str=image_item.base_64)+ user_image_item_name = f"{user_image_name}[{image_index}]"+ prompt_user_images[user_image_item_name] = user_imageelse:msg = (f"Could not find a valid user image or image collection named '{user_image_name}' from the provided context_provider"diff --git a/tests/cases/images.py b/tests/cases/images.pyindex f015728a5..1988a5291 100644--- a/tests/cases/images.py+++ b/tests/cases/images.py@@ -11,11 +11,13 @@ class ImageTestCases:# Individual file pathsIMAGE_FILE_PATH_PNG_1 = f"{TEST_IMAGE_DIRECTORY}/ai_lympics.png"- IMAGE_FILE_PATH_JPG_1 = f"{TEST_IMAGE_DIRECTORY}/solar_system.jpg"+ IMAGE_FILE_PATH_JPG_1 = f"{TEST_IMAGE_DIRECTORY}/animal_lympics.jpg"+ IMAGE_FILE_PATH_JPG_2 = f"{TEST_IMAGE_DIRECTORY}/solar_system.jpg"IMAGE_FILE_PATH_PNG_2 = f"{TEST_IMAGE_DIRECTORY}/solar_system.png"+ IMAGE_FILE_PATH_JPG_3 = f"{TEST_IMAGE_DIRECTORY}/eiffel_tower.jpg"# Remote URLs- IMAGE_URL_PNG = "https://www.python.org/static/community_logos/python-logo-master-v3-TM.png"+ IMAGE_URL_PNG = "https://pipelex-web.s3.amazonaws.com/tests/solar_system.png"# File path collectionsIMAGE_FILE_PATHS: ClassVar[list[str]] = [diff --git a/tests/integration/pipelex/pipes/controller/pipe_sequence/test_pipe_sequence_dry_run.py b/tests/integration/pipelex/pipes/controller/pipe_sequence/test_pipe_sequence_dry_run.pyindex 196c1b6e2..75af9a84a 100644--- a/tests/integration/pipelex/pipes/controller/pipe_sequence/test_pipe_sequence_dry_run.py+++ b/tests/integration/pipelex/pipes/controller/pipe_sequence/test_pipe_sequence_dry_run.py@@ -114,8 +114,6 @@ class TestPipeSequenceDryRun:# Verify each summary item (these should be proper ChannelSummary objects from the LLM mock)for i, summary in enumerate(channel_summaries_list.items):- print(summary)- print(type(summary))assert isinstance(summary, ChannelSummary), f"Summary {i} should be ChannelSummary"assert len(summary.channel_name) > 0, f"Summary {i} should have a non-empty channel name"assert isinstance(summary.summary_items, list), f"Summary {i} should have a list of summary items"diff --git a/tests/integration/pipelex/pipes/pipelines/multiple_images_input_to_llm.plx b/tests/integration/pipelex/pipes/pipelines/multiple_images_input_to_llm.plxnew file mode 100644index 000000000..f8d718ea9--- /dev/null+++ b/tests/integration/pipelex/pipes/pipelines/multiple_images_input_to_llm.plx@@ -0,0 +1,39 @@+domain = "test_multiple_images_input_to_llm"+description = "Test pipeline that takes multiple images as input to a PipeLLM."++[concept]+Analysis = "An analysis of the collection ofimages."++[pipe.analyze_image_collection]+type = "PipeLLM"+description = """+Analyze the collection of images: their common features, differences, and any other relevant information.+"""+inputs = { collection_of_images = "Image[]" }+output = "Analysis"+model = "gemini-2.5-flash"+prompt = """+Analyze this collection of images: their common features and differences.++$collection_of_images+"""++[pipe.compare_two_image_collections]+type = "PipeLLM"+description = """+Compare two collections of images: identify similarities and differences between the two groups.+"""+inputs = { collection_a = "Image[]", collection_b = "Image[]" }+output = "Analysis"+model = "gemini-2.5-flash"+prompt = """+Analyze these two collections of images.++First collection:+$collection_a++Second collection:+$collection_b++Describe the similarities and differences within each group and between the two groups.+"""diff --git a/tests/integration/pipelex/pipes/test_image_inputs.py b/tests/integration/pipelex/pipes/test_image_inputs.pyindex 29eb3ac4d..09a65d673 100644--- a/tests/integration/pipelex/pipes/test_image_inputs.py+++ b/tests/integration/pipelex/pipes/test_image_inputs.py@@ -8,6 +8,7 @@ from pipelex.core.concepts.native.concept_native import NativeConceptCodefrom pipelex.core.memory.working_memory_factory import WorkingMemoryFactoryfrom pipelex.core.pipes.pipe_factory import PipeFactoryfrom pipelex.core.stuffs.image_content import ImageContent+from pipelex.core.stuffs.list_content import ListContentfrom pipelex.core.stuffs.page_content import PageContentfrom pipelex.core.stuffs.stuff_factory import StuffFactoryfrom pipelex.core.stuffs.text_and_images_content import TextAndImagesContent@@ -125,3 +126,110 @@ class TestImageInputs:# Should find both the list of images in text_and_images and the single page_view imageassert pipe_llm.llm_prompt_spec.user_images == ["page.text_and_images.images", "page.page_view"]++ async def test_analyze_image_collection(+ self,+ pipe_run_mode: PipeRunMode,+ load_test_library: Callable[[list[Path]], None],+ ) -> None:+ """Test that a PipeLLM can process a ListContent of images (Image[] input)."""+ load_test_library([Path("tests/integration/pipelex/pipes/pipelines")])++ # Create 3 images for the collection+ image_contents = [+ ImageContent(url=f"{ImageTestCases.IMAGE_FILE_PATH_PNG_1}"),+ ImageContent(url=f"{ImageTestCases.IMAGE_FILE_PATH_JPG_1}"),+ ImageContent(url=f"{ImageTestCases.IMAGE_FILE_PATH_JPG_3}"),+ ]++ # Create a ListContent containing the images+ image_list_content = ListContent[ImageContent](items=image_contents)++ # Create a stuff with the list of images+ image_collection_stuff = StuffFactory.make_stuff(+ concept=ConceptFactory.make_native_concept(native_concept_code=NativeConceptCode.IMAGE),+ content=image_list_content,+ name="collection_of_images",+ )++ # Create working memory with the image collection+ working_memory = WorkingMemoryFactory.make_from_single_stuff(stuff=image_collection_stuff)++ # Run the pipe+ pipe_output = await get_pipe_router().run(+ pipe_job=PipeJobFactory.make_pipe_job(+ pipe=get_required_pipe(pipe_code="analyze_image_collection"),+ pipe_run_params=PipeRunParamsFactory.make_run_params(pipe_run_mode=pipe_run_mode),+ working_memory=working_memory,+ job_metadata=JobMetadata(),+ ),+ )++ # Verify the output+ assert pipe_output is not None+ assert pipe_output.working_memory is not None+ assert pipe_output.main_stuff is not None++ if pipe_run_mode != PipeRunMode.DRY:+ # Verify that the output is the Analysis concept from the PLX file+ assert pipe_output.main_stuff.concept.code == "Analysis"+ # Verify the content is some kind of text output (analysis result)+ assert pipe_output.main_stuff.content is not None++ async def test_compare_two_image_collections(+ self,+ pipe_run_mode: PipeRunMode,+ load_test_library: Callable[[list[Path]], None],+ ) -> None:+ """Test that a PipeLLM can process two ListContent of images (Image[] inputs)."""+ load_test_library([Path("tests/integration/pipelex/pipes/pipelines")])++ # Create collection_a with 2 images+ collection_a_images = [+ ImageContent(url=f"{ImageTestCases.IMAGE_FILE_PATH_PNG_1}"),+ ImageContent(url=f"{ImageTestCases.IMAGE_FILE_PATH_JPG_1}"),+ ]+ collection_a_list_content = ListContent[ImageContent](items=collection_a_images)+ collection_a_stuff = StuffFactory.make_stuff(+ concept=ConceptFactory.make_native_concept(native_concept_code=NativeConceptCode.IMAGE),+ content=collection_a_list_content,+ name="collection_a",+ )++ # Create collection_b with 2 images+ collection_b_images = [+ ImageContent(url=f"{ImageTestCases.IMAGE_FILE_PATH_JPG_2}"),+ ImageContent(url=f"{ImageTestCases.IMAGE_FILE_PATH_JPG_3}"),+ ]+ collection_b_list_content = ListContent[ImageContent](items=collection_b_images)+ collection_b_stuff = StuffFactory.make_stuff(+ concept=ConceptFactory.make_native_concept(native_concept_code=NativeConceptCode.IMAGE),+ content=collection_b_list_content,+ name="collection_b",+ )++ # Create working memory with both image collections+ working_memory = WorkingMemoryFactory.make_from_multiple_stuffs(+ stuff_list=[collection_a_stuff, collection_b_stuff],+ )++ # Run the pipe+ pipe_output = await get_pipe_router().run(+ pipe_job=PipeJobFactory.make_pipe_job(+ pipe=get_required_pipe(pipe_code="compare_two_image_collections"),+ pipe_run_params=PipeRunParamsFactory.make_run_params(pipe_run_mode=pipe_run_mode),+ working_memory=working_memory,+ job_metadata=JobMetadata(),+ ),+ )++ # Verify the output+ assert pipe_output is not None+ assert pipe_output.working_memory is not None+ assert pipe_output.main_stuff is not None++ if pipe_run_mode != PipeRunMode.DRY:+ # Verify that the output is the Analysis concept from the PLX file+ assert pipe_output.main_stuff.concept.code == "Analysis"+ # Verify the content is some kind of text output (analysis result)+ assert pipe_output.main_stuff.content is not Nonediff --git a/tests/integration/pipelex/test_data.py b/tests/integration/pipelex/test_data.pyindex 422b123a5..ea49e8d47 100644--- a/tests/integration/pipelex/test_data.py+++ b/tests/integration/pipelex/test_data.py@@ -156,7 +156,7 @@ class PipeTestCases:class PipeExtractTestCases:PIPE_OCR_IMAGE_TEST_CASES: ClassVar[list[str]] = [- ImageTestCases.IMAGE_FILE_PATH_PNG_1,+ ImageTestCases.IMAGE_FILE_PATH_PNG_2,ImageTestCases.IMAGE_URL_PNG,]PIPE_OCR_PDF_TEST_CASES: ClassVar[list[str]] = PDFTestCases.DOCUMENT_FILE_PATHS + PDFTestCases.DOCUMENT_URLS