Skip to content

Build AI methods with your coding agent, then run them anywhere: as an MCP for chatbots, as a webapp for people, or via API for your software.

Fix/multiple images input to llm (#520)

Louis Choquele86867e02132
diff --git a/.vscode/launch.json b/.vscode/launch.json
index a03eb1287..afb364211 100644
--- a/.vscode/launch.json
+++ b/.vscode/launch.json
@@ -41,7 +41,7 @@
"console": "integratedTerminal"
},
{
- "name": "Debug validate pipe",
+ "name": "Debug validate bundle",
"type": "debugpy",
"request": "launch",
"program": "${workspaceFolder}/.venv/bin/pipelex",
@@ -52,6 +52,18 @@
"console": "integratedTerminal",
"justMyCode": false
},
+ {
+ "name": "Debug validate pipe",
+ "type": "debugpy",
+ "request": "launch",
+ "program": "${workspaceFolder}/.venv/bin/pipelex",
+ "args": [
+ "validate",
+ "analyze_image_collection",
+ ],
+ "console": "integratedTerminal",
+ "justMyCode": false
+ },
{
"name": "Debug run pipe",
"type": "debugpy",
diff --git a/pipelex/builder/pipe/pipe_img_gen_spec.py b/pipelex/builder/pipe/pipe_img_gen_spec.py
index b3438495a..898453e85 100644
--- a/pipelex/builder/pipe/pipe_img_gen_spec.py
+++ b/pipelex/builder/pipe/pipe_img_gen_spec.py
@@ -36,7 +36,7 @@ class PipeImgGenSpec(PipeSpec):
type: Literal["PipeImgGen"] = "PipeImgGen"
pipe_category: Literal["PipeOperator"] = "PipeOperator"
img_gen_skill: ImgGenSkill | str = Field(description="Select the most adequate image generation skill according to the task to be performed.")
- prompt: str
+ prompt: str = Field(description="A finalized image generation prompt or prompt template: use `$` prefix for inline variables (e.g., `$topic`).")
@field_validator("img_gen_skill", mode="before")
@classmethod
diff --git a/pipelex/core/memory/working_memory.py b/pipelex/core/memory/working_memory.py
index 621f18c59..1b5e1765b 100644
--- a/pipelex/core/memory/working_memory.py
+++ b/pipelex/core/memory/working_memory.py
@@ -1,5 +1,5 @@
from operator import attrgetter
-from typing import Any
+from typing import Any, cast
from pydantic import BaseModel, Field, model_validator
from typing_extensions import override
@@ -210,6 +210,37 @@ class WorkingMemory(BaseModel, ContextProviderAbstract):
@override
def get_typed_object_or_attribute(self, name: str, wanted_type: type[Any] | None = None, accept_list: bool = False) -> Any:
+ """Retrieve a typed object or nested attribute from working memory.
+
+ This method is primarily used to:
+
+ - Find specific content types that require special handling when passed to LLMs,
+ such as Images and Documents (PDFs). These types need to be formatted differently
+ in LLM prompts compared to plain text content.
+ - Find the list of items to iterate over in a PipeBatch operation.
+
+ Supports dot-notation for accessing nested attributes (e.g., "stuff_name.attribute.sub_attr").
+ When the base stuff contains ListContent, extracts the specified attribute from each item.
+
+ Args:
+ name: The name of the stuff, optionally followed by dot-separated attribute path
+ (e.g., "my_stuff" or "my_stuff.nested_attr.deeper_attr")
+ wanted_type: Optional type to validate the retrieved content against. If provided
+ and the content doesn't match, raises WorkingMemoryTypeError.
+ Commonly used with ImageContent, PDFContent, etc.
+ accept_list: If True, allows retrieval from ListContent stuffs. If False and the
+ content is ListContent, raises WorkingMemoryTypeError
+
+ Returns:
+ The retrieved content or attribute value. For ListContent with dot-notation,
+ returns a flattened list of attribute values from each item.
+
+ Raises:
+ WorkingMemoryStuffNotFoundError: If the base stuff name doesn't exist
+ WorkingMemoryStuffAttributeNotFoundError: If the attribute path is invalid
+ WorkingMemoryTypeError: If type validation fails or ListContent is encountered
+ when accept_list is False
+ """
# TODO: Refactor this method. In the python paradigm, we should not have those ".", but arrays with field names.
if "." in name:
parts = name.split(".", 1) # Split only at the first dot
@@ -270,15 +301,75 @@ class WorkingMemory(BaseModel, ContextProviderAbstract):
)
return stuff_content
- content = self.get_stuff(name).content
- if wanted_type is not None and not isinstance(content, wanted_type):
+ top_level_content = self.get_stuff(name).content
+ if isinstance(top_level_content, ListContent):
+ top_level_content = cast("ListContent[Any]", top_level_content)
+ if wanted_type is ListContent:
+ return top_level_content.items
+ else:
+ if not accept_list:
+ raise WorkingMemoryTypeError(
+ variable_name=name,
+ message=f"Content of '{name}' is ListContent, but accept_list is False",
+ )
+ top_level_content_items = self._get_typed_items_from_list_content(list_content=top_level_content, wanted_type=wanted_type)
+ if top_level_content_items is None and wanted_type is not None:
+ raise WorkingMemoryTypeError(
+ variable_name=name,
+ message=f"Content of '{name}' is ListContent, but some of its items are not of type '{wanted_type.__name__}'",
+ )
+ if top_level_content_items is not None and len(top_level_content_items) == 0:
+ raise WorkingMemoryTypeError(
+ variable_name=name,
+ message=f"Content of '{name}' is ListContent, but it is empty",
+ )
+ return top_level_content_items
+
+ if wanted_type is not None and not isinstance(top_level_content, wanted_type):
raise WorkingMemoryTypeError(
variable_name=name,
- message=f"Content of '{name}' is of type '{type(content).__name__}', it should be '{wanted_type.__name__}'",
+ message=f"Content of '{name}' is of type '{type(top_level_content).__name__}', it should be '{wanted_type.__name__}'",
)
- return content
+ return top_level_content
+
+ def _get_typed_items_from_list_content(self, list_content: ListContent[Any], wanted_type: type[Any] | None) -> list[Any] | None:
+ """Extract items from ListContent with optional type validation.
+
+ Used to collect specific content types (e.g., Images, Documents) from lists
+ that require special handling when passed to LLMs.
+
+ Flattens nested lists and filters out None values. If any item fails type
+ validation, returns None to indicate failure.
+
+ Args:
+ list_content: The ListContent to extract items from
+ wanted_type: Optional type to validate each item against (e.g., ImageContent,
+ PDFContent). If None, no type checking is performed
+
+ Returns:
+ A list of extracted items if all items pass type validation,
+ or None if any item fails validation
+ """
+ the_items: list[Any] = []
+ for item in list_content.items:
+ item_content = item
+ # check type
+ if isinstance(item_content, list):
+ for item_content_item in item_content: # pyright: ignore[reportUnknownVariableType]
+ if item_content_item is None:
+ continue
+ if wanted_type and not isinstance(item_content_item, wanted_type):
+ return None
+ the_items.append(item_content_item)
+ else:
+ if item_content is None:
+ continue
+ if wanted_type and not isinstance(item_content, wanted_type):
+ return None
+ the_items.append(item_content)
+ return the_items
################################################################################################
# Stuff accessors
diff --git a/pipelex/pipe_operators/llm/llm_prompt_blueprint.py b/pipelex/pipe_operators/llm/llm_prompt_blueprint.py
index f2f9eea6e..035882cfe 100644
--- a/pipelex/pipe_operators/llm/llm_prompt_blueprint.py
+++ b/pipelex/pipe_operators/llm/llm_prompt_blueprint.py
@@ -1,4 +1,4 @@
-from typing import TYPE_CHECKING, Any
+from typing import TYPE_CHECKING, Any, cast
from pydantic import BaseModel
@@ -55,7 +55,7 @@ class LLMPromptBlueprint(BaseModel):
if self.user_images:
for user_image_name in self.user_images:
log.verbose(f"Getting user image '{user_image_name}' from context")
- # Try to get as a single ImageContent first
+ # First: Try to get as a single ImageContent or ListContent[ImageContent]
try:
prompt_image_content = context_provider.get_typed_object_or_attribute(
name=user_image_name,
@@ -66,12 +66,14 @@ class LLMPromptBlueprint(BaseModel):
user_image = PromptImageFactory.make_prompt_image(url=prompt_image_content.url, base_64_str=prompt_image_content.base_64)
prompt_user_images[user_image_name] = user_image
elif isinstance(prompt_image_content, list):
- for image_item in prompt_image_content: # pyright: ignore[reportUnknownVariableType]
- if not isinstance(image_item, ImageContent):
- msg = f"Item of '{user_image_name}' is of type '{type(image_item).__name__}', it should be ImageContent" # pyright: ignore[reportUnknownArgumentType]
+ prompt_image_content = cast("list[ImageContent]", prompt_image_content)
+ for image_index, image_item in enumerate(prompt_image_content, start=1):
+ if not isinstance(image_item, ImageContent): # pyright: ignore[reportUnnecessaryIsInstance]
+ msg = f"Item of '{user_image_name}' is of type '{type(image_item).__name__}', it should be ImageContent"
raise LLMPromptBlueprintValueError(msg)
user_image = PromptImageFactory.make_prompt_image(url=image_item.url, base_64_str=image_item.base_64)
- prompt_user_images[user_image_name] = user_image
+ user_image_item_name = f"{user_image_name}[{image_index}]"
+ prompt_user_images[user_image_item_name] = user_image
else:
msg = (
f"User image '{user_image_name}' is of type '{type(prompt_image_content).__name__}', "
@@ -84,10 +86,11 @@ class LLMPromptBlueprint(BaseModel):
image_collection = context_provider.get_typed_object_or_attribute(name=user_image_name, wanted_type=None)
# Check if it's a list or tuple
if isinstance(image_collection, (list, tuple)):
- for image_item in image_collection: # type: ignore[assignment]
- if isinstance(image_item, ImageContent):
- user_image = PromptImageFactory.make_prompt_image(url=image_item.url, base_64_str=image_item.base_64)
- prompt_user_images[user_image_name] = user_image
+ image_collection = cast("list[ImageContent] | tuple[ImageContent]", image_collection)
+ for image_index, image_item in enumerate(image_collection, start=1):
+ user_image = PromptImageFactory.make_prompt_image(url=image_item.url, base_64_str=image_item.base_64)
+ user_image_item_name = f"{user_image_name}[{image_index}]"
+ prompt_user_images[user_image_item_name] = user_image
else:
msg = (
f"Could not find a valid user image or image collection named '{user_image_name}' from the provided context_provider"
diff --git a/tests/cases/images.py b/tests/cases/images.py
index f015728a5..1988a5291 100644
--- a/tests/cases/images.py
+++ b/tests/cases/images.py
@@ -11,11 +11,13 @@ class ImageTestCases:
# Individual file paths
IMAGE_FILE_PATH_PNG_1 = f"{TEST_IMAGE_DIRECTORY}/ai_lympics.png"
- IMAGE_FILE_PATH_JPG_1 = f"{TEST_IMAGE_DIRECTORY}/solar_system.jpg"
+ IMAGE_FILE_PATH_JPG_1 = f"{TEST_IMAGE_DIRECTORY}/animal_lympics.jpg"
+ IMAGE_FILE_PATH_JPG_2 = f"{TEST_IMAGE_DIRECTORY}/solar_system.jpg"
IMAGE_FILE_PATH_PNG_2 = f"{TEST_IMAGE_DIRECTORY}/solar_system.png"
+ IMAGE_FILE_PATH_JPG_3 = f"{TEST_IMAGE_DIRECTORY}/eiffel_tower.jpg"
# Remote URLs
- IMAGE_URL_PNG = "https://www.python.org/static/community_logos/python-logo-master-v3-TM.png"
+ IMAGE_URL_PNG = "https://pipelex-web.s3.amazonaws.com/tests/solar_system.png"
# File path collections
IMAGE_FILE_PATHS: ClassVar[list[str]] = [
diff --git a/tests/integration/pipelex/pipes/controller/pipe_sequence/test_pipe_sequence_dry_run.py b/tests/integration/pipelex/pipes/controller/pipe_sequence/test_pipe_sequence_dry_run.py
index 196c1b6e2..75af9a84a 100644
--- a/tests/integration/pipelex/pipes/controller/pipe_sequence/test_pipe_sequence_dry_run.py
+++ b/tests/integration/pipelex/pipes/controller/pipe_sequence/test_pipe_sequence_dry_run.py
@@ -114,8 +114,6 @@ class TestPipeSequenceDryRun:
# Verify each summary item (these should be proper ChannelSummary objects from the LLM mock)
for i, summary in enumerate(channel_summaries_list.items):
- print(summary)
- print(type(summary))
assert isinstance(summary, ChannelSummary), f"Summary {i} should be ChannelSummary"
assert len(summary.channel_name) > 0, f"Summary {i} should have a non-empty channel name"
assert isinstance(summary.summary_items, list), f"Summary {i} should have a list of summary items"
diff --git a/tests/integration/pipelex/pipes/pipelines/multiple_images_input_to_llm.plx b/tests/integration/pipelex/pipes/pipelines/multiple_images_input_to_llm.plx
new file mode 100644
index 000000000..f8d718ea9
--- /dev/null
+++ b/tests/integration/pipelex/pipes/pipelines/multiple_images_input_to_llm.plx
@@ -0,0 +1,39 @@
+domain = "test_multiple_images_input_to_llm"
+description = "Test pipeline that takes multiple images as input to a PipeLLM."
+
+[concept]
+Analysis = "An analysis of the collection ofimages."
+
+[pipe.analyze_image_collection]
+type = "PipeLLM"
+description = """
+Analyze the collection of images: their common features, differences, and any other relevant information.
+"""
+inputs = { collection_of_images = "Image[]" }
+output = "Analysis"
+model = "gemini-2.5-flash"
+prompt = """
+Analyze this collection of images: their common features and differences.
+
+$collection_of_images
+"""
+
+[pipe.compare_two_image_collections]
+type = "PipeLLM"
+description = """
+Compare two collections of images: identify similarities and differences between the two groups.
+"""
+inputs = { collection_a = "Image[]", collection_b = "Image[]" }
+output = "Analysis"
+model = "gemini-2.5-flash"
+prompt = """
+Analyze these two collections of images.
+
+First collection:
+$collection_a
+
+Second collection:
+$collection_b
+
+Describe the similarities and differences within each group and between the two groups.
+"""
diff --git a/tests/integration/pipelex/pipes/test_image_inputs.py b/tests/integration/pipelex/pipes/test_image_inputs.py
index 29eb3ac4d..09a65d673 100644
--- a/tests/integration/pipelex/pipes/test_image_inputs.py
+++ b/tests/integration/pipelex/pipes/test_image_inputs.py
@@ -8,6 +8,7 @@ from pipelex.core.concepts.native.concept_native import NativeConceptCode
from pipelex.core.memory.working_memory_factory import WorkingMemoryFactory
from pipelex.core.pipes.pipe_factory import PipeFactory
from pipelex.core.stuffs.image_content import ImageContent
+from pipelex.core.stuffs.list_content import ListContent
from pipelex.core.stuffs.page_content import PageContent
from pipelex.core.stuffs.stuff_factory import StuffFactory
from pipelex.core.stuffs.text_and_images_content import TextAndImagesContent
@@ -125,3 +126,110 @@ class TestImageInputs:
# Should find both the list of images in text_and_images and the single page_view image
assert pipe_llm.llm_prompt_spec.user_images == ["page.text_and_images.images", "page.page_view"]
+
+ async def test_analyze_image_collection(
+ self,
+ pipe_run_mode: PipeRunMode,
+ load_test_library: Callable[[list[Path]], None],
+ ) -> None:
+ """Test that a PipeLLM can process a ListContent of images (Image[] input)."""
+ load_test_library([Path("tests/integration/pipelex/pipes/pipelines")])
+
+ # Create 3 images for the collection
+ image_contents = [
+ ImageContent(url=f"{ImageTestCases.IMAGE_FILE_PATH_PNG_1}"),
+ ImageContent(url=f"{ImageTestCases.IMAGE_FILE_PATH_JPG_1}"),
+ ImageContent(url=f"{ImageTestCases.IMAGE_FILE_PATH_JPG_3}"),
+ ]
+
+ # Create a ListContent containing the images
+ image_list_content = ListContent[ImageContent](items=image_contents)
+
+ # Create a stuff with the list of images
+ image_collection_stuff = StuffFactory.make_stuff(
+ concept=ConceptFactory.make_native_concept(native_concept_code=NativeConceptCode.IMAGE),
+ content=image_list_content,
+ name="collection_of_images",
+ )
+
+ # Create working memory with the image collection
+ working_memory = WorkingMemoryFactory.make_from_single_stuff(stuff=image_collection_stuff)
+
+ # Run the pipe
+ pipe_output = await get_pipe_router().run(
+ pipe_job=PipeJobFactory.make_pipe_job(
+ pipe=get_required_pipe(pipe_code="analyze_image_collection"),
+ pipe_run_params=PipeRunParamsFactory.make_run_params(pipe_run_mode=pipe_run_mode),
+ working_memory=working_memory,
+ job_metadata=JobMetadata(),
+ ),
+ )
+
+ # Verify the output
+ assert pipe_output is not None
+ assert pipe_output.working_memory is not None
+ assert pipe_output.main_stuff is not None
+
+ if pipe_run_mode != PipeRunMode.DRY:
+ # Verify that the output is the Analysis concept from the PLX file
+ assert pipe_output.main_stuff.concept.code == "Analysis"
+ # Verify the content is some kind of text output (analysis result)
+ assert pipe_output.main_stuff.content is not None
+
+ async def test_compare_two_image_collections(
+ self,
+ pipe_run_mode: PipeRunMode,
+ load_test_library: Callable[[list[Path]], None],
+ ) -> None:
+ """Test that a PipeLLM can process two ListContent of images (Image[] inputs)."""
+ load_test_library([Path("tests/integration/pipelex/pipes/pipelines")])
+
+ # Create collection_a with 2 images
+ collection_a_images = [
+ ImageContent(url=f"{ImageTestCases.IMAGE_FILE_PATH_PNG_1}"),
+ ImageContent(url=f"{ImageTestCases.IMAGE_FILE_PATH_JPG_1}"),
+ ]
+ collection_a_list_content = ListContent[ImageContent](items=collection_a_images)
+ collection_a_stuff = StuffFactory.make_stuff(
+ concept=ConceptFactory.make_native_concept(native_concept_code=NativeConceptCode.IMAGE),
+ content=collection_a_list_content,
+ name="collection_a",
+ )
+
+ # Create collection_b with 2 images
+ collection_b_images = [
+ ImageContent(url=f"{ImageTestCases.IMAGE_FILE_PATH_JPG_2}"),
+ ImageContent(url=f"{ImageTestCases.IMAGE_FILE_PATH_JPG_3}"),
+ ]
+ collection_b_list_content = ListContent[ImageContent](items=collection_b_images)
+ collection_b_stuff = StuffFactory.make_stuff(
+ concept=ConceptFactory.make_native_concept(native_concept_code=NativeConceptCode.IMAGE),
+ content=collection_b_list_content,
+ name="collection_b",
+ )
+
+ # Create working memory with both image collections
+ working_memory = WorkingMemoryFactory.make_from_multiple_stuffs(
+ stuff_list=[collection_a_stuff, collection_b_stuff],
+ )
+
+ # Run the pipe
+ pipe_output = await get_pipe_router().run(
+ pipe_job=PipeJobFactory.make_pipe_job(
+ pipe=get_required_pipe(pipe_code="compare_two_image_collections"),
+ pipe_run_params=PipeRunParamsFactory.make_run_params(pipe_run_mode=pipe_run_mode),
+ working_memory=working_memory,
+ job_metadata=JobMetadata(),
+ ),
+ )
+
+ # Verify the output
+ assert pipe_output is not None
+ assert pipe_output.working_memory is not None
+ assert pipe_output.main_stuff is not None
+
+ if pipe_run_mode != PipeRunMode.DRY:
+ # Verify that the output is the Analysis concept from the PLX file
+ assert pipe_output.main_stuff.concept.code == "Analysis"
+ # Verify the content is some kind of text output (analysis result)
+ assert pipe_output.main_stuff.content is not None
diff --git a/tests/integration/pipelex/test_data.py b/tests/integration/pipelex/test_data.py
index 422b123a5..ea49e8d47 100644
--- a/tests/integration/pipelex/test_data.py
+++ b/tests/integration/pipelex/test_data.py
@@ -156,7 +156,7 @@ class PipeTestCases:
class PipeExtractTestCases:
PIPE_OCR_IMAGE_TEST_CASES: ClassVar[list[str]] = [
- ImageTestCases.IMAGE_FILE_PATH_PNG_1,
+ ImageTestCases.IMAGE_FILE_PATH_PNG_2,
ImageTestCases.IMAGE_URL_PNG,
]
PIPE_OCR_PDF_TEST_CASES: ClassVar[list[str]] = PDFTestCases.DOCUMENT_FILE_PATHS + PDFTestCases.DOCUMENT_URLS