Skip to content

Build AI methods with your coding agent, then run them anywhere: as an MCP for chatbots, as a webapp for people, or via API for your software.

Update LLM model defaults and add E2E tests for structured object list generation

Louis Choquel3dc0555939b6
diff --git a/.pipelex/inference/deck/1_llm_deck.toml b/.pipelex/inference/deck/1_llm_deck.toml
index e83bf0825..360b07e2b 100644
--- a/.pipelex/inference/deck/1_llm_deck.toml
+++ b/.pipelex/inference/deck/1_llm_deck.toml
@@ -41,8 +41,8 @@ default-large-context-code = "gemini-3.0-pro"
default-large-context-text = "gemini-2.5-flash"
default-small = "gpt-4o-mini"
default-small-structured = "gpt-4o-mini"
-default-small-vision = "gemini-2.5-flash-lite"
-default-small-creative = "gemini-2.5-flash-lite"
+default-small-vision = "gemini-3.0-flash-preview"
+default-small-creative = "gemini-3.0-flash-preview"
####################################################################################################
# LLM Presets
diff --git a/tests/e2e/pipelex/pipes/pipe_operators/pipe_llm/pipe_llm_object_list.mthds b/tests/e2e/pipelex/pipes/pipe_operators/pipe_llm/pipe_llm_object_list.mthds
new file mode 100644
index 000000000..8e434ec55
--- /dev/null
+++ b/tests/e2e/pipelex/pipes/pipe_operators/pipe_llm/pipe_llm_object_list.mthds
@@ -0,0 +1,29 @@
+domain = "pipe_llm_object_list_e2e"
+description = "Test PipeLLM with structured object list output"
+
+# [concept.MoodboardPrompt]
+# description = "A vivid image generation prompt for a fashion moodboard panel"
+# refines = "Text"
+[concept]
+MoodboardPrompt = "A vivid image generation prompt for a fashion moodboard panel"
+
+[pipe.craft_prompts]
+type = "PipeLLM"
+description = "Generate diverse moodboard image prompts from fashion inspiration"
+inputs = { inspiration = "Text" }
+output = "MoodboardPrompt[]"
+model = "@default-small-creative"
+prompt = """
+You are a fashion art director creating a moodboard. Given the following fashion inspiration, generate exactly 5 distinct image generation prompts. Each prompt should capture a different visual facet of the aesthetic:
+
+1. A hero outfit or look
+2. A texture or fabric close-up
+3. A color palette scene
+4. A setting or environment that evokes the mood
+5. An accessory or detail shot
+
+Fashion inspiration:
+@inspiration
+
+For each prompt, write a vivid, detailed image generation prompt that would produce a beautiful moodboard panel. Focus on colors, textures, lighting, and composition.
+"""
diff --git a/tests/e2e/pipelex/pipes/pipe_operators/pipe_llm/test_pipe_llm_object_list.py b/tests/e2e/pipelex/pipes/pipe_operators/pipe_llm/test_pipe_llm_object_list.py
new file mode 100644
index 000000000..c9f8e4d44
--- /dev/null
+++ b/tests/e2e/pipelex/pipes/pipe_operators/pipe_llm/test_pipe_llm_object_list.py
@@ -0,0 +1,35 @@
+"""E2E test for PipeLLM with structured object list output."""
+
+import pytest
+
+from pipelex import pretty_print
+from pipelex.core.stuffs.structured_content import StructuredContent
+from pipelex.core.stuffs.text_content import TextContent
+from pipelex.pipe_run.pipe_run_mode import PipeRunMode
+from pipelex.pipeline.runner import PipelexRunner
+
+
+class TestPipeLLMObjectList:
+ async def test_craft_prompts(self, pipe_run_mode: PipeRunMode) -> None:
+ """Test a PipeLLM pipe that generates a list of structured MoodboardPrompt objects."""
+ pipeline_response = await PipelexRunner(
+ library_dirs=["tests/e2e/pipelex/pipes/pipe_operators"], pipe_run_mode=pipe_run_mode
+ ).execute_pipeline(
+ pipe_code="craft_prompts",
+ inputs={
+ "inspiration": TextContent(text="1970s bohemian chic with earthy tones and flowing fabrics"),
+ },
+ )
+
+ assert pipeline_response.pipe_output is not None
+ assert pipeline_response.pipe_output.working_memory is not None
+ assert pipeline_response.pipe_output.main_stuff is not None
+
+ items = pipeline_response.pipe_output.main_stuff_as_list(item_type=TextContent)
+ assert len(items) == 5
+
+ pretty_print(items, title="Moodboard prompts")
diff --git a/tests/integration/pipelex/cogt/test_data.py b/tests/integration/pipelex/cogt/test_data.py
index 91fcabcd5..0ac00d974 100644
--- a/tests/integration/pipelex/cogt/test_data.py
+++ b/tests/integration/pipelex/cogt/test_data.py
@@ -122,6 +122,7 @@ class LLMTestConstants:
USER_TEXT_SHORT = "In one short sentence, who is Bill Gates?"
USER_TEXT_SUPER_SHORT = "In one short sentence (< 5 words), who is Bill Gates?"
USER_TEXT_TO_EXTRACT_PERSON = "It's Robert, the nice plumber, he turns 57 next week."
+ USER_TEXT_TO_GEN_PERSON_LIST = "List the following people: Alice is 30, Bob is 25, and Charlie is 35."
USER_TEXT_TRICKY_1 = """
When my son was 7 he was 3ft tall. When he was 8 he was 4ft tall. When he was 9 he was 5ft tall.
How tall do you think he was when he was 12? and at 15?
diff --git a/tests/integration/pipelex/cogt/test_llm_inference.py b/tests/integration/pipelex/cogt/test_llm_inference.py
index 671fab64b..82b93a809 100644
--- a/tests/integration/pipelex/cogt/test_llm_inference.py
+++ b/tests/integration/pipelex/cogt/test_llm_inference.py
@@ -1,4 +1,5 @@
import pytest
+from pydantic import BaseModel
from pipelex import log, pretty_print
from pipelex.cogt.exceptions import PromptImageFormatError
@@ -54,6 +55,35 @@ class TestLLMInference:
pretty_print(generated_object)
# get_report_delegate().generate_report()
+ async def test_simple_gen_object_list_from_text(self, job_metadata: JobMetadata, llm_job_params: LLMJobParams, llm_combo: ModelCombo):
+ """Generate a list of structured objects from a text prompt."""
+ log.info(f"test_simple_gen_object_list_from_text: Testing llm_handle '{llm_combo.handle}'")
+ llm_worker = get_inference_manager().get_llm_worker(llm_handle=llm_combo.handle)
+ if not llm_worker.is_gen_object_supported:
+ msg = f"Object generation is not supported for this LLM worker: '{llm_worker.desc}'"
+ log.info(msg)
+ pytest.skip(msg)
+
+ class PersonList(BaseModel):
+ items: list[Person]
+
+ llm_job = LLMJobFactory.make_llm_job(
+ llm_prompt=LLMPrompt(
+ system_text=None,
+ user_text=LLMTestConstants.USER_TEXT_TO_GEN_PERSON_LIST,
+ ),
+ job_metadata=job_metadata,
+ llm_job_params=llm_job_params,
+ )
+ generated_list = await llm_worker.gen_object(llm_job=llm_job, schema=PersonList)
+ assert generated_list
+ assert generated_list.items
+ assert len(generated_list.items) >= 2
+ for person in generated_list.items:
+ assert person.name
+ assert person.age > 0
+ pretty_print(generated_list, title="Generated person list")
+
@pytest.mark.parametrize("image_path", [LLMVisionTestCases.PATH_IMG_PNG_1])
async def test_gen_text_from_image(self, job_metadata: JobMetadata, llm_job_params: LLMJobParams, llm_combo: ModelCombo, image_path: str):
log.info(f"test_gen_text_from_image: Testing llm_handle '{llm_combo.handle}'")