Build AI methods with your coding agent, then run them anywhere: as an MCP for chatbots, as a webapp for people, or via API for your software.
dev to release/v0.6.8 (#200)
Thomas Hebrardc540e0b7a076
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex fe3ac473e..36d478e5a 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -1,5 +1,25 @@# Changelog+## [v0.6.8] - 2025-07-25++### Added+- New method `make_stuff_using_concept_name_and_search_domains` in `StuffFactory` for creating stuff using concept names and search domains.+- New method `make_stuff_from_stuff_content_using_search_domains` in `StuffFactory` for creating stuff from stuff content using search domains.+- New method `make_from_implicit_memory` in `WorkingMemoryFactory` for creating working memory from implicit memory.+- New method `create_mock_content` in `WorkingMemoryFactory` for creating mock content for requirements.++### Changed+- Refactored `PipeInputSpec` to use `InputRequirement` and `TypedNamedInputRequirement` classes instead of plain strings for input specifications.+- Updated `WorkingMemoryFactory` to handle `ImplicitMemory` instead of `CompactMemory`.+- Replaced `ExecutePipelineException` with `PipelineInputError` in `execute_pipeline` function.+- Updated `PipeBatch`, `PipeCondition`, `PipeParallel`, `PipeSequence`, `PipeFunc`, `PipeImgGen`, `PipeJinja2`, `PipeLLM`, and `PipeOcr` classes to use `InputRequirement` for input handling.+- Updated `PipeInputSpec` creation in various test files to use `make_from_dict` method.+- Updated `pyproject.toml` to exclude `pypdfium2` version `4.30.1`.+- Updated `Jinja2TemplateCategory` to handle HTML and Markdown templates differently.++### Fixed+- Corrected error messages in `StuffFactory` and `StuffContentFactory` to provide more detailed information about exceptions.+## [v0.6.7] - 2025-07-24### Removeddiff --git a/pipelex/cli/_cli.py b/pipelex/cli/_cli.pyindex 93d6a83ca..5caa553ea 100644--- a/pipelex/cli/_cli.py+++ b/pipelex/cli/_cli.py@@ -36,7 +36,13 @@ def is_pipelex_libraries_folder(folder_path: str) -> bool:if not os.path.exists(folder_path) or not os.path.isdir(folder_path):return False- required_subdirs = ["pipelines", "llm_deck", "llm_integrations", "plugins", "templates"]+ required_subdirs = [+ "pipelines",+ "llm_deck",+ "llm_integrations",+ "plugins",+ "templates",+ ]for subdir in required_subdirs:subdir_path = os.path.join(folder_path, subdir)@@ -67,8 +73,18 @@ app = typer.Typer(@app.command("init-libraries")def init_libraries(- directory: Annotated[str, typer.Argument(help="Directory where to create the pipelex_libraries folder")] = ".",- overwrite: Annotated[bool, typer.Option("--overwrite", "-o", help="Warning: If set, existing files will be overwritten.")] = False,+ directory: Annotated[+ str,+ typer.Argument(help="Directory where to create the pipelex_libraries folder"),+ ] = ".",+ overwrite: Annotated[+ bool,+ typer.Option(+ "--overwrite",+ "-o",+ help="Warning: If set, existing files will be overwritten.",+ ),+ ] = False,) -> None:"""Initialize pipelex libraries in a pipelex_libraries folder in the specified directory.@@ -96,7 +112,10 @@ def init_libraries(@app.command("init-config")def init_config(- reset: Annotated[bool, typer.Option("--reset", "-r", help="Warning: If set, existing files will be overwritten.")] = False,+ reset: Annotated[+ bool,+ typer.Option("--reset", "-r", help="Warning: If set, existing files will be overwritten."),+ ] = False,) -> None:"""Initialize pipelex configuration in the current directory."""pipelex_template_path = os.path.join(config_manager.pipelex_root_dir, "pipelex_template.toml")@@ -116,9 +135,11 @@ def init_config(@app.command()def validate(relative_config_folder_path: Annotated[- str, typer.Option("--config-folder-path", "-c", help="Relative path to the config folder path")+ str,+ typer.Option("--config-folder-path", "-c", help="Relative path to the config folder path"),] = "./pipelex_libraries",) -> None:+ print("DEBUG: Entered validate()")"""Run the setup sequence."""# Check if pipelex libraries folder existsif not is_pipelex_libraries_folder(relative_config_folder_path):@@ -136,7 +157,8 @@ def validate(def dry_run_pipe(pipe_code: Annotated[str, typer.Argument(help="The pipe code to dry run")],relative_config_folder_path: Annotated[- str, typer.Option("--config-folder-path", "-c", help="Relative path to the config folder path")+ str,+ typer.Option("--config-folder-path", "-c", help="Relative path to the config folder path"),] = "./pipelex_libraries",) -> None:"""Dry run a single pipe."""@@ -165,7 +187,10 @@ def show_config() -> None:"""Show the pipelex configuration."""try:final_config = config_manager.load_config()- pretty_print(final_config, title=f"Pipelex configuration for project: {config_manager.get_project_name()}")+ pretty_print(+ final_config,+ title=f"Pipelex configuration for project: {config_manager.get_project_name()}",+ )except Exception as e:raise PipelexConfigError(f"Error loading configuration: {e}")@@ -173,7 +198,8 @@ def show_config() -> None:@app.command()def list_pipes(relative_config_folder_path: Annotated[- str, typer.Option("--config-folder-path", "-c", help="Relative path to the config folder path")+ str,+ typer.Option("--config-folder-path", "-c", help="Relative path to the config folder path"),] = "pipelex_libraries",) -> None:"""List all available pipes."""@@ -193,7 +219,8 @@ def show_pipe(typer.Argument(help="Pipeline code to show definition for"),],relative_config_folder_path: Annotated[- str, typer.Option("--config-folder-path", "-c", help="Relative path to the config folder path")+ str,+ typer.Option("--config-folder-path", "-c", help="Relative path to the config folder path"),] = "./pipelex_libraries",) -> None:"""Show pipe from the pipe library."""diff --git a/pipelex/client/protocol.py b/pipelex/client/protocol.pyindex 54ef7150b..8348b33c6 100644--- a/pipelex/client/protocol.py+++ b/pipelex/client/protocol.py@@ -1,13 +1,16 @@from abc import abstractmethod-from typing import Any, Dict, Optional, Protocol+from typing import Any, Dict, List, Optional, Protocolfrom pydantic import BaseModelfrom typing_extensions import runtime_checkablefrom pipelex.core.pipe_run_params import PipeOutputMultiplicity+from pipelex.core.stuff_content import StuffContentfrom pipelex.core.working_memory import WorkingMemoryfrom pipelex.types import StrEnum+StuffContentOrData = Dict[str, Any] | StuffContent | List[Any]+ImplicitMemory = Dict[str, StuffContentOrData]CompactMemory = Dict[str, Dict[str, Any]]COMPACT_MEMORY_KEY = "compact_memory"diff --git a/pipelex/core/concept.py b/pipelex/core/concept.pyindex 11e4fa9f7..e36b5e69b 100644--- a/pipelex/core/concept.py+++ b/pipelex/core/concept.py@@ -106,7 +106,7 @@ class Concept(BaseModel):if "." in concept_str:domain_code, concept_code = concept_str.split(".")return domain_code, concept_code- raise ConceptError(f"Could not extract domain and concept from concept code '{concept_str}'")+ raise ConceptError(f"Could not extract domain and concept from '{concept_str}'")@classmethoddef extract_concept_name_from_str(cls, concept_str: str) -> str:diff --git a/pipelex/core/concept_library.py b/pipelex/core/concept_library.pyindex ab094c7ed..788734a27 100644--- a/pipelex/core/concept_library.py+++ b/pipelex/core/concept_library.py@@ -5,6 +5,7 @@ from typing_extensions import overridefrom pipelex import logfrom pipelex.core.concept import Concept+from pipelex.core.concept_code_factory import ConceptCodeFactoryfrom pipelex.core.concept_factory import ConceptFactoryfrom pipelex.core.concept_native import NativeConceptfrom pipelex.core.concept_provider_abstract import ConceptProviderAbstract@@ -178,3 +179,12 @@ class ConceptLibrary(RootModel[ConceptLibraryRoot], ConceptProviderAbstract):is_image_class = bool(pydantic_model and issubclass(pydantic_model, ImageContent))refines_image = self.is_compatible_by_concept_code(tested_concept_code=concept.code, wanted_concept_code=NativeConcept.IMAGE.code)return is_image_class or refines_image++ @override+ def search_for_concept_in_domains(self, concept_name: str, search_domains: List[str]) -> Optional[Concept]:+ for domain in search_domains:+ concept_code = ConceptCodeFactory.make_concept_code(domain=domain, code=concept_name)+ if found_concept := self.get_concept(concept_code=concept_code):+ return found_concept++ return Nonediff --git a/pipelex/core/concept_provider_abstract.py b/pipelex/core/concept_provider_abstract.pyindex 77f89fa3d..379b96490 100644--- a/pipelex/core/concept_provider_abstract.py+++ b/pipelex/core/concept_provider_abstract.py@@ -54,3 +54,7 @@ class ConceptProviderAbstract(ABC):@abstractmethoddef is_concept_code_legal(self, concept_code: str) -> bool:pass++ @abstractmethod+ def search_for_concept_in_domains(self, concept_name: str, search_domains: List[str]) -> Optional[Concept]:+ passdiff --git a/pipelex/core/pipe_input_spec.py b/pipelex/core/pipe_input_spec.pyindex 57711b3a2..5e489fb47 100644--- a/pipelex/core/pipe_input_spec.py+++ b/pipelex/core/pipe_input_spec.py@@ -1,13 +1,38 @@-from typing import Callable, Dict, List, Set, Tuple+from typing import Callable, Dict, List, Optional, Set, Tuple, Type-from pydantic import Field, RootModel, field_validator+from pydantic import BaseModel, Field, RootModel, field_validatorfrom typing_extensions import Selffrom pipelex import logfrom pipelex.core.concept_code_factory import ConceptCodeFactory+from pipelex.core.pipe_run_params import PipeOutputMultiplicity+from pipelex.core.stuff_content import StuffContentfrom pipelex.exceptions import PipeInputNotFoundError-PipeInputSpecRoot = Dict[str, str]++class InputRequirement(BaseModel):+ concept_code: str+ multiplicity: Optional[PipeOutputMultiplicity] = None+++class NamedInputRequirement(InputRequirement):+ variable_name: str+ requirement_expression: Optional[str] = None+++class TypedNamedInputRequirement(NamedInputRequirement):+ structure_class: Type[StuffContent]++ @classmethod+ def make_from_named(+ cls,+ named: NamedInputRequirement,+ structure_class: Type[StuffContent],+ ) -> "TypedNamedInputRequirement":+ return cls(**named.model_dump(), structure_class=structure_class)+++PipeInputSpecRoot = Dict[str, InputRequirement]class PipeInputSpec(RootModel[PipeInputSpecRoot]):@@ -20,58 +45,70 @@ class PipeInputSpec(RootModel[PipeInputSpecRoot]):@field_validator("root", mode="wrap")@classmethod- def validate_concept_codes(cls, input_value: Dict[str, str], handler: Callable[[Dict[str, str]], Dict[str, str]]) -> Dict[str, str]:+ def validate_concept_codes(+ cls,+ input_value: Dict[str, InputRequirement],+ handler: Callable[[Dict[str, InputRequirement]], Dict[str, InputRequirement]],+ ) -> Dict[str, InputRequirement]:# First let Pydantic handle the basic type validation- validated_dict: Dict[str, str] = handler(input_value)+ validated_dict: Dict[str, InputRequirement] = handler(input_value)# Now we can transform and validate the keys and values- transformed_dict: Dict[str, str] = {}- for required_input, concept_str in validated_dict.items():+ transformed_dict: Dict[str, InputRequirement] = {}+ for required_input, requirement in validated_dict.items():# in case of sub-attribute, the variable name is the object name, before the 1st dottransformed_key: str = required_input.split(".", 1)[0]if transformed_key != required_input:log.verbose(f"Sub-attribute {required_input} detected, using {transformed_key} as variable name")# Validate concept_code- concept_code = ConceptCodeFactory.make_concept_code_from_str(concept_str=concept_str)+ concept_code = ConceptCodeFactory.make_concept_code_from_str(concept_str=requirement.concept_code)- if transformed_key in transformed_dict and transformed_dict[transformed_key] != concept_code:+ if transformed_key in transformed_dict and transformed_dict[transformed_key] != requirement:log.verbose(- f"Variable {transformed_key} already exists with a different concept code: {transformed_dict[transformed_key]} -> {concept_str}"+ f"Variable {transformed_key} already exists with a different concept code: {transformed_dict[transformed_key]} -> {requirement}")- transformed_dict[transformed_key] = concept_code+ transformed_dict[transformed_key] = InputRequirement(concept_code=concept_code, multiplicity=requirement.multiplicity)return transformed_dictdef set_default_domain(self, domain: str):- for input_name, input_concept_code in self.root.items():+ for input_name, requirement in self.root.items():+ input_concept_code = requirement.concept_codeif "." not in input_concept_code:- self.root[input_name] = f"{domain}.{input_concept_code}"+ requirement.concept_code = f"{domain}.{input_concept_code}"+ self.root[input_name] = requirementdef get_required_concept_code(self, variable_name: str) -> str:- concept_code = self.root.get(variable_name)- if not concept_code:+ requirement = self.root.get(variable_name)+ if not requirement:raise PipeInputNotFoundError(f"Variable '{variable_name}' not found in input spec")- return concept_code+ return requirement.concept_code- def add_requirement(self, variable_name: str, concept_code: str):- self.root[variable_name] = concept_code+ def add_requirement(self, variable_name: str, concept_code: str, multiplicity: Optional[PipeOutputMultiplicity] = None):+ self.root[variable_name] = InputRequirement(concept_code=concept_code, multiplicity=multiplicity)@classmethoddef make_empty(cls) -> Self:return cls(root={})@classmethod- def make_from_dict(cls, input_dict: Dict[str, str]) -> Self:- return cls(root=input_dict)+ def make_from_dict(cls, concepts_dict: Dict[str, str]) -> Self:+ the_dict: Dict[str, InputRequirement] = {+ var_name: InputRequirement(concept_code=concept_code) for var_name, concept_code in concepts_dict.items()+ }+ return cls(root=the_dict)@property- def items(self) -> List[Tuple[str, str]]:+ def items(self) -> List[Tuple[str, InputRequirement]]:return list(self.root.items())@propertydef concepts(self) -> Set[str]:- return set(self.root.values())+ all_concepts: Set[str] = set()+ for requirement in self.root.values():+ all_concepts.add(requirement.concept_code)+ return all_concepts@propertydef variables(self) -> List[str]:@@ -86,9 +123,17 @@ class PipeInputSpec(RootModel[PipeInputSpecRoot]):return the_required_names@property- def detailed_requirements(self) -> List[Tuple[str, str, str]]:- the_requirements: List[Tuple[str, str, str]] = []- for requirement_expression, concept_code in self.root.items():+ def named_input_requirements(self) -> List[NamedInputRequirement]:+ the_requirements: List[NamedInputRequirement] = []+ for requirement_expression, requirement in self.root.items():required_variable_name = requirement_expression.split(".", 1)[0]- the_requirements.append((required_variable_name, requirement_expression, concept_code))+ # TODO: refactor this with a proper class like InputRequirement+ the_requirements.append(+ NamedInputRequirement(+ variable_name=required_variable_name,+ requirement_expression=requirement_expression,+ concept_code=requirement.concept_code,+ multiplicity=requirement.multiplicity,+ )+ )return the_requirementsdiff --git a/pipelex/core/pipe_library.py b/pipelex/core/pipe_library.pyindex 98bc45387..0ef54eca9 100644--- a/pipelex/core/pipe_library.py+++ b/pipelex/core/pipe_library.py@@ -110,7 +110,7 @@ class PipeLibrary(RootModel[PipeLibraryRoot], PipeProviderAbstract):for pipe in domain_pipes:inputs = pipe.inputs- formatted_inputs = [f"{name}: {_format_concept_code(concept_code, domain)}" for name, concept_code in inputs.items]+ formatted_inputs = [f"{name}: {_format_concept_code(requirement.concept_code, domain)}" for name, requirement in inputs.items]formatted_inputs_str = ", ".join(formatted_inputs)output_code = _format_concept_code(pipe.output_concept_code, domain)diff --git a/pipelex/core/stuff.py b/pipelex/core/stuff.pyindex f7c3361a2..208c51701 100644--- a/pipelex/core/stuff.py+++ b/pipelex/core/stuff.py@@ -40,8 +40,9 @@ class Stuff(CustomBaseModel):if key in artefact_dict:stuff_name = self.stuff_name or f"unnamed using concept code {self.concept_code}"raise StuffError(- f"""Cannot create stuff artefact for stuff {stuff_name} because reserved field {key} already exists in the stuff content.- Forbidden fields are: `stuff_name`, `content_class`, `concept_code`, `stuff_code`, `content`"""+ f"""Cannot create stuff artefact for stuff '{stuff_name}' of concept '{self.concept_code}' because reserved field '{key}'+in the structured output '{self.content.__class__.__name__}' already exists in the stuff content.+Forbidden fields are: 'stuff_name', 'content_class', 'concept_code', 'stuff_code', 'content'.""")artefact_dict[key] = valuediff --git a/pipelex/core/stuff_factory.py b/pipelex/core/stuff_factory.pyindex 305178394..e5c7ca834 100644--- a/pipelex/core/stuff_factory.py+++ b/pipelex/core/stuff_factory.py@@ -1,13 +1,19 @@-from typing import Any, Dict, List, Optional, Tuple, Type+from typing import Any, Dict, List, Optional, Tuple, Type, castimport shortuuidfrom pydantic import BaseModel, ValidationError+from pipelex.client.protocol import StuffContentOrDatafrom pipelex.core.concept import Conceptfrom pipelex.core.concept_code_factory import ConceptCodeFactoryfrom pipelex.core.concept_native import NativeConceptfrom pipelex.core.stuff import Stuff-from pipelex.core.stuff_content import StuffContent, StuffContentInitableFromStr, TextContent+from pipelex.core.stuff_content import (+ ListContent,+ StuffContent,+ StuffContentInitableFromStr,+ TextContent,+)from pipelex.exceptions import ConceptError, PipelexErrorfrom pipelex.hub import get_class_registry, get_concept_provider, get_required_conceptfrom pipelex.tools.typing.pydantic_utils import format_pydantic_validation_error@@ -38,9 +44,9 @@ class StuffFactory:) -> Stuff:try:concept_code = ConceptCodeFactory.make_concept_code_from_str(concept_str=concept_str)- except ConceptError:+ except ConceptError as exc:stuff_ref = name or code or "unnamed"- raise StuffFactoryError(f"Concept '{concept_str}' does not contain a domain, could not make stuff '{stuff_ref}'")+ raise StuffFactoryError(f"Could not make stuff '{stuff_ref}': Could not make concept from '{concept_str}': {exc}") from excif not name:name = cls.make_stuff_name(concept_code)return Stuff(@@ -50,6 +56,24 @@ class StuffFactory:stuff_code=code or shortuuid.uuid()[:5],)+ @classmethod+ def make_stuff_using_concept_name_and_search_domains(+ cls,+ concept_name: str,+ search_domains: List[str],+ content: StuffContent,+ name: Optional[str] = None,+ code: Optional[str] = None,+ ) -> Stuff:+ concept_provider = get_concept_provider()+ concept = concept_provider.search_for_concept_in_domains(+ concept_name=concept_name,+ search_domains=search_domains,+ )+ if not concept:+ raise StuffFactoryError(f"Could not find a concept named '{concept_name}' in domains {search_domains}")+ return cls.make_stuff_using_concept(concept=concept, content=content, name=name, code=code)+@classmethoddef make_stuff_using_concept(cls,@@ -164,6 +188,79 @@ class StuffFactory:name=name,)+ @classmethod+ def make_stuff_from_stuff_content_using_search_domains(+ cls,+ name: str,+ stuff_content_or_data: StuffContentOrData,+ search_domains: List[str],+ code: Optional[str] = None,+ ) -> Stuff:+ content: StuffContent+ concept_name: str+ if isinstance(stuff_content_or_data, ListContent):+ content = cast(ListContent[Any], stuff_content_or_data)+ if len(content.items) == 0:+ raise StuffFactoryError("ListContent in compact memory has no items")+ concept_name = type(content.items[0]).__name__+ try:+ return cls.make_stuff_using_concept_name_and_search_domains(+ concept_name=concept_name,+ search_domains=search_domains,+ content=content,+ name=name,+ code=code,+ )+ except StuffFactoryError as exc:+ raise StuffFactoryError(f"Could not make stuff for ListContent '{name}': {exc}") from exc+ elif isinstance(stuff_content_or_data, StuffContent):+ content = stuff_content_or_data+ concept_name = type(content).__name__+ try:+ return cls.make_stuff_using_concept_name_and_search_domains(+ concept_name=concept_name,+ search_domains=search_domains,+ content=content,+ name=name,+ code=code,+ )+ except StuffFactoryError as exc:+ raise StuffFactoryError(f"Could not make stuff for StuffContent '{name}': {exc}") from exc+ elif isinstance(stuff_content_or_data, list):+ items = stuff_content_or_data+ if len(items) == 0:+ raise StuffFactoryError("List in compact memory has no items")+ first_item = items[0]+ concept_name = type(first_item).__name__+ content = ListContent[Any](items=items)+ try:+ return cls.make_stuff_using_concept_name_and_search_domains(+ concept_name=concept_name,+ search_domains=search_domains,+ content=content,+ name=name,+ code=code,+ )+ except StuffFactoryError as exc:+ raise StuffFactoryError(f"Could not make stuff for list of StuffContent '{name}': {exc}") from exc+ else:+ stuff_content_dict: Dict[str, Any] = stuff_content_or_data+ try:+ concept_code = stuff_content_dict["concept_code"]+ content_value = stuff_content_dict["content"]+ except KeyError as exc:+ raise StuffFactoryError(f"Stuff content data dict is badly formed: {exc}") from exc+ content = StuffContentFactory.make_stuffcontent_from_concept_code_with_fallback(+ concept_code=concept_code,+ value=content_value,+ )+ return StuffFactory.make_stuff(+ concept_str=concept_code,+ name=name,+ content=content,+ code=code,+ )+class StuffContentFactoryError(PipelexError):pass@@ -193,13 +290,12 @@ class StuffContentFactory:Create StuffContent from concept code, falling back to TextContent if no registry class is found."""concept = get_required_concept(concept_code=concept_code)- the_subclass_name = concept.structure_class_name- the_subclass = get_class_registry().get_class(name=the_subclass_name)+ the_structure_class = get_class_registry().get_class(name=concept.structure_class_name)- if the_subclass is None:+ if the_structure_class is None:return cls.make_content_from_value(stuff_content_subclass=TextContent, value=value)- if not issubclass(the_subclass, StuffContent):- raise StuffContentFactoryError(f"Concept '{concept_code}', subclass '{the_subclass}' is not a subclass of StuffContent")+ if not issubclass(the_structure_class, StuffContent):+ raise StuffContentFactoryError(f"Concept '{concept_code}', subclass '{the_structure_class}' is not a subclass of StuffContent")- return cls.make_content_from_value(stuff_content_subclass=the_subclass, value=value)+ return cls.make_content_from_value(stuff_content_subclass=the_structure_class, value=value)diff --git a/pipelex/core/working_memory_factory.py b/pipelex/core/working_memory_factory.pyindex 1ca389732..adaf6f287 100644--- a/pipelex/core/working_memory_factory.py+++ b/pipelex/core/working_memory_factory.py@@ -1,15 +1,16 @@-from typing import Any, Dict, List, Optional, Tuple, Type+from typing import Any, Dict, List, Optional, castimport shortuuidfrom polyfactory.factories.pydantic_factory import ModelFactoryfrom pydantic import BaseModelfrom pipelex import log-from pipelex.client.protocol import CompactMemory+from pipelex.client.protocol import CompactMemory, ImplicitMemoryfrom pipelex.core.concept_native import NativeConcept+from pipelex.core.pipe_input_spec import TypedNamedInputRequirementfrom pipelex.core.stuff import Stuff-from pipelex.core.stuff_content import ImageContent, PDFContent, StuffContent, TextContent-from pipelex.core.stuff_factory import StuffBlueprint, StuffContentFactory, StuffFactory+from pipelex.core.stuff_content import ImageContent, ListContent, PDFContent, StuffContent, TextContent+from pipelex.core.stuff_factory import StuffBlueprint, StuffFactoryfrom pipelex.core.working_memory import MAIN_STUFF_NAME, StuffDict, WorkingMemoryfrom pipelex.exceptions import WorkingMemoryFactoryErrorfrom pipelex.tools.misc.json_utils import load_json_dict_from_path@@ -126,7 +127,23 @@ class WorkingMemoryFactory(BaseModel):return working_memory@classmethod- def make_from_compact_memory(cls, compact_memory: CompactMemory) -> WorkingMemory:+ def make_from_compact_memory(+ cls,+ compact_memory: CompactMemory,+ search_domains: Optional[List[str]] = None,+ ) -> WorkingMemory:+ implicit_memory = cast(ImplicitMemory, compact_memory)+ return cls.make_from_implicit_memory(+ implicit_memory=implicit_memory,+ search_domains=search_domains,+ )++ @classmethod+ def make_from_implicit_memory(+ cls,+ implicit_memory: ImplicitMemory,+ search_domains: Optional[List[str]] = None,+ ) -> WorkingMemory:"""Create a WorkingMemory from a compact memory dictionary.@@ -138,20 +155,35 @@ class WorkingMemoryFactory(BaseModel):"""working_memory = cls.make_empty()- for stuff_key, stuff_data in compact_memory.items():- concept_code = stuff_data.get("concept_code", "")- content_value = stuff_data.get("content", {})-- content = StuffContentFactory.make_stuffcontent_from_concept_code_with_fallback(concept_code=concept_code, value=content_value)-- stuff = StuffFactory.make_stuff(concept_str=concept_code, name=stuff_key, content=content)+ for stuff_key, stuff_content_or_data in implicit_memory.items():+ stuff = StuffFactory.make_stuff_from_stuff_content_using_search_domains(+ name=stuff_key,+ stuff_content_or_data=stuff_content_or_data,+ search_domains=search_domains or [],+ )working_memory.add_new_stuff(name=stuff_key, stuff=stuff)return working_memory@classmethod- def make_for_dry_run(cls, needed_inputs: List[Tuple[str, str, Type[StuffContent]]]) -> "WorkingMemory":+ def create_mock_content(cls, requirement: TypedNamedInputRequirement) -> StuffContent:+ """Helper method to create mock content for a requirement."""+ if requirement.structure_class:+ # Create mock object using polyfactory+ class MockFactory(ModelFactory[requirement.structure_class]): # type: ignore+ __model__ = requirement.structure_class+ __check_model__ = True+ __use_examples__ = True+ __allow_none_optionals__ = False # Ensure Optional fields always get values++ return MockFactory.build() # type: ignore+ else:+ # Fallback to text content+ return TextContent(text=f"DRY RUN: Mock content for '{requirement.variable_name}' ({requirement.concept_code})")++ @classmethod+ def make_for_dry_run(cls, needed_inputs: List[TypedNamedInputRequirement]) -> "WorkingMemory":"""Create a WorkingMemory with mock objects for dry run mode.@@ -164,43 +196,63 @@ class WorkingMemoryFactory(BaseModel):working_memory = cls.make_empty()- for variable_name, concept_code, structure_class in needed_inputs:- log.debug(f"Creating dry run mock for '{variable_name}' with concept '{concept_code}' and class '{structure_class.__name__}'")+ for requirement in needed_inputs:+ log.debug(+ f"Creating dry run mock for '{requirement.variable_name}' with concept "+ f"'{requirement.concept_code}' and class '{requirement.structure_class.__name__}'"+ )try:- if structure_class:- # Create mock object using polyfactory- class MockFactory(ModelFactory[structure_class]): # type: ignore- __model__ = structure_class- __check_model__ = True- __use_examples__ = True- __allow_none_optionals__ = False # Ensure Optional fields always get values-- mock_content = MockFactory.build()+ if not requirement.multiplicity:+ mock_content = cls.create_mock_content(requirement)++ # Create stuff with mock content+ mock_stuff = Stuff(+ stuff_name=requirement.variable_name,+ stuff_code=shortuuid.uuid()[:5],+ concept_code=requirement.concept_code,+ content=mock_content,+ )++ working_memory.add_new_stuff(name=requirement.variable_name, stuff=mock_stuff)else:- # Fallback to text content- mock_content = TextContent(text=f"DRY RUN: Mock content for '{variable_name}' ({concept_code})")-- # Create stuff with mock content- mock_stuff = Stuff(- stuff_name=variable_name,- stuff_code=shortuuid.uuid()[:5],- concept_code=concept_code,- content=mock_content,- )-- working_memory.add_new_stuff(name=variable_name, stuff=mock_stuff)+ # Let's create a ListContent of multiple stuffs+ nb_stuffs: int+ if isinstance(requirement.multiplicity, bool):+ # TODO: make this configurable or use existing config variable+ nb_stuffs = 3+ else:+ nb_stuffs = requirement.multiplicity++ items: List[StuffContent] = []+ for _ in range(nb_stuffs):+ item_mock_content = cls.create_mock_content(requirement)+ items.append(item_mock_content)++ mock_list_content = ListContent[StuffContent](items=items)++ # Create stuff with mock content+ mock_stuff = Stuff(+ stuff_name=requirement.variable_name,+ stuff_code=shortuuid.uuid()[:5],+ concept_code=requirement.concept_code,+ content=mock_list_content,+ )++ working_memory.add_new_stuff(name=requirement.variable_name, stuff=mock_stuff)except Exception as e:- log.warning(f"Failed to create mock for '{variable_name}' ({concept_code}): {e}. Using fallback text content.")+ log.warning(+ f"Failed to create mock for '{requirement.variable_name}' ({requirement.concept_code}): {e}. Using fallback text content."+ )# Create fallback text content- fallback_content = TextContent(text=f"DRY RUN: Fallback mock for '{variable_name}' ({concept_code})")+ fallback_content = TextContent(text=f"DRY RUN: Fallback mock for '{requirement.variable_name}' ({requirement.concept_code})")fallback_stuff = Stuff(- stuff_name=variable_name,+ stuff_name=requirement.variable_name,stuff_code=shortuuid.uuid()[:5],- concept_code=concept_code,+ concept_code=requirement.concept_code,content=fallback_content,)- working_memory.add_new_stuff(name=variable_name, stuff=fallback_stuff)+ working_memory.add_new_stuff(name=requirement.variable_name, stuff=fallback_stuff)return working_memorydiff --git a/pipelex/exceptions.py b/pipelex/exceptions.pyindex 13bd1e339..90d70824c 100644--- a/pipelex/exceptions.py+++ b/pipelex/exceptions.py@@ -280,5 +280,5 @@ class StartPipelineException(Exception):pass-class ExecutePipelineException(Exception):+class PipelineInputError(Exception):passdiff --git a/pipelex/pipe_controllers/pipe_batch.py b/pipelex/pipe_controllers/pipe_batch.pyindex d95676d68..95d075f98 100644--- a/pipelex/pipe_controllers/pipe_batch.py+++ b/pipelex/pipe_controllers/pipe_batch.py@@ -14,7 +14,11 @@ from pipelex.core.stuff import Stufffrom pipelex.core.stuff_content import ListContent, StuffContentfrom pipelex.core.stuff_factory import StuffFactoryfrom pipelex.core.working_memory import MAIN_STUFF_NAME, WorkingMemory-from pipelex.exceptions import PipeInputError, PipeInputNotFoundError, WorkingMemoryStuffNotFoundError+from pipelex.exceptions import (+ PipeInputError,+ PipeInputNotFoundError,+ WorkingMemoryStuffNotFoundError,+)from pipelex.hub import get_pipeline_tracker, get_required_pipefrom pipelex.pipe_controllers.pipe_controller import PipeControllerfrom pipelex.pipeline.job_metadata import JobMetadata@@ -61,10 +65,7 @@ class PipeBatch(PipeController):@overridedef needed_inputs(self) -> PipeInputSpec:- needed_inputs = PipeInputSpec.make_empty()- for variable_name, concept_code in self.inputs.items:- needed_inputs.add_requirement(variable_name, concept_code)- return needed_inputs+ return self.inputsasync def _run_batch_pipe(self,diff --git a/pipelex/pipe_controllers/pipe_batch_factory.py b/pipelex/pipe_controllers/pipe_batch_factory.pyindex e633d51e9..82fee5325 100644--- a/pipelex/pipe_controllers/pipe_batch_factory.py+++ b/pipelex/pipe_controllers/pipe_batch_factory.py@@ -32,7 +32,7 @@ class PipeBatchFactory(PipeSpecificFactoryProtocol[PipeBatchBlueprint, PipeBatchdomain=domain_code,code=pipe_code,definition=pipe_blueprint.definition,- inputs=PipeInputSpec(root=pipe_blueprint.inputs or {}),+ inputs=PipeInputSpec.make_from_dict(concepts_dict=pipe_blueprint.inputs or {}),output_concept_code=pipe_blueprint.output,branch_pipe_code=pipe_blueprint.branch_pipe_code,batch_params=batch_params,diff --git a/pipelex/pipe_controllers/pipe_condition.py b/pipelex/pipe_controllers/pipe_condition.pyindex 40ae24723..be1c4c182 100644--- a/pipelex/pipe_controllers/pipe_condition.py+++ b/pipelex/pipe_controllers/pipe_condition.py@@ -118,8 +118,8 @@ class PipeCondition(PipeController):# 2. Add the inputs needed by all possible target pipesfor pipe_code in self.pipe_map.values():pipe = get_required_pipe(pipe_code=pipe_code)- for input_name, concept_code in pipe.needed_inputs().items:- needed_inputs.add_requirement(variable_name=input_name, concept_code=concept_code)+ for input_name, requirement in pipe.needed_inputs().items:+ needed_inputs.add_requirement(variable_name=input_name, concept_code=requirement.concept_code)return needed_inputs@@ -142,13 +142,13 @@ class PipeCondition(PipeController):the_needed_inputs = self.needed_inputs()# Check all required variables are in the inputs- for required_variable_name, _, _ in the_needed_inputs.detailed_requirements:- if required_variable_name not in self.inputs.variables:+ for named_input_requirement in the_needed_inputs.named_input_requirements:+ if named_input_requirement.variable_name not in self.inputs.variables:missing_input_var_error = StaticValidationError(error_type=StaticValidationErrorType.MISSING_INPUT_VARIABLE,domain_code=self.domain,pipe_code=self.code,- variable_names=[required_variable_name],+ variable_names=[named_input_requirement.variable_name],)match reactions.get(StaticValidationErrorType.MISSING_INPUT_VARIABLE, default_reaction):case StaticValidationReaction.IGNORE:@@ -310,9 +310,9 @@ class PipeCondition(PipeController):needed_inputs = self.needed_inputs()missing_input_names: List[str] = []- for required_variable_name, _, _ in needed_inputs.detailed_requirements:- if not working_memory.get_optional_stuff(required_variable_name):- missing_input_names.append(required_variable_name)+ for named_input_requirement in needed_inputs.named_input_requirements:+ if not working_memory.get_optional_stuff(named_input_requirement.variable_name):+ missing_input_names.append(named_input_requirement.variable_name)if missing_input_names:log.error(f"Dry run failed: missing required inputs: {missing_input_names}")diff --git a/pipelex/pipe_controllers/pipe_condition_factory.py b/pipelex/pipe_controllers/pipe_condition_factory.pyindex e9fca1be2..3578fde9e 100644--- a/pipelex/pipe_controllers/pipe_condition_factory.py+++ b/pipelex/pipe_controllers/pipe_condition_factory.py@@ -29,7 +29,7 @@ class PipeConditionFactory(PipeSpecificFactoryProtocol[PipeConditionBlueprint, Pdomain=domain_code,code=pipe_code,definition=pipe_blueprint.definition,- inputs=PipeInputSpec(root=pipe_blueprint.inputs or {}),+ inputs=PipeInputSpec.make_from_dict(concepts_dict=pipe_blueprint.inputs or {}),output_concept_code=pipe_blueprint.output,expression_template=pipe_blueprint.expression_template,expression=pipe_blueprint.expression,diff --git a/pipelex/pipe_controllers/pipe_parallel.py b/pipelex/pipe_controllers/pipe_parallel.pyindex ed81c7879..43caae75b 100644--- a/pipelex/pipe_controllers/pipe_parallel.py+++ b/pipelex/pipe_controllers/pipe_parallel.py@@ -51,14 +51,14 @@ class PipeParallel(PipeController):batch_as_input = sub_pipe.batch_params.input_item_stuff_name# Create a new PipeInputSpec without the batch_as inputfiltered_needed_inputs = PipeInputSpec.make_empty()- for var_name, concept_code in pipe_needed_inputs.root.items():+ for var_name, requirement in pipe_needed_inputs.root.items():if var_name != batch_as_input:- filtered_needed_inputs.add_requirement(variable_name=var_name, concept_code=concept_code)+ filtered_needed_inputs.add_requirement(variable_name=var_name, concept_code=requirement.concept_code)pipe_needed_inputs = filtered_needed_inputs# Add all inputs from this parallel pipe- for var_name, concept_code in pipe_needed_inputs.root.items():- needed_inputs.add_requirement(variable_name=var_name, concept_code=concept_code)+ for var_name, requirement in pipe_needed_inputs.root.items():+ needed_inputs.add_requirement(variable_name=var_name, concept_code=requirement.concept_code)return needed_inputs@@ -81,13 +81,13 @@ class PipeParallel(PipeController):the_needed_inputs = self.needed_inputs()# Check all required variables are in the inputs- for required_variable_name, _, _ in the_needed_inputs.detailed_requirements:- if required_variable_name not in self.inputs.variables:+ for named_input_requirement in the_needed_inputs.named_input_requirements:+ if named_input_requirement.variable_name not in self.inputs.variables:missing_input_var_error = StaticValidationError(error_type=StaticValidationErrorType.MISSING_INPUT_VARIABLE,domain_code=self.domain,pipe_code=self.code,- variable_names=[required_variable_name],+ variable_names=[named_input_requirement.variable_name],)match reactions.get(StaticValidationErrorType.MISSING_INPUT_VARIABLE, default_reaction):case StaticValidationReaction.IGNORE:@@ -224,9 +224,9 @@ class PipeParallel(PipeController):needed_inputs = self.needed_inputs()missing_input_names: List[str] = []- for required_variable_name, _, _ in needed_inputs.detailed_requirements:- if not working_memory.get_optional_stuff(required_variable_name):- missing_input_names.append(required_variable_name)+ for named_input_requirement in needed_inputs.named_input_requirements:+ if not working_memory.get_optional_stuff(named_input_requirement.variable_name):+ missing_input_names.append(named_input_requirement.variable_name)if missing_input_names:log.error(f"Dry run failed: missing required inputs: {missing_input_names}")diff --git a/pipelex/pipe_controllers/pipe_parallel_factory.py b/pipelex/pipe_controllers/pipe_parallel_factory.pyindex 4b8a94fcf..871e2ca0c 100644--- a/pipelex/pipe_controllers/pipe_parallel_factory.py+++ b/pipelex/pipe_controllers/pipe_parallel_factory.py@@ -43,7 +43,7 @@ class PipeParallelFactory(PipeSpecificFactoryProtocol[PipeParallelBlueprint, Pipdomain=domain_code,code=pipe_code,definition=pipe_blueprint.definition,- inputs=PipeInputSpec(root=pipe_blueprint.inputs or {}),+ inputs=PipeInputSpec.make_from_dict(concepts_dict=pipe_blueprint.inputs or {}),output_concept_code=pipe_blueprint.output,parallel_sub_pipes=parallel_sub_pipes,add_each_output=pipe_blueprint.add_each_output,diff --git a/pipelex/pipe_controllers/pipe_sequence.py b/pipelex/pipe_controllers/pipe_sequence.pyindex f1e24b5a7..05dccfd43 100644--- a/pipelex/pipe_controllers/pipe_sequence.py+++ b/pipelex/pipe_controllers/pipe_sequence.py@@ -9,7 +9,11 @@ from pipelex.core.pipe_input_spec import PipeInputSpecfrom pipelex.core.pipe_output import PipeOutputfrom pipelex.core.pipe_run_params import PipeRunMode, PipeRunParamsfrom pipelex.core.working_memory import WorkingMemory-from pipelex.exceptions import PipeRunParamsError, StaticValidationError, StaticValidationErrorType+from pipelex.exceptions import (+ PipeRunParamsError,+ StaticValidationError,+ StaticValidationErrorType,+)from pipelex.hub import get_required_pipefrom pipelex.pipe_controllers.pipe_controller import PipeControllerfrom pipelex.pipe_controllers.sub_pipe import SubPipe@@ -33,20 +37,29 @@ class PipeSequence(PipeController):sub_pipe_needed_inputs = get_required_pipe(pipe_code=sequential_sub_pipe.pipe_code).needed_inputs()# Handle batching: if this sub_pipe has batch_params, exclude the batch_as input- # since it's provided by the batching mechanism+ # since it's provided by the batching mechanism.+ # Instead we must add the batch_over variable nale.if sequential_sub_pipe.batch_params:batch_as_input = sequential_sub_pipe.batch_params.input_item_stuff_name+ batch_item_concept_code = sub_pipe_needed_inputs.get_required_concept_code(variable_name=batch_as_input)# Create a new PipeInputSpec without the batch_as input+ # TODO: this should be done with a simple remove_requirement() method on the PipeInputSpecfiltered_needed_inputs = PipeInputSpec.make_empty()- for var_name, concept_code in sub_pipe_needed_inputs.items:+ for var_name, requirement in sub_pipe_needed_inputs.items:if var_name != batch_as_input:- filtered_needed_inputs.add_requirement(variable_name=var_name, concept_code=concept_code)+ filtered_needed_inputs.add_requirement(variable_name=var_name, concept_code=requirement.concept_code)sub_pipe_needed_inputs = filtered_needed_inputs+ batch_over = sequential_sub_pipe.batch_params.input_list_stuff_name+ sub_pipe_needed_inputs.add_requirement(+ variable_name=batch_over,+ concept_code=batch_item_concept_code,+ multiplicity=True,+ )# Add inputs that haven't been generated by previous steps- for var_name, concept_code in sub_pipe_needed_inputs.items:+ for var_name, requirement in sub_pipe_needed_inputs.items:if var_name not in generated_outputs:- needed_inputs.add_requirement(variable_name=var_name, concept_code=concept_code)+ needed_inputs.add_requirement(var_name, requirement.concept_code, requirement.multiplicity)# Add this step's output to generated outputsif sequential_sub_pipe.output_name:@@ -82,13 +95,13 @@ class PipeSequence(PipeController):the_needed_inputs = self.needed_inputs()# Check all required variables are in the inputs- for required_variable_name, _, _ in the_needed_inputs.detailed_requirements:- if required_variable_name not in self.inputs.variables:+ for named_input_requirement in the_needed_inputs.named_input_requirements:+ if named_input_requirement.variable_name not in self.inputs.variables:missing_input_var_error = StaticValidationError(error_type=StaticValidationErrorType.MISSING_INPUT_VARIABLE,domain_code=self.domain,pipe_code=self.code,- variable_names=[required_variable_name],+ variable_names=[named_input_requirement.variable_name],)match reactions.get(StaticValidationErrorType.MISSING_INPUT_VARIABLE, default_reaction):case StaticValidationReaction.IGNORE:@@ -101,13 +114,17 @@ class PipeSequence(PipeController):# Check that all declared inputs are actually neededfor input_name in self.inputs.variables:if input_name not in the_needed_inputs.required_names:+ log.debug(f"the_needed_inputs.required_names: {the_needed_inputs.required_names}")extraneous_input_var_error = StaticValidationError(error_type=StaticValidationErrorType.EXTRANEOUS_INPUT_VARIABLE,domain_code=self.domain,pipe_code=self.code,variable_names=[input_name],)- match reactions.get(StaticValidationErrorType.EXTRANEOUS_INPUT_VARIABLE, default_reaction):+ match reactions.get(+ StaticValidationErrorType.EXTRANEOUS_INPUT_VARIABLE,+ default_reaction,+ ):case StaticValidationReaction.IGNORE:passcase StaticValidationReaction.LOG:@@ -169,6 +186,8 @@ class PipeSequence(PipeController):) -> PipeOutput:if pipe_run_params.run_mode != PipeRunMode.DRY:raise PipeRunParamsError(f"PipeSequence._dry_run_controller_pipe() called with run_mode = {pipe_run_params.run_mode} in pipe {self.code}")+ log.debug(f"PipeSequence._dry_run_controller_pipe() called with {self.code=} {pipe_run_params=}")+ working_memory.pretty_print_summary()return await self._run_controller_pipe(job_metadata=job_metadata,working_memory=working_memory,diff --git a/pipelex/pipe_controllers/pipe_sequence_factory.py b/pipelex/pipe_controllers/pipe_sequence_factory.pyindex 533ca9271..ce5500121 100644--- a/pipelex/pipe_controllers/pipe_sequence_factory.py+++ b/pipelex/pipe_controllers/pipe_sequence_factory.py@@ -26,7 +26,7 @@ class PipeSequenceFactory(PipeSpecificFactoryProtocol[PipeSequenceBlueprint, Pipdomain=domain_code,code=pipe_code,definition=pipe_blueprint.definition,- inputs=PipeInputSpec(root=pipe_blueprint.inputs or {}),+ inputs=PipeInputSpec.make_from_dict(concepts_dict=pipe_blueprint.inputs or {}),output_concept_code=pipe_blueprint.output,sequential_sub_pipes=pipe_steps,)diff --git a/pipelex/pipe_controllers/sub_pipe.py b/pipelex/pipe_controllers/sub_pipe.pyindex 5cbcb0e2c..77a003836 100644--- a/pipelex/pipe_controllers/sub_pipe.py+++ b/pipelex/pipe_controllers/sub_pipe.py@@ -28,6 +28,7 @@ class SubPipe(BaseModel):) -> PipeOutput:"""Run or dry run a single operation self."""log.debug(f"SubPipe {self.pipe_code} to generate {self.output_name}")+ working_memory.pretty_print_summary()# step_run_params.push_pipe_code(pipe_code=self.pipe_code)if self.output_multiplicity:sub_pipe_run_params.output_multiplicity = self.output_multiplicity@@ -102,7 +103,7 @@ class SubPipe(BaseModel):sub_pipe_path_str = ".".join(sub_pipe_path)error_details = f"SubPipe '{sub_pipe_path_str}', required_variables: {required_variables}, missing: '{exc.variable_name}'"raise PipeInputError(f"Some required stuff(s) not found: {error_details}") from exc- log.debug(required_stuffs, title=f"Required stuffs for {self.pipe_code}")+ log.verbose(required_stuffs, title=f"Required stuffs for {self.pipe_code}")# This is the only line that changes between run and dry_runif sub_pipe_run_params.run_mode == PipeRunMode.DRY:sub_pipe_run_params.run_mode = PipeRunMode.DRYdiff --git a/pipelex/pipe_operators/pipe_func.py b/pipelex/pipe_operators/pipe_func.pyindex 15d139db6..0336ca214 100644--- a/pipelex/pipe_operators/pipe_func.py+++ b/pipelex/pipe_operators/pipe_func.py@@ -1,14 +1,16 @@-from typing import List, Optional, Set, cast+from typing import List, Optional, Set, cast, get_type_hintsfrom typing_extensions import overridefrom pipelex import log-from pipelex.core.pipe_input_spec import PipeInputSpec+from pipelex.core.pipe_input_spec import PipeInputSpec, TypedNamedInputRequirementfrom pipelex.core.pipe_output import PipeOutputfrom pipelex.core.pipe_run_params import PipeRunParamsfrom pipelex.core.stuff_content import ListContent, StuffContent, TextContentfrom pipelex.core.stuff_factory import StuffFactoryfrom pipelex.core.working_memory import WorkingMemory+from pipelex.core.working_memory_factory import WorkingMemoryFactory+from pipelex.exceptions import DryRunErrorfrom pipelex.pipe_operators.pipe_operator import PipeOperatorfrom pipelex.pipeline.job_metadata import JobMetadatafrom pipelex.tools.func_registry import func_registry@@ -27,7 +29,7 @@ class PipeFunc(PipeOperator):@overridedef needed_inputs(self) -> PipeInputSpec:- return PipeInputSpec.make_empty()+ return self.inputs@overrideasync def _run_operator_pipe(@@ -80,7 +82,48 @@ class PipeFunc(PipeOperator):pipe_run_params: PipeRunParams,output_name: Optional[str] = None,) -> PipeOutput:- log.warning("Dry run not yet implemented for PipeFunc")+ log.debug(f"Dry run for PipeFunc '{self.function_name}'")++ function = func_registry.get_required_function(self.function_name)+ if not callable(function):+ raise ValueError(f"Function '{self.function_name}' is not callable")++ # Check that all needed inputs are present in working memory+ needed_inputs = self.needed_inputs()+ for input_name, _ in needed_inputs.items:+ if input_name not in working_memory.root:+ raise DryRunError(+ f"Required input '{input_name}' not found in working memory for function '{self.function_name}' in pipe '{self.code}'"+ )++ try:+ return_type = get_type_hints(function).get("return")++ if return_type is None:+ raise DryRunError(f"Function '{self.function_name}' has no return type annotation")+ else:+ if not issubclass(return_type, StuffContent):+ raise ValueError(f"Function '{self.function_name}' return type {return_type} is not a subclass of StuffContent")++ requirement = TypedNamedInputRequirement(+ variable_name="mock_output", concept_code=f"mock.{return_type.__name__}", structure_class=return_type, multiplicity=False+ )+ mock_content = WorkingMemoryFactory.create_mock_content(requirement)++ except Exception as e:+ raise DryRunError(f"Failed to get type hints for function '{self.function_name}' in pipe '{self.code}': {e}")++ output_stuff = StuffFactory.make_stuff(+ name=output_name,+ concept_str=self.output_concept_code,+ content=mock_content,+ )++ working_memory.set_new_main_stuff(+ stuff=output_stuff,+ name=output_name,+ )+return PipeFuncOutput(working_memory=working_memory,pipeline_run_id=job_metadata.pipeline_run_id,diff --git a/pipelex/pipe_operators/pipe_func_factory.py b/pipelex/pipe_operators/pipe_func_factory.pyindex 8000f5c9a..c38ac6c47 100644--- a/pipelex/pipe_operators/pipe_func_factory.py+++ b/pipelex/pipe_operators/pipe_func_factory.py@@ -24,7 +24,7 @@ class PipeFuncFactory(PipeSpecificFactoryProtocol[PipeFuncBlueprint, PipeFunc]):domain=domain_code,code=pipe_code,definition=pipe_blueprint.definition,- inputs=PipeInputSpec(root=pipe_blueprint.inputs or {}),+ inputs=PipeInputSpec.make_from_dict(concepts_dict=pipe_blueprint.inputs or {}),output_concept_code=pipe_blueprint.output,function_name=pipe_blueprint.function_name,)diff --git a/pipelex/pipe_operators/pipe_img_gen.py b/pipelex/pipe_operators/pipe_img_gen.pyindex ca2af1692..b0b97c012 100644--- a/pipelex/pipe_operators/pipe_img_gen.py+++ b/pipelex/pipe_operators/pipe_img_gen.py@@ -83,8 +83,8 @@ class PipeImgGen(PipeOperator):if self.imgg_prompt:needed_inputs.add_requirement(variable_name="imgg_prompt", concept_code=NativeConcept.TEXT.code)else:- for input_name, input_concept_code in self.inputs.items:- needed_inputs.add_requirement(variable_name=input_name, concept_code=input_concept_code)+ for input_name, requirement in self.inputs.items:+ needed_inputs.add_requirement(variable_name=input_name, concept_code=requirement.concept_code)return needed_inputs@override@@ -105,10 +105,10 @@ class PipeImgGen(PipeOperator):returncandidate_prompt_var_names: List[str] = []- for input_name, input_concept_code in self.inputs.items:- log.debug(f"Validating input '{input_name}' with concept code '{input_concept_code}'")+ for input_name, requirement in self.inputs.items:+ log.debug(f"Validating input '{input_name}' with concept code '{requirement.concept_code}'")if concept_provider.is_compatible_by_concept_code(- tested_concept_code=input_concept_code,+ tested_concept_code=requirement.concept_code,wanted_concept_code=NativeConcept.TEXT.code,):self.img_gen_prompt_var_name = input_name@@ -119,7 +119,7 @@ class PipeImgGen(PipeOperator):domain_code=self.domain,pipe_code=self.code,variable_names=[input_name],- provided_concept_code=input_concept_code,+ provided_concept_code=requirement.concept_code,explanation="Only a text input can be provided for image gen prompt",)match reactions.get(StaticValidationErrorType.INADEQUATE_INPUT_CONCEPT, default_reaction):diff --git a/pipelex/pipe_operators/pipe_img_gen_factory.py b/pipelex/pipe_operators/pipe_img_gen_factory.pyindex 4475ade36..2467681bc 100644--- a/pipelex/pipe_operators/pipe_img_gen_factory.py+++ b/pipelex/pipe_operators/pipe_img_gen_factory.py@@ -51,7 +51,7 @@ class PipeImgGenFactory(PipeSpecificFactoryProtocol[PipeImgGenBlueprint, PipeImgdomain=domain_code,code=pipe_code,definition=pipe_blueprint.definition,- inputs=PipeInputSpec(root=pipe_blueprint.inputs or {}),+ inputs=PipeInputSpec.make_from_dict(concepts_dict=pipe_blueprint.inputs or {}),output_concept_code=pipe_blueprint.output,output_multiplicity=output_multiplicity,imgg_prompt=pipe_blueprint.img_gen_prompt,diff --git a/pipelex/pipe_operators/pipe_jinja2.py b/pipelex/pipe_operators/pipe_jinja2.pyindex 2cacf32a8..8cb3d3d69 100644--- a/pipelex/pipe_operators/pipe_jinja2.py+++ b/pipelex/pipe_operators/pipe_jinja2.py@@ -80,8 +80,8 @@ class PipeJinja2(PipeOperator):@overridedef needed_inputs(self) -> PipeInputSpec:needed_inputs = PipeInputSpec.make_empty()- for input_name, concept_code in self.inputs.root.items():- needed_inputs.add_requirement(variable_name=input_name, concept_code=concept_code)+ for input_name, requirement in self.inputs.root.items():+ needed_inputs.add_requirement(variable_name=input_name, concept_code=requirement.concept_code)return needed_inputs@propertydiff --git a/pipelex/pipe_operators/pipe_jinja2_factory.py b/pipelex/pipe_operators/pipe_jinja2_factory.pyindex 0330bc337..13597ff3f 100644--- a/pipelex/pipe_operators/pipe_jinja2_factory.py+++ b/pipelex/pipe_operators/pipe_jinja2_factory.py@@ -42,7 +42,7 @@ class PipeJinja2Factory(PipeSpecificFactoryProtocol[PipeJinja2Blueprint, PipeJindomain=domain_code,code=pipe_code,definition=pipe_blueprint.definition,- inputs=PipeInputSpec(root=pipe_blueprint.inputs or {}),+ inputs=PipeInputSpec.make_from_dict(concepts_dict=pipe_blueprint.inputs or {}),output_concept_code=pipe_blueprint.output,jinja2_name=pipe_blueprint.jinja2_name,jinja2=preprocessed_template,diff --git a/pipelex/pipe_operators/pipe_llm.py b/pipelex/pipe_operators/pipe_llm.pyindex f1ce74bac..f92f8ac05 100644--- a/pipelex/pipe_operators/pipe_llm.py+++ b/pipelex/pipe_operators/pipe_llm.py@@ -102,11 +102,11 @@ class PipeLLM(PipeOperator):needed_inputs = PipeInputSpec.make_empty()concept_provider = get_concept_provider()- for input_name, concept_code in self.inputs.items:- if concept_provider.is_image_concept(concept_code=concept_code):+ for input_name, requirement in self.inputs.items:+ if concept_provider.is_image_concept(concept_code=requirement.concept_code):needed_inputs.add_requirement(variable_name=input_name, concept_code=NativeConcept.IMAGE.code)else:- needed_inputs.add_requirement(variable_name=input_name, concept_code=concept_code)+ needed_inputs.add_requirement(variable_name=input_name, concept_code=requirement.concept_code)return needed_inputs@@ -134,13 +134,13 @@ class PipeLLM(PipeOperator):the_needed_inputs = self.needed_inputs()# check all required variables are in the inputs- for required_variable_name, requirement_expression, concept_code in the_needed_inputs.detailed_requirements:- if required_variable_name not in self.inputs.variables:+ for named_input_requirement in the_needed_inputs.named_input_requirements:+ if named_input_requirement.variable_name not in self.inputs.variables:missing_input_var_error = StaticValidationError(error_type=StaticValidationErrorType.MISSING_INPUT_VARIABLE,domain_code=self.domain,pipe_code=self.code,- variable_names=[required_variable_name],+ variable_names=[named_input_requirement.variable_name],)match reactions.get(StaticValidationErrorType.MISSING_INPUT_VARIABLE, default_reaction):case StaticValidationReaction.IGNORE:@@ -151,18 +151,18 @@ class PipeLLM(PipeOperator):raise missing_input_var_error# there is one case where the needed input is of specific concept: the user_images- if concept_code == NativeConcept.IMAGE.code:+ if named_input_requirement.concept_code == NativeConcept.IMAGE.code:try:- concept_code_of_declared_input = self.inputs.get_required_concept_code(variable_name=required_variable_name)+ concept_code_of_declared_input = self.inputs.get_required_concept_code(variable_name=named_input_requirement.variable_name)except PipeInputNotFoundError as exc:raise PipeInputError(- f"Input variable '{required_variable_name}' is not in this PipeLLM '{self.code}' input spec: {self.inputs}"+ f"Input variable '{named_input_requirement.variable_name}' is not in this PipeLLM '{self.code}' input spec: {self.inputs}") from excif not concept_provider.is_compatible_by_concept_code(tested_concept_code=concept_code_of_declared_input,- wanted_concept_code=concept_code,+ wanted_concept_code=named_input_requirement.concept_code,):- if required_variable_name != requirement_expression:+ if named_input_requirement.variable_name != named_input_requirement.requirement_expression:# the required_input is a sub-attribute of the required variable# TODO: check that the sub-attribute is compatible with the concept code# let's check at least that the input is a structured concept@@ -181,7 +181,7 @@ class PipeLLM(PipeOperator):error_type=StaticValidationErrorType.INADEQUATE_INPUT_CONCEPT,domain_code=self.domain,pipe_code=self.code,- variable_names=[required_variable_name],+ variable_names=[named_input_requirement.variable_name],provided_concept_code=concept_code_of_declared_input,explanation=explanation,)diff --git a/pipelex/pipe_operators/pipe_llm_factory.py b/pipelex/pipe_operators/pipe_llm_factory.pyindex 434541c69..d5c3ce043 100644--- a/pipelex/pipe_operators/pipe_llm_factory.py+++ b/pipelex/pipe_operators/pipe_llm_factory.py@@ -119,7 +119,7 @@ class PipeLLMFactory(PipeSpecificFactoryProtocol[PipeLLMBlueprint, PipeLLM]):pipe_llm_prompt = PipeLLMPrompt(code="adhoc_for_pipe_llm_prompt",domain=domain_code,- inputs=PipeInputSpec(root=pipe_blueprint.inputs or {}),+ inputs=PipeInputSpec.make_from_dict(concepts_dict=pipe_blueprint.inputs or {}),system_prompt_pipe_jinja2=system_prompt_pipe_jinja2,system_prompt_verbatim_name=pipe_blueprint.system_prompt_name,system_prompt=pipe_blueprint.system_prompt or system_prompt,@@ -144,7 +144,7 @@ class PipeLLMFactory(PipeSpecificFactoryProtocol[PipeLLMBlueprint, PipeLLM]):domain=domain_code,code=pipe_code,definition=pipe_blueprint.definition,- inputs=PipeInputSpec(root=pipe_blueprint.inputs or {}),+ inputs=PipeInputSpec.make_from_dict(concepts_dict=pipe_blueprint.inputs or {}),output_concept_code=pipe_blueprint.output,pipe_llm_prompt=pipe_llm_prompt,llm_choices=llm_choices,diff --git a/pipelex/pipe_operators/pipe_ocr.py b/pipelex/pipe_operators/pipe_ocr.pyindex e01dc8cf0..b2eb568d3 100644--- a/pipelex/pipe_operators/pipe_ocr.py+++ b/pipelex/pipe_operators/pipe_ocr.py@@ -66,16 +66,18 @@ class PipeOcr(PipeOperator):# check that we have either an image or a pdf in inputs, at most one of them and nothing elsecandidate_prompt_var_names: List[str] = []- for input_name, input_concept_code in self.inputs.items:- log.debug(f"Validating input '{input_name}' with concept code '{input_concept_code}'")+ for input_name, requirement in self.inputs.items:+ log.debug(f"{input_name=}")+ log.debug(f"{requirement=}")+ log.debug(f"Validating input '{input_name}' with concept code '{requirement.concept_code}'")if concept_provider.is_compatible_by_concept_code(- tested_concept_code=input_concept_code,+ tested_concept_code=requirement.concept_code,wanted_concept_code=NativeConcept.IMAGE.code,):self.image_stuff_name = input_namecandidate_prompt_var_names.append(input_name)elif concept_provider.is_compatible_by_concept_code(- tested_concept_code=input_concept_code,+ tested_concept_code=requirement.concept_code,wanted_concept_code=NativeConcept.PDF.code,):self.pdf_stuff_name = input_name@@ -86,7 +88,7 @@ class PipeOcr(PipeOperator):domain_code=self.domain,pipe_code=self.code,variable_names=[input_name],- provided_concept_code=input_concept_code,+ provided_concept_code=requirement.concept_code,explanation="For OCR you must provide either a pdf or an image or a concept that refines one of them",)match reactions.get(StaticValidationErrorType.INADEQUATE_INPUT_CONCEPT, default_reaction):@@ -128,7 +130,7 @@ class PipeOcr(PipeOperator):@overridedef needed_inputs(self) -> PipeInputSpec:- return PipeInputSpec.make_from_dict({PIPE_OCR_INPUT_NAME: self.inputs.root[PIPE_OCR_INPUT_NAME]})+ return PipeInputSpec.make_from_dict({PIPE_OCR_INPUT_NAME: self.inputs.root[PIPE_OCR_INPUT_NAME].concept_code})@overrideasync def _run_operator_pipe(diff --git a/pipelex/pipe_operators/pipe_ocr_factory.py b/pipelex/pipe_operators/pipe_ocr_factory.pyindex fe8460d74..ad9d015a9 100644--- a/pipelex/pipe_operators/pipe_ocr_factory.py+++ b/pipelex/pipe_operators/pipe_ocr_factory.py@@ -40,7 +40,7 @@ class PipeOcrFactory(PipeSpecificFactoryProtocol[PipeOcrBlueprint, PipeOcr]):definition=pipe_blueprint.definition,ocr_engine=ocr_engine,output_concept_code=pipe_blueprint.output,- inputs=PipeInputSpec(root=pipe_blueprint.inputs or {}),+ inputs=PipeInputSpec.make_from_dict(concepts_dict=pipe_blueprint.inputs or {}),should_include_images=pipe_blueprint.page_images,should_caption_images=pipe_blueprint.page_image_captions,should_include_page_views=pipe_blueprint.page_views,diff --git a/pipelex/pipe_works/pipe_dry.py b/pipelex/pipe_works/pipe_dry.pyindex 8b7109b5f..def6f0dbd 100644--- a/pipelex/pipe_works/pipe_dry.py+++ b/pipelex/pipe_works/pipe_dry.py@@ -2,12 +2,12 @@ import asyncioimport functoolsimport timefrom concurrent.futures import ThreadPoolExecutor-from typing import Dict, List, Tuple, Type+from typing import Dict, List, Tuplefrom pipelex import logfrom pipelex.config import get_configfrom pipelex.core.pipe_abstract import PipeAbstract-from pipelex.core.pipe_input_spec import PipeInputSpec+from pipelex.core.pipe_input_spec import PipeInputSpec, TypedNamedInputRequirementfrom pipelex.core.pipe_run_params import PipeRunModefrom pipelex.core.pipe_run_params_factory import PipeRunParamsFactoryfrom pipelex.core.stuff_content import StuffContent, TextContent@@ -44,7 +44,6 @@ async def dry_run_single_pipe(pipe_code: str) -> str:return f"FAILED: {str(e)}"-# TODO: add a function to dry run a single pipe, make it callable as a param of `pipelex validate`async def dry_run_pipes(pipes: List[PipeAbstract]) -> Dict[str, str]:"""Dry run all pipes in the library using ThreadPoolExecutor for true parallelism.@@ -71,8 +70,12 @@ async def dry_run_pipes(pipes: List[PipeAbstract]) -> Dict[str, str]:"""Execute pipe.run_pipe in a thread and return its status."""try:# This function runs in a separate thread- needed_inputs_for_factory = _convert_to_working_memory_format(pipe.needed_inputs())+ needed_inputs = pipe.needed_inputs()+ log.debug(f"Needed inputs for {pipe.code}: {needed_inputs}")+ needed_inputs_for_factory = _convert_to_working_memory_format(needed_inputs_spec=needed_inputs)+ log.debug(f"Needed inputs for {pipe.code} converted to working memory format: {needed_inputs_for_factory}")working_memory = WorkingMemoryFactory.make_for_dry_run(needed_inputs=needed_inputs_for_factory)+ working_memory.pretty_print_summary()# Create a new event loop for this threadloop = asyncio.new_event_loop()@@ -94,14 +97,14 @@ async def dry_run_pipes(pipes: List[PipeAbstract]) -> Dict[str, str]:return result- except Exception as e:- error_msg = f"FAILED: {str(e)}"+ except Exception as exc:+ error_msg = f"FAILED: {str(exc)}"# Check if this pipe is allowed to failif pipe.code in allowed_to_fail_pipes:- log.debug(f"✗ Pipe {pipe.code} dry run failed: {e} (this is normal, allowed by config)")+ log.debug(f"✗ Pipe {pipe.code} dry run failed: {exc} (this is normal, allowed by config)")else:- log.error(f"✗ Pipe {pipe.code} dry run failed: {e}")+ log.error(f"✗ Pipe {pipe.code} dry run failed: {exc}")return (pipe.code, error_msg)@@ -118,16 +121,17 @@ async def dry_run_pipes(pipes: List[PipeAbstract]) -> Dict[str, str]:pipe_code, status = await futureresults[pipe_code] = status- successful_pipes = [code for code, status in results.items() if status == "SUCCESS"]- failed_pipes = [code for code, status in results.items() if status != "SUCCESS"]+ successful_pipes = [pipe_code for pipe_code, status in results.items() if status == "SUCCESS"]+ failed_pipes = [pipe_code for pipe_code, status in results.items() if status != "SUCCESS"]# Filter out pipes that are allowed to fail- unexpected_failures = [pipe for pipe in failed_pipes if pipe not in allowed_to_fail_pipes]+ unexpected_failures = {pipe_code: results[pipe_code] for pipe_code in failed_pipes if pipe_code not in allowed_to_fail_pipes}- log.info(f"Dry run completed: {len(successful_pipes)} successful, {len(failed_pipes)} failed, in {time.time() - start_time:.2f} seconds")+ log.info(f"Dry run completed: '{len(successful_pipes)}' successful, '{len(failed_pipes)}' failed, in '{time.time() - start_time:.2f}' seconds")if unexpected_failures:- raise Exception(f"Dry run failed with {len(unexpected_failures)} unexpected pipe failures: {', '.join(unexpected_failures)}")+ unexpected_failures_details = "\n".join([f"'{pipe_code}': {results[pipe_code]}" for pipe_code in unexpected_failures])+ raise Exception(f"Dry run failed with '{len(unexpected_failures)}' unexpected pipe failures:\n{unexpected_failures_details}")if failed_pipes and not unexpected_failures:log.info("All failures were expected (allowed by config)")@@ -135,7 +139,7 @@ async def dry_run_pipes(pipes: List[PipeAbstract]) -> Dict[str, str]:return results-def _convert_to_working_memory_format(needed_inputs_spec: PipeInputSpec) -> List[Tuple[str, str, Type[StuffContent]]]:+def _convert_to_working_memory_format(needed_inputs_spec: PipeInputSpec) -> List[TypedNamedInputRequirement]:"""Convert PipeInputSpec to the format needed by WorkingMemoryFactory.make_for_dry_run.@@ -145,29 +149,44 @@ def _convert_to_working_memory_format(needed_inputs_spec: PipeInputSpec) -> ListReturns:List of tuples (variable_name, concept_code, structure_class)"""- needed_inputs_for_factory: List[Tuple[str, str, Type[StuffContent]]] = []+ needed_inputs_for_factory: List[TypedNamedInputRequirement] = []concept_provider = get_concept_provider()class_registry = get_class_registry()- for required_variable_name, _, concept_code in needed_inputs_spec.detailed_requirements:+ for named_input_requirement in needed_inputs_spec.named_input_requirements:try:# Get the concept and its structure class- concept = concept_provider.get_required_concept(concept_code=concept_code)+ concept = concept_provider.get_required_concept(concept_code=named_input_requirement.concept_code)structure_class_name = concept.structure_class_name# Get the actual class from the registrystructure_class = class_registry.get_class(name=structure_class_name)if structure_class and issubclass(structure_class, StuffContent):- needed_inputs_for_factory.append((required_variable_name, concept_code, structure_class))+ typed_named_input_requirement = TypedNamedInputRequirement.make_from_named(+ named=named_input_requirement,+ structure_class=structure_class,+ )+ needed_inputs_for_factory.append(typed_named_input_requirement)else:# Fallback to TextContent if we can't get the proper class- log.warning(f"Could not get structure class '{structure_class_name}' for concept '{concept_code}', falling back to TextContent")- needed_inputs_for_factory.append((required_variable_name, concept_code, TextContent))+ log.warning(+ f"Could not get structure class '{structure_class_name}' for "+ f"concept '{named_input_requirement.concept_code}', falling back to TextContent"+ )+ text_typed_named_input_requirement = TypedNamedInputRequirement.make_from_named(+ named=named_input_requirement,+ structure_class=TextContent,+ )+ needed_inputs_for_factory.append(text_typed_named_input_requirement)except Exception as e:# Fallback to TextContent for any errors- log.warning(f"Error getting structure class for concept '{concept_code}': {e}, falling back to TextContent")- needed_inputs_for_factory.append((required_variable_name, concept_code, TextContent))+ log.warning(f"Error getting structure class for concept '{named_input_requirement.concept_code}': {e}, falling back to TextContent")+ text_typed_named_input_requirement = TypedNamedInputRequirement.make_from_named(+ named=named_input_requirement,+ structure_class=TextContent,+ )+ needed_inputs_for_factory.append(text_typed_named_input_requirement)return needed_inputs_for_factorydiff --git a/pipelex/pipeline/execute.py b/pipelex/pipeline/execute.pyindex 8bb4b5fa3..79981e5f4 100644--- a/pipelex/pipeline/execute.py+++ b/pipelex/pipeline/execute.py@@ -1,13 +1,22 @@-from typing import Optional+from typing import List, Optional-from pipelex.client.protocol import CompactMemory+from pipelex.client.protocol import ImplicitMemoryfrom pipelex.core.pipe_output import PipeOutput-from pipelex.core.pipe_run_params import FORCE_DRY_RUN_MODE_ENV_KEY, PipeOutputMultiplicity, PipeRunMode+from pipelex.core.pipe_run_params import (+ FORCE_DRY_RUN_MODE_ENV_KEY,+ PipeOutputMultiplicity,+ PipeRunMode,+)from pipelex.core.pipe_run_params_factory import PipeRunParamsFactoryfrom pipelex.core.working_memory import WorkingMemoryfrom pipelex.core.working_memory_factory import WorkingMemoryFactory-from pipelex.exceptions import ExecutePipelineException-from pipelex.hub import get_pipe_router, get_pipeline_manager, get_report_delegate, get_required_pipe+from pipelex.exceptions import PipelineInputError+from pipelex.hub import (+ get_pipe_router,+ get_pipeline_manager,+ get_report_delegate,+ get_required_pipe,+)from pipelex.pipe_works.pipe_job_factory import PipeJobFactoryfrom pipelex.pipeline.job_metadata import JobMetadatafrom pipelex.tools.environment import get_optional_env@@ -16,7 +25,8 @@ from pipelex.tools.environment import get_optional_envasync def execute_pipeline(pipe_code: str,working_memory: Optional[WorkingMemory] = None,- input_memory: Optional[CompactMemory] = None,+ input_memory: Optional[ImplicitMemory] = None,+ search_domains: Optional[List[str]] = None,output_name: Optional[str] = None,output_multiplicity: Optional[PipeOutputMultiplicity] = None,dynamic_output_concept_code: Optional[str] = None,@@ -53,12 +63,19 @@ async def execute_pipeline(Tuple[PipeOutput, str]A tuple containing the pipe output and the pipeline run ID."""- # Can be either working_memory or compact_memory, but not both- if working_memory and input_memory:- raise ExecutePipelineException(f"Cannot pass both working_memory and input_memory to `execute_pipeline` {pipe_code=}")+ search_domains = search_domains or []+ pipe = get_required_pipe(pipe_code=pipe_code)+ if pipe.domain not in search_domains:+ search_domains.insert(0, pipe.domain)- if input_memory:- working_memory = WorkingMemoryFactory.make_from_compact_memory(input_memory)+ # Can be either working_memory or compact_memory or neither, but not both+ if working_memory and input_memory:+ raise PipelineInputError(f"Cannot pass both working_memory and input_memory to `execute_pipeline` {pipe_code=}")+ elif input_memory:+ working_memory = WorkingMemoryFactory.make_from_implicit_memory(+ implicit_memory=input_memory,+ search_domains=search_domains,+ )if pipe_run_mode is None:if run_mode_from_env := get_optional_env(key=FORCE_DRY_RUN_MODE_ENV_KEY):@@ -69,7 +86,6 @@ async def execute_pipeline(pipeline = get_pipeline_manager().add_new_pipeline()pipeline_run_id = pipeline.pipeline_run_idget_report_delegate().open_registry(pipeline_run_id=pipeline_run_id)- pipe = get_required_pipe(pipe_code=pipe_code)job_metadata = JobMetadata(pipeline_run_id=pipeline_run_id,diff --git a/pipelex/tools/templating/jinja2_template_category.py b/pipelex/tools/templating/jinja2_template_category.pyindex 25a5293b9..dd609d6fd 100644--- a/pipelex/tools/templating/jinja2_template_category.py+++ b/pipelex/tools/templating/jinja2_template_category.py@@ -17,8 +17,13 @@ class Jinja2TemplateCategory(StrEnum):@propertydef filters(self) -> Dict[Jinja2FilterName, Callable[[Context, Any, Optional[TextFormat]], Any]]:match self:- case Jinja2TemplateCategory.HTML | Jinja2TemplateCategory.MARKDOWN | Jinja2TemplateCategory.MERMAID:+ case Jinja2TemplateCategory.MERMAID:return {}+ case Jinja2TemplateCategory.HTML | Jinja2TemplateCategory.MARKDOWN:+ return {+ Jinja2FilterName.FORMAT: text_format,+ Jinja2FilterName.TAG: tag,+ }case Jinja2TemplateCategory.LLM_PROMPT:return {Jinja2FilterName.FORMAT: text_format,diff --git a/pyproject.toml b/pyproject.tomlindex 6b9054366..2eb105ab5 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -1,6 +1,6 @@[project]name = "pipelex"-version = "0.6.7"+version = "0.6.8"description = "Pipelex is an open-source dev tool based on a simple declarative language that lets you define replicable, structured, composable LLM pipelines."authors = [{ name = "Evotis S.A.S.", email = "[email protected]" }]maintainers = [{ name = "Pipelex staff", email = "[email protected]" }]@@ -32,7 +32,7 @@ dependencies = ["pandas>=2.2.3","pillow>=11.2.1","polyfactory>=2.21.0",- "pypdfium2>=4.30.1",+ "pypdfium2>=4.30.0,!=4.30.1","pydantic==2.10.6","python-dotenv>=1.0.1","PyYAML>=6.0.2",diff --git a/tests/data/discord_newsletter/discord_sample.json b/tests/data/discord_newsletter/discord_sample.jsonnew file mode 100644index 000000000..ef9fa4c9e--- /dev/null+++ b/tests/data/discord_newsletter/discord_sample.json@@ -0,0 +1,163 @@+[+ {+ "id": "1320057977847283834",+ "name": "Introduce-Yourself",+ "type": "text",+ "position": 4,+ "parent": "1348223201812938782",+ "message_count": 5,+ "messages": [+ {+ "author": "John Smith (TechCorp)",+ "content": "## [**John Smith**](https://www.linkedin.com/in/john-smith/) (mostly on X https://x.com/johnsmith) \n\n*CTO @ [TechCorp](https://techcorp.com) · AI and search optimization.*\n\nI've been building software for 15 years now (love building also for fun), I am a dad, and I spend most of my free time doing fun stuff with the family. \n\nSpecial power: Been playing music for 20 years\n\n📍 Amsterdam, NL (CET)",+ "attachments": [],+ "embeds": [],+ "link": "https://discord.com/channels/1320057304199987230/1320057977847283834/1392231464141193318"+ },+ {+ "author": "Pierre Martin (DataFlow)",+ "content": "## [**Pierre Martin**](https://www.linkedin.com/in/pierre-martin/)\n*Chief Technology Officer [DataFlow](https://dataflow.sh) · Tech consulting with a focus on Data and AI stuff*\nI've been in Tech for more than 25 years. I am software engineer at heart. I am the producer of \"Tech Talk\" podcast, I appear regularly on Tech Business for \"The Tech Reviews\", I've been Director of TechInstitute in Paris, I created a community call tech-meetup (https://tech-meetup.com). I've also been very active in Paris with the AI meetup.\nSpecial power: I am a super networker person. I can sail a boat, fly an airplane, I was also involved in various communities.\n📍 Paris, France (CET)",+ "attachments": [],+ "embeds": [],+ "link": "https://discord.com/channels/1320057304199987230/1320057977847283834/1391681488709488711"+ },+ {+ "author": "Alex Johnson (WorkflowAI)",+ "content": "## [Alex Johnson](https://www.linkedin.com/in/alex-johnson/)\n\n*Co-founder & CTO @ [WorkflowAI](https://workflowai.com/)· WorkflowAI is an open-source language for repeatable AI workflows. Build modular pipes that connect like LEGO blocks, each using its optimal LLM. Turn custom AI code into shareable, versioned pipelines with guaranteed outputs. [Python library available now](https://github.com/WorkflowAI/workflowai), hosted API coming soon.*\nI'm a repeat entrepreneur in tech. Started in digital media distribution (music, VOD, peer-to-peer), then found success in social gaming (free-to-play games on Facebook and mobile). I was born in Sweden and speak Swedish fluently but I'm really a French guy.\n\nWorkflowAI is my first open-source project, so I'm in full learning mode.\n\nSpecial power: Modern rationality. Not Hollywood rationality (like Mr. Spock) but rather the art of fighting biases, seeking truth, and making decisions that work.\n\n📍 Paris, France (CET)",+ "attachments": [],+ "embeds": [],+ "link": "https://discord.com/channels/1320057304199987230/1320057977847283834/1391085336188424363"+ },+ {+ "author": "Mike Schmidt (AutoBot)",+ "content": "## [**Mike Schmidt**](https://www.linkedin.com/in/mike-schmidt-890b96303/)\n*CTO & Co-founder @ [AutoBot](https://autobot.ai) · We enable AI Agents to control mobile apps & phones.*\nI'm a Tech builder passionate about decentralized compute, now building AutoBot's AI automation platform for Android.\nPreviously worked on blockchain projects, especially on various networks.\nSpecial power: Once ran 100m in under 11 seconds as a track & field athlete. Not quite superhero speed, but fast enough to turn heads :fast: \n📍 Berlin, Germany (CET)",+ "attachments": [],+ "embeds": [],+ "link": "https://discord.com/channels/1320057304199987230/1320057977847283834/1390267363785441432"+ },+ {+ "author": "Chris Wilson (AutoBot)",+ "content": "## [**Chris Wilson**](https://www.linkedin.com/in/chris-wilson-22b4b8210/)\n*CEO & Co-founder @ [AutoBot](https://autobot.ai) · We enable AI Agents to control mobile apps & phones - We are open source with 3.2k GitHub stars in 9 weeks post-launch*\n\nOutside the startup grind, I'm passionate about travel and cooking - there's something therapeutic about finding fresh ingredients and turning them into a great meal. 🍄🟫\n\nWe recently incorporated in Delaware and successfully closed our Pre-Seed. Heading to SF/NY soon and excited to connect with some folks there. Please reach out 😄\n\n📍 Berlin, Germany (CET)",+ "attachments": [],+ "embeds": [],+ "link": "https://discord.com/channels/1320057304199987230/1320057977847283834/1390259441852481547"+ }+ ]+ },+ {+ "id": "1320059080857096214",+ "name": "🧠-Knowledge",+ "type": "text",+ "position": 7,+ "parent": "1320057304199987231",+ "message_count": 2,+ "messages": [+ {+ "author": "Robert Davis (AITech)",+ "content": "Nice presentation",+ "attachments": [],+ "embeds": [],+ "link": "https://discord.com/channels/1320057304199987230/1320059080857096214/1388797681568190514"+ },+ {+ "author": "Robert Davis (AITech)",+ "content": "https://youtube.com/watch?v=example123&feature=shared",+ "attachments": [],+ "embeds": [+ {+ "title": "Multimodal Open Source AI Models, From Online Demos to On-Device - ...",+ "description": "",+ "type": "video"+ }+ ],+ "link": "https://discord.com/channels/1320057304199987230/1320059080857096214/1388797614740475906"+ }+ ]+ },+ {+ "id": "1324834149965631569",+ "name": "🎉-Achievements",+ "type": "text",+ "position": 10,+ "parent": "1320057304199987231",+ "message_count": 5,+ "messages": [+ {+ "author": "Sam Moore (EdgeTech)",+ "content": "https://www.edgetech.cloud/blog/posts/announcing-edge-functions",+ "attachments": [],+ "embeds": [+ {+ "title": "Announcing the new Edge Functions Components powered by WebAssembly...",+ "description": "Developers can push edge functions to the EdgeTech Component Registry to generate HTTP responses at the edge.",+ "type": "article"+ }+ ],+ "link": "https://discord.com/channels/1320057304199987230/1324834149965631569/1392507684573675561"+ },+ {+ "author": "Kevin Thompson (DataAnalytics)",+ "content": "https://www.linkedin.com/posts/example-user_very-cool-to-see-our-joint-analysis-with-activity-7346527049613815808-8P9G?",+ "attachments": [],+ "embeds": [+ {+ "title": "Very cool to see our joint analysis with Cloud Services Provider...",+ "description": "Very cool to see our joint analysis with Cloud Services Provider about GraphRAG being featured in AI Company's blog.\n\nOverall, their article presents the approaches of GraphRAG and Agentic Search side by side. \n\nI think we should go a step further by pointing out that these two approaches are closely intertwined : an agentic system could use G...",+ "type": "article"+ }+ ],+ "link": "https://discord.com/channels/1320057304199987230/1324834149965631569/1390327059057999893"+ },+ {+ "author": "Julian Garcia (BackupTech)",+ "content": "https://backuptech.io/posts/2025-06-27/modern-data-format-introducing-new-archive/",+ "attachments": [],+ "embeds": [+ {+ "title": "It doesn't make sense to wrap modern data in old formats, introducing new archive format...",+ "description": "",+ "type": "article"+ }+ ],+ "link": "https://discord.com/channels/1320057304199987230/1324834149965631569/1388195527321194548"+ },+ {+ "author": "Peter Chen (MLTech)",+ "content": "No I didn't start as one, I did some for a startup which had a product that needed to integrate with CRMs 🙂\n\nIn reply to Kevin Thompson (DataAnalytics)'s message:\nDidn't you start as a CRM integrator 🙈",+ "attachments": [],+ "embeds": [],+ "link": "https://discord.com/channels/1320057304199987230/1324834149965631569/1388135901829267559"+ },+ {+ "author": "Kevin Thompson (DataAnalytics)",+ "content": "Didn't you start as a CRM integrator 🙈",+ "attachments": [],+ "embeds": [],+ "link": "https://discord.com/channels/1320057304199987230/1324834149965631569/1388124441887899648"+ }+ ]+ },+ {+ "id": "1348214314829615116",+ "name": "👁-Computer-Vision",+ "type": "text",+ "position": 16,+ "parent": "1320057304199987231",+ "message_count": 1,+ "messages": [+ {+ "author": "Ryan Miller (VisionAI)",+ "content": "Also this hasn't been posted enough either but I think https://example-ai.org/ is a really cool model and is open weights and really impressive performance as it lets you do anything essentially",+ "attachments": [],+ "embeds": [+ {+ "title": "EXAMPLE: Scalable Unified Multimodal Model",+ "description": "A scalable unified multimodal model, revolutionizing how AI interacts with complex systems.",+ "type": "article"+ }+ ],+ "link": "https://discord.com/channels/1320057304199987230/1348214314829615116/1389575075795177473"+ }+ ]+ }+]\ No newline at end of filediff --git a/tests/e2e/pipelex/pipes/pipe_controller/pipe_condition/test_pipe_condition_simple.py b/tests/e2e/pipelex/pipes/pipe_controller/pipe_condition/test_pipe_condition_simple.pyindex a53d77d42..0f151863a 100644--- a/tests/e2e/pipelex/pipes/pipe_controller/pipe_condition/test_pipe_condition_simple.py+++ b/tests/e2e/pipelex/pipes/pipe_controller/pipe_condition/test_pipe_condition_simple.py@@ -3,7 +3,7 @@import pytestfrom pipelex import pretty_print-from pipelex.core.pipe_input_spec import PipeInputSpec+from pipelex.core.pipe_input_spec import PipeInputSpec, TypedNamedInputRequirementfrom pipelex.core.pipe_run_params import PipeRunModefrom pipelex.core.pipe_run_params_factory import PipeRunParamsFactoryfrom pipelex.core.working_memory_factory import WorkingMemoryFactory@@ -23,7 +23,7 @@ class TestPipeConditionSimple:pipe_condition = PipeCondition(code="test_condition_fail",domain="test_domain",- inputs=PipeInputSpec(root={"user_category": "test_pipe_condition.CategoryInput"}),+ inputs=PipeInputSpec.make_from_dict(concepts_dict={"user_category": "test_pipe_condition.CategoryInput"}),output_concept_code="native.Text",expression_template="{{ user_category.category }}",pipe_map={"small": "process_small", "medium": "process_medium", "large": "process_large"},@@ -52,7 +52,7 @@ class TestPipeConditionSimple:pipe_condition = PipeCondition(code="test_condition_succeed",domain="test_domain",- inputs=PipeInputSpec(root={"user_status": "test_pipe_condition.CategoryInput"}),+ inputs=PipeInputSpec.make_from_dict(concepts_dict={"user_status": "test_pipe_condition.CategoryInput"}),output_concept_code="native.Text",expression_template="{{ user_status.category }}",pipe_map={@@ -64,7 +64,15 @@ class TestPipeConditionSimple:)# Test with proper working memory - should SUCCEED or fail at expression evaluation (not missing inputs)- working_memory = WorkingMemoryFactory.make_for_dry_run(needed_inputs=[("user_status", "test_pipe_condition.CategoryInput", CategoryInput)])+ working_memory = WorkingMemoryFactory.make_for_dry_run(+ needed_inputs=[+ TypedNamedInputRequirement(+ variable_name="user_status",+ concept_code="test_pipe_condition.CategoryInput",+ structure_class=CategoryInput,+ )+ ]+ )try:pipe_output = await pipe_condition.run_pipe(diff --git a/tests/e2e/pipelex/pipes/pipe_controller/pipe_sequence/test_pipe_sequence_batching.py b/tests/e2e/pipelex/pipes/pipe_controller/pipe_sequence/test_pipe_sequence_batching.pyindex 42fc03e6e..00d947ccf 100644--- a/tests/e2e/pipelex/pipes/pipe_controller/pipe_sequence/test_pipe_sequence_batching.py+++ b/tests/e2e/pipelex/pipes/pipe_controller/pipe_sequence/test_pipe_sequence_batching.py@@ -5,6 +5,7 @@ from typing import castimport pytestfrom pipelex import log+from pipelex.core.pipe_input_spec import TypedNamedInputRequirementfrom pipelex.core.pipe_run_params import PipeRunModefrom pipelex.core.pipe_run_params_factory import PipeRunParamsFactoryfrom pipelex.core.stuff_factory import StuffFactory@@ -24,7 +25,11 @@ async def test_review_analysis_sequence_with_batching(pipe_run_mode: PipeRunModeif pipe_run_mode == PipeRunMode.DRY:working_memory = WorkingMemoryFactory.make_for_dry_run(needed_inputs=[- ("document", "customer_feedback.Document", Document),+ TypedNamedInputRequirement(+ variable_name="document",+ concept_code="customer_feedback.Document",+ structure_class=Document,+ ),])pipe = get_required_pipe(pipe_code="analyze_reviews_sequence")diff --git a/tests/integration/pipelex/pipes/pipe_controller/pipe_batch/test_pipe_batch_simple.py b/tests/integration/pipelex/pipes/pipe_controller/pipe_batch/test_pipe_batch_simple.pyindex d84ff7678..7a7a0e7f9 100644--- a/tests/integration/pipelex/pipes/pipe_controller/pipe_batch/test_pipe_batch_simple.py+++ b/tests/integration/pipelex/pipes/pipe_controller/pipe_batch/test_pipe_batch_simple.py@@ -29,7 +29,7 @@ class TestPipeBatchSimple:domain="test_integration",code="simple_batch",branch_pipe_code="uppercase_transformer", # This exists in the TOML file- inputs=PipeInputSpec(root={"text_list": "Text", "text_item": "Text"}),+ inputs=PipeInputSpec.make_from_dict(concepts_dict={"text_list": "Text", "text_item": "Text"}),output_concept_code="test_integration.UppercaseText",batch_params=BatchParams(input_list_stuff_name="text_list", input_item_stuff_name="text_item"),)diff --git a/tests/integration/pipelex/pipes/pipe_controller/pipe_condition/test_pipe_condition_simple.py b/tests/integration/pipelex/pipes/pipe_controller/pipe_condition/test_pipe_condition_simple.pyindex c72fd6494..9179ded28 100644--- a/tests/integration/pipelex/pipes/pipe_controller/pipe_condition/test_pipe_condition_simple.py+++ b/tests/integration/pipelex/pipes/pipe_controller/pipe_condition/test_pipe_condition_simple.py@@ -31,7 +31,7 @@ class TestPipeConditionSimple:pipe_condition = PipeCondition(domain="test_integration",code="text_length_condition",- inputs=PipeInputSpec(root={"input_text": "Text"}),+ inputs=PipeInputSpec.make_from_dict(concepts_dict={"input_text": "Text"}),output_concept_code="Text",expression_template="{% if input_text.text|length > 5 %}long{% else %}short{% endif %}",pipe_map={"long": "capitalize_long_text", "short": "add_prefix_short_text"},@@ -100,7 +100,7 @@ class TestPipeConditionSimple:pipe_condition = PipeCondition(domain="test_integration",code="text_length_condition",- inputs=PipeInputSpec(root={"input_text": "Text"}),+ inputs=PipeInputSpec.make_from_dict(concepts_dict={"input_text": "Text"}),output_concept_code="Text",expression_template="{% if input_text.text|length > 5 %}long{% else %}short{% endif %}",pipe_map={"long": "capitalize_long_text", "short": "add_prefix_short_text"},diff --git a/tests/integration/pipelex/pipes/pipe_controller/pipe_parallel/test_pipe_parallel_simple.py b/tests/integration/pipelex/pipes/pipe_controller/pipe_parallel/test_pipe_parallel_simple.pyindex 620d14ed8..4ee80eea5 100644--- a/tests/integration/pipelex/pipes/pipe_controller/pipe_parallel/test_pipe_parallel_simple.py+++ b/tests/integration/pipelex/pipes/pipe_controller/pipe_parallel/test_pipe_parallel_simple.py@@ -28,7 +28,7 @@ class TestPipeParallelSimple:pipe_parallel = PipeParallel(domain="test_integration",code="parallel_text_analyzer",- inputs=PipeInputSpec(root={"input_text": "Text"}),+ inputs=PipeInputSpec.make_from_dict(concepts_dict={"input_text": "Text"}),output_concept_code="Text",parallel_sub_pipes=[SubPipe(pipe_code="analyze_sentiment", output_name="sentiment_result"),@@ -140,7 +140,7 @@ class TestPipeParallelSimple:pipe_parallel = PipeParallel(domain="test_integration",code="parallel_text_analyzer",- inputs=PipeInputSpec(root={"input_text": "Text"}),+ inputs=PipeInputSpec.make_from_dict(concepts_dict={"input_text": "Text"}),output_concept_code="Text",parallel_sub_pipes=[SubPipe(pipe_code="analyze_sentiment", output_name="sentiment_result"),diff --git a/tests/integration/pipelex/pipes/pipe_controller/pipe_sequence/test_pipe_sequence_dry_run.py b/tests/integration/pipelex/pipes/pipe_controller/pipe_sequence/test_pipe_sequence_dry_run.pynew file mode 100644index 000000000..cfc1daea9--- /dev/null+++ b/tests/integration/pipelex/pipes/pipe_controller/pipe_sequence/test_pipe_sequence_dry_run.py@@ -0,0 +1,114 @@+"""Dry run tests for PipeSequence controller with batching functionality."""++import pytest+from pytest import FixtureRequest++from pipelex import pretty_print+from pipelex.core.pipe_output import PipeOutput+from pipelex.core.pipe_run_params import PipeRunMode+from pipelex.core.pipe_run_params_factory import PipeRunParamsFactory+from pipelex.core.stuff import Stuff+from pipelex.core.stuff_content import ListContent+from pipelex.core.stuff_factory import StuffFactory+from pipelex.core.working_memory import WorkingMemory+from pipelex.core.working_memory_factory import WorkingMemoryFactory+from pipelex.hub import get_pipe_router, get_report_delegate+from pipelex.pipeline.job_metadata import JobMetadata+from pipelex.tools.misc.json_utils import load_json_list_from_path+from tests.test_pipelines.discord_newsletter import ChannelSummary, DiscordChannelUpdate++[email protected]_runnable[email protected](loop_scope="class")+class TestPipeSequenceDryRun:+ """Test PipeSequence dry run functionality with batching."""++ async def test_discord_newsletter_dry_run_working_memory(+ self,+ request: FixtureRequest,+ pipe_run_mode: PipeRunMode,+ ) -> None:+ """Test that the Discord newsletter pipeline creates correct working memory with ListContent for batched inputs."""+ # Load the discord channel updates data from JSON+ DISCORD_EXTRACT_PATH = "tests/data/discord_newsletter/discord_sample.json"+ discord_channel_updates_data = load_json_list_from_path(DISCORD_EXTRACT_PATH)++ # Create structured DiscordChannelUpdate objects+ discord_channel_updates = ListContent[DiscordChannelUpdate](+ items=[DiscordChannelUpdate.model_validate(article_data) for article_data in discord_channel_updates_data]+ )++ # Create Stuff object for the discord channel updates list+ discord_updates_stuff = StuffFactory.make_stuff(+ concept_str="discord_newsletter.DiscordChannelUpdate", content=discord_channel_updates, name="discord_channel_updates"+ )++ # Create working memory with the discord channel updates+ working_memory = WorkingMemoryFactory.make_from_single_stuff(stuff=discord_updates_stuff)++ # Run the Discord newsletter pipeline in dry run mode+ pipe_output: PipeOutput = await get_pipe_router().run_pipe_code(+ pipe_code="write_discord_newsletter",+ job_metadata=JobMetadata(job_name=request.node.originalname), # type: ignore+ working_memory=working_memory,+ pipe_run_params=PipeRunParamsFactory.make_run_params(pipe_run_mode=pipe_run_mode),+ )++ # Log output for debugging+ pretty_print(pipe_output, title="Discord Newsletter Dry Run Output")+ get_report_delegate().generate_report()++ # Basic assertions+ assert pipe_output is not None+ assert pipe_output.working_memory is not None+ assert pipe_output.main_stuff is not None++ # The pipeline only has one step (summarization), so the final output is ChannelSummary+ # This test is focused on verifying that batching works correctly in dry run mode+ assert pipe_output.main_stuff.concept_code == "discord_newsletter.ChannelSummary"+ if pipe_run_mode == PipeRunMode.DRY:+ assert isinstance(pipe_output.main_stuff.content, ListContent)++ # Verify working memory structure+ final_working_memory: WorkingMemory = pipe_output.working_memory++ # Check that discord_channel_updates was created as ListContent+ discord_updates_stuff_final = final_working_memory.get_stuff("discord_channel_updates")+ assert discord_updates_stuff_final.concept_code == "discord_newsletter.DiscordChannelUpdate"++ if pipe_run_mode == PipeRunMode.DRY:+ # The key assertion: verify it's a ListContent with multiple items+ discord_updates_list = discord_updates_stuff_final.as_list_of_fixed_content_type(item_type=DiscordChannelUpdate)+ assert isinstance(discord_updates_list, ListContent)+ assert len(discord_updates_list.items) > 1, "Should have multiple items for batching"++ # Verify each discord update item+ for i, update in enumerate(discord_updates_list.items):+ assert isinstance(update, DiscordChannelUpdate), f"Update {i} should be DiscordChannelUpdate"+ assert len(update.name) > 0, f"Update {i} should have a non-empty channel name"++ # Check that channel_summaries was created as ListContent (result of batched operation)+ channel_summaries_stuff: Stuff | None = final_working_memory.get_optional_stuff("channel_summaries")+ assert channel_summaries_stuff is not None, "channel_summaries should be in working memory"+ assert channel_summaries_stuff.concept_code == "discord_newsletter.ChannelSummary"++ # Verify channel_summaries is also a ListContent with multiple ChannelSummary items+ channel_summaries_list: ListContent[ChannelSummary] = channel_summaries_stuff.as_list_of_fixed_content_type(item_type=ChannelSummary)+ assert isinstance(channel_summaries_list, ListContent)+ assert len(channel_summaries_list.items) > 1, "Should have multiple ChannelSummary items from batch processing"++ # Verify each summary item (these should be proper ChannelSummary objects from the LLM mock)+ for i, summary in enumerate(channel_summaries_list.items):+ assert isinstance(summary, ChannelSummary), f"Summary {i} should be ChannelSummary"+ assert len(summary.channel_name) > 0, f"Summary {i} should have a non-empty channel name"+ assert isinstance(summary.summary_items, list), f"Summary {i} should have a list of summary items"++ # Verify that the number of summaries matches the number of original channel updates+ assert len(channel_summaries_list.items) == len(discord_updates_list.items), (+ "Number of summaries should match number of original channel updates"+ )++ print("✅ Successfully verified dry run working memory:")+ print(f" - discord_channel_updates: ListContent with {len(discord_updates_list.items)} items")+ print(f" - channel_summaries: ListContent with {len(channel_summaries_list.items)} items")+ print(f" - Final output: ChannelSummary list with concept code {pipe_output.main_stuff.concept_code}")diff --git a/tests/integration/pipelex/pipes/pipe_controller/pipe_sequence/test_pipe_sequence_simple.py b/tests/integration/pipelex/pipes/pipe_controller/pipe_sequence/test_pipe_sequence_simple.pyindex cbf66fec5..965565348 100644--- a/tests/integration/pipelex/pipes/pipe_controller/pipe_sequence/test_pipe_sequence_simple.py+++ b/tests/integration/pipelex/pipes/pipe_controller/pipe_sequence/test_pipe_sequence_simple.py@@ -29,7 +29,7 @@ class TestPipeSequenceSimple:pipe_sequence = PipeSequence(domain="test_integration",code="simple_sequence",- inputs=PipeInputSpec(root={"input_text": "Text"}),+ inputs=PipeInputSpec.make_from_dict(concepts_dict={"input_text": "Text"}),output_concept_code="Text",sequential_sub_pipes=[SubPipe(pipe_code="capitalize_text", output_name="capitalized_text"),diff --git a/tests/integration/pipelex/pipes/pipe_operator/pipe_llm/test_pipe_llm.py b/tests/integration/pipelex/pipes/pipe_operator/pipe_llm/test_pipe_llm.pyindex 7c6ba21ab..862a0f0d1 100644--- a/tests/integration/pipelex/pipes/pipe_operator/pipe_llm/test_pipe_llm.py+++ b/tests/integration/pipelex/pipes/pipe_operator/pipe_llm/test_pipe_llm.py@@ -67,7 +67,7 @@ class TestPipeLLM:pipe=PipeLLM(code="adhoc_for_test_pipe_llm_image",domain="generic",- inputs=PipeInputSpec(root={stuff_name: stuff.concept_code}),+ inputs=PipeInputSpec.make_from_dict(concepts_dict={stuff_name: stuff.concept_code}),output_concept_code=NativeConcept.TEXT.code,pipe_llm_prompt=PipeLLMPrompt(code="adhoc_for_test_pipe_llm_image",diff --git a/tests/integration/pipelex/pipes/pipe_operator/pipe_ocr/test_pipe_ocr.py b/tests/integration/pipelex/pipes/pipe_operator/pipe_ocr/test_pipe_ocr.pyindex 032addeff..7b11c43b0 100644--- a/tests/integration/pipelex/pipes/pipe_operator/pipe_ocr/test_pipe_ocr.py+++ b/tests/integration/pipelex/pipes/pipe_operator/pipe_ocr/test_pipe_ocr.py@@ -28,11 +28,7 @@ class TestPipeOCR:pipe=PipeOcr(code="adhoc_for_test_pipe_ocr_image",domain="generic",- inputs=PipeInputSpec(- root={- "page_scan": "Image",- }- ),+ inputs=PipeInputSpec.make_from_dict(concepts_dict={"page_scan": "Image"}),should_include_images=True,should_caption_images=False,should_include_page_views=True,@@ -62,11 +58,7 @@ class TestPipeOCR:pipe=PipeOcr(code="adhoc_for_test_pipe_ocr_pdf",domain="generic",- inputs=PipeInputSpec(- root={- PIPE_OCR_INPUT_NAME: "PDF",- }- ),+ inputs=PipeInputSpec.make_from_dict(concepts_dict={PIPE_OCR_INPUT_NAME: "PDF"}),should_include_images=True,should_caption_images=False,should_include_page_views=True,diff --git a/tests/pipelex_unit_test.toml b/tests/pipelex_unit_test.tomlindex 13f74ba3f..6edfb9564 100644--- a/tests/pipelex_unit_test.toml+++ b/tests/pipelex_unit_test.toml@@ -10,3 +10,4 @@ default_imgg_handle = "fal-ai/fast-lightning-sdxl"[pipelex.feature_config]is_reporting_enabled = false+diff --git a/tests/test_pipelines/discord_newsletter.py b/tests/test_pipelines/discord_newsletter.pynew file mode 100644index 000000000..62a12a059--- /dev/null+++ b/tests/test_pipelines/discord_newsletter.py@@ -0,0 +1,54 @@+from typing import List++from pydantic import Field++from pipelex.core.stuff_content import StructuredContent+++class Attachment(StructuredContent):+ """Represents a Discord message attachment"""++ name: str = Field(..., description="Name of the attachment file")+ url: str = Field(..., description="URL of the attachment")+++class Embed(StructuredContent):+ """Represents a Discord message embed"""++ title: str = Field(..., description="Title of the embed")+ description: str = Field(..., description="Description of the embed content")+ type: str = Field(..., description="Type of the embed (e.g., article, video)")+++class DiscordMessage(StructuredContent):+ """Represents a Discord message within a channel"""++ author: str = Field(..., description="Author of the message")+ content: str = Field(..., description="Content of the message")+ attachments: List[Attachment] = Field(default_factory=list, description="List of message attachments")+ embeds: List[Embed] = Field(default_factory=list, description="List of message embeds")+ link: str = Field(..., description="Link to the message")+++class DiscordChannelUpdate(StructuredContent):+ """Represents a Discord channel with its messages"""++ name: str = Field(..., description="Name of the Discord channel")+ position: int = Field(..., description="Position of the channel")+ messages: List[DiscordMessage] = Field(default_factory=list, description="List of messages in the channel")+++class ChannelSummary(StructuredContent):+ """Represents a summarized Discord channel for newsletter inclusion"""++ channel_name: str = Field(..., description="Name of the Discord channel")+ summary_items: List[str] = Field(..., description="Well-written summaries of the channel's activity")+++class Newsletter(StructuredContent):+ """Represents the final newsletter content"""++ weekly_summary: str = Field(..., description="200 character summary of weekly Share channel content")+ new_members: List[str] = Field(default_factory=list, description="New member introductions in bullet points")+ channel_sections: List[ChannelSummary] = Field(default_factory=list, description="Ordered channel summaries")+ geographic_hubs: List[ChannelSummary] = Field(default_factory=list, description="Geographic hub channels grouped at end")diff --git a/tests/test_pipelines/discord_newsletter.toml b/tests/test_pipelines/discord_newsletter.tomlnew file mode 100644index 000000000..600f6c780--- /dev/null+++ b/tests/test_pipelines/discord_newsletter.toml@@ -0,0 +1,32 @@+domain = "discord_newsletter"+definition = "Discord newsletter generation pipeline"++[concept]+DiscordChannelUpdate = "A Discord channel with its messages for newsletter generation"+ChannelSummary = "A summarized Discord channel for newsletter inclusion"+Newsletter = "The final newsletter content with organized channel summaries"++[pipe.write_discord_newsletter]+PipeSequence = "Create a newsletter from Discord articles by summarizing channels and organizing content"+inputs = { discord_channel_updates = "DiscordChannelUpdate" }+output = "Newsletter"+steps = [+ { pipe = "summarize_discord_channel_update", batch_over = "discord_channel_updates", batch_as = "discord_channel_update", result = "channel_summaries" },+]++[pipe.summarize_discord_channel_update]+PipeLLM = "Summarize a Discord channel's messages into newsletter-friendly content"+inputs = { discord_channel_update = "DiscordChannelUpdate" }+output = "ChannelSummary"+system_prompt = "You are a newsletter editor who creates engaging summaries of Discord channel content. You extract key information, preserve important links, and write in a clear, concise style suitable for newsletter readers."+prompt_template = """+Analyze this Discord channel update and create a newsletter-friendly summary.++Channel Information:+@discord_channel_update++The summary should be informative and engaging for newsletter readers who want to understand what happened in this channel during the week.+Generate one or more summary items: each one can correspond to a single message or to a bunch of messages that were part of the same conversation.+Each summary item should be in plain text, no bullet points. You can make some parts **bold** to highlight important information.+"""+diff --git a/tests/unit/pipelex/core/test_dry_working_memory.py b/tests/unit/pipelex/core/test_dry_working_memory.pyindex d3cc9c809..17b76a39d 100644--- a/tests/unit/pipelex/core/test_dry_working_memory.py+++ b/tests/unit/pipelex/core/test_dry_working_memory.py@@ -1,13 +1,14 @@"""Test dry working memory functionality."""-from typing import List, Tuple, Type, cast+from typing import Listimport pytestfrom pytest import FixtureRequestfrom pipelex import logfrom pipelex.core.concept_native import NativeConcept-from pipelex.core.stuff_content import PageContent, StuffContent, TextContent+from pipelex.core.pipe_input_spec import TypedNamedInputRequirement+from pipelex.core.stuff_content import PageContent, TextContentfrom pipelex.core.working_memory_factory import WorkingMemoryFactoryfrom tests.test_pipelines.tricky_questions import ThoughtfulAnswer@@ -25,12 +26,13 @@ class TestDryWorkingMemory:log.info("Testing dry run with PageContent")# Test the specific inputs requested by the user- needed_inputs = cast(- List[Tuple[str, str, Type[StuffContent]]],- [- ("page", NativeConcept.PAGE.code, PageContent),- ],- )+ needed_inputs = [+ TypedNamedInputRequirement(+ variable_name="page",+ concept_code=NativeConcept.PAGE.code,+ structure_class=PageContent,+ ),+ ]dry_memory = WorkingMemoryFactory.make_for_dry_run(needed_inputs=needed_inputs)# Verify working memory contains exactly this@@ -64,13 +66,18 @@ class TestDryWorkingMemory:log.info("Testing dry run with structured content (ThoughtfulAnswer)")# Use ThoughtfulAnswer from tricky questions domain- needed_inputs = cast(- List[Tuple[str, str, Type[StuffContent]]],- [- ("thoughtful_answer", "test_tricky_questions.ThoughtfulAnswer", ThoughtfulAnswer),- ("question", "answer.Question", TextContent),- ],- )+ needed_inputs = [+ TypedNamedInputRequirement(+ variable_name="thoughtful_answer",+ concept_code="test_tricky_questions.ThoughtfulAnswer",+ structure_class=ThoughtfulAnswer,+ ),+ TypedNamedInputRequirement(+ variable_name="question",+ concept_code="answer.Question",+ structure_class=TextContent,+ ),+ ]dry_memory = WorkingMemoryFactory.make_for_dry_run(needed_inputs=needed_inputs)@@ -110,13 +117,18 @@ class TestDryWorkingMemory:"""Test that make_for_dry_run falls back to TextContent when needed."""log.info("Testing dry run with TextContent fallback")- needed_inputs = cast(- List[Tuple[str, str, Type[StuffContent]]],- [- ("question_analysis", "QuestionAnalysis", TextContent),- ("conclusion", "test_tricky_questions.ThoughtfulAnswerConclusion", TextContent),- ],- )+ needed_inputs = [+ TypedNamedInputRequirement(+ variable_name="question_analysis",+ concept_code="QuestionAnalysis",+ structure_class=TextContent,+ ),+ TypedNamedInputRequirement(+ variable_name="conclusion",+ concept_code="test_tricky_questions.ThoughtfulAnswerConclusion",+ structure_class=TextContent,+ ),+ ]dry_memory = WorkingMemoryFactory.make_for_dry_run(needed_inputs=needed_inputs)@@ -144,14 +156,23 @@ class TestDryWorkingMemory:"""Test that make_for_dry_run handles mixed content types correctly."""log.info("Testing dry run with mixed content types")- needed_inputs = cast(- List[Tuple[str, str, Type[StuffContent]]],- [- ("thoughtful_answer", "test_tricky_questions.ThoughtfulAnswer", ThoughtfulAnswer),- ("raw_question", "answer.Question", TextContent),- ("analysis_result", "QuestionAnalysis", TextContent),- ],- )+ needed_inputs = [+ TypedNamedInputRequirement(+ variable_name="thoughtful_answer",+ concept_code="test_tricky_questions.ThoughtfulAnswer",+ structure_class=ThoughtfulAnswer,+ ),+ TypedNamedInputRequirement(+ variable_name="raw_question",+ concept_code="answer.Question",+ structure_class=TextContent,+ ),+ TypedNamedInputRequirement(+ variable_name="analysis_result",+ concept_code="QuestionAnalysis",+ structure_class=TextContent,+ ),+ ]dry_memory = WorkingMemoryFactory.make_for_dry_run(needed_inputs=needed_inputs)@@ -182,7 +203,7 @@ class TestDryWorkingMemory:"""Test that make_for_dry_run handles empty inputs gracefully."""log.info("Testing dry run with empty inputs")- needed_inputs = cast(List[Tuple[str, str, Type[StuffContent]]], [])+ needed_inputs: List[TypedNamedInputRequirement] = []dry_memory = WorkingMemoryFactory.make_for_dry_run(needed_inputs=needed_inputs)@@ -200,14 +221,23 @@ class TestDryWorkingMemory:log.info("Testing dry run with realistic tricky questions pipeline scenario")# Simulate the conclude_tricky_question_by_steps pipeline needs- needed_inputs = cast(- List[Tuple[str, str, Type[StuffContent]]],- [- ("question", "answer.Question", TextContent),- ("question_analysis", "QuestionAnalysis", TextContent),- ("thoughtful_answer", "test_tricky_questions.ThoughtfulAnswer", ThoughtfulAnswer),- ],- )+ needed_inputs = [+ TypedNamedInputRequirement(+ variable_name="question",+ concept_code="answer.Question",+ structure_class=TextContent,+ ),+ TypedNamedInputRequirement(+ variable_name="question_analysis",+ concept_code="QuestionAnalysis",+ structure_class=TextContent,+ ),+ TypedNamedInputRequirement(+ variable_name="thoughtful_answer",+ concept_code="test_tricky_questions.ThoughtfulAnswer",+ structure_class=ThoughtfulAnswer,+ ),+ ]dry_memory = WorkingMemoryFactory.make_for_dry_run(needed_inputs=needed_inputs)diff --git a/tests/unit/pipelex/core/test_pipe_condition_validation.py b/tests/unit/pipelex/core/test_pipe_condition_validation.pyindex f496199a0..18116b8e8 100644--- a/tests/unit/pipelex/core/test_pipe_condition_validation.py+++ b/tests/unit/pipelex/core/test_pipe_condition_validation.py@@ -10,7 +10,7 @@ class TestPipeConditionValidation:pipe_condition = PipeCondition(domain="test_domain",code="test_condition",- inputs=PipeInputSpec(root={"input_var": "test_domain.Text"}),+ inputs=PipeInputSpec.make_from_dict(concepts_dict={"input_var": "test_domain.Text"}),output_concept_code="test_domain.ProcessedText",expression="input_var",pipe_map={"value1": "pipe_a", "value2": "pipe_b"},@@ -29,7 +29,7 @@ class TestPipeConditionValidation:pipe_condition_template = PipeCondition(domain="test_domain",code="test_condition_template",- inputs=PipeInputSpec(root={"var": "test_domain.Text"}),+ inputs=PipeInputSpec.make_from_dict(concepts_dict={"var": "test_domain.Text"}),output_concept_code="test_domain.Result",expression_template="{{ var }}",pipe_map={"value": "target_pipe"},@@ -39,7 +39,7 @@ class TestPipeConditionValidation:pipe_condition_expr = PipeCondition(domain="test_domain",code="test_condition_expr",- inputs=PipeInputSpec(root={"var": "test_domain.Text"}),+ inputs=PipeInputSpec.make_from_dict(concepts_dict={"var": "test_domain.Text"}),output_concept_code="test_domain.Result",expression="var",pipe_map={"value": "target_pipe"},diff --git a/tests/unit/pipelex/core/test_pipe_parallel_validation.py b/tests/unit/pipelex/core/test_pipe_parallel_validation.pyindex fb50f5ce8..32186496a 100644--- a/tests/unit/pipelex/core/test_pipe_parallel_validation.py+++ b/tests/unit/pipelex/core/test_pipe_parallel_validation.py@@ -32,7 +32,7 @@ class TestPipeParallelValidation:pipe_parallel = PipeParallel(domain="test_domain",code="parallel_document_processor",- inputs=PipeInputSpec(root={"document": "test_domain.document", "context": "test_domain.context"}),+ inputs=PipeInputSpec.make_from_dict(concepts_dict={"document": "test_domain.document", "context": "test_domain.context"}),output_concept_code="test_domain.ProcessedAnalysis",parallel_sub_pipes=[SubPipe(pipe_code="analyze_document", output_name="analysis_result")],add_each_output=True,@@ -55,7 +55,7 @@ class TestPipeParallelValidation:pipe_parallel = PipeParallel(domain="test_domain",code="test_parallel",- inputs=PipeInputSpec(root={"input_var": "test_domain.Text"}),+ inputs=PipeInputSpec.make_from_dict(concepts_dict={"input_var": "test_domain.Text"}),output_concept_code="test_domain.ProcessedText",parallel_sub_pipes=[SubPipe(pipe_code="test_pipe_1", output_name="result_1")],add_each_output=True,@@ -66,7 +66,7 @@ class TestPipeParallelValidation:assert pipe_parallel.code == "test_parallel"assert pipe_parallel.domain == "test_domain"assert len(pipe_parallel.parallel_sub_pipes) == 1- assert pipe_parallel.inputs.root["input_var"] == "test_domain.Text"+ assert pipe_parallel.inputs.root["input_var"].concept_code == "test_domain.Text"assert pipe_parallel.output_concept_code == "test_domain.ProcessedText"assert pipe_parallel.add_each_output is Trueassert pipe_parallel.combined_output is None@@ -78,7 +78,7 @@ class TestPipeParallelValidation:pipe_parallel = PipeParallel(domain="test_domain",code="parallel_document_processor",- inputs=PipeInputSpec(root={"document": "test_domain.Document", "context": "test_domain.Context"}),+ inputs=PipeInputSpec.make_from_dict(concepts_dict={"document": "test_domain.Document", "context": "test_domain.Context"}),output_concept_code="test_domain.ProcessedAnalysis",parallel_sub_pipes=[], # No sub-pipes to avoid dependency issuesadd_each_output=True,diff --git a/tests/unit/pipelex/core/test_pipe_sequence_validation.py b/tests/unit/pipelex/core/test_pipe_sequence_validation.pyindex 26fe0450e..c92816f2a 100644--- a/tests/unit/pipelex/core/test_pipe_sequence_validation.py+++ b/tests/unit/pipelex/core/test_pipe_sequence_validation.py@@ -11,7 +11,7 @@ class TestPipeSequenceValidation:pipe_sequence = PipeSequence(domain="test_domain",code="test_sequence",- inputs=PipeInputSpec(root={"text": "test_domain.Text"}),+ inputs=PipeInputSpec.make_from_dict(concepts_dict={"text": "test_domain.Text"}),output_concept_code="test_domain.ProcessedText",sequential_sub_pipes=[SubPipe(pipe_code="test_pipe_1", output_name="intermediate_result")],)@@ -27,11 +27,11 @@ class TestPipeSequenceValidation:pipe_sequence = PipeSequence(domain="test_domain",code="test_sequence",- inputs=PipeInputSpec(root={"initial_input": "test_domain.Text"}),+ inputs=PipeInputSpec.make_from_dict(concepts_dict={"initial_input": "test_domain.Text"}),output_concept_code="test_domain.FinalOutput",sequential_sub_pipes=[SubPipe(pipe_code="step_1", output_name="intermediate"), SubPipe(pipe_code="step_2", output_name="final_output")],)assert pipe_sequence.code == "test_sequence"assert len(pipe_sequence.sequential_sub_pipes) == 2- assert pipe_sequence.inputs.root["initial_input"] == "test_domain.Text"+ assert pipe_sequence.inputs.root["initial_input"].concept_code == "test_domain.Text"diff --git a/uv.lock b/uv.lockindex 85b6ce24f..c56e16196 100644--- a/uv.lock+++ b/uv.lock@@ -1860,7 +1860,7 @@ requires-dist = [{ name = "pillow", specifier = ">=11.2.1" },{ name = "polyfactory", specifier = ">=2.21.0" },{ name = "pydantic", specifier = "==2.10.6" },- { name = "pypdfium2", specifier = ">=4.30.1" },+ { name = "pypdfium2", specifier = ">=4.30.0,!=4.30.1" },{ name = "pyright", marker = "extra == 'dev'", specifier = "==1.1.398" },{ name = "pytest", marker = "extra == 'dev'", specifier = ">=8.3.3" },{ name = "pytest-asyncio", marker = "extra == 'dev'", specifier = ">=0.24.0" },@@ -2203,22 +2203,22 @@ wheels = [[[package]]name = "pypdfium2"-version = "4.30.1"-source = { registry = "https://pypi.org/simple" }-sdist = { url = "https://files.pythonhosted.org/packages/55/d4/905e621c62598a08168c272b42fc00136c8861cfce97afb2a1ecbd99487a/pypdfium2-4.30.1.tar.gz", hash = "sha256:5f5c7c6d03598e107d974f66b220a49436aceb191da34cda5f692be098a814ce", size = 164854, upload-time = "2024-12-19T19:28:11.459Z" }-wheels = [- { url = "https://files.pythonhosted.org/packages/30/8e/3ce0856b3af0f058dd3655ce57d31d1dbde4d4bd0e172022ffbf1b58a4b9/pypdfium2-4.30.1-py3-none-macosx_10_13_x86_64.whl", hash = "sha256:e07c47633732cc18d890bb7e965ad28a9c5a932e548acb928596f86be2e5ae37", size = 2889836, upload-time = "2024-12-19T19:27:39.531Z" },- { url = "https://files.pythonhosted.org/packages/c2/6a/f6995b21f9c6c155487ce7df70632a2df1ba49efcb291b9943ea45f28b15/pypdfium2-4.30.1-py3-none-macosx_11_0_arm64.whl", hash = "sha256:5ea2d44e96d361123b67b00f527017aa9c847c871b5714e013c01c3eb36a79fe", size = 2769232, upload-time = "2024-12-19T19:27:43.227Z" },- { url = "https://files.pythonhosted.org/packages/53/91/79060923148e6d380b8a299b32bba46d70aac5fe1cd4f04320bcbd1a48d3/pypdfium2-4.30.1-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1de7a3a36803171b3f66911131046d65a732f9e7834438191cb58235e6163c4e", size = 2847531, upload-time = "2024-12-19T19:27:46.372Z" },- { url = "https://files.pythonhosted.org/packages/a8/6c/93507f87c159e747eaab54352c0fccbaec3f1b3749d0bb9085a47899f898/pypdfium2-4.30.1-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:b8a4231efb13170354f568c722d6540b8d5b476b08825586d48ef70c40d16e03", size = 2636266, upload-time = "2024-12-19T19:27:49.767Z" },- { url = "https://files.pythonhosted.org/packages/24/dc/d56f74a092f2091e328d6485f16562e2fc51cffb0ad6d5c616d80c1eb53c/pypdfium2-4.30.1-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6f434a4934e8244aa95343ffcf24e9ad9f120dbb4785f631bb40a88c39292493", size = 2919296, upload-time = "2024-12-19T19:27:51.767Z" },- { url = "https://files.pythonhosted.org/packages/be/d9/a2f1ee03d47fbeb48bcfde47ed7155772739622cfadf7135a84ba6a97824/pypdfium2-4.30.1-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f454032a0bc7681900170f67d8711b3942824531e765f91c2f5ce7937f999794", size = 2866119, upload-time = "2024-12-19T19:27:53.561Z" },- { url = "https://files.pythonhosted.org/packages/01/47/6aa019c32aa39d3f33347c458c0c5887e84096cbe444456402bc97e66704/pypdfium2-4.30.1-py3-none-musllinux_1_1_aarch64.whl", hash = "sha256:bbf9130a72370ee9d602e39949b902db669a2a1c24746a91e5586eb829055d9f", size = 6228684, upload-time = "2024-12-19T19:27:56.781Z" },- { url = "https://files.pythonhosted.org/packages/4c/07/2954c15b3f7c85ceb80cad36757fd41b3aba0dd14e68f4bed9ce3f2e7e74/pypdfium2-4.30.1-py3-none-musllinux_1_1_i686.whl", hash = "sha256:5cb52884b1583b96e94fd78542c63bb42e06df5e8f9e52f8f31f5ad5a1e53367", size = 6231815, upload-time = "2024-12-19T19:28:00.351Z" },- { url = "https://files.pythonhosted.org/packages/b4/9b/b4667e95754624f4af5a912001abba90c046e1c80d4a4e887f0af664ffec/pypdfium2-4.30.1-py3-none-musllinux_1_1_x86_64.whl", hash = "sha256:1a9e372bd4867ff223cc8c338e33fe11055dad12f22885950fc27646cc8d9122", size = 6313429, upload-time = "2024-12-19T19:28:02.536Z" },- { url = "https://files.pythonhosted.org/packages/43/38/f9e77cf55ba5546a39fa659404b78b97de2ca344848271e7731efb0954cd/pypdfium2-4.30.1-py3-none-win32.whl", hash = "sha256:421f1cf205e213e07c1f2934905779547f4f4a2ff2f59dde29da3d511d3fc806", size = 2834989, upload-time = "2024-12-19T19:28:04.657Z" },- { url = "https://files.pythonhosted.org/packages/a4/f3/8d3a350efb4286b5ebdabcf6736f51d8e3b10dbe68804c6930b00f5cf329/pypdfium2-4.30.1-py3-none-win_amd64.whl", hash = "sha256:598a7f20264ab5113853cba6d86c4566e4356cad037d7d1f849c8c9021007e05", size = 2960157, upload-time = "2024-12-19T19:28:07.772Z" },- { url = "https://files.pythonhosted.org/packages/e1/6b/2706497c86e8d69fb76afe5ea857fe1794621aa0f3b1d863feb953fe0f22/pypdfium2-4.30.1-py3-none-win_arm64.whl", hash = "sha256:c2b6d63f6d425d9416c08d2511822b54b8e3ac38e639fc41164b1d75584b3a8c", size = 2814810, upload-time = "2024-12-19T19:28:09.857Z" },+version = "4.30.0"+source = { registry = "https://pypi.org/simple" }+sdist = { url = "https://files.pythonhosted.org/packages/a1/14/838b3ba247a0ba92e4df5d23f2bea9478edcfd72b78a39d6ca36ccd84ad2/pypdfium2-4.30.0.tar.gz", hash = "sha256:48b5b7e5566665bc1015b9d69c1ebabe21f6aee468b509531c3c8318eeee2e16", size = 140239, upload-time = "2024-05-09T18:33:17.552Z" }+wheels = [+ { url = "https://files.pythonhosted.org/packages/c7/9a/c8ff5cc352c1b60b0b97642ae734f51edbab6e28b45b4fcdfe5306ee3c83/pypdfium2-4.30.0-py3-none-macosx_10_13_x86_64.whl", hash = "sha256:b33ceded0b6ff5b2b93bc1fe0ad4b71aa6b7e7bd5875f1ca0cdfb6ba6ac01aab", size = 2837254, upload-time = "2024-05-09T18:32:48.653Z" },+ { url = "https://files.pythonhosted.org/packages/21/8b/27d4d5409f3c76b985f4ee4afe147b606594411e15ac4dc1c3363c9a9810/pypdfium2-4.30.0-py3-none-macosx_11_0_arm64.whl", hash = "sha256:4e55689f4b06e2d2406203e771f78789bd4f190731b5d57383d05cf611d829de", size = 2707624, upload-time = "2024-05-09T18:32:51.458Z" },+ { url = "https://files.pythonhosted.org/packages/11/63/28a73ca17c24b41a205d658e177d68e198d7dde65a8c99c821d231b6ee3d/pypdfium2-4.30.0-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4e6e50f5ce7f65a40a33d7c9edc39f23140c57e37144c2d6d9e9262a2a854854", size = 2793126, upload-time = "2024-05-09T18:32:53.581Z" },+ { url = "https://files.pythonhosted.org/packages/d1/96/53b3ebf0955edbd02ac6da16a818ecc65c939e98fdeb4e0958362bd385c8/pypdfium2-4.30.0-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:3d0dd3ecaffd0b6dbda3da663220e705cb563918249bda26058c6036752ba3a2", size = 2591077, upload-time = "2024-05-09T18:32:55.99Z" },+ { url = "https://files.pythonhosted.org/packages/ec/ee/0394e56e7cab8b5b21f744d988400948ef71a9a892cbeb0b200d324ab2c7/pypdfium2-4.30.0-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:cc3bf29b0db8c76cdfaac1ec1cde8edf211a7de7390fbf8934ad2aa9b4d6dfad", size = 2864431, upload-time = "2024-05-09T18:32:57.911Z" },+ { url = "https://files.pythonhosted.org/packages/65/cd/3f1edf20a0ef4a212a5e20a5900e64942c5a374473671ac0780eaa08ea80/pypdfium2-4.30.0-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f1f78d2189e0ddf9ac2b7a9b9bd4f0c66f54d1389ff6c17e9fd9dc034d06eb3f", size = 2812008, upload-time = "2024-05-09T18:32:59.886Z" },+ { url = "https://files.pythonhosted.org/packages/c8/91/2d517db61845698f41a2a974de90762e50faeb529201c6b3574935969045/pypdfium2-4.30.0-py3-none-musllinux_1_1_aarch64.whl", hash = "sha256:5eda3641a2da7a7a0b2f4dbd71d706401a656fea521b6b6faa0675b15d31a163", size = 6181543, upload-time = "2024-05-09T18:33:02.597Z" },+ { url = "https://files.pythonhosted.org/packages/ba/c4/ed1315143a7a84b2c7616569dfb472473968d628f17c231c39e29ae9d780/pypdfium2-4.30.0-py3-none-musllinux_1_1_i686.whl", hash = "sha256:0dfa61421b5eb68e1188b0b2231e7ba35735aef2d867d86e48ee6cab6975195e", size = 6175911, upload-time = "2024-05-09T18:33:05.376Z" },+ { url = "https://files.pythonhosted.org/packages/7a/c4/9e62d03f414e0e3051c56d5943c3bf42aa9608ede4e19dc96438364e9e03/pypdfium2-4.30.0-py3-none-musllinux_1_1_x86_64.whl", hash = "sha256:f33bd79e7a09d5f7acca3b0b69ff6c8a488869a7fab48fdf400fec6e20b9c8be", size = 6267430, upload-time = "2024-05-09T18:33:08.067Z" },+ { url = "https://files.pythonhosted.org/packages/90/47/eda4904f715fb98561e34012826e883816945934a851745570521ec89520/pypdfium2-4.30.0-py3-none-win32.whl", hash = "sha256:ee2410f15d576d976c2ab2558c93d392a25fb9f6635e8dd0a8a3a5241b275e0e", size = 2775951, upload-time = "2024-05-09T18:33:10.567Z" },+ { url = "https://files.pythonhosted.org/packages/25/bd/56d9ec6b9f0fc4e0d95288759f3179f0fcd34b1a1526b75673d2f6d5196f/pypdfium2-4.30.0-py3-none-win_amd64.whl", hash = "sha256:90dbb2ac07be53219f56be09961eb95cf2473f834d01a42d901d13ccfad64b4c", size = 2892098, upload-time = "2024-05-09T18:33:13.107Z" },+ { url = "https://files.pythonhosted.org/packages/be/7a/097801205b991bc3115e8af1edb850d30aeaf0118520b016354cf5ccd3f6/pypdfium2-4.30.0-py3-none-win_arm64.whl", hash = "sha256:119b2969a6d6b1e8d55e99caaf05290294f2d0fe49c12a3f17102d01c441bd29", size = 2752118, upload-time = "2024-05-09T18:33:15.489Z" },][[package]]