Skip to content

Build AI methods with your coding agent, then run them anywhere: as an MCP for chatbots, as a webapp for people, or via API for your software.

Feature/api payload (#111)

Thomas Hebrardc345ff02e346
diff --git a/.cursor/rules/piping.mdc b/.cursor/rules/pipelex.mdc
similarity index 100%
rename from .cursor/rules/piping.mdc
rename to .cursor/rules/pipelex.mdc
diff --git a/.cursor/rules/prompt-templates.mdc b/.cursor/rules/prompt-templates.mdc
index aea74bd47..ed05224d8 100644
--- a/.cursor/rules/prompt-templates.mdc
+++ b/.cursor/rules/prompt-templates.mdc
@@ -1,6 +1,6 @@
---
description:
-globs: **/pipelex_libraries/pipelines/toml/**/*.toml
+globs: pipelex/libraries/pipelines/**/*.toml
alwaysApply: false
---
This rule explains how to write prompt templates in PipeLLM definitions. These prompts will be rendered and become the user_text or the system_prompt part of LLM chat completion queries.
diff --git a/Makefile b/Makefile
index b759c9552..6859154f1 100644
--- a/Makefile
+++ b/Makefile
@@ -406,11 +406,11 @@ li: lock install
check-TODOs: env
$(call PRINT_TITLE,"Checking for TODOs")
- $(VENV_RUFF) check --select=TD -v .
+ $(VENV_RUFF) check --select=TD .
fix-unused-imports: env
$(call PRINT_TITLE,"Fixing unused imports")
- $(VENV_RUFF) check --select=F401 --fix -v .
+ $(VENV_RUFF) check --select=F401 --fix .
doc: env
$(call PRINT_TITLE,"Serving documentation with mkdocs")
diff --git a/pipelex/client/api_serializer.py b/pipelex/client/api_serializer.py
new file mode 100644
index 000000000..0cbd56163
--- /dev/null
+++ b/pipelex/client/api_serializer.py
@@ -0,0 +1,124 @@
+from datetime import datetime
+from decimal import Decimal
+from enum import Enum
+from pathlib import Path
+from typing import Any, Dict, List, cast
+
+from pipelex.client.protocol import CompactMemory
+from pipelex.core.concept_native import NativeConcept
+from pipelex.core.pipe_output import PipeOutput
+from pipelex.core.stuff_content import StuffContent, TextContent
+from pipelex.core.stuff_factory import StuffContentFactory
+from pipelex.core.working_memory import WorkingMemory
+from pipelex.exceptions import ApiSerializationError
+
+
+class ApiSerializer:
+ """Handles API-specific serialization with kajson, datetime formatting, and cleanup."""
+
+ # Fixed datetime format for API consistency
+ API_DATETIME_FORMAT = "%Y-%m-%dT%H:%M:%S"
+ FIELDS_TO_SKIP = ("__class__", "__module__")
+
+ @classmethod
+ def serialize_working_memory_for_api(cls, working_memory: WorkingMemory) -> CompactMemory:
+ """
+ Convert WorkingMemory to API-ready format using kajson with proper datetime handling.
+
+ Args:
+ working_memory: The WorkingMemory to serialize
+
+ Returns:
+ Dict ready for API transmission with datetime strings and no __class__/__module__
+ """
+ compact_memory: CompactMemory = {}
+
+ for stuff_name, stuff in working_memory.root.items():
+ if stuff.concept_code == NativeConcept.TEXT.code:
+ stuff_content = cast(TextContent, stuff.content)
+ item_dict: Dict[str, Any] = {
+ "concept_code": stuff.concept_code,
+ "content": stuff_content.text,
+ }
+ else:
+ content_dict = stuff.content.model_dump(serialize_as_any=True)
+ clean_content = cls._clean_and_format_content(content_dict)
+
+ item_dict = {
+ "concept_code": stuff.concept_code,
+ "content": clean_content,
+ }
+
+ compact_memory[stuff_name] = item_dict
+
+ return compact_memory
+
+ @classmethod
+ def serialize_pipe_output_for_api(cls, pipe_output: PipeOutput) -> CompactMemory:
+ """
+ Convert PipeOutput to API-ready format.
+
+ Args:
+ pipe_output: The PipeOutput to serialize
+
+ Returns:
+ Dict ready for API transmission
+ """
+ return {"compact_memory": cls.serialize_working_memory_for_api(pipe_output.working_memory)}
+
+ @classmethod
+ def _clean_and_format_content(cls, content: Any) -> Any:
+ """
+ Recursively clean content by removing the fields in FIELDS_TO_SKIP and formatting datetimes.
+
+ Args:
+ content: Content to clean
+
+ Returns:
+ Cleaned content with formatted datetimes
+ """
+ if isinstance(content, dict):
+ cleaned: Dict[str, Any] = {}
+ content_dict = cast(Dict[str, Any], content)
+ for key in content_dict:
+ if key in cls.FIELDS_TO_SKIP:
+ continue
+ cleaned[key] = cls._clean_and_format_content(content_dict[key])
+ return cleaned
+ elif isinstance(content, list):
+ cleaned_list: List[Any] = []
+ content_list = cast(List[Any], content)
+ for idx in range(len(content_list)):
+ cleaned_list.append(cls._clean_and_format_content(content_list[idx]))
+ return cleaned_list
+ elif isinstance(content, datetime):
+ return content.strftime(cls.API_DATETIME_FORMAT)
+ elif isinstance(content, Enum):
+ return content.value # Convert enum to its value
+ elif isinstance(content, Decimal):
+ return float(content) # Convert Decimal to float for JSON compatibility
+ elif isinstance(content, Path):
+ return str(content) # Convert Path to string representation
+ else:
+ return content
+
+ @classmethod
+ def make_stuff_content_from_api_data(cls, concept_code: str, value: Dict[str, Any] | str) -> StuffContent:
+ """
+ Create StuffContent from API data using concept code.
+
+ Args:
+ concept_code: The concept code to determine the content type
+ value: The content value from API
+
+ Returns:
+ StuffContent instance
+
+ Raises:
+ ApiSerializationError: If concept cannot be resolved or content creation fails
+ """
+ try:
+ return StuffContentFactory.make_stuffcontent_from_concept_code_with_fallback(concept_code=concept_code, value=value)
+
+ except Exception as exc:
+ raise ApiSerializationError(f"Failed to create StuffContent for concept '{concept_code}': {exc}") from exc
diff --git a/pipelex/client/client.py b/pipelex/client/client.py
index 7f7d322ee..793507f0f 100644
--- a/pipelex/client/client.py
+++ b/pipelex/client/client.py
@@ -1,10 +1,11 @@
-from typing import Any, Optional, cast
+from typing import Any, Optional
import httpx
-from kajson import kajson
from typing_extensions import override
-from pipelex.client.protocol import PipelexProtocol, PipelineRequest, PipelineResponse
+from pipelex.client.pipeline_request_factory import PipelineRequestFactory
+from pipelex.client.pipeline_response_factory import PipelineResponseFactory
+from pipelex.client.protocol import PipelexProtocol, PipelineResponse
from pipelex.core.pipe_run_params import PipeOutputMultiplicity
from pipelex.core.working_memory import WorkingMemory
from pipelex.exceptions import ClientAuthenticationError
@@ -78,14 +79,14 @@ class PipelexClient(PipelexProtocol):
output_multiplicity: Optional[PipeOutputMultiplicity] = None,
dynamic_output_concept_code: Optional[str] = None,
) -> PipelineResponse:
- pipeline_request = PipelineRequest(
+ pipeline_request = PipelineRequestFactory.make_from_working_memory(
working_memory=working_memory,
output_name=output_name,
output_multiplicity=output_multiplicity,
dynamic_output_concept_code=dynamic_output_concept_code,
)
- response = await self._make_api_call(f"pipelex/v1/pipeline/{pipe_code}/execute", request=kajson.dumps(pipeline_request))
- return cast(PipelineResponse, kajson.loads(response))
+ response = await self._make_api_call(f"v1/pipeline/{pipe_code}/execute", request=pipeline_request.model_dump_json())
+ return PipelineResponseFactory.make_from_api_response(response)
@override
async def start_pipeline(
@@ -96,11 +97,11 @@ class PipelexClient(PipelexProtocol):
output_multiplicity: Optional[PipeOutputMultiplicity] = None,
dynamic_output_concept_code: Optional[str] = None,
) -> PipelineResponse:
- pipeline_request = PipelineRequest(
+ pipeline_request = PipelineRequestFactory.make_from_working_memory(
working_memory=working_memory,
output_name=output_name,
output_multiplicity=output_multiplicity,
dynamic_output_concept_code=dynamic_output_concept_code,
)
- response = await self._make_api_call(f"pipelex/v1/pipeline/{pipe_code}/start", request=kajson.dumps(pipeline_request))
- return cast(PipelineResponse, kajson.loads(response))
+ response = await self._make_api_call(f"v1/pipeline/{pipe_code}/start", request=pipeline_request.model_dump_json())
+ return PipelineResponseFactory.make_from_api_response(response)
diff --git a/pipelex/client/pipeline_request_factory.py b/pipelex/client/pipeline_request_factory.py
new file mode 100644
index 000000000..d209cca19
--- /dev/null
+++ b/pipelex/client/pipeline_request_factory.py
@@ -0,0 +1,89 @@
+from typing import Any, Dict, Optional
+
+from pipelex.client.api_serializer import ApiSerializer
+from pipelex.client.protocol import COMPACT_MEMORY_KEY, CompactMemory, PipelineRequest
+from pipelex.core.pipe_run_params import PipeOutputMultiplicity
+from pipelex.core.stuff_factory import StuffFactory
+from pipelex.core.working_memory import WorkingMemory
+from pipelex.core.working_memory_factory import WorkingMemoryFactory
+
+
+class PipelineRequestFactory:
+ """Factory class for creating PipelineRequest objects from WorkingMemory."""
+
+ @staticmethod
+ def make_from_working_memory(
+ working_memory: Optional[WorkingMemory] = None,
+ output_name: Optional[str] = None,
+ output_multiplicity: Optional[PipeOutputMultiplicity] = None,
+ dynamic_output_concept_code: Optional[str] = None,
+ ) -> PipelineRequest:
+ """
+ Create a PipelineRequest from a WorkingMemory object.
+
+ Args:
+ working_memory: The WorkingMemory to convert
+ output_name: Name of the output slot to write to
+ output_multiplicity: Output multiplicity setting
+ dynamic_output_concept_code: Override for the dynamic output concept code
+
+ Returns:
+ PipelineRequest with the working memory serialized to reduced format
+ """
+ compact_memory = None
+ if working_memory is not None:
+ compact_memory = ApiSerializer.serialize_working_memory_for_api(working_memory)
+
+ return PipelineRequest(
+ compact_memory=compact_memory,
+ output_name=output_name,
+ output_multiplicity=output_multiplicity,
+ dynamic_output_concept_code=dynamic_output_concept_code,
+ )
+
+ @staticmethod
+ def make_working_memory_from_reduced(compact_memory: Optional[CompactMemory]) -> WorkingMemory:
+ """
+ Create a WorkingMemory from a reduced memory dictionary.
+
+ Args:
+ compact_memory: Dictionary in the format from API
+
+ Returns:
+ WorkingMemory object reconstructed from the reduced format
+ """
+ working_memory = WorkingMemoryFactory.make_empty()
+ if compact_memory is None:
+ return working_memory
+
+ for stuff_key, stuff_data in compact_memory.items():
+ concept_code = stuff_data.get("concept_code", "")
+ content_value = stuff_data.get("content", {})
+
+ # Use API serializer to create content
+ content = ApiSerializer.make_stuff_content_from_api_data(concept_code=concept_code, value=content_value)
+
+ # Create stuff directly
+ stuff = StuffFactory.make_stuff(concept_str=concept_code, name=stuff_key, content=content)
+
+ working_memory.add_new_stuff(name=stuff_key, stuff=stuff)
+
+ return working_memory
+
+ @staticmethod
+ def make_request_from_body(request_body: Dict[str, Any]) -> PipelineRequest:
+ """
+ Create a PipelineRequest from raw request body dictionary.
+
+ Args:
+ request_body: Raw dictionary from API request body
+
+ Returns:
+ PipelineRequest object with dictionary working_memory
+ """
+ return PipelineRequest(
+ compact_memory=request_body.get(COMPACT_MEMORY_KEY),
+ output_name=request_body.get("output_name"),
+ output_multiplicity=request_body.get("output_multiplicity"),
+ dynamic_output_concept_code=request_body.get("dynamic_output_concept_code"),
+ )
diff --git a/pipelex/client/pipeline_response_factory.py b/pipelex/client/pipeline_response_factory.py
new file mode 100644
index 000000000..a21a3b499
--- /dev/null
+++ b/pipelex/client/pipeline_response_factory.py
@@ -0,0 +1,64 @@
+from typing import Any, Dict, Optional
+
+from pipelex.client.api_serializer import ApiSerializer
+from pipelex.client.protocol import PipelineResponse, PipelineState
+from pipelex.core.pipe_output import PipeOutput
+
+
+class PipelineResponseFactory:
+ """Factory class for creating PipelineResponse objects from PipeOutput."""
+
+ @staticmethod
+ def make_from_pipe_output(
+ pipe_output: Optional[PipeOutput] = None,
+ pipeline_run_id: str = "",
+ created_at: str = "",
+ pipeline_state: PipelineState = PipelineState.COMPLETED,
+ finished_at: Optional[str] = None,
+ status: Optional[str] = "success",
+ message: Optional[str] = None,
+ error: Optional[str] = None,
+ ) -> PipelineResponse:
+ """
+ Create a PipelineResponse from a PipeOutput object.
+
+ Args:
+ pipe_output: The PipeOutput to convert
+ pipeline_run_id: Unique identifier for the pipeline run
+ created_at: Timestamp when the pipeline was created
+ pipeline_state: Current state of the pipeline
+ finished_at: Timestamp when the pipeline finished
+ status: Status of the API call
+ message: Optional message providing additional information
+ error: Optional error message
+
+ Returns:
+ PipelineResponse with the pipe output serialized to reduced format
+ """
+ reduced_output = None
+ if pipe_output is not None:
+ reduced_output = ApiSerializer.serialize_pipe_output_for_api(pipe_output=pipe_output)
+
+ return PipelineResponse(
+ pipeline_run_id=pipeline_run_id,
+ created_at=created_at,
+ pipeline_state=pipeline_state,
+ finished_at=finished_at,
+ pipe_output=reduced_output,
+ status=status,
+ message=message,
+ error=error,
+ )
+
+ @staticmethod
+ def make_from_api_response(response: Dict[str, Any]) -> PipelineResponse:
+ """
+ Create a PipelineResponse from an API response dictionary.
+
+ Args:
+ response: Dictionary containing the API response data
+
+ Returns:
+ PipelineResponse instance created from the response data
+ """
+ return PipelineResponse(**response)
diff --git a/pipelex/client/protocol.py b/pipelex/client/protocol.py
index fb77ee4aa..9531b0aa5 100644
--- a/pipelex/client/protocol.py
+++ b/pipelex/client/protocol.py
@@ -1,14 +1,16 @@
from abc import abstractmethod
-from typing import Optional, Protocol
+from typing import Any, Dict, Optional, Protocol
from pydantic import BaseModel
from typing_extensions import runtime_checkable
-from pipelex.core.pipe_output import PipeOutput
from pipelex.core.pipe_run_params import PipeOutputMultiplicity
from pipelex.core.working_memory import WorkingMemory
from pipelex.types import StrEnum
+CompactMemory = Dict[str, Dict[str, Any]]
+COMPACT_MEMORY_KEY = "compact_memory"
+
class PipelineState(StrEnum):
"""
@@ -43,13 +45,13 @@ class PipelineRequest(BaseModel):
Request for executing a pipeline.
Attributes:
- working_memory (Optional[WorkingMemory]): WorkingMemory instance passed to the pipeline
+ compact_memory (Optional[CompactMemory]): In the format of WorkingMemory.to_compact_memory()
output_name (Optional[str]): Name of the output slot to write to
output_multiplicity (Optional[PipeOutputMultiplicity]): Output multiplicity setting
dynamic_output_concept_code (Optional[str]): Override for the dynamic output concept code
"""
- working_memory: Optional[WorkingMemory] = None
+ compact_memory: Optional[CompactMemory] = None
output_name: Optional[str] = None
output_multiplicity: Optional[PipeOutputMultiplicity] = None
dynamic_output_concept_code: Optional[str] = None
@@ -64,14 +66,42 @@ class PipelineResponse(ApiResponse):
created_at (str): Timestamp when the pipeline was created
pipeline_state (PipelineState): Current state of the pipeline
finished_at (Optional[str]): Timestamp when the pipeline finished, if completed
- pipe_output (Optional[PipeOutput]): Output data from the pipeline execution, if available
+ pipe_output (Optional[CompactMemory]): Output data from the pipeline execution as raw dict, if available
+
+ Example of pipe_output:
+ "pipe_output": {
+ "compact_memory": {
+ "text": {
+ "concept_code": "native.Text",
+ "content": "Some text........"
+ },
+ "question": {
+ "concept_code": "answer.Question",
+ "content": {
+ "text": "What are aerodynamic features?"
+ }
+ },
+ "main_stuff": {
+ "concept_code": "retrieve.RetrievedExcerpt",
+ "content": {
+ "items": [
+ {
+ "text": "What we're seeing isn't just an incremental...",
+ "justification": "This excerpt directly mentions the 'aerodynamic profile' of ...."
+ },
+ ...
+ ]
+ }
+ }
+ }
+ }
"""
pipeline_run_id: str
created_at: str
pipeline_state: PipelineState
finished_at: Optional[str] = None
- pipe_output: Optional[PipeOutput] = None
+ pipe_output: Optional[CompactMemory] = None
@runtime_checkable
diff --git a/pipelex/core/concept_library.py b/pipelex/core/concept_library.py
index b4e8701a0..6d489cf2a 100644
--- a/pipelex/core/concept_library.py
+++ b/pipelex/core/concept_library.py
@@ -8,6 +8,7 @@ from pipelex.core.concept import Concept
from pipelex.core.concept_factory import ConceptFactory
from pipelex.core.concept_native import NativeConcept
from pipelex.core.concept_provider_abstract import ConceptProviderAbstract
+from pipelex.core.domain import SpecialDomain
from pipelex.core.stuff_content import ImageContent
from pipelex.exceptions import ConceptLibraryConceptNotFoundError, ConceptLibraryError
from pipelex.hub import get_class_registry
@@ -107,6 +108,13 @@ class ConceptLibrary(RootModel[ConceptLibraryRoot], ConceptProviderAbstract):
@override
def get_required_concept(self, concept_code: str) -> Concept:
+ if self.is_native_concept(concept_str=concept_code):
+ if Concept.concept_str_contains_domain(concept_str=concept_code):
+ domain, concept_code = Concept.extract_domain_and_concept_from_str(concept_str=concept_code)
+ concept_code = f"{domain}.{concept_code}"
+ else:
+ concept_code = f"{SpecialDomain.NATIVE.value}.{concept_code}"
+
the_concept = self.get_concept(concept_code=concept_code)
if not the_concept:
if self.is_concept_implicit(concept_code=concept_code):
@@ -147,3 +155,10 @@ class ConceptLibrary(RootModel[ConceptLibraryRoot], ConceptProviderAbstract):
is_image_class = bool(pydantic_model and issubclass(pydantic_model, ImageContent))
refines_image = self.is_compatible_by_concept_code(tested_concept_code=concept.code, wanted_concept_code=NativeConcept.IMAGE.code)
return is_image_class or refines_image
+
+ def is_native_concept(self, concept_str: str) -> bool:
+ if Concept.concept_str_contains_domain(concept_str=concept_str):
+ domain = Concept.extract_domain_from_str(concept_str=concept_str)
+ return domain == SpecialDomain.NATIVE.value
+ else:
+ return concept_str in NativeConcept.names()
diff --git a/pipelex/core/stuff.py b/pipelex/core/stuff.py
index 3ef47b43f..dbef01262 100644
--- a/pipelex/core/stuff.py
+++ b/pipelex/core/stuff.py
@@ -1,6 +1,6 @@
from typing import Any, Dict, Optional, Type, Union
-from pydantic import BaseModel, ConfigDict, Field
+from pydantic import ConfigDict
from typing_extensions import override
from pipelex import log
@@ -23,16 +23,10 @@ from pipelex.tools.misc.string_utils import pascal_case_to_snake_case
from pipelex.tools.typing.pydantic_utils import CustomBaseModel
-class StuffCreationRecord(BaseModel):
- pass
-
-
class Stuff(CustomBaseModel):
model_config = ConfigDict(extra="ignore", strict=True)
stuff_code: str
- pipelex_session_id: Optional[str] = Field(default=None)
- creation_record: Optional[StuffCreationRecord] = None
stuff_name: Optional[str] = None
concept_code: str
content: StuffContent
@@ -81,7 +75,7 @@ class Stuff(CustomBaseModel):
@property
def short_desc(self) -> str:
- return f"""{self.pipelex_session_id}/{self.stuff_code}:
+ return f"""{self.stuff_code}:
{self.concept_code} — {type(self.content).__name__}:
{self.content.short_desc}"""
diff --git a/pipelex/core/stuff_factory.py b/pipelex/core/stuff_factory.py
index eaaca9d8c..305178394 100644
--- a/pipelex/core/stuff_factory.py
+++ b/pipelex/core/stuff_factory.py
@@ -1,16 +1,15 @@
-from typing import Any, Dict, List, Optional, Tuple
+from typing import Any, Dict, List, Optional, Tuple, Type
import shortuuid
-from pydantic import BaseModel, Field, ValidationError
+from pydantic import BaseModel, ValidationError
-from pipelex.config import get_config
from pipelex.core.concept import Concept
from pipelex.core.concept_code_factory import ConceptCodeFactory
from pipelex.core.concept_native import NativeConcept
-from pipelex.core.stuff import Stuff, StuffCreationRecord
-from pipelex.core.stuff_content import StuffContent, StuffContentInitableFromStr
+from pipelex.core.stuff import Stuff
+from pipelex.core.stuff_content import StuffContent, StuffContentInitableFromStr, TextContent
from pipelex.exceptions import ConceptError, PipelexError
-from pipelex.hub import get_class_registry, get_required_concept
+from pipelex.hub import get_class_registry, get_concept_provider, get_required_concept
from pipelex.tools.typing.pydantic_utils import format_pydantic_validation_error
@@ -19,9 +18,9 @@ class StuffFactoryError(PipelexError):
class StuffBlueprint(BaseModel):
- name: str
- concept_code: str = Field(alias="concept")
- value: str
+ stuff_name: str
+ concept_code: str
+ content: Dict[str, Any] | str
class StuffFactory:
@@ -36,8 +35,6 @@ class StuffFactory:
content: StuffContent,
name: Optional[str] = None,
code: Optional[str] = None,
- creation_record: Optional[StuffCreationRecord] = None,
- pipelex_session_id: Optional[str] = None,
) -> Stuff:
try:
concept_code = ConceptCodeFactory.make_concept_code_from_str(concept_str=concept_str)
@@ -51,8 +48,6 @@ class StuffFactory:
content=content,
stuff_name=name,
stuff_code=code or shortuuid.uuid()[:5],
- creation_record=creation_record,
- pipelex_session_id=pipelex_session_id or get_config().session_id,
)
@classmethod
@@ -62,8 +57,6 @@ class StuffFactory:
content: StuffContent,
name: Optional[str] = None,
code: Optional[str] = None,
- creation_record: Optional[StuffCreationRecord] = None,
- pipelex_session_id: Optional[str] = None,
) -> Stuff:
if not name:
name = cls.make_stuff_name(concept_str=concept.code)
@@ -72,37 +65,39 @@ class StuffFactory:
content=content,
stuff_name=name,
stuff_code=code or shortuuid.uuid()[:5],
- creation_record=creation_record,
- pipelex_session_id=pipelex_session_id or get_config().session_id,
)
@classmethod
- def make_from_blueprint_dict(cls, blueprint_dict: Dict[str, Any]) -> "Stuff":
- blueprint = StuffBlueprint.model_validate(obj=blueprint_dict)
- return cls.make_from_blueprint(blueprint=blueprint)
+ def make_from_blueprint(cls, blueprint: StuffBlueprint) -> "Stuff":
+ if isinstance(blueprint.content, str) and get_concept_provider().is_compatible_by_concept_code(
+ tested_concept_code=blueprint.concept_code, wanted_concept_code=NativeConcept.TEXT.code
+ ):
+ the_stuff = cls.make_from_str(
+ concept_str=NativeConcept.TEXT.code,
+ str_value=blueprint.content,
+ name=blueprint.stuff_name,
+ )
+ else:
+ the_stuff_content = StuffContentFactory.make_stuffcontent_from_concept_code_required(
+ concept_code=blueprint.concept_code, value=blueprint.content
+ )
+ the_stuff = cls.make_stuff(
+ concept_str=blueprint.concept_code,
+ content=the_stuff_content,
+ name=blueprint.stuff_name,
+ )
+ return the_stuff
@classmethod
- def make_from_blueprint_str(cls, blueprint_str: str) -> "Stuff":
- blueprint = StuffBlueprint.model_validate_json(json_data=blueprint_str)
+ def make_from_blueprint_dict(cls, blueprint: StuffBlueprint) -> "Stuff":
return cls.make_from_blueprint(blueprint=blueprint)
- @classmethod
- def make_from_blueprint(cls, blueprint: StuffBlueprint) -> "Stuff":
- the_stuff = cls.make_from_str(
- concept_str=blueprint.concept_code,
- str_value=blueprint.value,
- name=blueprint.name,
- pipelex_session_id="blueprint",
- )
- return the_stuff
-
@classmethod
def make_from_str(
cls,
str_value: str,
name: Optional[str] = None,
concept_str: str = NativeConcept.TEXT.code,
- pipelex_session_id: Optional[str] = None,
) -> Stuff:
try:
concept_code = ConceptCodeFactory.make_concept_code_from_str(concept_str=concept_str)
@@ -124,8 +119,6 @@ class StuffFactory:
content=stuff_content,
stuff_name=name,
stuff_code=shortuuid.uuid()[:5],
- creation_record=None,
- pipelex_session_id=pipelex_session_id or get_config().session_id,
)
@classmethod
@@ -170,3 +163,43 @@ class StuffFactory:
content=the_stuff_content,
name=name,
)
+
+
+class StuffContentFactoryError(PipelexError):
+ pass
+
+
+class StuffContentFactory:
+ @classmethod
+ def make_content_from_value(cls, stuff_content_subclass: Type[StuffContent], value: Dict[str, Any] | str) -> StuffContent:
+ if isinstance(value, str) and stuff_content_subclass == TextContent:
+ return TextContent(text=value)
+ return stuff_content_subclass.model_validate(obj=value)
+
+ @classmethod
+ def make_stuffcontent_from_concept_code_required(cls, concept_code: str, value: Dict[str, Any] | str) -> StuffContent:
+ """
+ Create StuffContent from concept code, requiring the concept to be linked to a class in the registry.
+ Raises StuffContentFactoryError if no registry class is found.
+ """
+ concept = get_required_concept(concept_code=concept_code)
+ the_subclass_name = concept.structure_class_name
+ the_subclass = get_class_registry().get_required_subclass(name=the_subclass_name, base_class=StuffContent)
+ return cls.make_content_from_value(stuff_content_subclass=the_subclass, value=value)
+
+ @classmethod
+ def make_stuffcontent_from_concept_code_with_fallback(cls, concept_code: str, value: Dict[str, Any] | str) -> StuffContent:
+ """
+ Create StuffContent from concept code, falling back to TextContent if no registry class is found.
+ """
+ concept = get_required_concept(concept_code=concept_code)
+ the_subclass_name = concept.structure_class_name
+ the_subclass = get_class_registry().get_class(name=the_subclass_name)
+
+ if the_subclass is None:
+ return cls.make_content_from_value(stuff_content_subclass=TextContent, value=value)
+
+ if not issubclass(the_subclass, StuffContent):
+ raise StuffContentFactoryError(f"Concept '{concept_code}', subclass '{the_subclass}' is not a subclass of StuffContent")
+
+ return cls.make_content_from_value(stuff_content_subclass=the_subclass, value=value)
diff --git a/pipelex/core/working_memory.py b/pipelex/core/working_memory.py
index ad2393aec..2f2e4b967 100644
--- a/pipelex/core/working_memory.py
+++ b/pipelex/core/working_memory.py
@@ -1,5 +1,5 @@
from operator import attrgetter
-from typing import Any, Dict, List, Optional, Set, Type, TypeVar
+from typing import Any, Dict, List, Optional, Set, Type
from pydantic import BaseModel, Field, model_validator
from typing_extensions import Self
@@ -34,9 +34,6 @@ BATCH_ITEM_STUFF_NAME = "BATCH_ITEM"
StuffDict = Dict[str, Stuff]
StuffArtefactDict = Dict[str, StuffArtefact]
-# Create a generic type variable
-T = TypeVar("T")
-
class WorkingMemory(BaseModel):
root: StuffDict = Field(default_factory=dict)
diff --git a/pipelex/exceptions.py b/pipelex/exceptions.py
index 613ad32d4..dac9a6d54 100644
--- a/pipelex/exceptions.py
+++ b/pipelex/exceptions.py
@@ -249,3 +249,9 @@ class PipeInputNotFoundError(PipelexError):
class PipeInputDetailsError(PipelexError):
pass
+
+
+class ApiSerializationError(Exception):
+ """Exception raised when API serialization fails."""
+
+ pass
diff --git a/pipelex/plugins/openai/openai_llms.py b/pipelex/plugins/openai/openai_llms.py
index 93718486b..7324fd314 100644
--- a/pipelex/plugins/openai/openai_llms.py
+++ b/pipelex/plugins/openai/openai_llms.py
@@ -10,7 +10,7 @@ from pipelex.plugins.openai.openai_factory import OpenAIFactory
async def openai_list_available_models(llm_platform: LLMPlatform) -> List[Model]:
openai_client_async = OpenAIFactory.make_openai_client(llm_platform=llm_platform)
match llm_platform:
- case LLMPlatform.VERTEXAI | LLMPlatform.PERPLEXITY:
+ case LLMPlatform.VERTEXAI | LLMPlatform.PERPLEXITY | LLMPlatform.CUSTOM_LLM:
raise LLMSDKError(f"Platform '{llm_platform}' does not support listing models with OpenAI SDK")
case _:
pass
diff --git a/tests/integration/pipelex/client/test_client.py b/tests/integration/pipelex/client/test_client.py
index 76fd4355a..24bf1f8aa 100644
--- a/tests/integration/pipelex/client/test_client.py
+++ b/tests/integration/pipelex/client/test_client.py
@@ -5,7 +5,7 @@ from pydantic import BaseModel
from pipelex import pretty_print
from pipelex.client.client import PipelexClient
-from pipelex.client.protocol import PipelineState
+from pipelex.client.protocol import COMPACT_MEMORY_KEY, PipelineState
from pipelex.core.stuff import Stuff
from pipelex.core.stuff_content import TextContent
from pipelex.core.stuff_factory import StuffFactory
@@ -96,18 +96,49 @@ class TestPipelexApiClient:
for example in examples:
# Create working memory from example data
memory = WorkingMemory()
- for stuff in example.memory:
- memory.add_new_stuff(name=stuff.stuff_name or stuff.concept_code, stuff=stuff)
+ question = example.memory[1]
+ text = example.memory[0]
+ memory.add_new_stuff(name=question.stuff_name or question.concept_code, stuff=question)
+ memory.add_new_stuff(name=text.stuff_name or text.concept_code, stuff=text)
# Execute pipe
client = PipelexClient()
- result = await client.execute_pipeline(
+ pipeline_reponse = await client.execute_pipeline(
pipe_code=example.pipe_code,
working_memory=memory,
)
- pretty_print(result)
+ pretty_print(pipeline_reponse, title="PIPELINE RESPONSE")
# Verify result
- assert result.pipeline_run_id is not None
- assert result.pipeline_state == PipelineState.COMPLETED
- assert result.pipe_output is not None
+ assert pipeline_reponse.pipeline_run_id is not None
+ assert pipeline_reponse.pipeline_state == PipelineState.COMPLETED
+ assert pipeline_reponse.pipe_output is not None
+
+ working_memory = pipeline_reponse.pipe_output[COMPACT_MEMORY_KEY]
+
+ # Verify question structure
+ assert working_memory["question"] == {
+ "concept_code": "answer.Question",
+ "content": {"text": "Aerodynamic features?"},
+ }
+
+ # Verify main_stuff structure
+ assert working_memory["main_stuff"] is not None
+ assert working_memory["main_stuff"]["concept_code"] == "retrieve.RetrievedExcerpt"
+ assert "content" in working_memory["main_stuff"]
+ assert "items" in working_memory["main_stuff"]["content"]
+ assert isinstance(working_memory["main_stuff"]["content"]["items"], list)
+ assert len(working_memory["main_stuff"]["content"]["items"]) > 0
+
+ # Verify each item has required fields
+ for item in working_memory["main_stuff"]["content"]["items"]:
+ assert "text" in item
+ assert "justification" in item
+ assert isinstance(item["text"], str)
+ assert isinstance(item["justification"], str)
+
+ # Verify text structure
+ assert working_memory["text"]["concept_code"] == "native.Text"
+ assert "content" in working_memory["text"]
+ assert isinstance(working_memory["text"]["content"], str)
+ assert "The Dawn of Ultra-Rapid Transit" in working_memory["text"]["content"]
diff --git a/tests/integration/pipelex/pipes/test_pipe_running_variants.py b/tests/integration/pipelex/pipes/test_pipe_running_variants.py
index 4cf432173..967692317 100644
--- a/tests/integration/pipelex/pipes/test_pipe_running_variants.py
+++ b/tests/integration/pipelex/pipes/test_pipe_running_variants.py
@@ -34,7 +34,7 @@ class TestPipeRunningVariants:
blueprint: StuffBlueprint,
pipe_code: str,
):
- log.verbose(blueprint, title=f"{topic}: start from '{blueprint.name}', run pipe '{pipe_code}'")
+ log.verbose(blueprint, title=f"{topic}: start from '{blueprint.stuff_name}', run pipe '{pipe_code}'")
working_memory = WorkingMemoryFactory.make_from_single_blueprint(blueprint=blueprint)
pipe_output: PipeOutput = await get_pipe_router().run_pipe_code(
pipe_code=pipe_code,
diff --git a/tests/integration/pipelex/test_data.py b/tests/integration/pipelex/test_data.py
index 0d7321877..77e2ec745 100644
--- a/tests/integration/pipelex/test_data.py
+++ b/tests/integration/pipelex/test_data.py
@@ -44,19 +44,16 @@ class PipeTestCases:
name="text",
concept_str="Text",
content=TextContent(text="Describe a t-shirt in 2 sentences"),
- pipelex_session_id="unit_test",
)
SIMPLE_STUFF_IMAGE = StuffFactory.make_stuff(
name="image",
concept_str="Image",
content=ImageContent(url=URL_IMG_FASHION_PHOTO_1),
- pipelex_session_id="unit_test",
)
SIMPLE_STUFF_PDF = StuffFactory.make_stuff(
name="pdf",
concept_str="PDF",
content=PDFContent(url=PDFTestCases.DOCUMENT_URLS[0]),
- pipelex_session_id="unit_test",
)
COMPLEX_STUFF = StuffFactory.make_stuff(
name="complex",
@@ -67,7 +64,6 @@ class PipeTestCases:
ImageContent(url=URL_IMG_GANTT_1),
]
),
- pipelex_session_id="unit_test",
)
STUFF_CONTENT_WITH_IMAGE_ATTRIBUTE_1 = SomeContentWithImageAttribute(image_attribute=ImageContent(url=URL_IMG_FASHION_PHOTO_1))
@@ -75,7 +71,6 @@ class PipeTestCases:
concept_str="Image",
content=STUFF_CONTENT_WITH_IMAGE_ATTRIBUTE_1,
name="stuff_with_image",
- pipelex_session_id="unit_test",
)
STUFF_CONTENT_WITH_IMAGE_ATTRIBUTE_IN_SUB_OBJECT = SomeContentWithImageSubObjectAttribute(
image_attribute=ImageContent(url=URL_IMG_FASHION_PHOTO_2),
@@ -85,7 +80,6 @@ class PipeTestCases:
concept_str="Image",
content=STUFF_CONTENT_WITH_IMAGE_ATTRIBUTE_IN_SUB_OBJECT,
name="stuff_with_image_in_sub_object",
- pipelex_session_id="unit_test",
)
STUFFS_IMAGE_ATTRIBUTES: ClassVar[List[Tuple[Stuff, List[str]]]] = [ # stuff, attribute_paths
(STUFF_WITH_IMAGE_ATTRIBUTE, ["stuff_with_image.image_attribute"]),
@@ -99,7 +93,7 @@ class PipeTestCases:
],
),
]
- TRICKY_QUESTION_BLUEPRINT = StuffBlueprint(name="question", concept="answer.Question", value=USER_TEXT_TRICKY_2)
+ TRICKY_QUESTION_BLUEPRINT = StuffBlueprint(stuff_name="question", concept_code="answer.Question", content=USER_TEXT_TRICKY_2)
BLUEPRINT_AND_PIPE: ClassVar[List[Tuple[str, StuffBlueprint, str]]] = [ # topic, blueprint, pipe
(
"Tricky question conclude",
@@ -148,7 +142,6 @@ class PipeTestCases:
TextContent(text="green"),
]
),
- pipelex_session_id="unit_test",
),
"colors",
"color",
diff --git a/tests/test_pipelines/datetime.py b/tests/test_pipelines/datetime.py
new file mode 100644
index 000000000..e4315e917
--- /dev/null
+++ b/tests/test_pipelines/datetime.py
@@ -0,0 +1,12 @@
+from datetime import datetime
+
+from pipelex.core.stuff_content import StructuredContent
+
+
+class DateTimeEvent(StructuredContent):
+ """Test model for datetime content."""
+
+ event_name: str
+ start_time: datetime
+ end_time: datetime
+ created_at: datetime
diff --git a/tests/test_pipelines/datetime.toml b/tests/test_pipelines/datetime.toml
new file mode 100644
index 000000000..b8f9af0ad
--- /dev/null
+++ b/tests/test_pipelines/datetime.toml
@@ -0,0 +1,4 @@
+domain = "event"
+
+[concept]
+DateTimeEvent = "A date time event with start and end time"
\ No newline at end of file
diff --git a/tests/test_pipelines/test_content_factory.py b/tests/test_pipelines/test_content_factory.py
new file mode 100644
index 000000000..e6a11cd34
--- /dev/null
+++ b/tests/test_pipelines/test_content_factory.py
@@ -0,0 +1,6 @@
+from pipelex.core.stuff_content import StructuredContent
+
+
+class MockRegisteredContent(StructuredContent):
+ title: str
+ description: str
diff --git a/tests/unit/pipelex/client/__init__.py b/tests/unit/pipelex/client/__init__.py
new file mode 100644
index 000000000..713f4401d
--- /dev/null
+++ b/tests/unit/pipelex/client/__init__.py
@@ -0,0 +1 @@
+# Client tests
diff --git a/tests/unit/pipelex/client/test_api_serialization.py b/tests/unit/pipelex/client/test_api_serialization.py
new file mode 100644
index 000000000..91abfceff
--- /dev/null
+++ b/tests/unit/pipelex/client/test_api_serialization.py
@@ -0,0 +1,497 @@
+import json
+from datetime import datetime
+from decimal import Decimal
+from enum import Enum
+from pathlib import Path
+
+# Import additional types for complex testing
+from typing import Any, Dict, List, Optional, cast
+
+import pytest
+from pydantic import BaseModel
+
+from pipelex.client.api_serializer import ApiSerializationError, ApiSerializer
+from pipelex.client.protocol import COMPACT_MEMORY_KEY
+from pipelex.core.concept_native import NativeConcept
+from pipelex.core.pipe_output import PipeOutput
+from pipelex.core.stuff_content import NumberContent, TextContent
+from pipelex.core.stuff_factory import StuffFactory
+from pipelex.core.working_memory import WorkingMemory
+from pipelex.core.working_memory_factory import WorkingMemoryFactory
+from tests.test_pipelines.datetime import DateTimeEvent
+
+
+# Test models for complex scenarios
+class Priority(Enum):
+ LOW = "low"
+ MEDIUM = "medium"
+ HIGH = "high"
+
+
+class TaskStatus(BaseModel):
+ is_complete: bool
+ completion_date: Optional[datetime] = None
+ notes: List[str] = []
+
+
+class ComplexTask(BaseModel):
+ task_id: str
+ title: str
+ priority: Priority
+ status: TaskStatus
+ due_dates: List[datetime]
+ metadata: Dict[str, Any]
+ score: Optional[Decimal] = None
+
+
+class Project(BaseModel):
+ name: str
+ created_at: datetime
+ tasks: List[ComplexTask]
+ settings: Dict[str, Any]
+
+
+class TestApiSerialization:
+ """Test API-specific serialization with kajson, datetime formatting, and cleanup."""
+
+ @pytest.fixture
+ def datetime_content_memory(self) -> WorkingMemory:
+ """Create WorkingMemory with datetime content."""
+ datetime_event = DateTimeEvent(
+ event_name="Project Kickoff Meeting",
+ start_time=datetime(2024, 1, 15, 10, 0, 0),
+ end_time=datetime(2024, 1, 15, 11, 30, 0),
+ created_at=datetime(2024, 1, 1, 9, 0, 0),
+ )
+
+ stuff = StuffFactory.make_stuff(concept_str="event.DateTimeEvent", name="project_meeting", content=datetime_event)
+ return WorkingMemoryFactory.make_from_single_stuff(stuff=stuff)
+
+ @pytest.fixture
+ def text_content_memory(self) -> WorkingMemory:
+ """Create WorkingMemory with text content."""
+ return WorkingMemoryFactory.make_from_text(text="Sample text content", concept_str=NativeConcept.TEXT.code, name="sample_text")
+
+ @pytest.fixture
+ def number_content_memory(self) -> WorkingMemory:
+ """Create WorkingMemory with number content."""
+ number_content = NumberContent(number=3.14159)
+ stuff = StuffFactory.make_stuff(concept_str="native.Number", name="pi_value", content=number_content)
+ return WorkingMemoryFactory.make_from_single_stuff(stuff=stuff)
+
+ def test_serialize_working_memory_with_datetime(self, datetime_content_memory: WorkingMemory):
+ """Test that datetime content is properly serialized to ISO format strings."""
+ compact_memory = ApiSerializer.serialize_working_memory_for_api(datetime_content_memory)
+
+ # Should have one entry for the datetime content
+ assert len(compact_memory) == 1
+ assert "project_meeting" in compact_memory
+
+ # Check the dict structure
+ datetime_blueprint = compact_memory["project_meeting"]
+ assert isinstance(datetime_blueprint, dict)
+ assert datetime_blueprint["concept_code"] == "event.DateTimeEvent"
+
+ # Check content is properly serialized
+ content = datetime_blueprint["content"]
+ assert isinstance(content, dict)
+ assert "event_name" in content
+ assert "start_time" in content
+ assert "end_time" in content
+ assert "created_at" in content
+
+ # Verify the event name
+ assert content["event_name"] == "Project Kickoff Meeting"
+
+ # Verify datetime objects are now formatted as ISO strings
+ assert content["start_time"] == "2024-01-15T10:00:00"
+ assert content["end_time"] == "2024-01-15T11:30:00"
+ assert content["created_at"] == "2024-01-01T09:00:00"
+
+ # Ensure no __module__ or __class__ fields are present
+ assert "__module__" not in content
+ assert "__class__" not in content
+
+ def test_api_serialized_memory_is_json_serializable(self, datetime_content_memory: WorkingMemory):
+ """Test that API serialized memory is JSON serializable."""
+ compact_memory = ApiSerializer.serialize_working_memory_for_api(datetime_content_memory)
+
+ # This should NOT raise an exception now
+ json_string = json.dumps(compact_memory)
+ roundtrip = json.loads(json_string)
+
+ # Verify roundtrip works
+ assert roundtrip == compact_memory
+
+ # Verify datetime fields are strings
+ content = roundtrip["project_meeting"]["content"]
+ assert isinstance(content["start_time"], str)
+ assert isinstance(content["end_time"], str)
+ assert isinstance(content["created_at"], str)
+
+ def test_serialize_text_content(self, text_content_memory: WorkingMemory):
+ """Test that text content is handled specially."""
+ compact_memory = ApiSerializer.serialize_working_memory_for_api(text_content_memory)
+
+ assert len(compact_memory) == 1
+ assert "sample_text" in compact_memory
+
+ text_blueprint = compact_memory["sample_text"]
+ assert text_blueprint["concept_code"] == NativeConcept.TEXT.code
+ assert isinstance(text_blueprint["content"], str)
+ assert text_blueprint["content"] == "Sample text content"
+
+ def test_serialize_number_content(self, number_content_memory: WorkingMemory):
+ """Test that number content is properly serialized."""
+ compact_memory = ApiSerializer.serialize_working_memory_for_api(number_content_memory)
+
+ assert len(compact_memory) == 1
+ assert "pi_value" in compact_memory
+
+ number_blueprint = compact_memory["pi_value"]
+ assert number_blueprint["concept_code"] == "native.Number"
+ assert isinstance(number_blueprint["content"], dict)
+ assert number_blueprint["content"]["number"] == 3.14159
+
+ def test_serialize_pipe_output(self, datetime_content_memory: WorkingMemory):
+ """Test that PipeOutput is properly serialized."""
+ pipe_output = PipeOutput(working_memory=datetime_content_memory)
+ reduced_output = ApiSerializer.serialize_pipe_output_for_api(pipe_output)
+
+ assert COMPACT_MEMORY_KEY in reduced_output
+
+ # Should contain the same structure as working memory serialization
+ working_memory_data = reduced_output[COMPACT_MEMORY_KEY]
+ assert "project_meeting" in working_memory_data
+
+ # Verify datetime formatting
+ content = working_memory_data["project_meeting"]["content"]
+ assert content["start_time"] == "2024-01-15T10:00:00"
+
+ def test_make_stuff_content_from_api_data_text(self):
+ """Test creating StuffContent from API data for text."""
+ content = ApiSerializer.make_stuff_content_from_api_data(concept_code=NativeConcept.TEXT.code, value="Test text content")
+
+ assert isinstance(content, TextContent)
+ assert content.text == "Test text content"
+
+ def test_make_stuff_content_from_api_data_datetime(self):
+ """Test creating StuffContent from API datetime data."""
+ api_data = {
+ "event_name": "Test Event",
+ "start_time": "2024-01-15T10:00:00",
+ "end_time": "2024-01-15T11:30:00",
+ "created_at": "2024-01-01T09:00:00",
+ }
+
+ content = ApiSerializer.make_stuff_content_from_api_data(concept_code="event.DateTimeEvent", value=api_data)
+ content = cast(DateTimeEvent, content)
+ assert content.event_name == "Test Event"
+ assert content.start_time == datetime(2024, 1, 15, 10, 0, 0)
+ assert content.end_time == datetime(2024, 1, 15, 11, 30, 0)
+ assert content.created_at == datetime(2024, 1, 1, 9, 0, 0)
+
+ def test_make_stuff_content_from_api_data_error(self):
+ """Test error handling for invalid concept codes."""
+ with pytest.raises(ApiSerializationError, match="Failed to create StuffContent"):
+ ApiSerializer.make_stuff_content_from_api_data(concept_code="invalid.ConceptCode", value={"some": "data"})
+
+ def test_make_stuff_content_from_api_data_text_concept_no_structure(self):
+ """Test creating StuffContent from API data for concept with no structure (should be TextContent)."""
+ # Test case for concept that has no structure - should be treated as TextContent
+ content = ApiSerializer.make_stuff_content_from_api_data(concept_code="answer.Question", value="What is the capital of France?")
+
+ assert isinstance(content, TextContent)
+ assert content.text == "What is the capital of France?"
+
+ def test_make_stuff_content_from_api_data_various_cases(self):
+ """Test make_stuff_content_from_api_data with various input cases."""
+
+ # Test 1: Native text concept
+ text_content = ApiSerializer.make_stuff_content_from_api_data(concept_code=NativeConcept.TEXT.code, value="Simple text")
+ assert isinstance(text_content, TextContent)
+ assert text_content.text == "Simple text"
+
+ # Test 2: Concept with no structure should become TextContent
+ question_content = ApiSerializer.make_stuff_content_from_api_data(concept_code="answer.Question", value="What is 2+2?")
+ assert isinstance(question_content, TextContent)
+ assert question_content.text == "What is 2+2?"
+
+ # Test 3: Number content (structured)
+ number_data = {"number": 42.0}
+ number_content = ApiSerializer.make_stuff_content_from_api_data(concept_code="native.Number", value=number_data)
+ assert number_content.__class__.__name__ == "NumberContent"
+ assert hasattr(number_content, "number")
+ assert number_content.number == 42.0 # type: ignore
+
+ def test_datetime_format_consistency(self):
+ """Test that the datetime format is consistent."""
+ test_datetime = datetime(2024, 12, 25, 15, 30, 45)
+ formatted = test_datetime.strftime(ApiSerializer.API_DATETIME_FORMAT)
+
+ assert formatted == "2024-12-25T15:30:45"
+
+ # Verify no microseconds or timezone info
+ assert "." not in formatted
+ assert "+" not in formatted
+ assert "Z" not in formatted
+
+ # ===== COMPLEX TESTS =====
+
+ def test_serialize_complex_nested_pydantic_models(self):
+ """Test serialization with deeply nested Pydantic models."""
+ # Create complex nested structure
+ task1 = ComplexTask(
+ task_id="TASK-001",
+ title="Implement Feature X",
+ priority=Priority.HIGH,
+ status=TaskStatus(is_complete=False, completion_date=None, notes=["Started implementation", "Need to review specs"]),
+ due_dates=[datetime(2024, 2, 15, 10, 0, 0), datetime(2024, 2, 28, 17, 0, 0)],
+ metadata={"tags": ["frontend", "ui"], "estimated_hours": 40.5, "assigned_team": "Alpha"},
+ score=Decimal("8.75"),
+ )
+
+ task2 = ComplexTask(
+ task_id="TASK-002",
+ title="Bug Fix",
+ priority=Priority.MEDIUM,
+ status=TaskStatus(is_complete=True, completion_date=datetime(2024, 1, 20, 14, 30, 0), notes=["Fixed authentication issue"]),
+ due_dates=[datetime(2024, 1, 25, 12, 0, 0)],
+ metadata={"severity": "medium", "component": "auth"},
+ score=Decimal("6.00"),
+ )
+
+ project = Project(
+ name="Q1 Development Sprint",
+ created_at=datetime(2024, 1, 1, 9, 0, 0),
+ tasks=[task1, task2],
+ settings={"auto_assign": True, "notification_hours": [9, 14, 18], "max_parallel_tasks": 5},
+ )
+
+ # Create stuff content from the complex model - use a proper StructuredContent subclass
+ from pipelex.core.stuff_content import StructuredContent
+
+ class ProjectContent(StructuredContent):
+ name: str
+ created_at: datetime
+ tasks: List[ComplexTask]
+ settings: Dict[str, Any]
+
+ project_content = ProjectContent(name=project.name, created_at=project.created_at, tasks=project.tasks, settings=project.settings)
+
+ stuff = StuffFactory.make_stuff(concept_str="project.Complex", name="complex_project", content=project_content)
+ memory = WorkingMemoryFactory.make_from_single_stuff(stuff=stuff)
+
+ # Test serialization
+ compact_memory = ApiSerializer.serialize_working_memory_for_api(memory)
+
+ # Verify JSON serializable - this is the key test!
+ json_string = json.dumps(compact_memory)
+ roundtrip = json.loads(json_string)
+ assert roundtrip == compact_memory
+
+ # Verify complex structure is preserved
+ content = compact_memory["complex_project"]["content"]
+ assert content["name"] == "Q1 Development Sprint"
+ assert len(content["tasks"]) == 2
+
+ # Verify nested datetime formatting
+ assert content["created_at"] == "2024-01-01T09:00:00"
+ assert content["tasks"][0]["due_dates"][0] == "2024-02-15T10:00:00"
+ assert content["tasks"][1]["status"]["completion_date"] == "2024-01-20T14:30:00"
+
+ # Verify enums are converted to values
+ assert content["tasks"][0]["priority"] == "high" # Priority.HIGH.value
+ assert content["tasks"][1]["priority"] == "medium" # Priority.MEDIUM.value
+
+ # Verify Decimal handling (converted to float)
+ assert content["tasks"][0]["score"] == 8.75 # float(Decimal("8.75"))
+ assert content["tasks"][1]["score"] == 6.0 # float(Decimal("6.00"))
+
+ def test_serialize_lists_of_datetimes(self):
+ """Test serialization of lists containing datetime objects."""
+ from pipelex.core.stuff_content import StructuredContent
+
+ class ScheduleContent(StructuredContent):
+ meeting_times: List[datetime]
+ deadlines: Dict[str, datetime]
+
+ schedule = ScheduleContent(
+ meeting_times=[datetime(2024, 1, 15, 10, 0, 0), datetime(2024, 1, 16, 14, 30, 0), datetime(2024, 1, 17, 9, 15, 0)],
+ deadlines={"milestone_1": datetime(2024, 2, 1, 23, 59, 59), "milestone_2": datetime(2024, 3, 15, 17, 0, 0)},
+ )
+
+ stuff = StuffFactory.make_stuff(concept_str="schedule.Complex", name="schedule", content=schedule)
+ memory = WorkingMemoryFactory.make_from_single_stuff(stuff=stuff)
+
+ compact_memory = ApiSerializer.serialize_working_memory_for_api(memory)
+
+ # Verify JSON serializable
+ json_string = json.dumps(compact_memory)
+ roundtrip = json.loads(json_string)
+ assert roundtrip == compact_memory
+
+ content = compact_memory["schedule"]["content"]
+
+ # Verify list of datetimes
+ assert len(content["meeting_times"]) == 3
+ assert content["meeting_times"][0] == "2024-01-15T10:00:00"
+ assert content["meeting_times"][1] == "2024-01-16T14:30:00"
+ assert content["meeting_times"][2] == "2024-01-17T09:15:00"
+
+ # Verify dict of datetimes
+ assert content["deadlines"]["milestone_1"] == "2024-02-01T23:59:59"
+ assert content["deadlines"]["milestone_2"] == "2024-03-15T17:00:00"
+
+ def test_serialize_with_none_values(self):
+ """Test serialization with None/Optional values."""
+ from pipelex.core.stuff_content import StructuredContent
+
+ class OptionalContent(StructuredContent):
+ required_field: str
+ optional_datetime: Optional[datetime]
+ optional_list: Optional[List[str]]
+ nullable_dict: Optional[Dict[str, Any]]
+
+ content = OptionalContent(required_field="test", optional_datetime=None, optional_list=None, nullable_dict=None)
+
+ stuff = StuffFactory.make_stuff(concept_str="optional.Test", name="optional_test", content=content)
+ memory = WorkingMemoryFactory.make_from_single_stuff(stuff=stuff)
+
+ compact_memory = ApiSerializer.serialize_working_memory_for_api(memory)
+
+ # Verify JSON serializable
+ json_string = json.dumps(compact_memory)
+ roundtrip = json.loads(json_string)
+ assert roundtrip == compact_memory
+
+ content_data = compact_memory["optional_test"]["content"]
+ assert content_data["required_field"] == "test"
+ assert content_data["optional_datetime"] is None
+ assert content_data["optional_list"] is None
+ assert content_data["nullable_dict"] is None
+
+ def test_serialize_deeply_nested_structure(self):
+ """Test serialization with deeply nested data structures."""
+ from pipelex.core.stuff_content import StructuredContent
+
+ class DeepContent(StructuredContent):
+ level1: Dict[str, Any]
+
+ deep_structure = {
+ "level2": {
+ "level3": {
+ "level4": {
+ "timestamps": [datetime(2024, 1, 1, 12, 0, 0), datetime(2024, 1, 2, 13, 30, 0)],
+ "metadata": {"created": datetime(2024, 1, 1, 10, 0, 0), "modified": datetime(2024, 1, 3, 15, 45, 0)},
+ }
+ }
+ }
+ }
+
+ content = DeepContent(level1=deep_structure)
+
+ stuff = StuffFactory.make_stuff(concept_str="deep.Nested", name="deep_test", content=content)
+ memory = WorkingMemoryFactory.make_from_single_stuff(stuff=stuff)
+
+ compact_memory = ApiSerializer.serialize_working_memory_for_api(memory)
+
+ # Verify JSON serializable
+ json_string = json.dumps(compact_memory)
+ roundtrip = json.loads(json_string)
+ assert roundtrip == compact_memory
+
+ # Navigate to deep nested datetime
+ content_data = compact_memory["deep_test"]["content"]
+ level4 = content_data["level1"]["level2"]["level3"]["level4"]
+
+ # Verify deep datetime formatting
+ assert level4["timestamps"][0] == "2024-01-01T12:00:00"
+ assert level4["timestamps"][1] == "2024-01-02T13:30:00"
+ assert level4["metadata"]["created"] == "2024-01-01T10:00:00"
+ assert level4["metadata"]["modified"] == "2024-01-03T15:45:00"
+
+ def test_serialize_with_class_module_fields_removal(self):
+ """Test that __class__ and __module__ fields are properly removed."""
+ from pipelex.core.stuff_content import StructuredContent
+
+ class TestContent(StructuredContent):
+ data: Dict[str, Any]
+
+ # Create content with intentional __class__ and __module__ fields
+ test_data = {
+ "normal_field": "value",
+ "__class__": "ShouldBeRemoved",
+ "__module__": "should.be.removed",
+ "nested": {"field": "nested_value", "__class__": "NestedShouldBeRemoved", "__module__": "nested.should.be.removed"},
+ "list_with_class": [{"item": "value1", "__class__": "ItemClass"}, {"item": "value2", "__module__": "item.module"}],
+ }
+
+ content = TestContent(data=test_data)
+
+ stuff = StuffFactory.make_stuff(concept_str="test.ClassModule", name="class_test", content=content)
+ memory = WorkingMemoryFactory.make_from_single_stuff(stuff=stuff)
+
+ compact_memory = ApiSerializer.serialize_working_memory_for_api(memory)
+
+ content_data = compact_memory["class_test"]["content"]
+
+ # Verify top-level __class__ and __module__ removal
+ assert "__class__" not in content_data
+ assert "__module__" not in content_data
+ assert content_data["data"]["normal_field"] == "value"
+
+ # Verify nested __class__ and __module__ removal
+ assert "__class__" not in content_data["data"]
+ assert "__module__" not in content_data["data"]
+ assert "__class__" not in content_data["data"]["nested"]
+ assert "__module__" not in content_data["data"]["nested"]
+ assert content_data["data"]["nested"]["field"] == "nested_value"
+
+ # Verify list item __class__ and __module__ removal
+ assert "__class__" not in content_data["data"]["list_with_class"][0]
+ assert "__module__" not in content_data["data"]["list_with_class"][1]
+ assert content_data["data"]["list_with_class"][0]["item"] == "value1"
+ assert content_data["data"]["list_with_class"][1]["item"] == "value2"
+
+ def test_serialize_with_path_objects(self):
+ """Test serialization with pathlib.Path objects."""
+ from pipelex.core.stuff_content import StructuredContent
+
+ class PathContent(StructuredContent):
+ file_path: Path
+ directory_path: Path
+ nested_paths: Dict[str, Path]
+ path_list: List[Path]
+
+ content = PathContent(
+ file_path=Path("/home/user/documents/file.txt"),
+ directory_path=Path("/var/log/"),
+ nested_paths={"config": Path("/etc/config.yaml"), "temp": Path("/tmp/temp.log")},
+ path_list=[Path("/usr/bin/python"), Path("/opt/app/main.py")],
+ )
+
+ stuff = StuffFactory.make_stuff(concept_str="path.Test", name="path_test", content=content)
+ memory = WorkingMemoryFactory.make_from_single_stuff(stuff=stuff)
+
+ compact_memory = ApiSerializer.serialize_working_memory_for_api(memory)
+
+ # Verify JSON serializable - this is the key test!
+ json_string = json.dumps(compact_memory)
+ roundtrip = json.loads(json_string)
+ assert roundtrip == compact_memory
+
+ # Verify Path objects were converted to strings
+ content_data = compact_memory["path_test"]["content"]
+
+ # Check that paths are now strings
+ assert content_data["file_path"] == "/home/user/documents/file.txt"
+ assert content_data["directory_path"] == "/var/log"
+
+ # Verify nested paths in dict
+ assert content_data["nested_paths"]["config"] == "/etc/config.yaml"
+ assert content_data["nested_paths"]["temp"] == "/tmp/temp.log"
+
+ # Verify paths in list
+ assert content_data["path_list"][0] == "/usr/bin/python"
+ assert content_data["path_list"][1] == "/opt/app/main.py"
diff --git a/tests/unit/pipelex/core/test_concept_library.py b/tests/unit/pipelex/core/test_concept_library.py
new file mode 100644
index 000000000..331378091
--- /dev/null
+++ b/tests/unit/pipelex/core/test_concept_library.py
@@ -0,0 +1,155 @@
+from pipelex.core.concept_library import ConceptLibrary
+from pipelex.core.concept_native import NativeConcept
+
+
+class TestConceptLibraryIsNativeConcept:
+ """Test ConceptLibrary.is_native_concept method."""
+
+ def test_is_native_concept_with_native_domain_prefix_true(self):
+ """Test concept strings with native domain prefix return True."""
+ concept_library = ConceptLibrary()
+
+ # Test with various native concepts that have explicit domain
+ assert concept_library.is_native_concept("native.Text") is True
+ assert concept_library.is_native_concept("native.Image") is True
+ assert concept_library.is_native_concept("native.PDF") is True
+ assert concept_library.is_native_concept("native.Number") is True
+ assert concept_library.is_native_concept("native.Dynamic") is True
+ assert concept_library.is_native_concept("native.LlmPrompt") is True
+ assert concept_library.is_native_concept("native.TextAndImages") is True
+ assert concept_library.is_native_concept("native.Page") is True
+ assert concept_library.is_native_concept("native.Anything") is True
+
+ def test_is_native_concept_with_non_native_domain_prefix_false(self):
+ """Test concept strings with non-native domain prefix return False."""
+ concept_library = ConceptLibrary()
+
+ # Test with various non-native domains
+ assert concept_library.is_native_concept("custom.Text") is False
+ assert concept_library.is_native_concept("documents.Page") is False
+ assert concept_library.is_native_concept("images.Photo") is False
+ assert concept_library.is_native_concept("test_domain.TestConcept") is False
+ assert concept_library.is_native_concept("my_domain.MyClass") is False
+
+ def test_is_native_concept_without_domain_native_concept_names_true(self):
+ """Test concept strings without domain that are native concept names return True."""
+ concept_library = ConceptLibrary()
+
+ # Test all native concept names without domain
+ native_names = NativeConcept.names()
+ for native_name in native_names:
+ assert concept_library.is_native_concept(native_name) is True, f"Failed for native concept: {native_name}"
+
+ def test_is_native_concept_without_domain_non_native_names_false(self):
+ """Test concept strings without domain that are not native concept names return False."""
+ concept_library = ConceptLibrary()
+
+ # Test various non-native concept names without domain
+ assert concept_library.is_native_concept("CustomConcept") is False
+ assert concept_library.is_native_concept("MyClass") is False
+ assert concept_library.is_native_concept("Document") is False
+ assert concept_library.is_native_concept("Photo") is False
+ assert concept_library.is_native_concept("UnknownConcept") is False
+
+ def test_is_native_concept_empty_string(self):
+ """Test empty string returns False."""
+ concept_library = ConceptLibrary()
+
+ assert concept_library.is_native_concept("") is False
+
+ def test_is_native_concept_invalid_domain_format(self):
+ """Test concept strings with invalid domain format (multiple dots)."""
+ concept_library = ConceptLibrary()
+
+ # These should be handled by the concept_str_contains_domain check
+ # Multiple dots should not be considered as having a domain
+ assert concept_library.is_native_concept("domain.concept.subconcept") is False
+ assert concept_library.is_native_concept("a.b.c.d") is False
+
+ def test_is_native_concept_just_dot(self):
+ """Test concept string that is just a dot."""
+ concept_library = ConceptLibrary()
+
+ assert concept_library.is_native_concept(".") is False
+
+ def test_is_native_concept_dot_at_start_or_end(self):
+ """Test concept strings with dot at start or end."""
+ concept_library = ConceptLibrary()
+
+ # .Text has domain="" and concept="Text", so domain != "native" -> False
+ assert concept_library.is_native_concept(".Text") is False
+ # Text. has domain="Text" and concept="", so domain != "native" -> False
+ assert concept_library.is_native_concept("Text.") is False
+ # .native has domain="" and concept="native", so domain != "native" -> False
+ assert concept_library.is_native_concept(".native") is False
+ # native. has domain="native" and concept="", so domain == "native" -> True
+ assert concept_library.is_native_concept("native.") is True
+
+ def test_is_native_concept_case_sensitivity(self):
+ """Test that the function is case sensitive for domain checking."""
+ concept_library = ConceptLibrary()
+
+ # Test case variations of native domain
+ assert concept_library.is_native_concept("Native.Text") is False # Capital N
+ assert concept_library.is_native_concept("NATIVE.Text") is False # All caps
+ # The concept part doesn't matter for domain validation, only the domain part matters
+ assert concept_library.is_native_concept("native.text") is True # lowercase concept but native domain
+
+ # Test case variations of native concept names without domain
+ assert concept_library.is_native_concept("text") is False # lowercase
+ assert concept_library.is_native_concept("TEXT") is False # uppercase
+ assert concept_library.is_native_concept("image") is False # lowercase
+
+ def test_is_native_concept_whitespace(self):
+ """Test concept strings with whitespace."""
+ concept_library = ConceptLibrary()
+
+ # Whitespace in concept names without domain should make it invalid (not in NativeConcept.names())
+ assert concept_library.is_native_concept(" Text") is False
+ assert concept_library.is_native_concept("Text ") is False
+ # Whitespace in domain part should make it invalid
+ assert concept_library.is_native_concept(" native.Text") is False
+ # Whitespace in concept part doesn't matter for domain validation, only domain matters
+ assert concept_library.is_native_concept("native.Text ") is True
+ assert concept_library.is_native_concept("native .Text") is False # whitespace in domain
+ assert concept_library.is_native_concept("native. Text") is True # whitespace in concept part
+
+ def test_is_native_concept_special_characters(self):
+ """Test concept strings with special characters."""
+ concept_library = ConceptLibrary()
+
+ # Special characters should make it invalid
+ assert concept_library.is_native_concept("native-Text") is False
+ assert concept_library.is_native_concept("native_Text") is False
+ assert concept_library.is_native_concept("native/Text") is False
+ assert concept_library.is_native_concept("native\\Text") is False
+
+ def test_native_concept_names_consistency(self):
+ """Test that the function uses the correct native concept names."""
+ concept_library = ConceptLibrary()
+
+ # Get the actual native concept names
+ native_names = NativeConcept.names()
+
+ # Verify some expected names are present
+ expected_names = ["Text", "Image", "PDF", "Number", "Dynamic", "LlmPrompt", "TextAndImages", "Page", "Anything"]
+ for expected_name in expected_names:
+ assert expected_name in native_names, f"Expected native concept name {expected_name} not found"
+ # Test that each expected name returns True
+ assert concept_library.is_native_concept(expected_name) is True
+
+ def test_is_native_concept_common_examples(self):
+ """Test that common native concept names work without domain prefix."""
+ concept_library = ConceptLibrary()
+
+ # Test the most commonly used native concept names
+ common_native_concepts = ["Text", "Image", "PDF", "Number", "Dynamic"]
+
+ for concept_name in common_native_concepts:
+ # These should return True without needing "native." prefix
+ assert concept_library.is_native_concept(concept_name) is True, f"'{concept_name}' should be recognized as native concept"
+
+ # And they should also work with the explicit "native." prefix
+ assert concept_library.is_native_concept(f"native.{concept_name}") is True, (
+ f"'native.{concept_name}' should also be recognized as native concept"
+ )
diff --git a/tests/unit/pipelex/core/test_stuff_content_factory.py b/tests/unit/pipelex/core/test_stuff_content_factory.py
new file mode 100644
index 000000000..38a4a4970
--- /dev/null
+++ b/tests/unit/pipelex/core/test_stuff_content_factory.py
@@ -0,0 +1,174 @@
+from typing import Any, ClassVar, Dict, List, Tuple
+
+import pytest
+
+from pipelex.core.stuff_content import StructuredContent, TextContent
+from pipelex.core.stuff_factory import StuffContentFactory
+
+
+class TestCases:
+ # Test cases for TextContent with string content
+ TEXT_STRING_BLUEPRINT: ClassVar[Dict[str, Any]] = {
+ "concept_code": "native.Text",
+ "content": "The Dawn of Ultra-Rapid Transit: NextGen High-Speed Trains Redefine Travel",
+ }
+
+ # Test cases for TextContent with dict content
+ TEXT_DICT_BLUEPRINT: ClassVar[Dict[str, Any]] = {"concept_code": "native.Text", "content": {"text": "Sample text content"}}
+
+ # Test cases for native concept without prefix (should work)
+ TEXT_NO_PREFIX_BLUEPRINT: ClassVar[Dict[str, Any]] = {"concept_code": "Text", "content": {"text": "Text content without native prefix"}}
+
+ # Test cases for registered class (using actual registered class)
+ REGISTERED_CLASS_BLUEPRINT: ClassVar[Dict[str, Any]] = {
+ "concept_code": "test.MockRegisteredContent",
+ "content": {"title": "Test Question", "description": "What are aerodynamic features?"},
+ }
+
+ # Test cases for unregistered class (creates implicit concept, returns TextContent)
+ UNREGISTERED_STRING_BLUEPRINT: ClassVar[Dict[str, Any]] = {
+ "concept_code": "unknown.NonExistentConcept",
+ "content": "This should create implicit concept and return TextContent",
+ }
+
+ # Test cases for unregistered class with dict content
+ UNREGISTERED_DICT_BLUEPRINT: ClassVar[Dict[str, Any]] = {
+ "concept_code": "unknown.NonExistentConcept",
+ "content": {"text": "Dict content for implicit concept"},
+ }
+
+ TEST_BLUEPRINTS: ClassVar[List[Tuple[str, Dict[str, Any]]]] = [
+ ("text_string", TEXT_STRING_BLUEPRINT),
+ ("text_dict", TEXT_DICT_BLUEPRINT),
+ ("text_no_prefix", TEXT_NO_PREFIX_BLUEPRINT),
+ ("registered_class", REGISTERED_CLASS_BLUEPRINT),
+ ("unregistered_string", UNREGISTERED_STRING_BLUEPRINT),
+ ("unregistered_dict", UNREGISTERED_DICT_BLUEPRINT),
+ ]
+
+
+class TestStuffContentFactory:
+ def test_make_content_from_value_text_string(self):
+ """Test make_content_from_value with TextContent and string value."""
+ result = StuffContentFactory.make_content_from_value(stuff_content_subclass=TextContent, value="Test string content")
+
+ assert isinstance(result, TextContent)
+ assert result.text == "Test string content"
+
+ def test_make_content_from_value_text_dict(self):
+ """Test make_content_from_value with TextContent and dict value."""
+ result = StuffContentFactory.make_content_from_value(stuff_content_subclass=TextContent, value={"text": "Dict text content"})
+
+ assert isinstance(result, TextContent)
+ assert result.text == "Dict text content"
+
+ def test_make_content_from_value_structured_dict(self):
+ """Test make_content_from_value with StructuredContent and dict value."""
+
+ class MockStructuredContent(StructuredContent):
+ title: str
+ description: str
+
+ result = StuffContentFactory.make_content_from_value(
+ stuff_content_subclass=MockStructuredContent, value={"title": "Test Title", "description": "Test Description"}
+ )
+
+ assert isinstance(result, MockStructuredContent)
+ assert result.title == "Test Title"
+ assert result.description == "Test Description"
+
+ def test_make_stuffcontent_from_concept_code_required_text_content(self):
+ """Test required method with native.Text concept (should work)."""
+ result = StuffContentFactory.make_stuffcontent_from_concept_code_required(concept_code="native.Text", value="Test text content")
+
+ assert isinstance(result, TextContent)
+ assert result.text == "Test text content"
+
+ def test_make_stuffcontent_from_concept_code_required_implicit_concept(self):
+ """Test required method with implicit concept (should create TextContent)."""
+ result = StuffContentFactory.make_stuffcontent_from_concept_code_required(
+ concept_code="unknown.NonExistentConcept", value="Test content for implicit concept"
+ )
+
+ assert isinstance(result, TextContent)
+ assert result.text == "Test content for implicit concept"
+
+ def test_make_stuffcontent_from_concept_code_with_fallback_text_success(self):
+ """Test fallback method with native.Text concept."""
+ result = StuffContentFactory.make_stuffcontent_from_concept_code_with_fallback(concept_code="native.Text", value="Test text content")
+
+ assert isinstance(result, TextContent)
+ assert result.text == "Test text content"
+
+ def test_make_stuffcontent_from_concept_code_with_fallback_implicit_string(self):
+ """Test fallback method with implicit concept and string value."""
+ result = StuffContentFactory.make_stuffcontent_from_concept_code_with_fallback(
+ concept_code="unknown.NonExistentConcept", value="Fallback text content"
+ )
+
+ assert isinstance(result, TextContent)
+ assert result.text == "Fallback text content"
+
+ def test_make_stuffcontent_from_concept_code_with_fallback_implicit_dict(self):
+ """Test fallback method with implicit concept and dict value."""
+ result = StuffContentFactory.make_stuffcontent_from_concept_code_with_fallback(
+ concept_code="unknown.NonExistentConcept", value={"text": "Dict fallback content"}
+ )
+
+ assert isinstance(result, TextContent)
+ assert result.text == "Dict fallback content"
+
+ @pytest.mark.parametrize(
+ "test_name, blueprint",
+ TestCases.TEST_BLUEPRINTS,
+ )
+ def test_blueprint_scenarios(self, test_name: str, blueprint: Dict[str, Any]):
+ """Test various blueprint scenarios with parametrized test cases."""
+ concept_code = blueprint["concept_code"]
+ content = blueprint["content"]
+
+ if test_name.startswith("text_"):
+ # Test native.Text concept with required method
+ result = StuffContentFactory.make_stuffcontent_from_concept_code_required(concept_code=concept_code, value=content)
+ assert isinstance(result, TextContent)
+ if isinstance(content, str):
+ assert result.text == content
+ else:
+ assert result.text == content["text"]
+
+ elif test_name == "registered_class":
+ # Test with registered class - since MockRegisteredContent isn't actually registered,
+ # it will be treated as an implicit concept and return TextContent
+ # But the content dict format is incompatible with TextContent's expected structure
+
+ # Test required method - it will succeed but create TextContent
+ # The dict will be passed through model_validate which should fail for TextContent
+ try:
+ result_required = StuffContentFactory.make_stuffcontent_from_concept_code_required(concept_code=concept_code, value=content)
+ # If it succeeds, it should be TextContent (due to implicit concept)
+ assert isinstance(result_required, TextContent)
+ except Exception:
+ # If it fails due to validation error, that's also expected
+ pass
+
+ # Test fallback method - same behavior expected
+ try:
+ result_fallback = StuffContentFactory.make_stuffcontent_from_concept_code_with_fallback(concept_code=concept_code, value=content)
+ # If it succeeds, it should be TextContent (due to implicit concept)
+ assert isinstance(result_fallback, TextContent)
+ except Exception:
+ # If it fails due to validation error, that's also expected
+ pass
+
+ elif test_name.startswith("unregistered_"):
+ # Test behavior for unregistered/implicit concepts
+ # Both methods should work and return TextContent due to implicit concept creation
+
+ result_required = StuffContentFactory.make_stuffcontent_from_concept_code_required(concept_code=concept_code, value=content)
+ assert isinstance(result_required, TextContent)
+
+ result_fallback = StuffContentFactory.make_stuffcontent_from_concept_code_with_fallback(concept_code=concept_code, value=content)
+ assert isinstance(result_fallback, TextContent)
+
+ # Both should produce the same result
+ assert result_required.text == result_fallback.text
diff --git a/tests/unit/pipelex/core/test_working_memory.py b/tests/unit/pipelex/core/test_working_memory.py
new file mode 100644
index 000000000..0e023b08b
--- /dev/null
+++ b/tests/unit/pipelex/core/test_working_memory.py
@@ -0,0 +1,181 @@
+from typing import ClassVar, List, Tuple
+
+import pytest
+
+from pipelex.core.concept_native import NativeConcept
+from pipelex.core.stuff_content import HtmlContent, ImageContent, ListContent, NumberContent, TextAndImagesContent, TextContent
+from pipelex.core.stuff_factory import StuffFactory
+from pipelex.core.working_memory import WorkingMemory
+from pipelex.core.working_memory_factory import WorkingMemoryFactory
+
+
+class TestWorkingMemoryData:
+ """Test data for WorkingMemory tests."""
+
+ # Sample text content
+ SAMPLE_TEXT = """
+ The Dawn of Ultra-Rapid Transit: NextGen High-Speed Trains Redefine Travel
+ By Eliza Montgomery, Transportation Technology Reporter
+
+ In an era where time is increasingly precious, a revolution in rail transportation is quietly
+ transforming how we connect cities and regions. The emergence of ultra-high-speed train
+ networks, capable of speeds exceeding 350 mph, promises to render certain short-haul
+ flights obsolete while dramatically reducing carbon emissions.
+ """
+
+ # Sample PDF and image URLs
+ SAMPLE_PDF_URL = "assets/extract_dpe/dpe_single_page.pdf"
+ SAMPLE_IMAGE_URL = "assets/gantt_charts/sample_gantt.png"
+
+ # Test cases for different content types
+ SINGLE_TEXT_CASE = "single_text"
+ SINGLE_IMAGE_CASE = "single_image"
+ SINGLE_PDF_CASE = "single_pdf"
+ MULTIPLE_STUFF_CASE = "multiple_stuff"
+ WITH_ALIASES_CASE = "with_aliases"
+ COMPLEX_LIST_CASE = "complex_list"
+ TEXT_AND_IMAGES_CASE = "text_and_images"
+ HTML_CONTENT_CASE = "html_content"
+ NUMBER_CONTENT_CASE = "number_content"
+
+ TEST_CASES: ClassVar[List[Tuple[str, str]]] = [
+ ("Single text content", SINGLE_TEXT_CASE),
+ ("Single image content", SINGLE_IMAGE_CASE),
+ ("Single PDF content", SINGLE_PDF_CASE),
+ ("Multiple stuff items", MULTIPLE_STUFF_CASE),
+ ("WorkingMemory with aliases", WITH_ALIASES_CASE),
+ ("Complex list content", COMPLEX_LIST_CASE),
+ ("Text and images content", TEXT_AND_IMAGES_CASE),
+ ("HTML content", HTML_CONTENT_CASE),
+ ("Number content", NUMBER_CONTENT_CASE),
+ ]
+
+
+class TestWorkingMemory:
+ """Unit tests for WorkingMemory class - focus on core functionality."""
+
+ @pytest.fixture
+ def single_text_memory(self) -> WorkingMemory:
+ """Create WorkingMemory with single text content."""
+ return WorkingMemoryFactory.make_from_text(text=TestWorkingMemoryData.SAMPLE_TEXT, concept_str=NativeConcept.TEXT.code, name="sample_text")
+
+ @pytest.fixture
+ def single_image_memory(self) -> WorkingMemory:
+ """Create WorkingMemory with single image content."""
+ return WorkingMemoryFactory.make_from_image(
+ image_url=TestWorkingMemoryData.SAMPLE_IMAGE_URL, concept_str="gantt.GanttImage", name="gantt_chart_image"
+ )
+
+ @pytest.fixture
+ def single_pdf_memory(self) -> WorkingMemory:
+ """Create WorkingMemory with single PDF content."""
+ return WorkingMemoryFactory.make_from_pdf(pdf_url=TestWorkingMemoryData.SAMPLE_PDF_URL, concept_str="PDF", name="pdf_document")
+
+ @pytest.fixture
+ def multiple_stuff_memory(self) -> WorkingMemory:
+ """Create WorkingMemory with multiple stuff items."""
+ text_stuff = StuffFactory.make_stuff(
+ concept_str="native.Text", name="question", content=TextContent(text="What are the aerodynamic features?")
+ )
+
+ document_stuff = StuffFactory.make_stuff(
+ concept_str="native.Text", name="document", content=TextContent(text=TestWorkingMemoryData.SAMPLE_TEXT)
+ )
+
+ image_stuff = StuffFactory.make_stuff(
+ concept_str="native.Image", name="diagram", content=ImageContent(url=TestWorkingMemoryData.SAMPLE_IMAGE_URL)
+ )
+
+ return WorkingMemoryFactory.make_from_multiple_stuffs(stuff_list=[text_stuff, document_stuff, image_stuff], main_name="document")
+
+ @pytest.fixture
+ def memory_with_aliases(self) -> WorkingMemory:
+ """Create WorkingMemory with aliases."""
+ text_stuff = StuffFactory.make_stuff(concept_str="native.Text", name="primary_text", content=TextContent(text="Primary content"))
+
+ secondary_stuff = StuffFactory.make_stuff(concept_str="native.Text", name="secondary_text", content=TextContent(text="Secondary content"))
+
+ memory = WorkingMemory()
+ memory.add_new_stuff(name="primary_text", stuff=text_stuff)
+ memory.add_new_stuff(name="secondary_text", stuff=secondary_stuff)
+ memory.set_alias(alias="main_text", target="primary_text")
+ memory.set_alias(alias="backup_text", target="secondary_text")
+
+ return memory
+
+ @pytest.fixture
+ def complex_list_memory(self) -> WorkingMemory:
+ """Create WorkingMemory with complex list content containing mixed types."""
+ complex_content: ListContent[TextContent | ImageContent | NumberContent] = ListContent(
+ items=[
+ TextContent(text="The quick brown fox jumps over the lazy dog"),
+ ImageContent(url=TestWorkingMemoryData.SAMPLE_IMAGE_URL),
+ NumberContent(number=42.5),
+ ]
+ )
+
+ complex_stuff = StuffFactory.make_stuff(concept_str="native.List", name="mixed_list", content=complex_content)
+
+ return WorkingMemoryFactory.make_from_single_stuff(stuff=complex_stuff)
+
+ @pytest.fixture
+ def text_and_images_memory(self) -> WorkingMemory:
+ """Create WorkingMemory with text and images content."""
+ text_and_images_content = TextAndImagesContent(
+ text=TextContent(text="Project overview with diagrams"),
+ images=[ImageContent(url=TestWorkingMemoryData.SAMPLE_IMAGE_URL), ImageContent(url="assets/diagrams/architecture.png")],
+ )
+
+ stuff = StuffFactory.make_stuff(concept_str="native.TextAndImages", name="project_overview", content=text_and_images_content)
+
+ return WorkingMemoryFactory.make_from_single_stuff(stuff=stuff)
+
+ @pytest.fixture
+ def html_content_memory(self) -> WorkingMemory:
+ """Create WorkingMemory with HTML content."""
+ html_content = HtmlContent(
+ inner_html="<h1>Test Report</h1><p>This is a <strong>test</strong> report.</p><ul><li>Item 1</li><li>Item 2</li></ul>",
+ css_class="report-content",
+ )
+
+ stuff = StuffFactory.make_stuff(concept_str="native.Html", name="test_report", content=html_content)
+
+ return WorkingMemoryFactory.make_from_single_stuff(stuff=stuff)
+
+ @pytest.fixture
+ def number_content_memory(self) -> WorkingMemory:
+ """Create WorkingMemory with number content."""
+ number_content = NumberContent(number=3.14159)
+
+ stuff = StuffFactory.make_stuff(concept_str="native.Number", name="pi_value", content=number_content)
+
+ return WorkingMemoryFactory.make_from_single_stuff(stuff=stuff)
+
+ def test_working_memory_basic_functionality(self, single_text_memory: WorkingMemory):
+ """Test basic WorkingMemory functionality."""
+ # Should have one entry for the text content
+ assert len(single_text_memory.root) == 1
+ assert "sample_text" in single_text_memory.root
+
+ # Check stuff retrieval
+ stuff = single_text_memory.get_stuff("sample_text")
+ assert stuff.concept_code == NativeConcept.TEXT.code
+ assert isinstance(stuff.content, TextContent)
+ assert stuff.content.text == TestWorkingMemoryData.SAMPLE_TEXT
+
+ def test_working_memory_aliases(self, memory_with_aliases: WorkingMemory):
+ """Test WorkingMemory alias functionality."""
+ # Should have two root items and two aliases
+ assert len(memory_with_aliases.root) == 2
+ assert len(memory_with_aliases.aliases) == 2
+
+ # Check aliases work
+ primary_stuff = memory_with_aliases.get_stuff("primary_text")
+ alias_stuff = memory_with_aliases.get_stuff("main_text")
+ assert primary_stuff == alias_stuff
+
+ def test_working_memory_empty(self):
+ """Test empty WorkingMemory."""
+ empty_memory = WorkingMemoryFactory.make_empty()
+ assert len(empty_memory.root) == 0
+ assert len(empty_memory.aliases) == 0