Build AI methods with your coding agent, then run them anywhere: as an MCP for chatbots, as a webapp for people, or via API for your software.
Fix/type checkers (#293)
Thomas Hebrardd3bd43f50263
diff --git a/.badges/tests.json b/.badges/tests.jsonindex 00505132c..d43b5371e 100644--- a/.badges/tests.json+++ b/.badges/tests.json@@ -1,7 +1,7 @@{"schemaVersion": 1,"label": "tests",- "message": "1088",+ "message": "1121","color": "blue","cacheSeconds": 300}\ No newline at end of filediff --git a/.env.example b/.env.exampleindex 76865ecf5..a82594c92 100644--- a/.env.example+++ b/.env.example@@ -1,6 +1,5 @@-PYRIGHT_PYTHON_FORCE_VERSION=1.1.398--PIPELEX_INFERENCE_API_KEY=+PIPELEX_API_TOKEN=+PIPELEX_API_BASE_URL=http://localhost:8081/apiOPENAI_API_KEY=@@ -12,10 +11,6 @@ AZURE_API_BASE=AZURE_API_VERSION=AZURE_API_KEY=-GCP_PROJECT_ID=-GCP_LOCATION=-GCP_CREDENTIALS_FILE_PATH=path/to/credentials.json-ANTHROPIC_API_KEY=MISTRAL_API_KEY=@@ -23,9 +18,20 @@ MISTRAL_API_KEY=PERPLEXITY_API_KEY=PERPLEXITY_API_ENDPOINT=https://api.perplexity.ai+FAL_API_KEY=++GCP_PROJECT_ID=+GCP_LOCATION=+GCP_CREDENTIALS_FILE_PATH=gcp_credentials.json++CUSTOM_ENDPOINT_BASE_URL=http://localhost:11434/v1+CUSTOM_ENDPOINT_API_KEY=+XAI_API_KEY=XAI_API_ENDPOINT=https://api.x.ai/v1/-FAL_API_KEY=+PIPELEX_INFERENCE_API_KEY=BLACKBOX_API_KEY=++GOOGLE_API_KEY=\ No newline at end of filediff --git a/CHANGELOG.md b/CHANGELOG.mdindex 120b3334b..c51115498 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -1,6 +1,6 @@# Changelog-## [v0.10.3] - 2025-09-26+## [Unreleased] - 2025-09-26### Highlights@@ -633,7 +633,7 @@ Simplified input memory:### Changed-- **Refactored Cognitive Workers**: The abstraction for `LLM`, `Imgg`, and `Ocr` workers has been elegantly simplified. The old decorator-based approach (`..._job_func`) has been replaced with a more robust pattern: a public base method now handles pre- and post-execution logic while calling a private abstract method that each worker implements.+- **Refactored Cognitive Workers**: The abstraction for `LLM`, `ImgGen`, and `Ocr` workers has been elegantly simplified. The old decorator-based approach (`..._job_func`) has been replaced with a more robust pattern: a public base method now handles pre- and post-execution logic while calling a private abstract method that each worker implements.- The `b64_image_bytes` field in `PromptImageBytes` was renamed to `base_64` for better consistency.### Fixed@@ -729,15 +729,15 @@ is_reporting_enabled = true### Tests- TestTemplatePreprocessor-- TestImggByOpenAIGpt+- TestImgGenByOpenAIGpt- TestImageGeneration-- TestPipeImgg+- TestPipeImgGen## [v0.2.9] - 2025-05-30- Include `pyproject.toml` inside the project build.-- Fix `ImggEngineFactory`: image generation (imgg) handle required format is `platform/model_name`+- Fix `ImgGenEngineFactory`: image generation (imgg) handle required format is `platform/model_name`- pipelex cli: Added `list-pipes` method that can list all the available pipes along with their descriptions.- Use a minimum version for `uv` instead of a fixed version- Implement `AGENTS.md` for Codexdiff --git a/Makefile b/Makefileindex 15f51017b..d9b38e0e3 100644--- a/Makefile+++ b/Makefile@@ -14,6 +14,7 @@ VENV_PYRIGHT := $(VIRTUAL_ENV)/bin/pyrightVENV_MYPY := $(VIRTUAL_ENV)/bin/mypyVENV_PIPELEX := $(VIRTUAL_ENV)/bin/pipelexVENV_MKDOCS := $(VIRTUAL_ENV)/bin/mkdocs+VENV_PYLINT := $(VIRTUAL_ENV)/bin/pylintUV_MIN_VERSION = $(shell grep -m1 'required-version' pyproject.toml | sed -E 's/.*= *"([^<>=, ]+).*/\1/')@@ -101,7 +102,7 @@ export HELP.PHONY: \all help env lock install update build \- format lint pyright mypy \+ format lint pyright mypy pylint \cleanderived cleanenv cleanlibraries cleanall \test test-xdist t test-quiet tq test-with-prints tp test-inference ti \test-img-gen tg test-ocr to codex-tests gha-tests \@@ -380,13 +381,16 @@ lint: envpyright: env$(call PRINT_TITLE,"Typechecking with pyright")- @$(VENV_PYRIGHT) --pythonpath $(VIRTUAL_ENV)/bin/python3 && \- echo "Done typechecking with pyright — disregard warning about latest version, it's giving us false positives"+ @$(VENV_PYRIGHT) . --project pyproject.tomlmypy: env$(call PRINT_TITLE,"Typechecking with mypy")$(VENV_MYPY)+pylint: env+ $(call PRINT_TITLE,"Linting with pylint")+ $(VENV_PYLINT) --rcfile pyproject.toml pipelex tests+############################################################################################# MERGE CHECKS@@ -408,6 +412,10 @@ merge-check-mypy: env$(call PRINT_TITLE,"Typechecking with mypy")$(VENV_MYPY) --config-file pyproject.toml+merge-check-pylint: env+ $(call PRINT_TITLE,"Linting with pylint")+ $(VENV_PYLINT) --rcfile pyproject.toml .+############################################################################################# MISCELLANEOUS##########################################################################################@@ -447,11 +455,11 @@ docs-deploy: env### SHORTHANDS##########################################################################################-c: format lint pyright mypy+c: format lint pyright pylint mypy@echo "> done: c = check"cc: cleanderived c- @echo "> done: cc = cleanderived format lint pyright mypy"+ @echo "> done: cc = cleanderived format lint pyright pylint mypy"check: cc check-unused-imports@echo "> done: check"diff --git a/pipelex/cli/_cli.py b/pipelex/cli/_cli.pyindex 8ca16df2e..b7dc649dd 100644--- a/pipelex/cli/_cli.py+++ b/pipelex/cli/_cli.py@@ -1,5 +1,3 @@-from typing import Optional-import typerfrom click import Command, Contextfrom typer.core import TyperGroup@@ -11,7 +9,7 @@ from pipelex.cli.commands.build_cmd import build_appclass PipelexCLI(TyperGroup):@override- def get_command(self, ctx: Context, cmd_name: str) -> Optional[Command]:+ def get_command(self, ctx: Context, cmd_name: str) -> Command | None:cmd = super().get_command(ctx, cmd_name)if cmd is None:typer.echo(f"Unknown command: {cmd_name}")diff --git a/pipelex/cli/commands/build_cmd.py b/pipelex/cli/commands/build_cmd.pyindex cdd4ea3bf..fb64deadb 100644--- a/pipelex/cli/commands/build_cmd.py+++ b/pipelex/cli/commands/build_cmd.py@@ -1,6 +1,6 @@import asyncioimport time-from typing import Annotated, Optional+from typing import Annotatedimport typer@@ -30,7 +30,7 @@ def build_pipe_cmd(typer.Argument(help="Brief description of what the pipeline should do"),],output_path: Annotated[- Optional[str],+ str | None,typer.Option("--output", "-o", help="Path to save the generated PLX file (use --output='' to skip saving)"),] = "./generated_pipeline.plx",) -> None:diff --git a/pipelex/cli/commands/init_cmd.py b/pipelex/cli/commands/init_cmd.pyindex b1b8b68f0..60861e604 100644--- a/pipelex/cli/commands/init_cmd.py+++ b/pipelex/cli/commands/init_cmd.py@@ -1,7 +1,7 @@import osimport shutilfrom importlib.metadata import metadata-from typing import Annotated, List+from typing import Annotatedimport typer@@ -26,7 +26,8 @@ def do_init_libraries(directory: str = ".", overwrite: bool = False) -> None:else:typer.echo(f"✅ Successfully initialized pipelex libraries at '{target_dir}' (only created non-existing files)")except Exception as exc:- raise PipelexCLIError(f"Failed to initialize libraries at '{directory}': {exc}") from exc+ msg = f"Failed to initialize libraries at '{directory}': {exc}"+ raise PipelexCLIError(msg) from excdef do_init_config(reset: bool = False) -> None:@@ -37,8 +38,8 @@ def do_init_config(reset: bool = False) -> None:os.makedirs(target_config_dir, exist_ok=True)try:- copied_files: List[str] = []- existing_files: List[str] = []+ copied_files: list[str] = []+ existing_files: list[str] = []def copy_directory_structure(src_dir: str, dst_dir: str, relative_path: str = "") -> None:"""Recursively copy directory structure, handling existing files."""@@ -50,12 +51,11 @@ def do_init_config(reset: bool = False) -> None:if os.path.isdir(src_item):os.makedirs(dst_item, exist_ok=True)copy_directory_structure(src_item, dst_item, relative_item)+ elif os.path.exists(dst_item) and not reset:+ existing_files.append(relative_item)else:- if os.path.exists(dst_item) and not reset:- existing_files.append(relative_item)- else:- shutil.copy2(src_item, dst_item)- copied_files.append(relative_item)+ shutil.copy2(src_item, dst_item)+ copied_files.append(relative_item)copy_directory_structure(config_template_dir, target_config_dir)@@ -74,7 +74,8 @@ def do_init_config(reset: bool = False) -> None:typer.echo(f"✅ Configuration directory {target_config_dir} is already up to date")except Exception as exc:- raise PipelexCLIError(f"Failed to initialize configuration: {exc}") from exc+ msg = f"Failed to initialize configuration: {exc}"+ raise PipelexCLIError(msg) from exc# Typer group for init commandsdiff --git a/pipelex/cli/commands/migrate_cmd.py b/pipelex/cli/commands/migrate_cmd.pyindex c6cd93cb9..28482710d 100644--- a/pipelex/cli/commands/migrate_cmd.py+++ b/pipelex/cli/commands/migrate_cmd.py@@ -1,7 +1,7 @@from __future__ import annotationsfrom pathlib import Path-from typing import Annotated+from typing import TYPE_CHECKING, Annotatedimport typer@@ -9,6 +9,9 @@ from pipelex.cli.commands.common import is_pipelex_libraries_folderfrom pipelex.migration.migrate_v0_1_0_to_v0_2_0 import TOMLMigrator, migrate_concept_syntaxfrom pipelex.migration.migration_result import MigrationResult+if TYPE_CHECKING:+ from pipelex.migration.migration_result import MigrationResult+def do_migrate(target_path: str = "./pipelex_libraries",@@ -59,7 +62,7 @@ def do_migrate(base_for_print = pipelines_dir_or_file if pipelines_dir_or_file.is_dir() else pipelines_dir_or_file.parentfor file_path in result.modified_files:try:- with open(file_path, "r", encoding="utf-8") as f:+ with open(file_path, encoding="utf-8") as f:content = f.read()changes = migrator.get_migration_preview(content)@@ -94,10 +97,10 @@ def do_migrate(typer.echo(" Run 'pipelex validate all -c pipelex/libraries' to verify the migration")except FileNotFoundError as exc:typer.echo(f"❌ {exc}")- raise typer.Exit(1)+ raise typer.Exit(1) from excexcept Exception as exc:typer.echo(f"❌ Migration failed: {exc}")- raise typer.Exit(1)+ raise typer.Exit(1) from exc# Typer group for migration commandsdiff --git a/pipelex/cli/commands/show_cmd.py b/pipelex/cli/commands/show_cmd.pyindex 4c23ec10d..e74f3ffc2 100644--- a/pipelex/cli/commands/show_cmd.py+++ b/pipelex/cli/commands/show_cmd.py@@ -20,7 +20,8 @@ def do_show_config() -> None:title=f"Pipelex configuration for project: {config_manager.get_project_name()}",)except Exception as exc:- raise PipelexConfigError(f"Error loading configuration: {exc}") from exc+ msg = f"Error loading configuration: {exc}"+ raise PipelexConfigError(msg) from excdef do_list_pipes(relative_config_folder_path: str = "pipelex_libraries") -> None:@@ -30,7 +31,8 @@ def do_list_pipes(relative_config_folder_path: str = "pipelex_libraries") -> Nontry:get_pipe_provider().pretty_list_pipes()except Exception as exc:- raise PipelexCLIError(f"Failed to list pipes: {exc}") from exc+ msg = f"Failed to list pipes: {exc}"+ raise PipelexCLIError(msg) from excdef do_show_pipe(pipe_code: str, relative_config_folder_path: str = "./pipelex_libraries") -> None:@@ -52,7 +54,8 @@ def show_config_cmd() -> None:@show_app.command("pipes")def list_pipes_cmd(relative_config_folder_path: Annotated[- str, typer.Option("--config-folder-path", "-c", help="Relative path to the config folder path")+ str,+ typer.Option("--config-folder-path", "-c", help="Relative path to the config folder path"),] = "pipelex_libraries",) -> None:do_list_pipes(relative_config_folder_path=relative_config_folder_path)@@ -62,7 +65,8 @@ def list_pipes_cmd(def show_pipe_cmd(pipe_code: Annotated[str, typer.Argument(help="Pipeline code to show definition for")],relative_config_folder_path: Annotated[- str, typer.Option("--config-folder-path", "-c", help="Relative path to the config folder path")+ str,+ typer.Option("--config-folder-path", "-c", help="Relative path to the config folder path"),] = "./pipelex_libraries",) -> None:do_show_pipe(pipe_code=pipe_code, relative_config_folder_path=relative_config_folder_path)diff --git a/pipelex/cli/commands/validate_cmd.py b/pipelex/cli/commands/validate_cmd.pyindex bf99797ce..fcab31d02 100644--- a/pipelex/cli/commands/validate_cmd.py+++ b/pipelex/cli/commands/validate_cmd.py@@ -39,7 +39,7 @@ def do_dry_run_pipe(pipe_code: str, relative_config_folder_path: str = "./pipeledry_run_pipe(get_pipe_provider().get_required_pipe(pipe_code=pipe_code),raise_on_failure=True,- )+ ),)get_pipeline_tracker().output_flowchart()@@ -51,7 +51,8 @@ validate_app = typer.Typer(help="Validation and dry-run commands", no_args_is_he@validate_app.command("all")def validate_all_cmd(relative_config_folder_path: Annotated[- str, typer.Option("--config-folder-path", "-c", help="Relative path to the config folder path")+ str,+ typer.Option("--config-folder-path", "-c", help="Relative path to the config folder path"),] = "./pipelex_libraries",) -> None:do_validate_all_libraries_and_dry_run(relative_config_folder_path=relative_config_folder_path)@@ -61,7 +62,8 @@ def validate_all_cmd(def dry_run_pipe_cmd(pipe_code: Annotated[str, typer.Argument(help="The pipe code to dry run")],relative_config_folder_path: Annotated[- str, typer.Option("--config-folder-path", "-c", help="Relative path to the config folder path")+ str,+ typer.Option("--config-folder-path", "-c", help="Relative path to the config folder path"),] = "./pipelex_libraries",) -> None:do_dry_run_pipe(pipe_code=pipe_code, relative_config_folder_path=relative_config_folder_path)diff --git a/tests/e2e/pipelex/tools/__init__.py b/pipelex/client/__init__.pysimilarity index 100%rename from tests/e2e/pipelex/tools/__init__.pyrename to pipelex/client/__init__.pydiff --git a/pipelex/client/api_serializer.py b/pipelex/client/api_serializer.pyindex 7d8ef22ee..9cec0aa91 100644--- a/pipelex/client/api_serializer.py+++ b/pipelex/client/api_serializer.py@@ -2,13 +2,15 @@ from datetime import datetimefrom decimal import Decimalfrom enum import Enumfrom pathlib import Path-from typing import Any, Dict, List, Optional, cast+from typing import TYPE_CHECKING, Any, castfrom pipelex.client.protocol import CompactMemoryfrom pipelex.core.concepts.concept_native import NativeConceptEnumfrom pipelex.core.memory.working_memory import WorkingMemoryfrom pipelex.core.pipes.pipe_output import PipeOutput-from pipelex.core.stuffs.stuff_content import TextContent++if TYPE_CHECKING:+ from pipelex.core.stuffs.stuff_content import TextContentclass ApiSerializer:@@ -19,15 +21,15 @@ class ApiSerializer:FIELDS_TO_SKIP = ("__class__", "__module__")@classmethod- def serialize_working_memory_for_api(cls, working_memory: Optional[WorkingMemory] = None) -> CompactMemory:- """- Convert WorkingMemory to API-ready format using kajson with proper datetime handling.+ def serialize_working_memory_for_api(cls, working_memory: WorkingMemory | None = None) -> CompactMemory:+ """Convert WorkingMemory to API-ready format using kajson with proper datetime handling.Args:working_memory: The WorkingMemory to serializeReturns:Dict ready for API transmission with datetime strings and no __class__/__module__+"""compact_memory: CompactMemory = {}if working_memory is None:@@ -35,8 +37,8 @@ class ApiSerializer:for stuff_name, stuff in working_memory.root.items():if NativeConceptEnum.is_text(concept_code=stuff.concept.code):- stuff_content = cast(TextContent, stuff.content)- item_dict: Dict[str, Any] = {+ stuff_content = cast("TextContent", stuff.content)+ item_dict: dict[str, Any] = {"concept_code": stuff.concept.code,"content": stuff_content.text,}@@ -55,41 +57,40 @@ class ApiSerializer:@classmethoddef serialize_pipe_output_for_api(cls, pipe_output: PipeOutput) -> CompactMemory:- """- Convert PipeOutput to API-ready format.+ """Convert PipeOutput to API-ready format.Args:pipe_output: The PipeOutput to serializeReturns:Dict ready for API transmission+"""return {"compact_memory": cls.serialize_working_memory_for_api(pipe_output.working_memory)}@classmethoddef _clean_and_format_content(cls, content: Any) -> Any:- """- Recursively clean content by removing the fields in FIELDS_TO_SKIP and formatting datetimes.+ """Recursively clean content by removing the fields in FIELDS_TO_SKIP and formatting datetimes.Args:content: Content to cleanReturns:Cleaned content with formatted datetimes+"""if isinstance(content, dict):- cleaned: Dict[str, Any] = {}- content_dict = cast(Dict[str, Any], content)+ cleaned: dict[str, Any] = {}+ content_dict = cast("dict[str, Any]", content)for key in content_dict:if key in cls.FIELDS_TO_SKIP:continuecleaned[key] = cls._clean_and_format_content(content_dict[key])return cleanedelif isinstance(content, list):- cleaned_list: List[Any] = []- content_list = cast(List[Any], content)- for idx in range(len(content_list)):- cleaned_list.append(cls._clean_and_format_content(content_list[idx]))+ cleaned_list: list[Any] = []+ content_list = cast("list[Any]", content)+ cleaned_list.extend(cls._clean_and_format_content(content_list[idx]) for idx in range(len(content_list)))return cleaned_listelif isinstance(content, datetime):return content.strftime(cls.API_DATETIME_FORMAT)diff --git a/pipelex/client/client.py b/pipelex/client/client.pyindex 7da3480ed..99f133027 100644--- a/pipelex/client/client.py+++ b/pipelex/client/client.py@@ -1,4 +1,4 @@-from typing import Any, Optional+from typing import Anyimport httpxfrom typing_extensions import override@@ -14,31 +14,34 @@ from pipelex.tools.environment import get_required_envclass PipelexClient(PipelexProtocol):- """- A client for interacting with Pipelex pipelines through the API.+ """A client for interacting with Pipelex pipelines through the API.This client provides a user-friendly interface for executing pipelines throughthe remote API.+Args:api_token: The API token to use for authentication. If not provided, it will be loaded from the PIPELEX_API_TOKEN environment variable.If the environment variable is not set, an error will be raised.+"""def __init__(self,- api_token: Optional[str] = None,- api_base_url: Optional[str] = None,+ api_token: str | None = None,+ api_base_url: str | None = None,):self.api_token = api_token or get_required_env("PIPELEX_API_TOKEN")if not self.api_token:- raise ClientAuthenticationError("API token is required for API execution")+ msg = "API token is required for API execution"+ raise ClientAuthenticationError(msg)self.api_base_url = api_base_url or get_required_env("PIPELEX_API_BASE_URL")if not self.api_base_url:- raise ClientAuthenticationError("API base URL is required for API execution")+ msg = "API base URL is required for API execution"+ raise ClientAuthenticationError(msg)- self.client: Optional[httpx.AsyncClient] = None+ self.client: httpx.AsyncClient | None = Nonedef start_client(self) -> "PipelexClient":"""Initialize the HTTP client for API calls."""@@ -51,8 +54,9 @@ class PipelexClient(PipelexProtocol):await self.client.aclose()self.client = None- async def _make_api_call(self, endpoint: str, request: Optional[str] = None) -> Any:+ async def _make_api_call(self, endpoint: str, request: str | None = None) -> Any:"""Make an API call to the Pipelex server.+Args:endpoint: The API endpoint to call, relative to the base URLrequest: A JSON-formatted string to send as the request body, or None if no body is needed@@ -60,6 +64,7 @@ class PipelexClient(PipelexProtocol):Any: The JSON-decoded response from the serverRaises:httpx.HTTPError: If the request fails or returns a non-200 status code+"""if not self.client:self.start_client()@@ -75,14 +80,15 @@ class PipelexClient(PipelexProtocol):async def execute_pipeline(self,pipe_code: str,- working_memory: Optional[WorkingMemory] = None,- input_memory: Optional[CompactMemory] = None,- output_name: Optional[str] = None,- output_multiplicity: Optional[PipeOutputMultiplicity] = None,- dynamic_output_concept_code: Optional[str] = None,+ working_memory: WorkingMemory | None = None,+ input_memory: CompactMemory | None = None,+ output_name: str | None = None,+ output_multiplicity: PipeOutputMultiplicity | None = None,+ dynamic_output_concept_code: str | None = None,) -> PipelineResponse:if working_memory and input_memory:- raise ValueError(f"working_memory and input_memory cannot be provided together to the API execute_pipeline {pipe_code=}")+ msg = f"working_memory and input_memory cannot be provided together to the API execute_pipeline {pipe_code=}"+ raise ValueError(msg)if input_memory is not None:working_memory = WorkingMemoryFactory.make_from_compact_memory(input_memory)@@ -99,14 +105,15 @@ class PipelexClient(PipelexProtocol):async def start_pipeline(self,pipe_code: str,- working_memory: Optional[WorkingMemory] = None,- input_memory: Optional[CompactMemory] = None,- output_name: Optional[str] = None,- output_multiplicity: Optional[PipeOutputMultiplicity] = None,- dynamic_output_concept_code: Optional[str] = None,+ working_memory: WorkingMemory | None = None,+ input_memory: CompactMemory | None = None,+ output_name: str | None = None,+ output_multiplicity: PipeOutputMultiplicity | None = None,+ dynamic_output_concept_code: str | None = None,) -> PipelineResponse:if working_memory and input_memory:- raise ValueError(f"working_memory and input_memory cannot be provided together to the API start_pipeline {pipe_code=}")+ msg = f"working_memory and input_memory cannot be provided together to the API start_pipeline {pipe_code=}"+ raise ValueError(msg)if input_memory is not None:working_memory = WorkingMemoryFactory.make_from_compact_memory(input_memory)diff --git a/pipelex/client/pipeline_request_factory.py b/pipelex/client/pipeline_request_factory.pyindex 51b732806..9cc57c2a2 100644--- a/pipelex/client/pipeline_request_factory.py+++ b/pipelex/client/pipeline_request_factory.py@@ -1,4 +1,4 @@-from typing import Any, Dict, Optional+from typing import Anyfrom pipelex.client.api_serializer import ApiSerializerfrom pipelex.client.protocol import COMPACT_MEMORY_KEY, PipelineRequest@@ -11,13 +11,12 @@ class PipelineRequestFactory:@staticmethoddef make_from_working_memory(- working_memory: Optional[WorkingMemory] = None,- output_name: Optional[str] = None,- output_multiplicity: Optional[PipeOutputMultiplicity] = None,- dynamic_output_concept_code: Optional[str] = None,+ working_memory: WorkingMemory | None = None,+ output_name: str | None = None,+ output_multiplicity: PipeOutputMultiplicity | None = None,+ dynamic_output_concept_code: str | None = None,) -> PipelineRequest:- """- Create a PipelineRequest from a WorkingMemory object.+ """Create a PipelineRequest from a WorkingMemory object.Args:working_memory: The WorkingMemory to convert@@ -27,8 +26,8 @@ class PipelineRequestFactory:Returns:PipelineRequest with the working memory serialized to reduced format- """+ """return PipelineRequest(input_memory=ApiSerializer.serialize_working_memory_for_api(working_memory),output_name=output_name,@@ -37,15 +36,15 @@ class PipelineRequestFactory:)@staticmethod- def make_from_body(request_body: Dict[str, Any]) -> PipelineRequest:- """- Create a PipelineRequest from raw request body dictionary.+ def make_from_body(request_body: dict[str, Any]) -> PipelineRequest:+ """Create a PipelineRequest from raw request body dictionary.Args:request_body: Raw dictionary from API request bodyReturns:PipelineRequest object with dictionary working_memory+"""return PipelineRequest(input_memory=request_body.get(COMPACT_MEMORY_KEY),diff --git a/pipelex/client/pipeline_response_factory.py b/pipelex/client/pipeline_response_factory.pyindex f0e70db84..1d1d947fa 100644--- a/pipelex/client/pipeline_response_factory.py+++ b/pipelex/client/pipeline_response_factory.py@@ -1,4 +1,4 @@-from typing import Any, Dict, Optional+from typing import Anyfrom pipelex.client.api_serializer import ApiSerializerfrom pipelex.client.protocol import PipelineResponse, PipelineState@@ -10,17 +10,16 @@ class PipelineResponseFactory:@staticmethoddef make_from_pipe_output(- pipe_output: Optional[PipeOutput] = None,+ pipe_output: PipeOutput | None = None,pipeline_run_id: str = "",created_at: str = "",pipeline_state: PipelineState = PipelineState.COMPLETED,- finished_at: Optional[str] = None,- status: Optional[str] = "success",- message: Optional[str] = None,- error: Optional[str] = None,+ finished_at: str | None = None,+ status: str | None = "success",+ message: str | None = None,+ error: str | None = None,) -> PipelineResponse:- """- Create a PipelineResponse from a PipeOutput object.+ """Create a PipelineResponse from a PipeOutput object.Args:pipe_output: The PipeOutput to convert@@ -34,6 +33,7 @@ class PipelineResponseFactory:Returns:PipelineResponse with the pipe output serialized to reduced format+"""compact_output = Noneif pipe_output is not None:@@ -51,14 +51,14 @@ class PipelineResponseFactory:)@staticmethod- def make_from_api_response(response: Dict[str, Any]) -> PipelineResponse:- """- Create a PipelineResponse from an API response dictionary.+ def make_from_api_response(response: dict[str, Any]) -> PipelineResponse:+ """Create a PipelineResponse from an API response dictionary.Args:response: Dictionary containing the API response dataReturns:PipelineResponse instance created from the response data+"""- return PipelineResponse(**response)+ return PipelineResponse.model_validate(response)diff --git a/pipelex/client/protocol.py b/pipelex/client/protocol.pyindex cddce3300..f47172cf5 100644--- a/pipelex/client/protocol.py+++ b/pipelex/client/protocol.py@@ -1,5 +1,5 @@from abc import abstractmethod-from typing import Any, Dict, List, Optional, Protocol+from typing import Any, Protocolfrom pydantic import BaseModelfrom typing_extensions import runtime_checkable@@ -9,16 +9,14 @@ from pipelex.core.pipes.pipe_run_params import PipeOutputMultiplicityfrom pipelex.core.stuffs.stuff_content import StuffContentfrom pipelex.types import StrEnum-StuffContentOrData = Dict[str, Any] | StuffContent | List[Any] | str-ImplicitMemory = Dict[str, StuffContentOrData]-CompactMemory = Dict[str, Dict[str, Any]]+StuffContentOrData = dict[str, Any] | StuffContent | list[Any] | str+ImplicitMemory = dict[str, StuffContentOrData]+CompactMemory = dict[str, dict[str, Any]]COMPACT_MEMORY_KEY = "compact_memory"class PipelineState(StrEnum):- """- Enum representing the possible states of a pipe execution.- """+ """Enum representing the possible states of a pipe execution."""RUNNING = "RUNNING"COMPLETED = "COMPLETED"@@ -29,47 +27,46 @@ class PipelineState(StrEnum):class ApiResponse(BaseModel):- """- Base response class for Pipelex API calls.+ """Base response class for Pipelex API calls.Attributes:- status (Optional[str]): Status of the API call ("success", "error", etc.)- message (Optional[str]): Optional message providing additional information- error (Optional[str]): Optional error message when status is not "success"+ status (str | None): Status of the API call ("success", "error", etc.)+ message (str | None): Optional message providing additional information+ error (str | None): Optional error message when status is not "success"+"""- status: Optional[str] = None- message: Optional[str] = None- error: Optional[str] = None+ status: str | None = None+ message: str | None = None+ error: str | None = Noneclass PipelineRequest(BaseModel):- """- Request for executing a pipeline.+ """Request for executing a pipeline.Attributes:- input_memory (Optional[CompactMemory]): In the format of WorkingMemory.to_compact_memory()- output_name (Optional[str]): Name of the output slot to write to- output_multiplicity (Optional[PipeOutputMultiplicity]): Output multiplicity setting- dynamic_output_concept_code (Optional[str]): Override for the dynamic output concept code+ input_memory (CompactMemory | None): In the format of WorkingMemory.to_compact_memory()+ output_name (str | None): Name of the output slot to write to+ output_multiplicity (PipeOutputMultiplicity | None): Output multiplicity setting+ dynamic_output_concept_code (str | None): Override for the dynamic output concept code+"""- input_memory: Optional[CompactMemory] = None- output_name: Optional[str] = None- output_multiplicity: Optional[PipeOutputMultiplicity] = None- dynamic_output_concept_code: Optional[str] = None+ input_memory: CompactMemory | None = None+ output_name: str | None = None+ output_multiplicity: PipeOutputMultiplicity | None = None+ dynamic_output_concept_code: str | None = Noneclass PipelineResponse(ApiResponse):- """- Response for pipeline execution requests.+ """Response for pipeline execution requests.Attributes:pipeline_run_id (str): Unique identifier for the pipeline runcreated_at (str): Timestamp when the pipeline was createdpipeline_state (PipelineState): Current state of the pipeline- finished_at (Optional[str]): Timestamp when the pipeline finished, if completed- pipe_output (Optional[CompactMemory]): Output data from the pipeline execution as raw dict, if available+ finished_at (str | None): Timestamp when the pipeline finished, if completed+ pipe_output (CompactMemory | None): Output data from the pipeline execution as raw dict, if availableExample of pipe_output:"pipe_output": {@@ -98,26 +95,27 @@ class PipelineResponse(ApiResponse):}}}+"""pipeline_run_id: strcreated_at: strpipeline_state: PipelineState- finished_at: Optional[str] = None- pipe_output: Optional[CompactMemory] = None+ finished_at: str | None = None+ pipe_output: CompactMemory | None = None@runtime_checkableclass PipelexProtocol(Protocol):- """- Protocol defining the contract for the Pipelex API.+ """Protocol defining the contract for the Pipelex API.This protocol specifies the interface that any Pipelex API implementation must adhere to.All methods are asynchronous and handle pipeline execution, monitoring, and control.Attributes:- api_token (Optional[str]): Authentication token for API access- api_base_url (Optional[str]): Base URL for the API+ api_token (str): Authentication token for API access+ api_base_url (str): Base URL for the API+"""api_token: str@@ -127,28 +125,28 @@ class PipelexProtocol(Protocol):async def execute_pipeline(self,pipe_code: str,- working_memory: Optional[WorkingMemory] = None,- input_memory: Optional[CompactMemory] = None,- output_name: Optional[str] = None,- output_multiplicity: Optional[PipeOutputMultiplicity] = None,- dynamic_output_concept_code: Optional[str] = None,+ working_memory: WorkingMemory | None = None,+ input_memory: CompactMemory | None = None,+ output_name: str | None = None,+ output_multiplicity: PipeOutputMultiplicity | None = None,+ dynamic_output_concept_code: str | None = None,) -> PipelineResponse:- """- Execute a pipeline synchronously and wait for its completion.+ """Execute a pipeline synchronously and wait for its completion.Args:pipe_code (str): The code identifying the pipeline to execute- working_memory (Optional[WorkingMemory]): Memory context passed to the pipeline- input_memory (Optional[CompactMemory]): Input memory passed to the pipeline- output_name (Optional[str]): Target output slot name- output_multiplicity (Optional[PipeOutputMultiplicity]): Output multiplicity setting- dynamic_output_concept_code (Optional[str]): Override for dynamic output concept+ working_memory (WorkingMemory | None): Memory context passed to the pipeline+ input_memory (CompactMemory | None): Input memory passed to the pipeline+ output_name (str | None): Target output slot name+ output_multiplicity (PipeOutputMultiplicity | None): Output multiplicity setting+ dynamic_output_concept_code (str | None): Override for dynamic output conceptReturns:PipelineResponse: Complete execution results including pipeline state and outputRaises:HTTPException: On execution failure or errorClientAuthenticationError: If API token is missing for API execution+"""...@@ -156,22 +154,21 @@ class PipelexProtocol(Protocol):async def start_pipeline(self,pipe_code: str,- working_memory: Optional[WorkingMemory] = None,- input_memory: Optional[CompactMemory] = None,- output_name: Optional[str] = None,- output_multiplicity: Optional[PipeOutputMultiplicity] = None,- dynamic_output_concept_code: Optional[str] = None,+ working_memory: WorkingMemory | None = None,+ input_memory: CompactMemory | None = None,+ output_name: str | None = None,+ output_multiplicity: PipeOutputMultiplicity | None = None,+ dynamic_output_concept_code: str | None = None,) -> PipelineResponse:- """- Start a pipeline execution asynchronously without waiting for completion.+ """Start a pipeline execution asynchronously without waiting for completion.Args:pipe_code (str): The code identifying the pipeline to execute- working_memory (Optional[WorkingMemory]): Memory context passed to the pipeline- input_memory (Optional[CompactMemory]): Input memory passed to the pipeline- output_name (Optional[str]): Target output slot name- output_multiplicity (Optional[PipeOutputMultiplicity]): Output multiplicity setting- dynamic_output_concept_code (Optional[str]): Override for dynamic output concept+ working_memory (WorkingMemory | None): Memory context passed to the pipeline+ input_memory (CompactMemory | None): Input memory passed to the pipeline+ output_name (str | None): Target output slot name+ output_multiplicity (PipeOutputMultiplicity | None): Output multiplicity setting+ dynamic_output_concept_code (str | None): Override for dynamic output conceptReturns:PipelineResponse: Initial response with pipeline_run_id and created_at timestamp@@ -179,5 +176,6 @@ class PipelexProtocol(Protocol):Raises:HTTPException: On pipeline start failureClientAuthenticationError: If API token is missing for API execution+"""...diff --git a/pipelex/cogt/config_cogt.py b/pipelex/cogt/config_cogt.pyindex b29539574..478e108e3 100644--- a/pipelex/cogt/config_cogt.py+++ b/pipelex/cogt/config_cogt.py@@ -1,5 +1,3 @@-from typing import List-from pipelex.cogt.img_gen.img_gen_job_components import ImgGenJobConfig, ImgGenJobParams, ImgGenJobParamsDefaultsfrom pipelex.cogt.llm.llm_job_components import LLMJobConfigfrom pipelex.plugins.fal.fal_config import FalConfig@@ -52,7 +50,7 @@ class InferenceConfig(ConfigModel):def model_specs_path(self, backend_name: str) -> str:return f"{self.inference_config_path}/backends/{backend_name}.toml"- def get_model_deck_paths(self) -> List[str]:+ def get_model_deck_paths(self) -> list[str]:"""Get all LLM deck TOML file paths sorted alphabetically."""model_deck_paths = [str(path)diff --git a/pipelex/cogt/content_generation/assignment_models.py b/pipelex/cogt/content_generation/assignment_models.pyindex 3a3e5e0e7..efd3da823 100644--- a/pipelex/cogt/content_generation/assignment_models.py+++ b/pipelex/cogt/content_generation/assignment_models.py@@ -1,4 +1,4 @@-from typing import Any, Dict, Optional, Type+from typing import Anyfrom pydantic import BaseModelfrom typing_extensions import override@@ -25,8 +25,8 @@ class LLMAssignmentFactory(BaseModel):async def make_llm_assignment(self,- job_metadata: Optional[JobMetadata] = None,- llm_setting: Optional[LLMSetting] = None,+ job_metadata: JobMetadata | None = None,+ llm_setting: LLMSetting | None = None,**prompt_arguments: Any,) -> "LLMAssignment":llm_prompt = await self.llm_prompt_factory.make_llm_prompt_from_args(**prompt_arguments)@@ -95,7 +95,7 @@ class ObjectAssignment(BaseModel):@staticmethoddef make_for_class(- object_class: Type[BaseModel],+ object_class: type[BaseModel],llm_assignment: LLMAssignment,) -> "ObjectAssignment":object_class_name = object_class.__name__@@ -127,10 +127,10 @@ class ImgGenAssignment(BaseModel):class Jinja2Assignment(BaseModel):- context: Dict[str, Any]- jinja2_name: Optional[str] = None- jinja2: Optional[str] = None- prompting_style: Optional[PromptingStyle] = None+ context: dict[str, Any]+ jinja2_name: str | None = None+ jinja2: str | None = None+ prompting_style: PromptingStyle | None = Nonetemplate_category: Jinja2TemplateCategory = Jinja2TemplateCategory.LLM_PROMPTdiff --git a/pipelex/cogt/content_generation/content_generator.py b/pipelex/cogt/content_generation/content_generator.pyindex 5da23a8e2..0280a4ef8 100644--- a/pipelex/cogt/content_generation/content_generator.py+++ b/pipelex/cogt/content_generation/content_generator.py@@ -1,4 +1,4 @@-from typing import Any, Dict, List, Optional, Type, cast+from typing import Any, castfrom typing_extensions import override@@ -60,7 +60,7 @@ class ContentGenerator(ContentGeneratorProtocol):async def make_object_direct( # pyright: ignore[reportIncompatibleMethodOverride]self,job_metadata: JobMetadata,- object_class: Type[BaseModelTypeVar],+ object_class: type[BaseModelTypeVar],llm_setting_for_object: LLMSetting,llm_prompt_for_object: LLMPrompt,) -> BaseModelTypeVar:@@ -76,18 +76,18 @@ class ContentGenerator(ContentGeneratorProtocol):)obj = await llm_gen_object(object_assignment=object_assignment)log.verbose(f"{self.__class__.__name__} generated object direct: {obj}")- return cast(BaseModelTypeVar, obj)+ return cast("BaseModelTypeVar", obj)@override@update_job_metadataasync def make_text_then_object( # pyright: ignore[reportIncompatibleMethodOverride]self,job_metadata: JobMetadata,- object_class: Type[BaseModelTypeVar],+ object_class: type[BaseModelTypeVar],llm_setting_main: LLMSetting,llm_setting_for_object: LLMSetting,llm_prompt_for_text: LLMPrompt,- llm_prompt_factory_for_object: Optional[LLMPromptFactoryAbstract] = None,+ llm_prompt_factory_for_object: LLMPromptFactoryAbstract | None = None,) -> BaseModelTypeVar:log.verbose(llm_prompt_for_text.user_text, title="llm_prompt_for_text")llm_assignment_for_text = LLMAssignment.make_from_prompt(@@ -123,18 +123,18 @@ class ContentGenerator(ContentGeneratorProtocol):obj = await llm_gen_object(object_assignment=fup_obj_assignment)log.verbose(f"{self.__class__.__name__} generated object after text: {obj}")- return cast(BaseModelTypeVar, obj)+ return cast("BaseModelTypeVar", obj)@override@update_job_metadataasync def make_object_list_direct( # pyright: ignore[reportIncompatibleMethodOverride]self,job_metadata: JobMetadata,- object_class: Type[BaseModelTypeVar],+ object_class: type[BaseModelTypeVar],llm_setting_for_object_list: LLMSetting,llm_prompt_for_object_list: LLMPrompt,- nb_items: Optional[int] = None,- ) -> List[BaseModelTypeVar]:+ nb_items: int | None = None,+ ) -> list[BaseModelTypeVar]:llm_assignment_for_object = LLMAssignment.make_from_prompt(job_metadata=job_metadata,llm_setting=llm_setting_for_object_list,@@ -146,20 +146,20 @@ class ContentGenerator(ContentGeneratorProtocol):)obj_list = await llm_gen_object_list(object_assignment=object_assignment)log.verbose(f"{self.__class__.__name__} generated object list direct: {obj_list}")- return cast(List[BaseModelTypeVar], obj_list)+ return cast("list[BaseModelTypeVar]", obj_list)@override@update_job_metadataasync def make_text_then_object_list( # pyright: ignore[reportIncompatibleMethodOverride]self,job_metadata: JobMetadata,- object_class: Type[BaseModelTypeVar],+ object_class: type[BaseModelTypeVar],llm_setting_main: LLMSetting,llm_setting_for_object_list: LLMSetting,llm_prompt_for_text: LLMPrompt,- llm_prompt_factory_for_object_list: Optional[LLMPromptFactoryAbstract] = None,- nb_items: Optional[int] = None,- ) -> List[BaseModelTypeVar]:+ llm_prompt_factory_for_object_list: LLMPromptFactoryAbstract | None = None,+ nb_items: int | None = None,+ ) -> list[BaseModelTypeVar]:llm_assignment_for_text = LLMAssignment.make_from_prompt(job_metadata=job_metadata,llm_setting=llm_setting_main,@@ -192,7 +192,7 @@ class ContentGenerator(ContentGeneratorProtocol):obj_list = await llm_gen_object_list(object_assignment=fup_obj_assignment)log.verbose(f"{self.__class__.__name__} generated object list after text: {obj_list}")- return cast(List[BaseModelTypeVar], obj_list)+ return cast("list[BaseModelTypeVar]", obj_list)@override@update_job_metadata@@ -201,8 +201,8 @@ class ContentGenerator(ContentGeneratorProtocol):job_metadata: JobMetadata,img_gen_handle: str,img_gen_prompt: ImgGenPrompt,- img_gen_job_params: Optional[ImgGenJobParams] = None,- img_gen_job_config: Optional[ImgGenJobConfig] = None,+ img_gen_job_params: ImgGenJobParams | None = None,+ img_gen_job_config: ImgGenJobConfig | None = None,) -> GeneratedImage:img_gen_config = get_config().cogt.img_gen_configimg_gen_assignment = ImgGenAssignment(@@ -225,9 +225,9 @@ class ContentGenerator(ContentGeneratorProtocol):img_gen_handle: str,img_gen_prompt: ImgGenPrompt,nb_images: int,- img_gen_job_params: Optional[ImgGenJobParams] = None,- img_gen_job_config: Optional[ImgGenJobConfig] = None,- ) -> List[GeneratedImage]:+ img_gen_job_params: ImgGenJobParams | None = None,+ img_gen_job_config: ImgGenJobConfig | None = None,+ ) -> list[GeneratedImage]:img_gen_config = get_config().cogt.img_gen_configimg_gen_assignment = ImgGenAssignment(job_metadata=job_metadata,@@ -244,10 +244,10 @@ class ContentGenerator(ContentGeneratorProtocol):@overrideasync def make_jinja2_text(self,- context: Dict[str, Any],- jinja2_name: Optional[str] = None,- jinja2: Optional[str] = None,- prompting_style: Optional[PromptingStyle] = None,+ context: dict[str, Any],+ jinja2_name: str | None = None,+ jinja2: str | None = None,+ prompting_style: PromptingStyle | None = None,template_category: Jinja2TemplateCategory = Jinja2TemplateCategory.LLM_PROMPT,) -> str:jinja2_assignment = Jinja2Assignment(@@ -267,8 +267,8 @@ class ContentGenerator(ContentGeneratorProtocol):job_metadata: JobMetadata,ocr_input: OcrInput,ocr_handle: str,- ocr_job_params: Optional[OcrJobParams] = None,- ocr_job_config: Optional[OcrJobConfig] = None,+ ocr_job_params: OcrJobParams | None = None,+ ocr_job_config: OcrJobConfig | None = None,) -> OcrOutput:ocr_assignment = OcrAssignment(job_metadata=job_metadata,@@ -277,5 +277,4 @@ class ContentGenerator(ContentGeneratorProtocol):ocr_job_params=ocr_job_params or OcrJobParams.make_default_ocr_job_params(),ocr_job_config=ocr_job_config or OcrJobConfig(),)- ocr_output = await ocr_gen_extract_pages(ocr_assignment=ocr_assignment)- return ocr_output+ return await ocr_gen_extract_pages(ocr_assignment=ocr_assignment)diff --git a/pipelex/cogt/content_generation/content_generator_dry.py b/pipelex/cogt/content_generation/content_generator_dry.pyindex d7df93572..baa3fb44b 100644--- a/pipelex/cogt/content_generation/content_generator_dry.py+++ b/pipelex/cogt/content_generation/content_generator_dry.py@@ -1,4 +1,4 @@-from typing import Any, Dict, List, Optional, Type+from typing import Anyfrom polyfactory.factories.pydantic_factory import ModelFactoryfrom typing_extensions import override@@ -22,8 +22,7 @@ from pipelex.tools.typing.pydantic_utils import BaseModelTypeVarclass ContentGeneratorDry(ContentGeneratorProtocol):- """- This class is used to generate mock content for testing purposes.+ """This class is used to generate mock content for testing purposes.It does not use any inference."""@@ -42,22 +41,21 @@ class ContentGeneratorDry(ContentGeneratorProtocol):func_name = "make_llm_text"log.dev(f"🤡 DRY RUN: {self.__class__.__name__}.{func_name}")prompt_truncated = llm_prompt_for_text.desc(truncate_text_length=self._text_gen_truncate_length)- generated_text = f"DRY RUN: {func_name} • llm_setting={llm_setting_main.desc()} • prompt={prompt_truncated}"- return generated_text+ return f"DRY RUN: {func_name} • llm_setting={llm_setting_main.desc()} • prompt={prompt_truncated}"@override@update_job_metadataasync def make_object_direct( # pyright: ignore[reportIncompatibleMethodOverride]self,job_metadata: JobMetadata,- object_class: Type[BaseModelTypeVar],+ object_class: type[BaseModelTypeVar],llm_setting_for_object: LLMSetting,llm_prompt_for_object: LLMPrompt,) -> BaseModelTypeVar:func_name = "make_object_direct"log.dev(f"🤡 DRY RUN: {self.__class__.__name__}.{func_name}")- class ObjectFactory(ModelFactory[object_class]): # type: ignore+ class ObjectFactory(ModelFactory[object_class]): # type: ignore[valid-type]__model__ = object_class__check_model__ = True__use_examples__ = True@@ -67,19 +65,18 @@ class ContentGeneratorDry(ContentGeneratorProtocol):# It is that way because the dry run was failing a lot of pipes that had validation test on the# field values. For example, if a string requires to be a snake_case, the ObjectFactory would# generate something like `DOIJZjoDoIJDZOjDZJo` which is... not a snake_case.- obj = ObjectFactory.build(factory_use_construct=True)- return obj+ return ObjectFactory.build(factory_use_construct=True)@override@update_job_metadataasync def make_text_then_object( # pyright: ignore[reportIncompatibleMethodOverride]self,job_metadata: JobMetadata,- object_class: Type[BaseModelTypeVar],+ object_class: type[BaseModelTypeVar],llm_setting_main: LLMSetting,llm_setting_for_object: LLMSetting,llm_prompt_for_text: LLMPrompt,- llm_prompt_factory_for_object: Optional[LLMPromptFactoryAbstract] = None,+ llm_prompt_factory_for_object: LLMPromptFactoryAbstract | None = None,) -> BaseModelTypeVar:func_name = "make_text_then_object"log.dev(f"🤡 DRY RUN: {self.__class__.__name__}.{func_name}")@@ -96,15 +93,15 @@ class ContentGeneratorDry(ContentGeneratorProtocol):async def make_object_list_direct( # pyright: ignore[reportIncompatibleMethodOverride]self,job_metadata: JobMetadata,- object_class: Type[BaseModelTypeVar],+ object_class: type[BaseModelTypeVar],llm_setting_for_object_list: LLMSetting,llm_prompt_for_object_list: LLMPrompt,- nb_items: Optional[int] = None,- ) -> List[BaseModelTypeVar]:+ nb_items: int | None = None,+ ) -> list[BaseModelTypeVar]:func_name = "make_object_list_direct"log.dev(f"🤡 DRY RUN: {self.__class__.__name__}.{func_name}")nb_list_items = nb_items or get_config().pipelex.dry_run_config.nb_list_items- objects = [+ return [await self.make_object_direct(job_metadata=job_metadata,object_class=object_class,@@ -113,20 +110,19 @@ class ContentGeneratorDry(ContentGeneratorProtocol):)for _ in range(nb_list_items)]- return objects@override@update_job_metadataasync def make_text_then_object_list( # pyright: ignore[reportIncompatibleMethodOverride]self,job_metadata: JobMetadata,- object_class: Type[BaseModelTypeVar],+ object_class: type[BaseModelTypeVar],llm_setting_main: LLMSetting,llm_setting_for_object_list: LLMSetting,llm_prompt_for_text: LLMPrompt,- llm_prompt_factory_for_object_list: Optional[LLMPromptFactoryAbstract] = None,- nb_items: Optional[int] = None,- ) -> List[BaseModelTypeVar]:+ llm_prompt_factory_for_object_list: LLMPromptFactoryAbstract | None = None,+ nb_items: int | None = None,+ ) -> list[BaseModelTypeVar]:func_name = "make_text_then_object_list"log.dev(f"🤡 DRY RUN: {self.__class__.__name__}.{func_name}")return await self.make_object_list_direct(@@ -144,19 +140,18 @@ class ContentGeneratorDry(ContentGeneratorProtocol):job_metadata: JobMetadata,img_gen_handle: str,img_gen_prompt: ImgGenPrompt,- img_gen_job_params: Optional[ImgGenJobParams] = None,- img_gen_job_config: Optional[ImgGenJobConfig] = None,+ img_gen_job_params: ImgGenJobParams | None = None,+ img_gen_job_config: ImgGenJobConfig | None = None,) -> GeneratedImage:func_name = "make_single_image"log.dev(f"🤡 DRY RUN: {self.__class__.__name__}.{func_name}")image_urls = get_config().pipelex.dry_run_config.image_urlsimage_url = image_urls[0]- generated_image = GeneratedImage(+ return GeneratedImage(url=image_url,width=1536,height=2752,)- return generated_image@override@update_job_metadata@@ -166,13 +161,13 @@ class ContentGeneratorDry(ContentGeneratorProtocol):img_gen_handle: str,img_gen_prompt: ImgGenPrompt,nb_images: int,- img_gen_job_params: Optional[ImgGenJobParams] = None,- img_gen_job_config: Optional[ImgGenJobConfig] = None,- ) -> List[GeneratedImage]:+ img_gen_job_params: ImgGenJobParams | None = None,+ img_gen_job_config: ImgGenJobConfig | None = None,+ ) -> list[GeneratedImage]:func_name = "make_image_list"log.dev(f"🤡 DRY RUN: {self.__class__.__name__}.{func_name}")image_urls = get_config().pipelex.dry_run_config.image_urls- generated_image_list = [+ return [GeneratedImage(url=image_urls[image_index % len(image_urls)],width=1536,@@ -180,26 +175,24 @@ class ContentGeneratorDry(ContentGeneratorProtocol):)for image_index in range(nb_images)]- return generated_image_list@overrideasync def make_jinja2_text(self,- context: Dict[str, Any],- jinja2_name: Optional[str] = None,- jinja2: Optional[str] = None,- prompting_style: Optional[PromptingStyle] = None,+ context: dict[str, Any],+ jinja2_name: str | None = None,+ jinja2: str | None = None,+ prompting_style: PromptingStyle | None = None,template_category: Jinja2TemplateCategory = Jinja2TemplateCategory.LLM_PROMPT,) -> str:# TODO: Use the code that checks if the jinja2 is a valid templatefunc_name = "make_jinja2_text"log.dev(f"🤡 DRY RUN: {self.__class__.__name__}.{func_name}")jinja2_truncated = jinja2[: self._text_gen_truncate_length] if jinja2 else None- jinja2_text = (+ return (f"DRY RUN: {func_name} • context={context} • jinja2_name={jinja2_name} • "f"jinja2={jinja2_truncated} • prompting_style={prompting_style} • template_category={template_category}")- return jinja2_text@overrideasync def make_ocr_extract_pages(@@ -207,8 +200,8 @@ class ContentGeneratorDry(ContentGeneratorProtocol):job_metadata: JobMetadata,ocr_input: OcrInput,ocr_handle: str,- ocr_job_params: Optional[OcrJobParams] = None,- ocr_job_config: Optional[OcrJobConfig] = None,+ ocr_job_params: OcrJobParams | None = None,+ ocr_job_config: OcrJobConfig | None = None,) -> OcrOutput:func_name = "make_ocr_extract_pages"log.dev(f"🤡 DRY RUN: {self.__class__.__name__}.{func_name}")diff --git a/pipelex/cogt/content_generation/content_generator_protocol.py b/pipelex/cogt/content_generation/content_generator_protocol.pyindex c52c139a7..936d3a3ac 100644--- a/pipelex/cogt/content_generation/content_generator_protocol.py+++ b/pipelex/cogt/content_generation/content_generator_protocol.py@@ -1,5 +1,6 @@+from collections.abc import Awaitable, Callable, Coroutinefrom functools import wraps-from typing import Any, Awaitable, Callable, Coroutine, Dict, List, Optional, ParamSpec, Protocol, Type, TypeVar+from typing import Any, ParamSpec, Protocol, TypeVarfrom pipelex.cogt.image.generated_image import GeneratedImagefrom pipelex.cogt.img_gen.img_gen_job_components import ImgGenJobConfig, ImgGenJobParams@@ -26,10 +27,12 @@ def update_job_metadata(func: Callable[P, Coroutine[Any, Any, R]]) -> Callable[P# Attempt to get job_metadata from kwargs or from argsjob_metadata = kwargs.get("job_metadata")if job_metadata is None:- raise RuntimeError("job_metadata argument is required for this decorated function.")+ msg = "job_metadata argument is required for this decorated function."+ raise RuntimeError(msg)if not isinstance(job_metadata, JobMetadata):- raise TypeError("The job_metadata argument must be of type JobMetadata.")+ msg = "The job_metadata argument must be of type JobMetadata."+ raise TypeError(msg)updated_metadata = JobMetadata(content_generation_job_id=func.__name__,@@ -52,7 +55,7 @@ class ContentGeneratorProtocol(Protocol):async def make_object_direct(self,job_metadata: JobMetadata,- object_class: Type[BaseModelTypeVar],+ object_class: type[BaseModelTypeVar],llm_setting_for_object: LLMSetting,llm_prompt_for_object: LLMPrompt,) -> BaseModelTypeVar: ...@@ -60,40 +63,40 @@ class ContentGeneratorProtocol(Protocol):async def make_text_then_object(self,job_metadata: JobMetadata,- object_class: Type[BaseModelTypeVar],+ object_class: type[BaseModelTypeVar],llm_setting_main: LLMSetting,llm_setting_for_object: LLMSetting,llm_prompt_for_text: LLMPrompt,- llm_prompt_factory_for_object: Optional[LLMPromptFactoryAbstract] = None,+ llm_prompt_factory_for_object: LLMPromptFactoryAbstract | None = None,) -> BaseModelTypeVar: ...async def make_object_list_direct(self,job_metadata: JobMetadata,- object_class: Type[BaseModelTypeVar],+ object_class: type[BaseModelTypeVar],llm_setting_for_object_list: LLMSetting,llm_prompt_for_object_list: LLMPrompt,- nb_items: Optional[int] = None,- ) -> List[BaseModelTypeVar]: ...+ nb_items: int | None = None,+ ) -> list[BaseModelTypeVar]: ...async def make_text_then_object_list(self,job_metadata: JobMetadata,- object_class: Type[BaseModelTypeVar],+ object_class: type[BaseModelTypeVar],llm_setting_main: LLMSetting,llm_setting_for_object_list: LLMSetting,llm_prompt_for_text: LLMPrompt,- llm_prompt_factory_for_object_list: Optional[LLMPromptFactoryAbstract] = None,- nb_items: Optional[int] = None,- ) -> List[BaseModelTypeVar]: ...+ llm_prompt_factory_for_object_list: LLMPromptFactoryAbstract | None = None,+ nb_items: int | None = None,+ ) -> list[BaseModelTypeVar]: ...async def make_single_image(self,job_metadata: JobMetadata,img_gen_handle: str,img_gen_prompt: ImgGenPrompt,- img_gen_job_params: Optional[ImgGenJobParams] = None,- img_gen_job_config: Optional[ImgGenJobConfig] = None,+ img_gen_job_params: ImgGenJobParams | None = None,+ img_gen_job_config: ImgGenJobConfig | None = None,) -> GeneratedImage: ...async def make_image_list(@@ -102,16 +105,16 @@ class ContentGeneratorProtocol(Protocol):img_gen_handle: str,img_gen_prompt: ImgGenPrompt,nb_images: int,- img_gen_job_params: Optional[ImgGenJobParams] = None,- img_gen_job_config: Optional[ImgGenJobConfig] = None,- ) -> List[GeneratedImage]: ...+ img_gen_job_params: ImgGenJobParams | None = None,+ img_gen_job_config: ImgGenJobConfig | None = None,+ ) -> list[GeneratedImage]: ...async def make_jinja2_text(self,- context: Dict[str, Any],- jinja2_name: Optional[str] = None,- jinja2: Optional[str] = None,- prompting_style: Optional[PromptingStyle] = None,+ context: dict[str, Any],+ jinja2_name: str | None = None,+ jinja2: str | None = None,+ prompting_style: PromptingStyle | None = None,template_category: Jinja2TemplateCategory = Jinja2TemplateCategory.LLM_PROMPT,) -> str: ...diff --git a/pipelex/cogt/content_generation/img_gen_generate.py b/pipelex/cogt/content_generation/img_gen_generate.pyindex b7ae6a7e5..66dec1ba7 100644--- a/pipelex/cogt/content_generation/img_gen_generate.py+++ b/pipelex/cogt/content_generation/img_gen_generate.py@@ -1,5 +1,3 @@-from typing import List, Union-from pipelex import logfrom pipelex.cogt.content_generation.assignment_models import ImgGenAssignmentfrom pipelex.cogt.image.generated_image import GeneratedImage@@ -20,7 +18,7 @@ async def img_gen_single_image(img_gen_assignment: ImgGenAssignment) -> Generatereturn generated_image-async def img_gen_image_list(img_gen_assignment: ImgGenAssignment) -> List[GeneratedImage]:+async def img_gen_image_list(img_gen_assignment: ImgGenAssignment) -> list[GeneratedImage]:img_gen_worker = get_img_gen_worker(img_gen_handle=img_gen_assignment.img_gen_handle)img_gen_job = ImgGenJobFactory.make_img_gen_job_from_prompt(img_gen_prompt=img_gen_assignment.img_gen_prompt,@@ -36,8 +34,7 @@ async def img_gen_image_list(img_gen_assignment: ImgGenAssignment) -> List[Generreturn generated_image_list-async def img_gen_image(img_gen_assignment: ImgGenAssignment) -> Union[GeneratedImage, List[GeneratedImage]]:+async def img_gen_image(img_gen_assignment: ImgGenAssignment) -> GeneratedImage | list[GeneratedImage]:if img_gen_assignment.nb_images > 1:return await img_gen_image_list(img_gen_assignment)- else:- return await img_gen_single_image(img_gen_assignment)+ return await img_gen_single_image(img_gen_assignment)diff --git a/pipelex/cogt/content_generation/llm_generate.py b/pipelex/cogt/content_generation/llm_generate.pyindex 21ddbce2c..424597d38 100644--- a/pipelex/cogt/content_generation/llm_generate.py+++ b/pipelex/cogt/content_generation/llm_generate.py@@ -1,4 +1,4 @@-from typing import List, cast+from typing import castfrom pydantic import BaseModel@@ -37,7 +37,7 @@ async def llm_gen_object(object_assignment: ObjectAssignment) -> BaseModel:return generated_object-async def llm_gen_object_list(object_assignment: ObjectAssignment) -> List[BaseModel]:+async def llm_gen_object_list(object_assignment: ObjectAssignment) -> list[BaseModel]:llm_assignment = object_assignment.llm_assignment_for_objectlog.verbose(f"llm_gen_object_list to generate a list of '{object_assignment.object_class_name}'")llm_worker = get_llm_worker(llm_handle=llm_assignment.llm_handle)@@ -50,11 +50,11 @@ async def llm_gen_object_list(object_assignment: ObjectAssignment) -> List[BaseMitem_class = get_class_registry().get_required_class(name=item_class_name)class ListSchema(BaseModel):- items: List[item_class] # type: ignore+ items: list[item_class] # type: ignore[valid-type] # pyright: ignore[reportInvalidTypeForm]wrapped_list: ListSchema = await llm_worker.gen_object(llm_job=llm_job,schema=ListSchema,)- generated_list: List[BaseModel] = cast(List[BaseModel], wrapped_list.items) # pyright: ignore[reportUnknownMemberType]+ generated_list: list[BaseModel] = cast("list[BaseModel]", wrapped_list.items) # pyright: ignore[reportUnknownMemberType]return generated_listdiff --git a/pipelex/cogt/content_generation/ocr_generate.py b/pipelex/cogt/content_generation/ocr_generate.pyindex 91c3205ee..66254b7fd 100644--- a/pipelex/cogt/content_generation/ocr_generate.py+++ b/pipelex/cogt/content_generation/ocr_generate.py@@ -12,5 +12,4 @@ async def ocr_gen_extract_pages(ocr_assignment: OcrAssignment) -> OcrOutput:ocr_job_config=ocr_assignment.ocr_job_config,job_metadata=ocr_assignment.job_metadata,)- ocr_output = await ocr_worker.ocr_extract_pages(ocr_job=ocr_job)- return ocr_output+ return await ocr_worker.ocr_extract_pages(ocr_job=ocr_job)diff --git a/pipelex/cogt/exceptions.py b/pipelex/cogt/exceptions.pyindex a8c893aa6..075930c73 100644--- a/pipelex/cogt/exceptions.py+++ b/pipelex/cogt/exceptions.py@@ -1,5 +1,3 @@-from typing import Optional-from pipelex.tools.exceptions import FatalError, RootExceptionfrom pipelex.types import StrEnum@@ -127,7 +125,7 @@ class ImgGenGeneratedTypeError(ImgGenGenerationError):class MissingDependencyError(CogtError):"""Raised when a required dependency is not installed."""- def __init__(self, dependency_name: str, extra_name: str, message: Optional[str] = None):+ def __init__(self, dependency_name: str, extra_name: str, message: str | None = None):self.dependency_name = dependency_nameself.extra_name = extra_nameerror_msg = f"Required dependency '{dependency_name}' is not installed."diff --git a/pipelex/cogt/image/__init__.py b/pipelex/cogt/image/__init__.pyindex 8b1378917..e69de29bb 100644--- a/pipelex/cogt/image/__init__.py+++ b/pipelex/cogt/image/__init__.py@@ -1 +0,0 @@-diff --git a/pipelex/cogt/image/prompt_image_factory.py b/pipelex/cogt/image/prompt_image_factory.pyindex e5b4a4df9..b0b608042 100644--- a/pipelex/cogt/image/prompt_image_factory.py+++ b/pipelex/cogt/image/prompt_image_factory.py@@ -1,5 +1,3 @@-from typing import Optional-from pipelex.cogt.exceptions import PromptImageFactoryErrorfrom pipelex.cogt.image.prompt_image import PromptImage, PromptImageBase64, PromptImageBinary, PromptImagePath, PromptImageUrlfrom pipelex.tools.misc.base_64_utils import (@@ -15,9 +13,9 @@ class PromptImageFactory:@classmethoddef make_prompt_image(cls,- file_path: Optional[str] = None,- url: Optional[str] = None,- base_64: Optional[bytes] = None,+ file_path: str | None = None,+ url: str | None = None,+ base_64: bytes | None = None,) -> PromptImage:if file_path:return PromptImagePath(file_path=file_path)@@ -26,7 +24,8 @@ class PromptImageFactory:elif base_64:return PromptImageBase64(base_64=base_64)else:- raise PromptImageFactoryError("PromptImageFactory requires one of file_path, url, or image_bytes")+ msg = "PromptImageFactory requires one of file_path, url, or image_bytes"+ raise PromptImageFactoryError(msg)@classmethoddef make_prompt_image_from_uri(@@ -58,24 +57,28 @@ class PromptImageFactory:@classmethodasync def promptimage_to_b64_async(cls, prompt_image: PromptImage) -> bytes:- if isinstance(prompt_image, PromptImagePath):- return await load_binary_as_base64_async(prompt_image.file_path)- elif isinstance(prompt_image, PromptImageBase64):- return prompt_image.base_64- elif isinstance(prompt_image, PromptImageUrl):- image_bytes = await cls.make_promptimagebase64_from_url_async(prompt_image)- return image_bytes.base_64- else:- raise PromptImageFactoryError(f"Unknown PromptImage type: {prompt_image}")+ match prompt_image:+ case PromptImagePath():+ return await load_binary_as_base64_async(prompt_image.file_path)+ case PromptImageBase64():+ return prompt_image.base_64+ case PromptImageUrl():+ image_bytes = await cls.make_promptimagebase64_from_url_async(prompt_image)+ return image_bytes.base_64+ case _:+ msg = f"Unknown PromptImage type: {prompt_image}"+ raise PromptImageFactoryError(msg)@classmethodasync def promptimage_to_bytes_async(cls, prompt_image: PromptImage) -> bytes:- if isinstance(prompt_image, PromptImagePath):- return await load_binary_async(prompt_image.file_path)- elif isinstance(prompt_image, PromptImageBase64):- return prompt_image.get_decoded_bytes()- elif isinstance(prompt_image, PromptImageUrl):- image_bytes = await cls.make_promptimagebinary_from_url_async(prompt_image)- return image_bytes.binary- else:- raise PromptImageFactoryError(f"Unknown PromptImage type: {prompt_image}")+ match prompt_image:+ case PromptImagePath():+ return await load_binary_async(prompt_image.file_path)+ case PromptImageBase64():+ return prompt_image.get_decoded_bytes()+ case PromptImageUrl():+ image_bytes = await cls.make_promptimagebinary_from_url_async(prompt_image)+ return image_bytes.binary+ case _:+ msg = f"Unknown PromptImage type: {prompt_image}"+ raise PromptImageFactoryError(msg)diff --git a/pipelex/cogt/img_gen/__init__.py b/pipelex/cogt/img_gen/__init__.pyindex 8b1378917..e69de29bb 100644--- a/pipelex/cogt/img_gen/__init__.py+++ b/pipelex/cogt/img_gen/__init__.py@@ -1 +0,0 @@-diff --git a/pipelex/cogt/img_gen/img_gen_job_components.py b/pipelex/cogt/img_gen/img_gen_job_components.pyindex a272e8aef..53ed07cc6 100644--- a/pipelex/cogt/img_gen/img_gen_job_components.py+++ b/pipelex/cogt/img_gen/img_gen_job_components.py@@ -1,4 +1,4 @@-from typing import Literal, Optional, Union+from typing import Literalfrom pydantic import BaseModel, Field@@ -39,30 +39,30 @@ class Background(StrEnum):class ImgGenJobParams(BaseModel):aspect_ratio: AspectRatio = Field(strict=False)background: Background = Field(strict=False)- quality: Optional[Quality] = Field(default=None, strict=False)- nb_steps: Optional[int] = Field(default=None, gt=0)- guidance_scale: Optional[float] = Field(default=None, gt=0)+ quality: Quality | None = Field(default=None, strict=False)+ nb_steps: int | None = Field(default=None, gt=0)+ guidance_scale: float | None = Field(default=None, gt=0)is_moderated: bool = False- safety_tolerance: Optional[int] = Field(default=None, ge=1, le=6)+ safety_tolerance: int | None = Field(default=None, ge=1, le=6)is_raw: booloutput_format: OutputFormat = Field(strict=False)- seed: Optional[int] = Field(None, ge=0)+ seed: int | None = Field(None, ge=0)class ImgGenJobParamsDefaults(ConfigModel):aspect_ratio: AspectRatio = Field(strict=False)background: Background = Field(strict=False)- quality: Optional[Quality] = Field(default=None, strict=False)- nb_steps: Optional[int] = Field(default=None, gt=0)+ quality: Quality | None = Field(default=None, strict=False)+ nb_steps: int | None = Field(default=None, gt=0)guidance_scale: float = Field(..., gt=0)is_moderated: boolsafety_tolerance: int = Field(..., ge=1, le=6)is_raw: booloutput_format: OutputFormat = Field(strict=False)- seed: Union[int, Literal["auto"]]+ seed: int | Literal["auto"]def make_img_gen_job_params(self) -> ImgGenJobParams:- seed: Optional[int]+ seed: int | Noneif isinstance(self.seed, str) and self.seed == "auto":seed = Noneelse:diff --git a/pipelex/cogt/img_gen/img_gen_job_factory.py b/pipelex/cogt/img_gen/img_gen_job_factory.pyindex 166c55521..e358c887f 100644--- a/pipelex/cogt/img_gen/img_gen_job_factory.py+++ b/pipelex/cogt/img_gen/img_gen_job_factory.py@@ -1,5 +1,3 @@-from typing import Optional-from pipelex.cogt.img_gen.img_gen_job import ImgGenJobfrom pipelex.cogt.img_gen.img_gen_job_components import ImgGenJobConfig, ImgGenJobParams, ImgGenJobReportfrom pipelex.cogt.img_gen.img_gen_prompt import ImgGenPrompt@@ -12,9 +10,9 @@ class ImgGenJobFactory:def make_img_gen_job_from_prompt(cls,img_gen_prompt: ImgGenPrompt,- img_gen_job_params: Optional[ImgGenJobParams] = None,- img_gen_job_config: Optional[ImgGenJobConfig] = None,- job_metadata: Optional[JobMetadata] = None,+ img_gen_job_params: ImgGenJobParams | None = None,+ img_gen_job_config: ImgGenJobConfig | None = None,+ job_metadata: JobMetadata | None = None,) -> ImgGenJob:img_gen_config = get_config().cogt.img_gen_configjob_metadata = job_metadata or JobMetadata(@@ -36,9 +34,9 @@ class ImgGenJobFactory:def make_img_gen_job_from_prompt_contents(cls,positive_text: str,- img_gen_job_params: Optional[ImgGenJobParams] = None,- img_gen_job_config: Optional[ImgGenJobConfig] = None,- job_metadata: Optional[JobMetadata] = None,+ img_gen_job_params: ImgGenJobParams | None = None,+ img_gen_job_config: ImgGenJobConfig | None = None,+ job_metadata: JobMetadata | None = None,) -> ImgGenJob:img_gen_config = get_config().cogt.img_gen_configjob_metadata = job_metadata or JobMetadata(diff --git a/pipelex/cogt/img_gen/img_gen_prompt.py b/pipelex/cogt/img_gen/img_gen_prompt.pyindex 83e3558e5..ea7859f65 100644--- a/pipelex/cogt/img_gen/img_gen_prompt.py+++ b/pipelex/cogt/img_gen/img_gen_prompt.py@@ -17,7 +17,8 @@ class ImgGenPrompt(BaseModel):passcase ProblemReaction.RAISE:if self.positive_text == "":- raise ImgGenPromptError("ImgGen prompt positive_text must not be an empty string")+ msg = "ImgGen prompt positive_text must not be an empty string"+ raise ImgGenPromptError(msg)case ProblemReaction.LOG:if self.positive_text == "":log.warning("ImgGen prompt positive_text should not be an empty string")diff --git a/pipelex/cogt/img_gen/img_gen_setting.py b/pipelex/cogt/img_gen/img_gen_setting.pyindex 5245dc66c..ad62e6962 100644--- a/pipelex/cogt/img_gen/img_gen_setting.py+++ b/pipelex/cogt/img_gen/img_gen_setting.py@@ -1,25 +1,26 @@-from typing import Optional, Union+from typing import Unionfrom pydantic import Field, model_validator-from typing_extensions import Selffrom pipelex.cogt.exceptions import ImgGenSettingsValidationErrorfrom pipelex.cogt.img_gen.img_gen_job_components import Qualityfrom pipelex.tools.config.config_model import ConfigModel+from pipelex.types import Selfclass ImgGenSetting(ConfigModel):img_gen_handle: str- quality: Optional[Quality] = Field(default=None, strict=False)- nb_steps: Optional[int] = Field(default=None, gt=0)- guidance_scale: Optional[float] = Field(default=None, gt=0)+ quality: Quality | None = Field(default=None, strict=False)+ nb_steps: int | None = Field(default=None, gt=0)+ guidance_scale: float | None = Field(default=None, gt=0)is_moderated: bool = False- safety_tolerance: Optional[int] = Field(default=None, ge=1, le=6)+ safety_tolerance: int | None = Field(default=None, ge=1, le=6)@model_validator(mode="after")def validate_quality_or_nb_steps(self) -> Self:if self.quality is not None and self.nb_steps is not None:- raise ImgGenSettingsValidationError("ImgGenSetting cannot have both 'quality' and 'nb_steps' specified. Use one or the other.")+ msg = "ImgGenSetting cannot have both 'quality' and 'nb_steps' specified. Use one or the other."+ raise ImgGenSettingsValidationError(msg)return selfdef desc(self) -> str:diff --git a/pipelex/cogt/img_gen/img_gen_worker_abstract.py b/pipelex/cogt/img_gen/img_gen_worker_abstract.pyindex 1e4432ae9..d55630df5 100644--- a/pipelex/cogt/img_gen/img_gen_worker_abstract.py+++ b/pipelex/cogt/img_gen/img_gen_worker_abstract.py@@ -1,5 +1,4 @@from abc import abstractmethod-from typing import List, Optionalfrom typing_extensions import override@@ -16,7 +15,7 @@ class ImgGenWorkerAbstract(InferenceWorkerAbstract):def __init__(self,inference_model: InferenceModelSpec,- reporting_delegate: Optional[ReportingProtocol] = None,+ reporting_delegate: ReportingProtocol | None = None,):InferenceWorkerAbstract.__init__(self, reporting_delegate=reporting_delegate)self.inference_model = inference_model@@ -73,7 +72,7 @@ class ImgGenWorkerAbstract(InferenceWorkerAbstract):self,img_gen_job: ImgGenJob,nb_images: int,- ) -> List[GeneratedImage]:+ ) -> list[GeneratedImage]:log.debug(f"Image gen worker gen_image_list using {self.desc}")# Verify that the job is valid@@ -103,5 +102,5 @@ class ImgGenWorkerAbstract(InferenceWorkerAbstract):self,img_gen_job: ImgGenJob,nb_images: int,- ) -> List[GeneratedImage]:+ ) -> list[GeneratedImage]:passdiff --git a/pipelex/cogt/img_gen/img_gen_worker_factory.py b/pipelex/cogt/img_gen/img_gen_worker_factory.pyindex 97e7c9ed9..f04584cdc 100644--- a/pipelex/cogt/img_gen/img_gen_worker_factory.py+++ b/pipelex/cogt/img_gen/img_gen_worker_factory.py@@ -1,5 +1,3 @@-from typing import Optional-from pipelex.cogt.exceptions import CogtError, MissingDependencyErrorfrom pipelex.cogt.img_gen.img_gen_worker_abstract import ImgGenWorkerAbstractfrom pipelex.cogt.model_backends.model_spec import InferenceModelSpec@@ -17,7 +15,7 @@ class ImgGenWorkerFactory:def make_img_gen_worker(self,inference_model: InferenceModelSpec,- reporting_delegate: Optional[ReportingProtocol] = None,+ reporting_delegate: ReportingProtocol | None = None,) -> ImgGenWorkerAbstract:plugin = Plugin.make_for_inference_model(inference_model=inference_model)backend = get_models_manager().get_required_inference_backend(inference_model.backend_name)@@ -28,16 +26,22 @@ class ImgGenWorkerFactory:try:fal_api_key = get_secret(secret_id="FAL_API_KEY")except SecretNotFoundError as exc:- raise FalCredentialsError("FAL_API_KEY not found") from exc+ msg = "FAL_API_KEY not found"+ raise FalCredentialsError(msg) from exctry:- from fal_client import AsyncClient as FalAsyncClient+ from fal_client import AsyncClient as FalAsyncClient # noqa: PLC0415except ImportError as exc:+ lib_name = "fal-client"+ lib_extra_name = "fal"+ msg = "The fal-client SDK is required to use FAL models (generation of images)."raise MissingDependencyError(- "fal-client", "fal", "The fal-client SDK is required to use FAL models (generation of images)."+ lib_name,+ lib_extra_name,+ msg,) from exc- from pipelex.plugins.fal.fal_img_gen_worker import FalImgGenWorker+ from pipelex.plugins.fal.fal_img_gen_worker import FalImgGenWorker # noqa: PLC0415img_gen_sdk_instance = plugin_sdk_registry.get_sdk_instance(plugin=plugin) or plugin_sdk_registry.set_sdk_instance(plugin=plugin,@@ -50,8 +54,8 @@ class ImgGenWorkerFactory:reporting_delegate=reporting_delegate,)case "openai":- from pipelex.plugins.openai.openai_factory import OpenAIFactory- from pipelex.plugins.openai.openai_img_gen_worker import OpenAIImgGenWorker+ from pipelex.plugins.openai.openai_factory import OpenAIFactory # noqa: PLC0415+ from pipelex.plugins.openai.openai_img_gen_worker import OpenAIImgGenWorker # noqa: PLC0415img_gen_sdk_instance = plugin_sdk_registry.get_sdk_instance(plugin=plugin) or plugin_sdk_registry.set_sdk_instance(plugin=plugin,@@ -67,6 +71,7 @@ class ImgGenWorkerFactory:reporting_delegate=reporting_delegate,)case _:- raise NotImplementedError(f"Plugin '{plugin}' is not supported for image generation")+ msg = f"Plugin '{plugin}' is not supported for image generation"+ raise NotImplementedError(msg)return img_gen_workerdiff --git a/pipelex/cogt/inference/__init__.py b/pipelex/cogt/inference/__init__.pyindex 8b1378917..e69de29bb 100644--- a/pipelex/cogt/inference/__init__.py+++ b/pipelex/cogt/inference/__init__.py@@ -1 +0,0 @@-diff --git a/pipelex/cogt/inference/inference_manager.py b/pipelex/cogt/inference/inference_manager.pyindex fec461379..b3f96b475 100644--- a/pipelex/cogt/inference/inference_manager.py+++ b/pipelex/cogt/inference/inference_manager.py@@ -1,5 +1,3 @@-from typing import Dict, Type-from typing_extensions import overridefrom pipelex import log@@ -21,9 +19,9 @@ class InferenceManager(InferenceManagerProtocol):def __init__(self):self.img_gen_worker_factory = ImgGenWorkerFactory()self.ocr_worker_factory = OcrWorkerFactory()- self.llm_workers: Dict[str, LLMWorkerAbstract] = {}- self.img_gen_workers: Dict[str, ImgGenWorkerAbstract] = {}- self.ocr_workers: Dict[str, OcrWorkerAbstract] = {}+ self.llm_workers: dict[str, LLMWorkerAbstract] = {}+ self.img_gen_workers: dict[str, ImgGenWorkerAbstract] = {}+ self.ocr_workers: dict[str, OcrWorkerAbstract] = {}@overridedef teardown(self):@@ -75,28 +73,24 @@ class InferenceManager(InferenceManagerProtocol):if llm_worker := self.llm_workers.get(llm_handle):return llm_workerif not get_config().cogt.inference_manager_config.is_auto_setup_preset_llm:- raise InferenceManagerWorkerSetupError(- f"No LLM worker for '{llm_handle}', set it up or enable cogt.inference_manager_config.is_auto_setup_preset_llm"- )+ msg = f"No LLM worker for '{llm_handle}', set it up or enable cogt.inference_manager_config.is_auto_setup_preset_llm"+ raise InferenceManagerWorkerSetupError(msg)inference_model = get_models_manager().get_inference_model(model_handle=llm_handle)- llm_worker = self._setup_one_internal_llm_worker(+ return self._setup_one_internal_llm_worker(inference_model=inference_model,llm_handle=llm_handle,)- return llm_worker-@overridedef set_llm_worker_from_external_plugin(self,llm_handle: str,- llm_worker_class: Type[LLMWorkerAbstract],+ llm_worker_class: type[LLMWorkerAbstract],should_warn_if_already_registered: bool = True,):- if llm_handle in self.llm_workers:- if should_warn_if_already_registered:- log.warning(f"LLM worker for '{llm_handle}' already registered, skipping")+ if llm_handle in self.llm_workers and should_warn_if_already_registered:+ log.warning(f"LLM worker for '{llm_handle}' already registered, skipping")self.llm_workers[llm_handle] = llm_worker_class(reporting_delegate=get_report_delegate())####################################################################################################@@ -118,9 +112,8 @@ class InferenceManager(InferenceManagerProtocol):img_gen_worker = self.img_gen_workers.get(img_gen_handle)if img_gen_worker is None:if not get_config().cogt.inference_manager_config.is_auto_setup_preset_img_gen:- raise InferenceManagerWorkerSetupError(- f"Found no Imgg worker for '{img_gen_handle}', set it up or enable cogt.inference_manager_config.is_auto_setup_preset_img_gen"- )+ msg = f"Found no ImgGen worker for '{img_gen_handle}', set it up or enable cogt.inference_manager_config.is_auto_setup_preset_img_gen"+ raise InferenceManagerWorkerSetupError(msg)img_gen_worker = self._setup_one_img_gen_worker(img_gen_handle=img_gen_handle)return img_gen_worker@@ -146,13 +139,11 @@ class InferenceManager(InferenceManagerProtocol):if ocr_worker := self.ocr_workers.get(model_handle):return ocr_workerif not get_config().cogt.inference_manager_config.is_auto_setup_preset_ocr:- raise InferenceManagerWorkerSetupError(- f"Found no OCR worker for '{model_handle}', set it up or enable cogt.inference_manager_config.is_auto_setup_preset_ocr"- )+ msg = f"Found no OCR worker for '{model_handle}', set it up or enable cogt.inference_manager_config.is_auto_setup_preset_ocr"+ raise InferenceManagerWorkerSetupError(msg)inference_model = get_models_manager().get_inference_model(model_handle=model_handle)- ocr_worker = self._setup_one_ocr_worker(+ return self._setup_one_ocr_worker(inference_model=inference_model,model_handle=model_handle,)- return ocr_workerdiff --git a/pipelex/cogt/inference/inference_manager_protocol.py b/pipelex/cogt/inference/inference_manager_protocol.pyindex 30d4b82c9..e6ea5d4c8 100644--- a/pipelex/cogt/inference/inference_manager_protocol.py+++ b/pipelex/cogt/inference/inference_manager_protocol.py@@ -1,4 +1,4 @@-from typing import Protocol, Type+from typing import Protocolfrom pipelex.cogt.img_gen.img_gen_worker_abstract import ImgGenWorkerAbstractfrom pipelex.cogt.llm.llm_worker_abstract import LLMWorkerAbstract@@ -6,8 +6,7 @@ from pipelex.cogt.ocr.ocr_worker_abstract import OcrWorkerAbstractclass InferenceManagerProtocol(Protocol):- """- This is the protocol for the inference manager.+ """This is the protocol for the inference manager.Its point is only to avoid a circular import."""@@ -22,7 +21,7 @@ class InferenceManagerProtocol(Protocol):def set_llm_worker_from_external_plugin(self,llm_handle: str,- llm_worker_class: Type[LLMWorkerAbstract],+ llm_worker_class: type[LLMWorkerAbstract],should_warn_if_already_registered: bool = True,): ...diff --git a/pipelex/cogt/inference/inference_worker_abstract.py b/pipelex/cogt/inference/inference_worker_abstract.pyindex 22f4510d4..ff442b53d 100644--- a/pipelex/cogt/inference/inference_worker_abstract.py+++ b/pipelex/cogt/inference/inference_worker_abstract.py@@ -1,5 +1,4 @@from abc import ABC, abstractmethod-from typing import Optionalfrom pipelex.reporting.reporting_protocol import ReportingProtocol@@ -7,7 +6,7 @@ from pipelex.reporting.reporting_protocol import ReportingProtocolclass InferenceWorkerAbstract(ABC):def __init__(self,- reporting_delegate: Optional[ReportingProtocol] = None,+ reporting_delegate: ReportingProtocol | None = None,):self.reporting_delegate = reporting_delegatediff --git a/pipelex/cogt/llm/llm_job_components.py b/pipelex/cogt/llm/llm_job_components.pyindex 9dfb1b062..d43af1f43 100644--- a/pipelex/cogt/llm/llm_job_components.py+++ b/pipelex/cogt/llm/llm_job_components.py@@ -1,5 +1,3 @@-from typing import Optional-from pydantic import BaseModel, Fieldfrom pipelex.cogt.llm.llm_report import LLMTokensUsage@@ -11,8 +9,8 @@ from pipelex.cogt.llm.llm_report import LLMTokensUsageclass LLMJobParams(BaseModel):temperature: float = Field(..., ge=0, le=1)- max_tokens: Optional[int] = Field(None, gt=0)- seed: Optional[int] = Field(None, ge=0)+ max_tokens: int | None = Field(None, gt=0)+ seed: int | None = Field(None, ge=0)class LLMJobConfig(BaseModel):@@ -26,4 +24,4 @@ class LLMJobConfig(BaseModel):class LLMJobReport(BaseModel):- llm_tokens_usage: Optional[LLMTokensUsage] = None+ llm_tokens_usage: LLMTokensUsage | None = Nonediff --git a/pipelex/cogt/llm/llm_job_factory.py b/pipelex/cogt/llm/llm_job_factory.pyindex 0e0cc7e9f..930a3b3b5 100644--- a/pipelex/cogt/llm/llm_job_factory.py+++ b/pipelex/cogt/llm/llm_job_factory.py@@ -1,4 +1,4 @@-from typing import Any, Dict, List, Optional+from typing import Anyfrom pipelex import logfrom pipelex.cogt.image.prompt_image import PromptImage@@ -18,8 +18,8 @@ class LLMJobFactory:cls,llm_prompt: LLMPrompt,llm_job_params: LLMJobParams,- llm_job_config: Optional[LLMJobConfig] = None,- job_metadata: Optional[JobMetadata] = None,+ llm_job_config: LLMJobConfig | None = None,+ job_metadata: JobMetadata | None = None,) -> LLMJob:config = get_config()llm_config = config.cogt.llm_config@@ -27,7 +27,7 @@ class LLMJobFactory:job_metadata.update(JobMetadata(job_category=JobCategory.LLM_JOB,- )+ ),)else:job_metadata = JobMetadata(@@ -49,10 +49,10 @@ class LLMJobFactory:async def make_llm_job_from_prompt_factory(cls,llm_prompt_factory: LLMPromptFactoryAbstract,- llm_prompt_arguments: Dict[str, Any],+ llm_prompt_arguments: dict[str, Any],llm_job_params: LLMJobParams,- llm_job_config: Optional[LLMJobConfig] = None,- job_metadata: Optional[JobMetadata] = None,+ llm_job_config: LLMJobConfig | None = None,+ job_metadata: JobMetadata | None = None,) -> LLMJob:log.debug(llm_prompt_arguments, title="llm_prompt_arguments")llm_prompt = await llm_prompt_factory.make_llm_prompt_from_args(**llm_prompt_arguments)@@ -69,10 +69,10 @@ class LLMJobFactory:async def make_llm_job_from_prompt_template(cls,llm_prompt_template: LLMPromptTemplate,- llm_prompt_arguments: Dict[str, Any],+ llm_prompt_arguments: dict[str, Any],llm_job_params: LLMJobParams,- llm_job_config: Optional[LLMJobConfig] = None,- job_metadata: Optional[JobMetadata] = None,+ llm_job_config: LLMJobConfig | None = None,+ job_metadata: JobMetadata | None = None,) -> LLMJob:log.debug(llm_prompt_arguments, title="llm_prompt_arguments")llm_prompt = await llm_prompt_template.make_llm_prompt_from_args(**llm_prompt_arguments)@@ -88,11 +88,11 @@ class LLMJobFactory:def make_llm_job_from_prompt_contents(cls,llm_job_params: LLMJobParams,- user_text: Optional[str] = None,- system_text: Optional[str] = None,- user_images: Optional[List[PromptImage]] = None,- llm_job_config: Optional[LLMJobConfig] = None,- job_metadata: Optional[JobMetadata] = None,+ user_text: str | None = None,+ system_text: str | None = None,+ user_images: list[PromptImage] | None = None,+ llm_job_config: LLMJobConfig | None = None,+ job_metadata: JobMetadata | None = None,) -> LLMJob:config = get_config()llm_config = config.cogt.llm_config@@ -100,7 +100,7 @@ class LLMJobFactory:job_metadata.update(JobMetadata(job_category=JobCategory.LLM_JOB,- )+ ),)else:job_metadata = JobMetadata(diff --git a/pipelex/cogt/llm/llm_prompt.py b/pipelex/cogt/llm/llm_prompt.pyindex 38aa757af..fbcbe1d56 100644--- a/pipelex/cogt/llm/llm_prompt.py+++ b/pipelex/cogt/llm/llm_prompt.py@@ -1,5 +1,3 @@-from typing import List, Optional-from pydantic import BaseModelfrom typing_extensions import override@@ -11,9 +9,9 @@ from pipelex.tools.runtime_manager import ProblemReaction, runtime_managerclass LLMPrompt(BaseModel):- system_text: Optional[str] = None- user_text: Optional[str] = None- user_images: List[PromptImage] = []+ system_text: str | None = None+ user_text: str | None = None+ user_images: list[PromptImage] = []def validate_before_execution(self):reaction = runtime_manager.problem_reactions.job@@ -25,9 +23,11 @@ class LLMPrompt(BaseModel):if self.system_text == "":log.warning(f"system_text should be None or contain text. system_text = '{self.system_text}'")else:- raise LLMPromptParameterError("system_text should be None or contain text")+ msg = "system_text should be None or contain text"+ raise LLMPromptParameterError(msg)if not is_not_none_and_has_text(text=self.user_text):- raise LLMPromptParameterError("user_text should contain text")+ msg = "user_text should contain text"+ raise LLMPromptParameterError(msg)case ProblemReaction.LOG:if not is_none_or_has_text(text=self.system_text):if self.system_text == "":@@ -51,7 +51,7 @@ class LLMPrompt(BaseModel):def __format__(self, format_spec: str) -> str:return self.desc()- def desc(self, truncate_text_length: Optional[int] = None) -> str:+ def desc(self, truncate_text_length: int | None = None) -> str:description = "LLM Prompt:"if truncate_text_length:if self.system_text:diff --git a/pipelex/cogt/llm/llm_prompt_spec.py b/pipelex/cogt/llm/llm_prompt_spec.pyindex 8b4298f7f..59da2e258 100644--- a/pipelex/cogt/llm/llm_prompt_spec.py+++ b/pipelex/cogt/llm/llm_prompt_spec.py@@ -1,11 +1,9 @@-from typing import Any, Dict, List, Optional, Set+from typing import TYPE_CHECKING, Anyfrom pydantic import BaseModel, model_validator-from typing_extensions import Selffrom pipelex import logfrom pipelex.cogt.exceptions import LLMPromptSpecError-from pipelex.cogt.image.prompt_image import PromptImagefrom pipelex.cogt.image.prompt_image_factory import PromptImageFactoryfrom pipelex.cogt.llm.llm_prompt import LLMPromptfrom pipelex.core.stuffs.stuff_content import ImageContent@@ -15,20 +13,24 @@ from pipelex.tools.templating.jinja2_blueprint import Jinja2Blueprintfrom pipelex.tools.templating.jinja2_required_variables import detect_jinja2_required_variablesfrom pipelex.tools.templating.templating_models import PromptingStylefrom pipelex.tools.typing.validation_utils import has_exactly_one_among_attributes_from_list, has_more_than_one_among_attributes_from_list+from pipelex.types import Self++if TYPE_CHECKING:+ from pipelex.cogt.image.prompt_image import PromptImageclass LLMPromptSpec(BaseModel):- prompting_style: Optional[PromptingStyle] = None+ prompting_style: PromptingStyle | None = None- system_prompt_jinja2_blueprint: Optional[Jinja2Blueprint] = None- system_prompt_verbatim_name: Optional[str] = None- system_prompt: Optional[str] = None+ system_prompt_jinja2_blueprint: Jinja2Blueprint | None = None+ system_prompt_verbatim_name: str | None = None+ system_prompt: str | None = None- user_text_jinja2_blueprint: Optional[Jinja2Blueprint] = None- user_prompt_verbatim_name: Optional[str] = None- user_text: Optional[str] = None+ user_text_jinja2_blueprint: Jinja2Blueprint | None = None+ user_prompt_verbatim_name: str | None = None+ user_text: str | None = None- user_images: Optional[List[str]] = None+ user_images: list[str] | None = None@model_validator(mode="after")def validate_user_text(self) -> Self:@@ -40,9 +42,8 @@ class LLMPromptSpec(BaseModel):"user_text",],):- raise LLMPromptSpecError(- f"LLMPromptSpec user text must have exactly one of user_text, user_text_jinja2_blueprint or user_prompt_verbatim_name: {self}"- )+ msg = f"LLMPromptSpec user text must have exactly one of user_text, user_text_jinja2_blueprint or user_prompt_verbatim_name: {self}"+ raise LLMPromptSpecError(msg)if has_more_than_one_among_attributes_from_list(obj=self,attributes_list=[@@ -51,9 +52,8 @@ class LLMPromptSpec(BaseModel):"system_prompt",],):- raise LLMPromptSpecError(- f"LLMPromptSpec system got more than one of system_prompt, system_prompt_jinja2_blueprint, system_prompt_verbatim_name: {self}"- )+ msg = f"LLMPromptSpec system got more than one of system_prompt, system_prompt_jinja2_blueprint, system_prompt_verbatim_name: {self}"+ raise LLMPromptSpecError(msg)return selfdef validate_with_libraries(self):@@ -69,8 +69,8 @@ class LLMPromptSpec(BaseModel):the_template = get_template(template_name=self.system_prompt_jinja2_blueprint.jinja2_name)log.debug(f"Validated jinja2 template '{self.system_prompt_jinja2_blueprint.jinja2_name}':\n{the_template}")- def required_variables(self) -> Set[str]:- required_variables: Set[str] = set()+ def required_variables(self) -> set[str]:+ required_variables: set[str] = set()if self.user_images:user_images_top_object_name = [user_image.split(".", 1)[0] for user_image in self.user_images]required_variables.update(user_images_top_object_name)@@ -85,7 +85,7 @@ class LLMPromptSpec(BaseModel):return {variable_namefor variable_name in required_variables- if not variable_name.startswith("_") and variable_name != "preliminary_text" and variable_name != "place_holder"+ if not variable_name.startswith("_") and variable_name not in ("preliminary_text", "place_holder")}# TODO: make this consistent with `LLMPromptFactoryAbstract` or `LLMPromptTemplate`,@@ -94,22 +94,21 @@ class LLMPromptSpec(BaseModel):self,output_concept_string: str,context_provider: ContextProviderAbstract,- output_structure_prompt: Optional[str] = None,- extra_params: Optional[Dict[str, Any]] = None,+ output_structure_prompt: str | None = None,+ extra_params: dict[str, Any] | None = None,) -> LLMPrompt:############################################################# User images############################################################- prompt_user_images: List[PromptImage] = []+ prompt_user_images: list[PromptImage] = []if self.user_images:for user_image_name in self.user_images:log.debug(f"Getting user image '{user_image_name}' from context")try:prompt_image_content = context_provider.get_typed_object_or_attribute(name=user_image_name, wanted_type=ImageContent)except ContextProviderException as exc:- raise LLMPromptSpecError(- f"Could not find a valid user image named '{user_image_name}' from the provided context_provider: {exc}"- ) from exc+ msg = f"Could not find a valid user image named '{user_image_name}' from the provided context_provider: {exc}"+ raise LLMPromptSpecError(msg) from excif prompt_image_content is not None: # An ImageContent can be optional..if base_64 := prompt_image_content.base_64:@@ -130,7 +129,8 @@ class LLMPromptSpec(BaseModel):extra_params=extra_params,)if not user_text:- raise ValueError("For user_text we need either a pipe_jinja2, a text_verbatim_name or a fixed user_text")+ msg = "For user_text we need either a pipe_jinja2, a text_verbatim_name or a fixed user_text"+ raise ValueError(msg)if output_structure_prompt:user_text += output_structure_prompt@@ -151,30 +151,28 @@ class LLMPromptSpec(BaseModel):############################################################# Full LLMPrompt############################################################- llm_prompt = LLMPrompt(+ return LLMPrompt(system_text=system_text,user_text=user_text,user_images=prompt_user_images,)- return llm_prompt-async def _unravel_text(self,context_provider: ContextProviderAbstract,- jinja2_blueprint: Optional[Jinja2Blueprint],- text_verbatim_name: Optional[str],- fixed_text: Optional[str],- extra_params: Optional[Dict[str, Any]] = None,- ) -> Optional[str]:- the_text: Optional[str]+ jinja2_blueprint: Jinja2Blueprint | None,+ text_verbatim_name: str | None,+ fixed_text: str | None,+ extra_params: dict[str, Any] | None = None,+ ) -> str | None:+ the_text: str | Noneif jinja2_blueprint:log.verbose(f"Working with Jinja2 pipe '{jinja2_blueprint.jinja2_name}'")if (prompting_style := self.prompting_style) and not jinja2_blueprint.prompting_style:jinja2_blueprint.prompting_style = prompting_stylelog.verbose(f"Setting prompting style to {prompting_style}")- context: Dict[str, Any] = context_provider.generate_context()+ context: dict[str, Any] = context_provider.generate_context()if extra_params:context.update(**extra_params)if jinja2_blueprint.extra_context:@@ -192,7 +190,8 @@ class LLMPromptSpec(BaseModel):template_name=text_verbatim_name,)if not user_text_verbatim:- raise ValueError(f"Could not find text_verbatim template '{text_verbatim_name}'")+ msg = f"Could not find text_verbatim template '{text_verbatim_name}'"+ raise ValueError(msg)the_text = user_text_verbatimelif fixed_text:the_text = fixed_textdiff --git a/pipelex/cogt/llm/llm_prompt_template.py b/pipelex/cogt/llm/llm_prompt_template.pyindex d5fe01206..6be640f18 100644--- a/pipelex/cogt/llm/llm_prompt_template.py+++ b/pipelex/cogt/llm/llm_prompt_template.py@@ -1,4 +1,4 @@-from typing import Any, List, Optional+from typing import Anyfrom typing_extensions import override@@ -19,8 +19,8 @@ class LLMPromptTemplate(LLMPromptFactoryAbstract):base_template_inputs: LLMPromptTemplateInputs = LLMPromptTemplateInputs()# fields kept for reference and debugging only (they have no effect)- source_system_template_name: Optional[str] = None- source_user_template_name: Optional[str] = None+ source_system_template_name: str | None = None+ source_user_template_name: str | None = None@property@override@@ -41,18 +41,18 @@ class LLMPromptTemplate(LLMPromptFactoryAbstract):arguments_dict = prompt_arguments.copy()# pop the base fields and then use the templating method- system_text: Optional[str] = arguments_dict.pop("system_text", None)- user_text: Optional[str] = arguments_dict.pop("user_text", None)+ system_text: str | None = arguments_dict.pop("system_text", None)+ user_text: str | None = arguments_dict.pop("user_text", None)if not user_text:user_text = self.proto_prompt.user_text# user_images is Optional here: None means the template is not altering the user_images field- user_images: Optional[List[PromptImage]] = None+ user_images: list[PromptImage] | None = Noneif "user_images" in arguments_dict:user_images = arguments_dict.pop("user_images")elif "user_image" in arguments_dict:user_images = [arguments_dict.pop("user_image")]- is_user_images_append: Optional[bool] = arguments_dict.pop("is_user_images_append", None)+ is_user_images_append: bool | None = arguments_dict.pop("is_user_images_append", None)return self.make_llm_prompt(system_text=system_text,@@ -64,19 +64,21 @@ class LLMPromptTemplate(LLMPromptFactoryAbstract):def make_llm_prompt(self,- system_text: Optional[str] = None,- user_text: Optional[str] = None,- user_images: Optional[List[PromptImage]] = None,- is_user_images_append: Optional[bool] = None,- template_inputs: Optional[LLMPromptTemplateInputs] = None,+ system_text: str | None = None,+ user_text: str | None = None,+ user_images: list[PromptImage] | None = None,+ is_user_images_append: bool | None = None,+ template_inputs: LLMPromptTemplateInputs | None = None,) -> LLMPrompt:if not is_none_or_has_text(system_text):if system_text == "":log.warning(f"Prompt template system_text should be None or contain text. system_text = '{system_text}'")else:- raise LLMPromptTemplateInputsError(f"Prompt template system_text should be None or contain text. system_text = '{system_text}'")+ msg = f"Prompt template system_text should be None or contain text. system_text = '{system_text}'"+ raise LLMPromptTemplateInputsError(msg)if not is_none_or_has_text(user_text):- raise LLMPromptTemplateInputsError(f"Prompt template user_text should be None or contain text. system_text = '{user_text}'")+ msg = f"Prompt template user_text should be None or contain text. system_text = '{user_text}'"+ raise LLMPromptTemplateInputsError(msg)all_template_inputs = self.base_template_inputs.complemented_by(additional_template_inputs=template_inputs)@@ -120,11 +122,11 @@ class LLMPromptTemplate(LLMPromptFactoryAbstract):def from_template_name(cls,template_provider: TemplateProviderAbstract,- system_text: Optional[str] = None,- user_text: Optional[str] = None,- user_images: Optional[List[PromptImage]] = None,- system_template_name: Optional[str] = None,- user_template_name: Optional[str] = None,+ system_text: str | None = None,+ user_text: str | None = None,+ user_images: list[PromptImage] | None = None,+ system_template_name: str | None = None,+ user_template_name: str | None = None,) -> "LLMPromptTemplate":proto_prompt: LLMPrompt = LLMPrompt(system_text=system_text,@@ -145,9 +147,9 @@ class LLMPromptTemplate(LLMPromptFactoryAbstract):@classmethoddef from_template_contents(cls,- system_text: Optional[str] = None,- user_text: Optional[str] = None,- user_images: Optional[List[PromptImage]] = None,+ system_text: str | None = None,+ user_text: str | None = None,+ user_images: list[PromptImage] | None = None,) -> "LLMPromptTemplate":proto_prompt = LLMPrompt(user_text=user_text,diff --git a/pipelex/cogt/llm/llm_prompt_template_inputs.py b/pipelex/cogt/llm/llm_prompt_template_inputs.pyindex 835a1c951..ff0b735fe 100644--- a/pipelex/cogt/llm/llm_prompt_template_inputs.py+++ b/pipelex/cogt/llm/llm_prompt_template_inputs.py@@ -1,7 +1,9 @@-from typing import Any, Dict, ItemsView, List, Optional+from __future__ import annotations++from typing import Anyfrom pydantic import ConfigDict, Field, RootModel, model_validator-from typing_extensions import Self, override+from typing_extensions import TYPE_CHECKING, overridefrom pipelex import logfrom pipelex.cogt.exceptions import LLMPromptTemplateInputsError@@ -9,7 +11,12 @@ from pipelex.tools.misc.json_utils import json_strfrom pipelex.tools.misc.string_utils import can_inject_textfrom pipelex.tools.runtime_manager import ProblemReaction, runtime_manager-LLMPromptTemplateInputsDict = Dict[str, Any]+if TYPE_CHECKING:+ from collections.abc import ItemsView++ from pipelex.types import Self++LLMPromptTemplateInputsDict = dict[str, Any]class LLMPromptTemplateInputs(RootModel[LLMPromptTemplateInputsDict]):@@ -37,16 +44,16 @@ class LLMPromptTemplateInputs(RootModel[LLMPromptTemplateInputsDict]):def items(self) -> ItemsView[str, Any]:return self.root.items()- def list_keys(self) -> List[str]:+ def list_keys(self) -> list[str]:return list(self.root.keys())- def complemented_by(self, additional_template_inputs: Optional["LLMPromptTemplateInputs"]) -> "LLMPromptTemplateInputs":+ def complemented_by(self, additional_template_inputs: LLMPromptTemplateInputs | None) -> LLMPromptTemplateInputs:all_template_inputs = self.root.copy()if additional_template_inputs:all_template_inputs.update(additional_template_inputs.root)return LLMPromptTemplateInputs(root=all_template_inputs)- def complemented_by_dict(self, additional_inputs_dict: Optional[LLMPromptTemplateInputsDict]) -> "LLMPromptTemplateInputs":+ def complemented_by_dict(self, additional_inputs_dict: LLMPromptTemplateInputsDict | None) -> LLMPromptTemplateInputs:all_template_inputs = self.root.copy()if additional_inputs_dict:all_template_inputs.update(additional_inputs_dict)@@ -57,5 +64,5 @@ class LLMPromptTemplateInputs(RootModel[LLMPromptTemplateInputsDict]):return json_str(self.root, title="llm_prompt_template_inputs", is_spaced=True)@staticmethod- def from_args(**template_inputs: Any) -> "LLMPromptTemplateInputs":+ def from_args(**template_inputs: Any) -> LLMPromptTemplateInputs:return LLMPromptTemplateInputs(root=template_inputs)diff --git a/pipelex/cogt/llm/llm_report.py b/pipelex/cogt/llm/llm_report.pyindex 5febe49a4..24c29ec77 100644--- a/pipelex/cogt/llm/llm_report.py+++ b/pipelex/cogt/llm/llm_report.py@@ -1,4 +1,4 @@-from typing import Any, Dict+from typing import Anyfrom pydantic import BaseModel@@ -38,11 +38,11 @@ class LLMTokenCostReport(BaseModel):nb_tokens_by_category: NbTokensByCategoryDictcosts_by_token_category: CostsByCategoryDict- def as_flat_dictionary(self) -> Dict[str, Any]:- the_dict: Dict[str, Any] = {}+ def as_flat_dictionary(self) -> dict[str, Any]:+ the_dict: dict[str, Any] = {}dict_for_job_metadata = self.job_metadata.model_dump(serialize_as_any=True)the_dict.update(dict_for_job_metadata)- dict_for_llm: Dict[str, Any] = {+ dict_for_llm: dict[str, Any] = {LLMTokenCostReportField.LLM_NAME: self.inference_model_name,LLMTokenCostReportField.PLATFORM_LLM_ID: self.platform_llm_id,}diff --git a/pipelex/cogt/llm/llm_setting.py b/pipelex/cogt/llm/llm_setting.pyindex 495499b4b..f01b21a78 100644--- a/pipelex/cogt/llm/llm_setting.py+++ b/pipelex/cogt/llm/llm_setting.py@@ -1,31 +1,26 @@-from typing import Literal, Optional, Set, Union+from typing import Literal, Unionfrom pydantic import Field, field_validator-from typing_extensions import Selffrom pipelex.cogt.llm.llm_job_components import LLMJobParamsfrom pipelex.cogt.model_backends.prompting_target import PromptingTargetfrom pipelex.tools.config.config_model import ConfigModel-from pipelex.tools.exceptions import ConfigValidationError+from pipelex.types import Selfclass LLMSetting(ConfigModel):llm_handle: strtemperature: float = Field(..., ge=0, le=1)- max_tokens: Optional[int] = None- prompting_target: Optional[PromptingTarget] = Field(default=None, strict=False)+ max_tokens: int | None = None+ prompting_target: PromptingTarget | None = Field(default=None, strict=False)@field_validator("max_tokens", mode="before")@classmethod- def validate_max_tokens(cls, value: Union[int, Literal["auto"], None]) -> Optional[int]:- if value is None:+ def validate_max_tokens(cls, value: int | Literal["auto"] | None) -> int | None:+ if value is None or (isinstance(value, str) and value == "auto"):return None- elif isinstance(value, str) and value == "auto":- return None- elif isinstance(value, int): # pyright: ignore[reportUnnecessaryIsInstance]+ if isinstance(value, int): # pyright: ignore[reportUnnecessaryIsInstance]return value- else:- raise ConfigValidationError(f'Invalid max_tokens shoubd be an int or "auto" but it is a {type(value)}: {value}')def make_llm_job_params(self) -> LLMJobParams:return LLMJobParams(@@ -50,26 +45,17 @@ class LLMSettingChoicesDefaults(ConfigModel):class LLMSettingChoices(ConfigModel):- for_text: Optional[LLMChoice]- for_object: Optional[LLMChoice]+ for_text: LLMChoice | None+ for_object: LLMChoice | None- def list_choices(self) -> Set[str]:- return set(- [- choice- for choice in [- self.for_text,- self.for_object,- ]- if isinstance(choice, str)- ]- )+ def list_choices(self) -> set[str]:+ return {c for c in (self.for_text, self.for_object) if isinstance(c, str)}@classmethoddef make_completed_with_defaults(cls,- for_text: Optional[LLMChoice] = None,- for_object: Optional[LLMChoice] = None,+ for_text: LLMChoice | None = None,+ for_object: LLMChoice | None = None,) -> Self:return cls(for_text=for_text,diff --git a/pipelex/cogt/llm/llm_worker_abstract.py b/pipelex/cogt/llm/llm_worker_abstract.pyindex d8569aea1..12851364e 100644--- a/pipelex/cogt/llm/llm_worker_abstract.py+++ b/pipelex/cogt/llm/llm_worker_abstract.py@@ -1,26 +1,30 @@+from __future__ import annotations+from abc import ABC, abstractmethod-from typing import Optional, Type+from typing import TYPE_CHECKINGfrom typing_extensions import overridefrom pipelex import logfrom pipelex.cogt.inference.inference_worker_abstract import InferenceWorkerAbstract-from pipelex.cogt.llm.llm_job import LLMJobfrom pipelex.pipeline.job_metadata import UnitJobId-from pipelex.reporting.reporting_protocol import ReportingProtocol-from pipelex.tools.typing.pydantic_utils import BaseModelTypeVar++if TYPE_CHECKING:+ from pipelex.cogt.llm.llm_job import LLMJob+ from pipelex.reporting.reporting_protocol import ReportingProtocol+ from pipelex.tools.typing.pydantic_utils import BaseModelTypeVarclass LLMWorkerAbstract(InferenceWorkerAbstract, ABC):def __init__(self,- reporting_delegate: Optional[ReportingProtocol] = None,+ reporting_delegate: ReportingProtocol | None = None,):- """- Initialize the LLMWorker.+ """Initialize the LLMWorker.Args:- reporting_delegate (Optional[ReportingProtocol]): An optional report delegate for reporting unit jobs.+ reporting_delegate (ReportingProtocol | None): An optional report delegate for reporting unit jobs.+"""InferenceWorkerAbstract.__init__(self, reporting_delegate=reporting_delegate)@@ -89,7 +93,7 @@ class LLMWorkerAbstract(InferenceWorkerAbstract, ABC):async def gen_object(self,llm_job: LLMJob,- schema: Type[BaseModelTypeVar],+ schema: type[BaseModelTypeVar],) -> BaseModelTypeVar:log.debug(f"LLM Worker gen_object using {self.desc}")log.verbose(llm_job.llm_prompt.desc(), title="llm_prompt")@@ -114,6 +118,6 @@ class LLMWorkerAbstract(InferenceWorkerAbstract, ABC):async def _gen_object(self,llm_job: LLMJob,- schema: Type[BaseModelTypeVar],+ schema: type[BaseModelTypeVar],) -> BaseModelTypeVar:passdiff --git a/pipelex/cogt/llm/llm_worker_factory.py b/pipelex/cogt/llm/llm_worker_factory.pyindex bd7ac90ef..c239c69a1 100644--- a/pipelex/cogt/llm/llm_worker_factory.py+++ b/pipelex/cogt/llm/llm_worker_factory.py@@ -1,5 +1,3 @@-from typing import Optional-from pipelex.cogt.exceptions import MissingDependencyErrorfrom pipelex.cogt.llm.llm_worker_internal_abstract import LLMWorkerInternalAbstractfrom pipelex.cogt.llm.structured_output import StructureMethod@@ -14,7 +12,7 @@ class LLMWorkerFactory:@staticmethoddef make_llm_worker(inference_model: InferenceModelSpec,- reporting_delegate: Optional[ReportingProtocol] = None,+ reporting_delegate: ReportingProtocol | None = None,) -> LLMWorkerInternalAbstract:plugin = Plugin.make_for_inference_model(inference_model=inference_model)backend = get_models_manager().get_required_inference_backend(inference_model.backend_name)@@ -22,13 +20,13 @@ class LLMWorkerFactory:llm_worker: LLMWorkerInternalAbstractmatch plugin.sdk:case "openai" | "azure_openai":- from pipelex.plugins.openai.openai_factory import OpenAIFactory+ from pipelex.plugins.openai.openai_factory import OpenAIFactory # noqa: PLC0415- structure_method: Optional[StructureMethod] = None+ structure_method: StructureMethod | None = Noneif get_config().cogt.llm_config.instructor_config.is_openai_structured_output_enabled:structure_method = StructureMethod.INSTRUCTOR_OPENAI_STRUCTURED- from pipelex.plugins.openai.openai_llm_worker import OpenAILLMWorker+ from pipelex.plugins.openai.openai_llm_worker import OpenAILLMWorker # noqa: PLC0415sdk_instance = plugin_sdk_registry.get_sdk_instance(plugin=plugin) or plugin_sdk_registry.set_sdk_instance(plugin=plugin,@@ -46,20 +44,23 @@ class LLMWorkerFactory:)case "anthropic" | "bedrock_anthropic":try:- import anthropic # noqa: F401+ import anthropic # noqa: PLC0415,F401except ImportError as exc:+ lib_name = "anthropic"+ lib_extra_name = "anthropic"+ msg = (+ "The anthropic SDK is required to use Anthropic models via the anthropic client. "+ "However, you can use Anthropic models through bedrock directly "+ "by using the 'bedrock-anthropic-claude' llm family. (eg: bedrock-anthropic-claude)"+ )raise MissingDependencyError(- "anthropic",- "anthropic",- (- "The anthropic SDK is required to use Anthropic models via the anthropic client. "- "However, you can use Anthropic models through bedrock directly "- "by using the 'bedrock-anthropic-claude' llm family. (eg: bedrock-anthropic-claude)"- ),+ lib_name,+ lib_extra_name,+ msg,) from exc- from pipelex.plugins.anthropic.anthropic_factory import AnthropicFactory- from pipelex.plugins.anthropic.anthropic_llm_worker import AnthropicLLMWorker+ from pipelex.plugins.anthropic.anthropic_factory import AnthropicFactory # noqa: PLC0415+ from pipelex.plugins.anthropic.anthropic_llm_worker import AnthropicLLMWorker # noqa: PLC0415sdk_instance = plugin_sdk_registry.get_sdk_instance(plugin=plugin) or plugin_sdk_registry.set_sdk_instance(plugin=plugin,@@ -75,20 +76,23 @@ class LLMWorkerFactory:)case "mistral":try:- import mistralai # noqa: F401+ import mistralai # noqa: PLC0415,F401except ImportError as exc:+ lib_name = "mistralai"+ lib_extra_name = "mistral"+ msg = (+ "The mistralai SDK is required to use Mistral models through the mistralai client. "+ "However, you can use Mistral models through bedrock directly "+ "by using the 'bedrock-mistral' llm family. (eg: bedrock-mistral-large)"+ )raise MissingDependencyError(- "mistralai",- "mistral",- (- "The mistralai SDK is required to use Mistral models through the mistralai client. "- "However, you can use Mistral models through bedrock directly "- "by using the 'bedrock-mistral' llm family. (eg: bedrock-mistral-large)"- ),+ lib_name,+ lib_extra_name,+ msg,) from exc- from pipelex.plugins.mistral.mistral_factory import MistralFactory- from pipelex.plugins.mistral.mistral_llm_worker import MistralLLMWorker+ from pipelex.plugins.mistral.mistral_factory import MistralFactory # noqa: PLC0415+ from pipelex.plugins.mistral.mistral_llm_worker import MistralLLMWorker # noqa: PLC0415sdk_instance = plugin_sdk_registry.get_sdk_instance(plugin=plugin) or plugin_sdk_registry.set_sdk_instance(plugin=plugin,@@ -103,15 +107,20 @@ class LLMWorkerFactory:)case "bedrock_boto3" | "bedrock_aioboto3":try:- import aioboto3 # noqa: F401- import boto3 # noqa: F401+ import aioboto3 # noqa: PLC0415,F401+ import boto3 # noqa: PLC0415,F401except ImportError as exc:+ lib_name = "boto3,aioboto3"+ lib_extra_name = "bedrock"+ msg = "The boto3 and aioboto3 SDKs are required to use Bedrock models."raise MissingDependencyError(- "boto3,aioboto3", "bedrock", "The boto3 and aioboto3 SDKs are required to use Bedrock models."+ lib_name,+ lib_extra_name,+ msg,) from exc- from pipelex.plugins.bedrock.bedrock_factory import BedrockFactory- from pipelex.plugins.bedrock.bedrock_llm_worker import BedrockLLMWorker+ from pipelex.plugins.bedrock.bedrock_factory import BedrockFactory # noqa: PLC0415+ from pipelex.plugins.bedrock.bedrock_llm_worker import BedrockLLMWorker # noqa: PLC0415sdk_instance = plugin_sdk_registry.get_sdk_instance(plugin=plugin) or plugin_sdk_registry.set_sdk_instance(plugin=plugin,@@ -125,16 +134,19 @@ class LLMWorkerFactory:)case "google":try:- import google.genai # noqa: F401+ import google.genai # noqa: PLC0415,F401except ImportError as exc:+ lib_name = "google-genai"+ lib_extra_name = "google"+ msg = "The google-genai SDK is required to use Google Gemini API directly. You can install it with 'pip install google-genai'."raise MissingDependencyError(- "google-genai",- "google",- ("The google-genai SDK is required to use Google Gemini API directly. You can install it with 'pip install google-genai'."),+ lib_name,+ lib_extra_name,+ msg,) from exc- from pipelex.plugins.google.google_factory import GoogleFactory- from pipelex.plugins.google.google_llm_worker import GoogleLLMWorker+ from pipelex.plugins.google.google_factory import GoogleFactory # noqa: PLC0415+ from pipelex.plugins.google.google_llm_worker import GoogleLLMWorker # noqa: PLC0415sdk_instance = plugin_sdk_registry.get_sdk_instance(plugin=plugin) or plugin_sdk_registry.set_sdk_instance(plugin=plugin,@@ -148,5 +160,6 @@ class LLMWorkerFactory:reporting_delegate=reporting_delegate,)case _:- raise NotImplementedError(f"Plugin '{plugin}' is not supported")+ msg = f"Plugin '{plugin}' is not supported"+ raise NotImplementedError(msg)return llm_workerdiff --git a/pipelex/cogt/llm/llm_worker_internal_abstract.py b/pipelex/cogt/llm/llm_worker_internal_abstract.pyindex 4ac2b935b..dd422171f 100644--- a/pipelex/cogt/llm/llm_worker_internal_abstract.py+++ b/pipelex/cogt/llm/llm_worker_internal_abstract.py@@ -1,5 +1,3 @@-from typing import Optional-from typing_extensions import overridefrom pipelex import log@@ -15,16 +13,16 @@ class LLMWorkerInternalAbstract(LLMWorkerAbstract):def __init__(self,inference_model: InferenceModelSpec,- structure_method: Optional[StructureMethod] = None,- reporting_delegate: Optional[ReportingProtocol] = None,+ structure_method: StructureMethod | None = None,+ reporting_delegate: ReportingProtocol | None = None,):- """- Initialize the LLMWorker.+ """Initialize the LLMWorker.Args:inference_model (InferenceModelSpec): The inference model to be used by the worker.- structure_method (Optional[StructureMethod]): The structure method to be used by the worker.- reporting_delegate (Optional[ReportingProtocol]): An optional report delegate for reporting unit jobs.+ structure_method (StructureMethod | None): The structure method to be used by the worker.+ reporting_delegate (ReportingProtocol | None): An optional report delegate for reporting unit jobs.+"""LLMWorkerAbstract.__init__(self, reporting_delegate=reporting_delegate)self.inference_model = inference_model@@ -61,9 +59,11 @@ class LLMWorkerInternalAbstract(LLMWorkerAbstract):def _check_vision_support(self, llm_job: LLMJob):if llm_job.llm_prompt.user_images:if not self.inference_model.is_vision_supported:- raise LLMCapabilityError(f"LLM Engine '{self.inference_model.tag}' does not support vision.")+ msg = f"LLM Engine '{self.inference_model.tag}' does not support vision."+ raise LLMCapabilityError(msg)nb_images = len(llm_job.llm_prompt.user_images)max_prompt_images = self.inference_model.max_prompt_images or 5000if nb_images > max_prompt_images:- raise LLMCapabilityError(f"LLM Engine '{self.inference_model.tag}' does not accept that many images: {nb_images}.")+ msg = f"LLM Engine '{self.inference_model.tag}' does not accept that many images: {nb_images}."+ raise LLMCapabilityError(msg)diff --git a/pipelex/cogt/model_backends/backend.py b/pipelex/cogt/model_backends/backend.pyindex 38046a638..94ec65cd5 100644--- a/pipelex/cogt/model_backends/backend.py+++ b/pipelex/cogt/model_backends/backend.py@@ -1,4 +1,4 @@-from typing import Any, Dict, List, Optional+from typing import Anyfrom pydantic import Field@@ -8,20 +8,19 @@ from pipelex.tools.config.config_model import ConfigModelclass InferenceBackend(ConfigModel):name: str- endpoint: Optional[str] = None- api_key: Optional[str] = None- extra_config: Dict[str, Any] = Field(default_factory=dict)- model_specs: Dict[str, InferenceModelSpec] = Field(default_factory=dict)+ endpoint: str | None = None+ api_key: str | None = None+ extra_config: dict[str, Any] = Field(default_factory=dict)+ model_specs: dict[str, InferenceModelSpec] = Field(default_factory=dict)- def list_model_names(self) -> List[str]:+ def list_model_names(self) -> list[str]:"""List the names of all models in the backend."""return list(self.model_specs.keys())- def get_model_spec(self, model_name: str) -> Optional[InferenceModelSpec]:+ def get_model_spec(self, model_name: str) -> InferenceModelSpec | None:"""Get a model spec by name."""- model_spec = self.model_specs.get(model_name)- return model_spec+ return self.model_specs.get(model_name)- def get_extra_config(self, key: str) -> Optional[Any]:+ def get_extra_config(self, key: str) -> Any | None:"""Get an extra config by key."""return self.extra_config.get(key)diff --git a/pipelex/cogt/model_backends/backend_factory.py b/pipelex/cogt/model_backends/backend_factory.pyindex 17c943fbf..352fe5b75 100644--- a/pipelex/cogt/model_backends/backend_factory.py+++ b/pipelex/cogt/model_backends/backend_factory.py@@ -1,4 +1,4 @@-from typing import Any, Dict, Optional+from typing import Anyfrom pydantic import Field@@ -10,9 +10,9 @@ from pipelex.tools.config.config_model import ConfigModelclass InferenceBackendBlueprint(ConfigModel):enabled: bool = True- endpoint: Optional[str] = None- api_key: Optional[str] = None- extra_config: Dict[str, Any] = Field(default_factory=dict)+ endpoint: str | None = None+ api_key: str | None = None+ extra_config: dict[str, Any] = Field(default_factory=dict)class InferenceBackendFactory:@@ -21,8 +21,8 @@ class InferenceBackendFactory:cls,name: str,blueprint: InferenceBackendBlueprint,- extra_config: Dict[str, Any],- model_specs: Dict[str, InferenceModelSpec],+ extra_config: dict[str, Any],+ model_specs: dict[str, InferenceModelSpec],) -> InferenceBackend:endpoint = blueprint.endpointapi_key = blueprint.api_keydiff --git a/pipelex/cogt/model_backends/backend_library.py b/pipelex/cogt/model_backends/backend_library.pyindex 6d52df12f..d9d27249c 100644--- a/pipelex/cogt/model_backends/backend_library.py+++ b/pipelex/cogt/model_backends/backend_library.py@@ -1,7 +1,6 @@-from typing import Any, Dict, List, Optional, Set, cast+from typing import TYPE_CHECKING, Any, castfrom pydantic import Field, RootModel, ValidationError-from typing_extensions import Selffrom pipelex import logfrom pipelex.cogt.exceptions import (@@ -12,15 +11,18 @@ from pipelex.cogt.exceptions import ()from pipelex.cogt.model_backends.backend import InferenceBackendfrom pipelex.cogt.model_backends.backend_factory import InferenceBackendBlueprint, InferenceBackendFactory-from pipelex.cogt.model_backends.model_spec import InferenceModelSpecfrom pipelex.cogt.model_backends.model_spec_factory import InferenceModelSpecBlueprint, InferenceModelSpecFactoryfrom pipelex.config import get_configfrom pipelex.tools.misc.dict_utils import apply_to_strings_recursivefrom pipelex.tools.misc.toml_utils import load_toml_from_pathfrom pipelex.tools.runtime_manager import runtime_managerfrom pipelex.tools.secrets.secrets_utils import UnknownVarPrefixError, VarFallbackPatternError, VarNotFoundError, substitute_vars+from pipelex.types import Self-InferenceBackendLibraryRoot = Dict[str, InferenceBackend]+if TYPE_CHECKING:+ from pipelex.cogt.model_backends.model_spec import InferenceModelSpec++InferenceBackendLibraryRoot = dict[str, InferenceBackend]class InferenceBackendLibrary(RootModel[InferenceBackendLibraryRoot]):@@ -38,11 +40,12 @@ class InferenceBackendLibrary(RootModel[InferenceBackendLibraryRoot]):try:backends_dict = load_toml_from_path(path=backends_library_path)except (FileNotFoundError, InferenceBackendLibraryError) as exc:- raise InferenceBackendLibraryError(f"Failed to load inference backend library from file '{backends_library_path}': {exc}") from exc+ msg = f"Failed to load inference backend library from file '{backends_library_path}': {exc}"+ raise InferenceBackendLibraryError(msg) from excfor backend_name, backend_dict in backends_dict.items():# We'll split the read settings into standard fields and extra configstandard_fields = InferenceBackendBlueprint.model_fields.keys()- extra_config: Dict[str, Any] = {}+ extra_config: dict[str, Any] = {}inference_backend_blueprint_dict_raw = backend_dict.copy()if not inference_backend_blueprint_dict_raw.get("enabled", True):continue@@ -51,20 +54,23 @@ class InferenceBackendLibrary(RootModel[InferenceBackendLibraryRoot]):try:inference_backend_blueprint_dict = apply_to_strings_recursive(inference_backend_blueprint_dict_raw, substitute_vars)except VarFallbackPatternError as var_fallback_pattern_exc:+ msg = f"Variable substitution failed due to a pattern error in file '{backends_library_path}':\n{var_fallback_pattern_exc}"+ key_name = "unknown"raise InferenceBackendCredentialsError(error_type=InferenceBackendCredentialsErrorType.VAR_FALLBACK_PATTERN,backend_name=backend_name,- message=f"Variable substitution failed due to a pattern error in file '{backends_library_path}':\n{var_fallback_pattern_exc}",- key_name="unknown",+ message=msg,+ key_name=key_name,) from var_fallback_pattern_excexcept VarNotFoundError as var_not_found_exc:+ msg = (+ f"Variable substitution failed due to a variable not found error in file '{backends_library_path}':"+ "\n{var_not_found_exc}\nRun mode: '{runtime_manager.run_mode}'"+ )raise InferenceBackendCredentialsError(error_type=InferenceBackendCredentialsErrorType.VAR_NOT_FOUND,backend_name=backend_name,- message=(- f"Variable substitution failed due to a variable not found error in file '{backends_library_path}':\n"- f"{var_not_found_exc}\nRun mode: '{runtime_manager.run_mode}'"- ),+ message=msg,key_name=var_not_found_exc.var_name,) from var_not_found_excexcept UnknownVarPrefixError as unknown_var_prefix_exc:@@ -78,7 +84,7 @@ class InferenceBackendLibrary(RootModel[InferenceBackendLibraryRoot]):key_name=unknown_var_prefix_exc.var_name,) from unknown_var_prefix_exc- for key in backend_dict.keys():+ for key in backend_dict:if key not in standard_fields:extra_config[key] = inference_backend_blueprint_dict.pop(key)backend_blueprint = InferenceBackendBlueprint.model_validate(inference_backend_blueprint_dict)@@ -91,17 +97,18 @@ class InferenceBackendLibrary(RootModel[InferenceBackendLibraryRoot]):try:model_specs_dict = apply_to_strings_recursive(model_specs_dict_raw, substitute_vars)except (VarNotFoundError, UnknownVarPrefixError) as exc:- raise InferenceModelSpecError(f"Variable substitution failed in file '{path_to_model_specs_toml}': {exc}") from exc+ msg = f"Variable substitution failed in file '{path_to_model_specs_toml}': {exc}"+ raise InferenceModelSpecError(msg) from excexcept (FileNotFoundError, InferenceModelSpecError) as exc:- raise InferenceBackendLibraryError(f"Failed to load inference model specs from file '{path_to_model_specs_toml}': {exc}") from exc- defaults_dict: Dict[str, Any] = model_specs_dict.pop("defaults", {})- backend_model_specs: Dict[str, InferenceModelSpec] = {}+ msg = f"Failed to load inference model specs from file '{path_to_model_specs_toml}': {exc}"+ raise InferenceBackendLibraryError(msg) from exc+ defaults_dict: dict[str, Any] = model_specs_dict.pop("defaults", {})+ backend_model_specs: dict[str, InferenceModelSpec] = {}for model_spec_name, value in model_specs_dict.items():if not isinstance(value, dict):- raise InferenceModelSpecError(- f"Model spec '{model_spec_name}' for backend '{backend_name}' at path '{path_to_model_specs_toml}' is not a dictionary"- )- model_spec_dict: Dict[str, Any] = cast(Dict[str, Any], value)+ msg = f"Model spec '{model_spec_name}' for backend '{backend_name}' at path '{path_to_model_specs_toml}' is not a dictionary"+ raise InferenceModelSpecError(msg)+ model_spec_dict: dict[str, Any] = cast("dict[str, Any]", value)try:# Start from the defaultsmodel_spec_blueprint_dict = defaults_dict.copy()@@ -115,10 +122,10 @@ class InferenceBackendLibrary(RootModel[InferenceBackendLibraryRoot]):)backend_model_specs[model_spec_name] = model_specexcept (InferenceModelSpecError, ValidationError) as exc:- raise InferenceBackendLibraryError(- f"Failed to load inference model spec '{model_spec_name}' for backend '{backend_name}' "- f"from file '{path_to_model_specs_toml}': {exc}"+ msg = (+ f"Failed to load inference model spec '{model_spec_name}' for backend '{backend_name}' from file '{path_to_model_specs_toml}'")+ raise InferenceBackendLibraryError(msg) from excbackend = InferenceBackendFactory.make_inference_backend(name=backend_name,blueprint=backend_blueprint,@@ -128,19 +135,19 @@ class InferenceBackendLibrary(RootModel[InferenceBackendLibraryRoot]):self.root[backend_name] = backendlog.debug(f"Loaded inference backend '{backend_name}'")- def list_backend_names(self) -> List[str]:+ def list_backend_names(self) -> list[str]:return list(self.root.keys())- def list_all_model_names(self) -> List[str]:+ def list_all_model_names(self) -> list[str]:"""List the names of all models in all backends."""- all_model_names: Set[str] = set()+ all_model_names: set[str] = set()for backend in self.root.values():all_model_names.update(backend.list_model_names())return sorted(all_model_names)- def get_all_models_and_possible_backends(self) -> Dict[str, List[str]]:+ def get_all_models_and_possible_backends(self) -> dict[str, list[str]]:"""Get a dictionary of all models and their possible backends."""- all_models_and_possible_backends: Dict[str, List[str]] = {}+ all_models_and_possible_backends: dict[str, list[str]] = {}for backend in self.root.values():for model_name in backend.list_model_names():if model_name not in all_models_and_possible_backends:@@ -148,7 +155,5 @@ class InferenceBackendLibrary(RootModel[InferenceBackendLibraryRoot]):all_models_and_possible_backends[model_name].append(backend.name)return all_models_and_possible_backends- def get_inference_backend(self, backend_name: str) -> Optional[InferenceBackend]:- """Get a backend by name."""- backend = self.root.get(backend_name)- return backend+ def get_inference_backend(self, backend_name: str) -> InferenceBackend | None:+ return self.root.get(backend_name)diff --git a/pipelex/cogt/model_backends/model_spec.py b/pipelex/cogt/model_backends/model_spec.pyindex 6a70a0cad..30568f829 100644--- a/pipelex/cogt/model_backends/model_spec.py+++ b/pipelex/cogt/model_backends/model_spec.py@@ -1,5 +1,3 @@-from typing import List, Optional-from pydantic import Fieldfrom pipelex.cogt.model_backends.model_constraints import ModelConstraints@@ -7,6 +5,7 @@ from pipelex.cogt.model_backends.model_type import ModelTypefrom pipelex.cogt.model_backends.prompting_target import PromptingTargetfrom pipelex.cogt.usage.cost_category import CostsByCategoryDictfrom pipelex.tools.config.config_model import ConfigModel+from pipelex.tools.typing.pydantic_utils import empty_list_factory_ofclass InferenceModelSpec(ConfigModel):@@ -15,13 +14,13 @@ class InferenceModelSpec(ConfigModel):sdk: strmodel_type: ModelType = Field(strict=False)model_id: str- inputs: List[str] = Field(default_factory=list)- outputs: List[str] = Field(default_factory=list)+ inputs: list[str] = Field(default_factory=list)+ outputs: list[str] = Field(default_factory=list)costs: CostsByCategoryDict = Field(strict=False)- max_tokens: Optional[int]- max_prompt_images: Optional[int]- prompting_target: Optional[PromptingTarget] = Field(default=None, strict=False)- constraints: List[ModelConstraints] = Field(default_factory=list)+ max_tokens: int | None+ max_prompt_images: int | None+ prompting_target: PromptingTarget | None = Field(default=None, strict=False)+ constraints: list[ModelConstraints] = Field(default_factory=empty_list_factory_of(ModelConstraints))@propertydef tag(self) -> str:diff --git a/pipelex/cogt/model_backends/model_spec_factory.py b/pipelex/cogt/model_backends/model_spec_factory.pyindex ed74d5ea9..c251282d8 100644--- a/pipelex/cogt/model_backends/model_spec_factory.py+++ b/pipelex/cogt/model_backends/model_spec_factory.py@@ -1,5 +1,3 @@-from typing import Dict, List, Optional-from pydantic import BaseModel, Field, field_validatorfrom pipelex.cogt.model_backends.model_constraints import ModelConstraints@@ -8,6 +6,7 @@ from pipelex.cogt.model_backends.model_type import ModelTypefrom pipelex.cogt.model_backends.prompting_target import PromptingTargetfrom pipelex.cogt.usage.cost_category import CostCategory, CostsByCategoryDictfrom pipelex.tools.config.config_model import ConfigModel+from pipelex.tools.typing.pydantic_utils import empty_list_factory_ofclass InferenceModelSpecBlueprint(ConfigModel):@@ -15,23 +14,25 @@ class InferenceModelSpecBlueprint(ConfigModel):sdk: strmodel_type: ModelType = Field(default=ModelType.LLM, strict=False)model_id: str- inputs: List[str] = Field(default_factory=list)- outputs: List[str] = Field(default_factory=list)+ inputs: list[str] = Field(default_factory=list)+ outputs: list[str] = Field(default_factory=list)costs: CostsByCategoryDict = Field(strict=False)- max_tokens: Optional[int] = None- max_prompt_images: Optional[int] = None- prompting_target: Optional[PromptingTarget] = Field(default=None, strict=False)- constraints: List[ModelConstraints] = Field(default_factory=list)+ max_tokens: int | None = None+ max_prompt_images: int | None = None+ prompting_target: PromptingTarget | None = Field(default=None, strict=False)+ constraints: list[ModelConstraints] = Field(default_factory=empty_list_factory_of(ModelConstraints))@field_validator("costs", mode="before")- def validate_costs(cls, value: Dict[str, float]) -> CostsByCategoryDict:+ @staticmethod+ def validate_costs(value: dict[str, float]) -> CostsByCategoryDict:return ConfigModel.transform_dict_of_floats_str_to_enum(input_dict=value,key_enum_cls=CostCategory,)@field_validator("constraints", mode="before")- def validate_constraints(cls, value: List[str]) -> List[ModelConstraints]:+ @staticmethod+ def validate_constraints(value: list[str]) -> list[ModelConstraints]:return ConfigModel.transform_list_of_str_to_enum(input_list=value,enum_cls=ModelConstraints,diff --git a/pipelex/cogt/model_routing/routing_models.py b/pipelex/cogt/model_routing/routing_models.pyindex e7826aa28..b289b8509 100644--- a/pipelex/cogt/model_routing/routing_models.py+++ b/pipelex/cogt/model_routing/routing_models.py@@ -1,5 +1,3 @@-from typing import Optional-from pydantic import BaseModelfrom pipelex.types import StrEnum@@ -16,4 +14,4 @@ class BackendMatchForModel(BaseModel):backend_name: strrouting_profile_name: strmatching_method: BackendMatchingMethod- matched_pattern: Optional[str]+ matched_pattern: str | Nonediff --git a/pipelex/cogt/model_routing/routing_profile.py b/pipelex/cogt/model_routing/routing_profile.pyindex fc3081412..109dfbf7a 100644--- a/pipelex/cogt/model_routing/routing_profile.py+++ b/pipelex/cogt/model_routing/routing_profile.py@@ -1,5 +1,3 @@-from typing import Dict, Optional-from pydantic import Fieldfrom pipelex.cogt.model_routing.routing_models import BackendMatchForModel, BackendMatchingMethod@@ -11,11 +9,11 @@ class RoutingProfile(ConfigModel):"""Configuration for model routing to backends."""name: str- description: Optional[str] = None- default: Optional[str] = None- routes: Dict[str, str] = Field(default_factory=dict) # Pattern -> Backend mapping+ description: str | None = None+ default: str | None = None+ routes: dict[str, str] = Field(default_factory=dict) # Pattern -> Backend mapping- def get_backend_match_for_model(self, model_name: str) -> Optional[BackendMatchForModel]:+ def get_backend_match_for_model(self, model_name: str) -> BackendMatchForModel | None:"""Get the backend name for a given model name.Args:@@ -23,6 +21,7 @@ class RoutingProfile(ConfigModel):Returns:Backend name to use for this model+"""# Check exact matches firstif model_name in self.routes:@@ -54,5 +53,4 @@ class RoutingProfile(ConfigModel):matching_method=BackendMatchingMethod.DEFAULT,matched_pattern=None,)- else:- return None+ return Nonediff --git a/pipelex/cogt/model_routing/routing_profile_factory.py b/pipelex/cogt/model_routing/routing_profile_factory.pyindex a08872690..42f225678 100644--- a/pipelex/cogt/model_routing/routing_profile_factory.py+++ b/pipelex/cogt/model_routing/routing_profile_factory.py@@ -1,5 +1,3 @@-from typing import Dict, Optional-from pydantic import Fieldfrom pipelex.cogt.model_routing.routing_profile import RoutingProfile@@ -10,15 +8,15 @@ class RoutingProfileBlueprint(ConfigModel):"""Blueprint for creating RoutingProfile instances."""description: str- default: Optional[str] = None- routes: Dict[str, str] = Field(default_factory=dict)+ default: str | None = None+ routes: dict[str, str] = Field(default_factory=dict)class RoutingProfileLibraryBlueprint(ConfigModel):"""Blueprint for the entire routing profile library."""active: str- profiles: Dict[str, RoutingProfileBlueprint] = Field(default_factory=dict)+ profiles: dict[str, RoutingProfileBlueprint] = Field(default_factory=dict)class RoutingProfileFactory:@@ -38,6 +36,7 @@ class RoutingProfileFactory:Returns:RoutingProfile instance+"""return RoutingProfile(name=name,diff --git a/pipelex/cogt/model_routing/routing_profile_library.py b/pipelex/cogt/model_routing/routing_profile_library.pyindex 8e7ec0473..949e8a60c 100644--- a/pipelex/cogt/model_routing/routing_profile_library.py+++ b/pipelex/cogt/model_routing/routing_profile_library.py@@ -1,7 +1,4 @@-from typing import Dict, Optional-from pydantic import Field, RootModel, ValidationError-from typing_extensions import Selffrom pipelex import logfrom pipelex.cogt.exceptions import RoutingProfileLibraryError, RoutingProfileLibraryNotFoundError@@ -13,22 +10,25 @@ from pipelex.cogt.model_routing.routing_profile_factory import ()from pipelex.config import get_configfrom pipelex.tools.misc.toml_utils import TOMLValidationError, load_toml_from_path+from pipelex.types import Self-RoutingProfileLibraryRoot = Dict[str, RoutingProfile]+RoutingProfileLibraryRoot = dict[str, RoutingProfile]class RoutingProfileLibrary(RootModel[RoutingProfileLibraryRoot]):"""Library for managing routing profile configurations."""root: RoutingProfileLibraryRoot = Field(default_factory=dict)- _active_config: Optional[str] = None+ _active_config: str | None = None@propertydef active_profile(self) -> RoutingProfile:if not self._active_config:- raise RoutingProfileLibraryError("No active routing profile loaded")+ msg = "No active routing profile loaded"+ raise RoutingProfileLibraryError(msg)if self._active_config not in self.root:- raise RoutingProfileLibraryError(f"Active routing profile '{self._active_config}' not found in loaded routing profile library")+ msg = f"Active routing profile '{self._active_config}' not found in loaded routing profile library"+ raise RoutingProfileLibraryError(msg)return self.root[self._active_config]@classmethod@@ -45,24 +45,22 @@ class RoutingProfileLibrary(RootModel[RoutingProfileLibraryRoot]):try:catalog_dict = load_toml_from_path(path=routing_profile_library_path)except FileNotFoundError as not_found_exc:- raise RoutingProfileLibraryNotFoundError(- f"Failed to load routing profile library from file '{routing_profile_library_path}': {not_found_exc}"- ) from not_found_exc+ msg = f"Failed to load routing profile library from file '{routing_profile_library_path}': {not_found_exc}"+ raise RoutingProfileLibraryNotFoundError(msg) from not_found_excexcept TOMLValidationError as toml_validation_exc:- raise RoutingProfileLibraryError(- f"Failed to load routing profile library from file '{routing_profile_library_path}': {toml_validation_exc}"- ) from toml_validation_exc+ msg = f"Failed to load routing profile library from file '{routing_profile_library_path}': {toml_validation_exc}"+ raise RoutingProfileLibraryError(msg) from toml_validation_exctry:catalog_blueprint = RoutingProfileLibraryBlueprint.model_validate(catalog_dict)except ValidationError as exc:- raise RoutingProfileLibraryError(f"Invalid routing profile library configuration in '{routing_profile_library_path}': {exc}") from exc+ msg = f"Invalid routing profile library configuration in '{routing_profile_library_path}': {exc}"+ raise RoutingProfileLibraryError(msg) from exc# Validate that the active config existsif catalog_blueprint.active not in catalog_blueprint.profiles:- raise RoutingProfileLibraryError(- f"Active profile '{catalog_blueprint.active}' not found in library. Available profiles: {list(catalog_blueprint.profiles.keys())}"- )+ msg = f"Active profile '{catalog_blueprint.active}' not found in library. Available profiles: {list(catalog_blueprint.profiles.keys())}"+ raise RoutingProfileLibraryError(msg)# Load all profilesself.root = {}@@ -76,7 +74,7 @@ class RoutingProfileLibrary(RootModel[RoutingProfileLibraryRoot]):log.debug(f"Loaded routing profile library with active profile: '{self._active_config}'")log.debug(f"Available profiles: {list(self.root.keys())}")- def get_backend_match_for_model_from_active_routing_profile(self, model_name: str) -> Optional[BackendMatchForModel]:+ def get_backend_match_for_model_from_active_routing_profile(self, model_name: str) -> BackendMatchForModel | None:"""Get the backend name for a given model.Args:@@ -87,10 +85,10 @@ class RoutingProfileLibrary(RootModel[RoutingProfileLibraryRoot]):Raises:RoutingProfileLibraryError: If no active profile is set or config not found+"""profile = self.active_profile- backend_match_for_model = profile.get_backend_match_for_model(model_name)- return backend_match_for_model+ return profile.get_backend_match_for_model(model_name)def list_routing_profile_names(self) -> list[str]:"""Get a list of all available routing profile names."""@@ -99,5 +97,6 @@ class RoutingProfileLibrary(RootModel[RoutingProfileLibraryRoot]):def get_required_routing_profile(self, routing_profile_name: str) -> RoutingProfile:routing_profile = self.root.get(routing_profile_name)if not routing_profile:- raise RoutingProfileLibraryError(f"Routing profile '{routing_profile_name}' not found in loaded routing profile library")+ msg = f"Routing profile '{routing_profile_name}' not found in loaded routing profile library"+ raise RoutingProfileLibraryError(msg)return routing_profilediff --git a/pipelex/cogt/models/model_deck.py b/pipelex/cogt/models/model_deck.pyindex 45ac09e87..93e780771 100644--- a/pipelex/cogt/models/model_deck.py+++ b/pipelex/cogt/models/model_deck.py@@ -1,7 +1,6 @@-from typing import Dict, List, Optional, Union+from typing import Unionfrom pydantic import Field, field_validator, model_validator-from typing_extensions import Selffrom pipelex import logfrom pipelex.cogt.exceptions import (@@ -19,14 +18,15 @@ from pipelex.cogt.model_backends.model_spec import InferenceModelSpecfrom pipelex.cogt.ocr.ocr_setting import OcrChoice, OcrSettingfrom pipelex.tools.config.config_model import ConfigModelfrom pipelex.tools.exceptions import ConfigValidationError+from pipelex.types import SelfLLM_PRESET_DISABLED = "disabled"-Waterfall = Union[str, List[str]]+Waterfall = Union[str, list[str]]class LLMDeckBlueprint(ConfigModel):- presets: Dict[str, LLMSetting] = Field(default_factory=dict)+ presets: dict[str, LLMSetting] = Field(default_factory=dict)choice_defaults: LLMSettingChoicesDefaultschoice_overrides: LLMSettingChoices = LLMSettingChoices(for_text=None,@@ -35,17 +35,17 @@ class LLMDeckBlueprint(ConfigModel):class OCRDeckBlueprint(ConfigModel):- presets: Dict[str, OcrSetting] = Field(default_factory=dict)+ presets: dict[str, OcrSetting] = Field(default_factory=dict)choice_default: OcrChoiceclass ImgGenDeckBlueprint(ConfigModel):- presets: Dict[str, ImgGenSetting] = Field(default_factory=dict)+ presets: dict[str, ImgGenSetting] = Field(default_factory=dict)choice_default: ImgGenChoiceclass ModelDeckBlueprint(ConfigModel):- aliases: Dict[str, Waterfall] = Field(default_factory=dict)+ aliases: dict[str, Waterfall] = Field(default_factory=dict)llm: LLMDeckBlueprintocr: OCRDeckBlueprint@@ -53,20 +53,20 @@ class ModelDeckBlueprint(ConfigModel):class ModelDeck(ConfigModel):- inference_models: Dict[str, InferenceModelSpec] = Field(default_factory=dict)- aliases: Dict[str, Waterfall] = Field(default_factory=dict)+ inference_models: dict[str, InferenceModelSpec] = Field(default_factory=dict)+ aliases: dict[str, Waterfall] = Field(default_factory=dict)- llm_presets: Dict[str, LLMSetting] = Field(default_factory=dict)+ llm_presets: dict[str, LLMSetting] = Field(default_factory=dict)llm_choice_defaults: LLMSettingChoicesDefaultsllm_choice_overrides: LLMSettingChoices = LLMSettingChoices(for_text=None,for_object=None,)- ocr_presets: Dict[str, OcrSetting] = Field(default_factory=dict)+ ocr_presets: dict[str, OcrSetting] = Field(default_factory=dict)ocr_choice_default: OcrChoice- img_gen_presets: Dict[str, ImgGenSetting] = Field(default_factory=dict)+ img_gen_presets: dict[str, ImgGenSetting] = Field(default_factory=dict)img_gen_choice_default: ImgGenChoicedef check_llm_setting(self, llm_setting_or_preset_id: LLMChoice, is_disabled_allowed: bool = False):@@ -77,43 +77,41 @@ class ModelDeck(ConfigModel):returnif preset_id == LLM_PRESET_DISABLED and is_disabled_allowed:return- raise LLMChoiceNotFoundError(f"llm preset id '{preset_id}' not found in deck")+ msg = f"llm preset id '{preset_id}' not found in deck"+ raise LLMChoiceNotFoundError(msg)def get_llm_setting(self, llm_choice: LLMChoice) -> LLMSetting:if isinstance(llm_choice, LLMSetting):return llm_choice- else:- # it's a string, so either an llm preset id or an llm handle- if llm_preset := self.llm_presets.get(llm_choice):- return llm_preset- elif self.is_handle_defined(model_handle=llm_choice):- return LLMSetting(llm_handle=llm_choice, temperature=0.7, max_tokens=None)- else:- raise LLMChoiceNotFoundError(f"LLM choice '{llm_choice}' not found in deck")+ # it's a string, so either an llm preset id or an llm handle+ if llm_preset := self.llm_presets.get(llm_choice):+ return llm_preset+ if self.is_handle_defined(model_handle=llm_choice):+ return LLMSetting(llm_handle=llm_choice, temperature=0.7, max_tokens=None)+ msg = f"LLM choice '{llm_choice}' not found in deck"+ raise LLMChoiceNotFoundError(msg)def get_ocr_setting(self, ocr_choice: OcrChoice) -> OcrSetting:if isinstance(ocr_choice, OcrSetting):return ocr_choice- else:- # it's a string, so either an ocr preset id or an ocr handle- if ocr_preset := self.ocr_presets.get(ocr_choice):- return ocr_preset- elif self.is_handle_defined(model_handle=ocr_choice):- return OcrSetting(ocr_handle=ocr_choice)- else:- raise OcrChoiceNotFoundError(f"OCR choice '{ocr_choice}' not found in deck")+ # it's a string, so either an ocr preset id or an ocr handle+ if ocr_preset := self.ocr_presets.get(ocr_choice):+ return ocr_preset+ if self.is_handle_defined(model_handle=ocr_choice):+ return OcrSetting(ocr_handle=ocr_choice)+ msg = f"OCR choice '{ocr_choice}' not found in deck"+ raise OcrChoiceNotFoundError(msg)def get_img_gen_setting(self, img_gen_choice: ImgGenChoice) -> ImgGenSetting:if isinstance(img_gen_choice, ImgGenSetting):return img_gen_choice- else:- # it's a string, so either an img gen preset id or an img gen handle- if img_gen_preset := self.img_gen_presets.get(img_gen_choice):- return img_gen_preset- elif self.is_handle_defined(model_handle=img_gen_choice):- return ImgGenSetting(img_gen_handle=img_gen_choice)- else:- raise ImgGenChoiceNotFoundError(f"Image generation choice '{img_gen_choice}' not found in deck")+ # it's a string, so either an img gen preset id or an img gen handle+ if img_gen_preset := self.img_gen_presets.get(img_gen_choice):+ return img_gen_preset+ if self.is_handle_defined(model_handle=img_gen_choice):+ return ImgGenSetting(img_gen_handle=img_gen_choice)+ msg = f"Image generation choice '{img_gen_choice}' not found in deck"+ raise ImgGenChoiceNotFoundError(msg)@classmethoddef final_validate(cls, deck: Self): # pyright: ignore[reportIncompatibleMethodOverride]@@ -122,7 +120,8 @@ class ModelDeck(ConfigModel):try:cls._validate_llm_setting(llm_setting=llm_setting, inference_model=inference_model)except ConfigValidationError as exc:- raise ModelDeckValidatonError(f"LLM preset '{llm_preset_id}' is invalid: {exc}")+ msg = f"LLM preset '{llm_preset_id}' is invalid: {exc}"+ raise ModelDeckValidatonError(msg) from exc################################################################ ModelDeck validations@@ -132,25 +131,27 @@ class ModelDeck(ConfigModel):def _validate_llm_setting(cls, llm_setting: LLMSetting, inference_model: InferenceModelSpec):if inference_model.max_tokens is not None and (llm_setting_max_tokens := llm_setting.max_tokens):if llm_setting_max_tokens > inference_model.max_tokens:- raise LLMSettingsValidationError(- (- f"LLM setting '{llm_setting.llm_handle}' has a max_tokens of {llm_setting_max_tokens}, "- f"which is greater than the model's max_tokens of {inference_model.max_tokens}"- )+ msg = (+ f"LLM setting '{llm_setting.llm_handle}' has a max_tokens of {llm_setting_max_tokens}, "+ f"which is greater than the model's max_tokens of {inference_model.max_tokens}")+ raise LLMSettingsValidationError(msg)if ModelConstraints.TEMPERATURE_MUST_BE_1 in inference_model.constraints and llm_setting.temperature != 1:- raise LLMSettingsValidationError(+ msg = (f"LLM setting '{llm_setting.llm_handle}' has a temperature of {llm_setting.temperature}, "f"which is not allowed by the model's constraints: it must be 1")+ raise LLMSettingsValidationError(msg)@field_validator("llm_choice_defaults", mode="after")@classmethoddef validate_llm_choice_defaults(cls, llm_choice_defaults: LLMSettingChoices) -> LLMSettingChoices:if llm_choice_defaults.for_text is None:- raise ConfigValidationError("llm_choice_defaults.for_text cannot be None")+ msg = "llm_choice_defaults.for_text cannot be None"+ raise ConfigValidationError(msg)if llm_choice_defaults.for_object is None:- raise ConfigValidationError("llm_choice_defaults.for_object cannot be None")+ msg = "llm_choice_defaults.for_object cannot be None"+ raise ConfigValidationError(msg)return llm_choice_defaults@field_validator("llm_choice_overrides", mode="after")@@ -165,7 +166,8 @@ class ModelDeck(ConfigModel):def validate_llm_presets(self) -> Self:for llm_preset_id, llm_setting in self.llm_presets.items():if llm_setting.llm_handle not in self.inference_models:- raise LLMHandleNotFoundError(f"llm_handle '{llm_setting.llm_handle}' for llm_preset '{llm_preset_id}' not found in deck")+ msg = f"llm_handle '{llm_setting.llm_handle}' for llm_preset '{llm_preset_id}' not found in deck"+ raise LLMHandleNotFoundError(msg)return self@model_validator(mode="after")@@ -176,12 +178,11 @@ class ModelDeck(ConfigModel):def _validate_llm_choices(self, llm_choices: LLMSettingChoices):for llm_setting in llm_choices.list_choices():self.check_llm_setting(llm_setting_or_preset_id=llm_setting)- return- def get_optional_inference_model(self, model_handle: str) -> Optional[InferenceModelSpec]:+ def get_optional_inference_model(self, model_handle: str) -> InferenceModelSpec | None:if inference_model := self.inference_models.get(model_handle):return inference_model- elif redirection := self.aliases.get(model_handle):+ if redirection := self.aliases.get(model_handle):log.debug(f"Redirection for '{model_handle}': {redirection}")if isinstance(redirection, str):alias_list = [redirection]@@ -199,7 +200,8 @@ class ModelDeck(ConfigModel):def get_required_inference_model(self, model_handle: str) -> InferenceModelSpec:inference_model = self.get_optional_inference_model(model_handle=model_handle)if inference_model is None:- raise LLMHandleNotFoundError(f"Model handle '{model_handle}' not found in deck")+ msg = f"Model handle '{model_handle}' not found in deck"+ raise LLMHandleNotFoundError(msg)if model_handle not in self.inference_models:log.dev(f"Model handle '{model_handle}' is an alias which resolves to '{inference_model.name}'")return inference_modeldiff --git a/pipelex/cogt/models/model_deck_check.py b/pipelex/cogt/models/model_deck_check.pyindex 74891c158..4faf808aa 100644--- a/pipelex/cogt/models/model_deck_check.py+++ b/pipelex/cogt/models/model_deck_check.py@@ -8,31 +8,30 @@ from pipelex.hub import get_model_deckdef check_llm_choice_with_deck(llm_choice: LLMChoice):if isinstance(llm_choice, LLMSetting):return+llm_deck = get_model_deck()- if llm_choice in llm_deck.llm_presets:- return- elif llm_deck.is_handle_defined(model_handle=llm_choice):++ if llm_choice in llm_deck.llm_presets or llm_deck.is_handle_defined(model_handle=llm_choice):return- raise LLMChoiceNotFoundError(f"LLM choice '{llm_choice}' not found in deck")+ msg = f"LLM choice '{llm_choice}' not found in deck"+ raise LLMChoiceNotFoundError(msg)def check_ocr_choice_with_deck(ocr_choice: OcrChoice):if isinstance(ocr_choice, OcrSetting):returnocr_deck = get_model_deck()- if ocr_choice in ocr_deck.ocr_presets:+ if ocr_choice in ocr_deck.ocr_presets or ocr_deck.is_handle_defined(model_handle=ocr_choice):return- elif ocr_deck.is_handle_defined(model_handle=ocr_choice):- return- raise OcrChoiceNotFoundError(f"OCR choice '{ocr_choice}' not found in deck")+ msg = f"OCR choice '{ocr_choice}' not found in deck"+ raise OcrChoiceNotFoundError(msg)def check_img_gen_choice_with_deck(img_gen_choice: ImgGenChoice):if isinstance(img_gen_choice, ImgGenSetting):returnimg_gen_deck = get_model_deck()- if img_gen_choice in img_gen_deck.img_gen_presets:- return- elif img_gen_deck.is_handle_defined(model_handle=img_gen_choice):+ if img_gen_choice in img_gen_deck.img_gen_presets or img_gen_deck.is_handle_defined(model_handle=img_gen_choice):return- raise ImgGenChoiceNotFoundError(f"Image generation choice '{img_gen_choice}' not found in deck")+ msg = f"Image generation choice '{img_gen_choice}' not found in deck"+ raise ImgGenChoiceNotFoundError(msg)diff --git a/pipelex/cogt/models/model_manager.py b/pipelex/cogt/models/model_manager.pyindex ba7966f29..a314ac2ff 100644--- a/pipelex/cogt/models/model_manager.py+++ b/pipelex/cogt/models/model_manager.py@@ -1,5 +1,5 @@import os-from typing import Any, Dict, Optional+from typing import Anyfrom typing_extensions import override@@ -22,12 +22,13 @@ class ModelManager(ModelManagerAbstract):def __init__(self) -> None:self.routing_profile_library = RoutingProfileLibrary.make_empty()self.inference_backend_library = InferenceBackendLibrary.make_empty()- self.model_deck: Optional[ModelDeck] = None+ self.model_deck: ModelDeck | None = None@overridedef get_model_deck(self) -> ModelDeck:if self.model_deck is None:- raise RuntimeError("Model deck is not initialized")+ msg = "Model deck is not initialized"+ raise RuntimeError(msg)return self.model_deck@override@@ -53,33 +54,36 @@ class ModelManager(ModelManagerAbstract):try:validate_toml_file(deck_path)except TOMLValidationError as exc:- log.error(f"TOML formatting issues in LLM deck file '{deck_path}': {exc}")- raise LibraryError(f"TOML validation failed for LLM deck file '{deck_path}': {exc}") from exc+ msg = f"TOML validation failed for LLM deck file '{deck_path}': {exc}"+ log.error(msg)+ raise LibraryError(msg) from exc@classmethoddef load_deck_blueprint(cls) -> ModelDeckBlueprint:deck_paths = get_config().cogt.inference_config.get_model_deck_paths()- full_deck_dict: Dict[str, Any] = {}+ full_deck_dict: dict[str, Any] = {}if not deck_paths:- raise ModelDeckNotFoundError("No LLM deck paths found. Please run `pipelex init-libraries` to create it.")+ msg = "No LLM deck paths found. Please run `pipelex init-libraries` to create it."+ raise ModelDeckNotFoundError(msg)for deck_path in deck_paths:if not os.path.exists(deck_path):- raise ModelDeckNotFoundError(f"LLM deck path `{deck_path}` not found. Please run `pipelex init-libraries` to create it.")+ msg = f"LLM deck path `{deck_path}` not found. Please run `pipelex init-libraries` to create it."+ raise ModelDeckNotFoundError(msg)try:deck_dict = load_toml_from_path(path=deck_path)log.debug(f"Loaded LLM deck from {deck_path}")deep_update(full_deck_dict, deck_dict)except Exception as exc:- log.error(f"Failed to load LLM deck file '{deck_path}': {exc}")+ msg = f"Failed to load LLM deck file '{deck_path}': {exc}"+ log.error(msg)raise- model_deck_blueprint = ModelDeckBlueprint.model_validate(full_deck_dict)- return model_deck_blueprint+ return ModelDeckBlueprint.model_validate(full_deck_dict)def build_deck(self, model_deck_blueprint: ModelDeckBlueprint) -> ModelDeck:all_models_and_possible_backends = self.inference_backend_library.get_all_models_and_possible_backends()- inference_models: Dict[str, InferenceModelSpec] = {}+ inference_models: dict[str, InferenceModelSpec] = {}for model_name, available_backends in all_models_and_possible_backends.items():backend_match_for_model = self.routing_profile_library.get_backend_match_for_model_from_active_routing_profile(@@ -91,20 +95,22 @@ class ModelManager(ModelManagerAbstract):matched_backend_name = backend_match_for_model.backend_namebackend = self.inference_backend_library.get_inference_backend(backend_name=matched_backend_name)if backend is None:- raise ModelsManagerError(f"Backend '{matched_backend_name}', requested for model '{model_name}', could not be found")+ msg = f"Backend '{matched_backend_name}', requested for model '{model_name}', could not be found"+ raise ModelsManagerError(msg)model_spec = backend.get_model_spec(model_name)if model_spec is None:# Not finding the model spec can be an error or not according to the matching methodmatch backend_match_for_model.matching_method:case BackendMatchingMethod.EXACT_MATCH:- raise ModelsManagerError(+ msg = (f"Model spec '{model_name}' not found in backend '{matched_backend_name}' "f"which was matched exactly in routing profile '{backend_match_for_model.routing_profile_name}'")+ raise ModelsManagerError(msg)case BackendMatchingMethod.PATTERN_MATCH:log.verbose(f"Model spec '{model_name}' not found in backend '{matched_backend_name}' but it's OK because "- f"it was only matched by pattern in routing profile '{backend_match_for_model.routing_profile_name}'"+ f"it was only matched by pattern in routing profile '{backend_match_for_model.routing_profile_name}'",)# We can skip it because it was only a pattern matchcontinue@@ -118,18 +124,20 @@ class ModelManager(ModelManagerAbstract):continuebackend = self.inference_backend_library.get_inference_backend(backend_name=available_backend)if backend is None:- raise ModelsManagerError(f"Backend '{available_backend}' not found for model '{model_name}'")+ msg = f"Backend '{available_backend}' not found for model '{model_name}'"+ raise ModelsManagerError(msg)model_spec = backend.get_model_spec(model_name)if model_spec is not None:breakif model_spec is None:- raise ModelsManagerError(+ msg = (f"Model spec '{model_name}' not found in any of the available backends '{available_backends}' "f"which was set as default in routing profile '{backend_match_for_model.routing_profile_name}'")+ raise ModelsManagerError(msg)inference_models[model_name] = model_spec- model_deck = ModelDeck(+ return ModelDeck(inference_models=inference_models,aliases=model_deck_blueprint.aliases,llm_presets=model_deck_blueprint.llm.presets,@@ -140,17 +148,18 @@ class ModelManager(ModelManagerAbstract):img_gen_presets=model_deck_blueprint.img_gen.presets,img_gen_choice_default=model_deck_blueprint.img_gen.choice_default,)- return model_deck@overridedef get_inference_model(self, model_handle: str) -> InferenceModelSpec:if self.model_deck is None:- raise RuntimeError("Model deck is not initialized")+ msg = "Model deck is not initialized"+ raise RuntimeError(msg)return self.model_deck.get_required_inference_model(model_handle=model_handle)@overridedef get_required_inference_backend(self, backend_name: str) -> InferenceBackend:backend = self.inference_backend_library.get_inference_backend(backend_name)if backend is None:- raise ModelsManagerError(f"Inference backend '{backend_name}' not found")+ msg = f"Inference backend '{backend_name}' not found"+ raise ModelsManagerError(msg)return backenddiff --git a/pipelex/cogt/ocr/ocr_input.py b/pipelex/cogt/ocr/ocr_input.pyindex 8c6dbb420..a364aa014 100644--- a/pipelex/cogt/ocr/ocr_input.py+++ b/pipelex/cogt/ocr/ocr_input.py@@ -1,10 +1,8 @@-from typing import Optional-from pydantic import BaseModel, model_validator-from typing_extensions import Selffrom pipelex.cogt.exceptions import CogtErrorfrom pipelex.tools.typing.validation_utils import has_exactly_one_among_attributes_from_list+from pipelex.types import Selfclass OcrInputError(CogtError):@@ -12,11 +10,12 @@ class OcrInputError(CogtError):class OcrInput(BaseModel):- image_uri: Optional[str] = None- pdf_uri: Optional[str] = None+ image_uri: str | None = None+ pdf_uri: str | None = None@model_validator(mode="after")def validate_at_exactly_one_input(self) -> Self:if not has_exactly_one_among_attributes_from_list(self, attributes_list=["image_uri", "pdf_uri"]):- raise OcrInputError("Exactly one of 'image_uri' or 'pdf_uri' must be provided")+ msg = "Exactly one of 'image_uri' or 'pdf_uri' must be provided"+ raise OcrInputError(msg)return selfdiff --git a/pipelex/cogt/ocr/ocr_job.py b/pipelex/cogt/ocr/ocr_job.pyindex 8a2edd17a..3047498bf 100644--- a/pipelex/cogt/ocr/ocr_job.py+++ b/pipelex/cogt/ocr/ocr_job.py@@ -3,7 +3,6 @@ from datetime import datetimefrom typing_extensions import overridefrom pipelex.cogt.inference.inference_job_abstract import InferenceJobAbstract-from pipelex.cogt.model_backends.model_spec import InferenceModelSpecfrom pipelex.cogt.ocr.ocr_input import OcrInputfrom pipelex.cogt.ocr.ocr_job_components import OcrJobConfig, OcrJobParams, OcrJobReport@@ -18,7 +17,7 @@ class OcrJob(InferenceJobAbstract):def validate_before_execution(self):pass- def ocr_job_before_start(self, inference_model: InferenceModelSpec):+ def ocr_job_before_start(self):# Reset metadataself.job_metadata.started_at = datetime.now()diff --git a/pipelex/cogt/ocr/ocr_job_components.py b/pipelex/cogt/ocr/ocr_job_components.pyindex 788258065..e07317231 100644--- a/pipelex/cogt/ocr/ocr_job_components.py+++ b/pipelex/cogt/ocr/ocr_job_components.py@@ -1,5 +1,3 @@-from typing import Optional-from pydantic import BaseModelfrom pipelex.tools.config.config_model import ConfigModel@@ -7,11 +5,11 @@ from pipelex.tools.config.config_model import ConfigModelclass OcrJobParams(BaseModel):should_include_images: bool- max_nb_images: Optional[int]- image_min_size: Optional[int]+ max_nb_images: int | None+ image_min_size: int | Noneshould_caption_images: boolshould_include_page_views: bool- page_views_dpi: Optional[int]+ page_views_dpi: int | None@classmethoddef make_default_ocr_job_params(cls) -> "OcrJobParams":diff --git a/pipelex/cogt/ocr/ocr_job_factory.py b/pipelex/cogt/ocr/ocr_job_factory.pyindex 2633b0d0a..c89601f4c 100644--- a/pipelex/cogt/ocr/ocr_job_factory.py+++ b/pipelex/cogt/ocr/ocr_job_factory.py@@ -1,5 +1,3 @@-from typing import Optional-from pipelex.cogt.ocr.ocr_input import OcrInputfrom pipelex.cogt.ocr.ocr_job import OcrJobfrom pipelex.cogt.ocr.ocr_job_components import OcrJobConfig, OcrJobParams, OcrJobReport@@ -11,9 +9,9 @@ class OcrJobFactory:def make_ocr_job(cls,ocr_input: OcrInput,- ocr_job_params: Optional[OcrJobParams] = None,- ocr_job_config: Optional[OcrJobConfig] = None,- job_metadata: Optional[JobMetadata] = None,+ ocr_job_params: OcrJobParams | None = None,+ ocr_job_config: OcrJobConfig | None = None,+ job_metadata: JobMetadata | None = None,) -> OcrJob:# TODO: manage the param default through the config# ocr_config = get_config().cogt.ocr_configdiff --git a/pipelex/cogt/ocr/ocr_output.py b/pipelex/cogt/ocr/ocr_output.pyindex d3d038551..186e5600c 100644--- a/pipelex/cogt/ocr/ocr_output.py+++ b/pipelex/cogt/ocr/ocr_output.py@@ -1,17 +1,15 @@-from typing import Dict, List, Optional-from pydantic import Fieldfrom pipelex import logfrom pipelex.tools.misc.base_64_utils import save_base64_to_binary_filefrom pipelex.tools.misc.file_utils import ensure_directory_exists, save_text_to_path-from pipelex.tools.typing.pydantic_utils import CustomBaseModel+from pipelex.tools.typing.pydantic_utils import CustomBaseModel, empty_list_factory_ofclass ExtractedImage(CustomBaseModel):image_id: str- base_64: Optional[str] = None- caption: Optional[str] = None+ base_64: str | None = None+ caption: str | None = Nonedef save_to_directory(self, directory: str):ensure_directory_exists(directory)@@ -23,16 +21,16 @@ class ExtractedImage(CustomBaseModel):class ExtractedImageFromPage(ExtractedImage):- top_left_x: Optional[int] = None- top_left_y: Optional[int] = None- bottom_right_x: Optional[int] = None- bottom_right_y: Optional[int] = None+ top_left_x: int | None = None+ top_left_y: int | None = None+ bottom_right_x: int | None = None+ bottom_right_y: int | None = Noneclass Page(CustomBaseModel):- text: Optional[str] = None- extracted_images: List[ExtractedImageFromPage] = Field(default_factory=list)- page_view: Optional[ExtractedImageFromPage] = None+ text: str | None = None+ extracted_images: list[ExtractedImageFromPage] = Field(default_factory=empty_list_factory_of(ExtractedImageFromPage))+ page_view: ExtractedImageFromPage | None = Nonedef save_to_directory(self, directory: str, page_text_file_name: str):ensure_directory_exists(directory)@@ -47,7 +45,7 @@ class Page(CustomBaseModel):class OcrOutput(CustomBaseModel):- pages: Dict[int, Page]+ pages: dict[int, Page]@propertydef concatenated_text(self) -> str:diff --git a/pipelex/cogt/ocr/ocr_setting.py b/pipelex/cogt/ocr/ocr_setting.pyindex 942320362..9ec34a603 100644--- a/pipelex/cogt/ocr/ocr_setting.py+++ b/pipelex/cogt/ocr/ocr_setting.py@@ -1,4 +1,4 @@-from typing import Optional, Union+from typing import Unionfrom pydantic import Field@@ -7,8 +7,8 @@ from pipelex.tools.config.config_model import ConfigModelclass OcrSetting(ConfigModel):ocr_handle: str- max_nb_images: Optional[int] = Field(default=None, ge=0)- image_min_size: Optional[int] = Field(default=None, ge=0)+ max_nb_images: int | None = Field(default=None, ge=0)+ image_min_size: int | None = Field(default=None, ge=0)def desc(self) -> str:return f"OcrSetting(ocr_handle={self.ocr_handle}, max_nb_images={self.max_nb_images}, image_min_size={self.image_min_size})"diff --git a/pipelex/cogt/ocr/ocr_worker_abstract.py b/pipelex/cogt/ocr/ocr_worker_abstract.pyindex 42570da61..e80464a5d 100644--- a/pipelex/cogt/ocr/ocr_worker_abstract.py+++ b/pipelex/cogt/ocr/ocr_worker_abstract.py@@ -1,5 +1,5 @@from abc import abstractmethod-from typing import Any, Dict, Optional+from typing import Anyfrom typing_extensions import override@@ -15,9 +15,9 @@ from pipelex.reporting.reporting_protocol import ReportingProtocolclass OcrWorkerAbstract(InferenceWorkerAbstract):def __init__(self,- extra_config: Dict[str, Any],+ extra_config: dict[str, Any],inference_model: InferenceModelSpec,- reporting_delegate: Optional[ReportingProtocol] = None,+ reporting_delegate: ReportingProtocol | None = None,):InferenceWorkerAbstract.__init__(self, reporting_delegate=reporting_delegate)self.extra_config = extra_config@@ -53,7 +53,7 @@ class OcrWorkerAbstract(InferenceWorkerAbstract):ocr_job.job_metadata.unit_job_id = UnitJobId.OCR_EXTRACT_PAGES# Prepare job- ocr_job.ocr_job_before_start(inference_model=self.inference_model)+ ocr_job.ocr_job_before_start()# Execute jobresult = await self._ocr_extract_pages(ocr_job=ocr_job)diff --git a/pipelex/cogt/ocr/ocr_worker_factory.py b/pipelex/cogt/ocr/ocr_worker_factory.pyindex 718d50417..d66c563b1 100644--- a/pipelex/cogt/ocr/ocr_worker_factory.py+++ b/pipelex/cogt/ocr/ocr_worker_factory.py@@ -1,5 +1,3 @@-from typing import Optional-from pipelex.cogt.exceptions import MissingDependencyErrorfrom pipelex.cogt.model_backends.model_spec import InferenceModelSpecfrom pipelex.cogt.ocr.ocr_worker_abstract import OcrWorkerAbstract@@ -12,7 +10,7 @@ class OcrWorkerFactory:def make_ocr_worker(self,inference_model: InferenceModelSpec,- reporting_delegate: Optional[ReportingProtocol] = None,+ reporting_delegate: ReportingProtocol | None = None,) -> OcrWorkerAbstract:plugin = Plugin.make_for_inference_model(inference_model=inference_model)backend = get_models_manager().get_required_inference_backend(inference_model.backend_name)@@ -21,16 +19,19 @@ class OcrWorkerFactory:match plugin.sdk:case "mistral":try:- import mistralai # noqa: F401+ import mistralai # noqa: PLC0415,F401except ImportError as exc:+ lib_name = "mistralai"+ lib_extra_name = "mistral"+ msg = "The mistralai SDK is required to use Mistral OCR models through the mistralai client."raise MissingDependencyError(- "mistralai",- "mistral",- "The mistralai SDK is required to use Mistral OCR models through the mistralai client.",+ lib_name,+ lib_extra_name,+ msg,) from exc- from pipelex.plugins.mistral.mistral_factory import MistralFactory- from pipelex.plugins.mistral.mistral_ocr_worker import MistralOcrWorker+ from pipelex.plugins.mistral.mistral_factory import MistralFactory # noqa: PLC0415+ from pipelex.plugins.mistral.mistral_ocr_worker import MistralOcrWorker # noqa: PLC0415ocr_sdk_instance = plugin_sdk_registry.get_sdk_instance(plugin=plugin) or plugin_sdk_registry.set_sdk_instance(plugin=plugin,@@ -44,15 +45,15 @@ class OcrWorkerFactory:reporting_delegate=reporting_delegate,)case "pypdfium2":- from pipelex.plugins.pypdfium2.pypdfium2_worker import Pypdfium2Worker+ from pipelex.plugins.pypdfium2.pypdfium2_worker import Pypdfium2Worker # noqa: PLC0415ocr_worker = Pypdfium2Worker(- sdk_instance=None,extra_config=backend.extra_config,inference_model=inference_model,reporting_delegate=reporting_delegate,)case _:- raise NotImplementedError(f"Plugin '{plugin}' is not supported")+ msg = f"Plugin '{plugin}' is not supported"+ raise NotImplementedError(msg)return ocr_workerdiff --git a/pipelex/cogt/usage/__init__.py b/pipelex/cogt/usage/__init__.pynew file mode 100644index 000000000..e69de29bbdiff --git a/pipelex/cogt/usage/cost_category.py b/pipelex/cogt/usage/cost_category.pyindex 3330090aa..bdbf1be69 100644--- a/pipelex/cogt/usage/cost_category.py+++ b/pipelex/cogt/usage/cost_category.py@@ -1,5 +1,3 @@-from typing import Dict-from pipelex.types import StrEnum@@ -16,4 +14,4 @@ class CostCategory(StrEnum):OUTPUT_REJECTED_PREDICTION = "output_rejected_prediction"-CostsByCategoryDict = Dict[CostCategory, float]+CostsByCategoryDict = dict[CostCategory, float]diff --git a/pipelex/cogt/usage/cost_registry.py b/pipelex/cogt/usage/cost_registry.pyindex 18b807c21..36538c696 100644--- a/pipelex/cogt/usage/cost_registry.py+++ b/pipelex/cogt/usage/cost_registry.py@@ -1,4 +1,4 @@-from typing import Any, Dict, List, Optional+from typing import Anyimport pandas as pdfrom pydantic import Field, RootModel@@ -12,28 +12,28 @@ from pipelex.cogt.llm.llm_report import LLMTokenCostReport, LLMTokenCostReportFifrom pipelex.cogt.usage.cost_category import CostCategory, CostsByCategoryDictfrom pipelex.cogt.usage.costs_per_token import model_cost_per_tokenfrom pipelex.cogt.usage.token_category import TokenCategory+from pipelex.tools.typing.pydantic_utils import empty_list_factory_of-CostRegistryRoot = List[LLMTokenCostReport]+CostRegistryRoot = list[LLMTokenCostReport]class CostRegistry(RootModel[CostRegistryRoot]):- root: CostRegistryRoot = Field(default_factory=list)+ root: CostRegistryRoot = Field(default_factory=empty_list_factory_of(LLMTokenCostReport))def to_dataframe(self) -> pd.DataFrame:- records: List[Dict[str, Any]] = []+ records: list[dict[str, Any]] = []for token_cost_report in self.root:record_dict = token_cost_report.as_flat_dictionary()records.append(record_dict)- df = pd.DataFrame(records)- return df+ return pd.DataFrame(records)@classmethoddef generate_report(cls,pipeline_run_id: str,- llm_tokens_usages: List[LLMTokensUsage],+ llm_tokens_usages: list[LLMTokensUsage],unit_scale: float,- cost_report_file_path: Optional[str] = None,+ cost_report_file_path: str | None = None,):if not llm_tokens_usages:if pipeline_run_id != "untitled":@@ -75,10 +75,11 @@ class CostRegistry(RootModel[CostRegistryRoot]):LLMTokenCostReportField.COST_INPUT_NON_CACHED: "sum",LLMTokenCostReportField.COST_INPUT_JOINED: "sum",LLMTokenCostReportField.COST_OUTPUT: "sum",- }+ },).reset_index()if agg_by_llm_name is None or agg_by_llm_name.empty: # pyright: ignore[reportUnnecessaryComparison]- raise CostRegistryError("Empty report aggregation by LLM name")+ msg = "Empty report aggregation by LLM name"+ raise CostRegistryError(msg)console = Console()title = "Costs by LLM model"@@ -160,21 +161,21 @@ class CostRegistry(RootModel[CostRegistryRoot]):cost_category=token_type.to_cost_category,)costs_by_token_category[token_type.to_cost_category] = cost_per_token * nb_tokens- token_cost_report = LLMTokenCostReport(+ return LLMTokenCostReport(job_metadata=llm_tokens_usage.job_metadata,inference_model_name=llm_tokens_usage.inference_model_name,platform_llm_id=llm_tokens_usage.inference_model_id,nb_tokens_by_category=llm_tokens_usage.nb_tokens_by_category,costs_by_token_category=costs_by_token_category,)- return token_cost_report@classmethoddef complete_cost_report(cls, llm_tokens_usage: LLMTokensUsage) -> LLMTokenCostReport:cost_report = cls.compute_cost_report(llm_tokens_usage=llm_tokens_usage)# compute the input_non_cached tokensif cost_report.nb_tokens_by_category.get(TokenCategory.INPUT_NON_CACHED) is not None:- raise CostRegistryError("CostCategory.INPUT_NON_CACHED already exists in the cost report")+ msg = "CostCategory.INPUT_NON_CACHED already exists in the cost report"+ raise CostRegistryError(msg)# we use pop to remove input tokens which will be replaced by "input joined"nb_tokens_input_joined = cost_report.nb_tokens_by_category.pop(TokenCategory.INPUT, 0)cost_report.costs_by_token_category.pop(CostCategory.INPUT, None)@@ -186,7 +187,8 @@ class CostRegistry(RootModel[CostRegistryRoot]):cost_report.nb_tokens_by_category[TokenCategory.INPUT_CACHED] = nb_tokens_input_cachedcost_report.costs_by_token_category[CostCategory.INPUT_NON_CACHED] = nb_tokens_input_non_cached * model_cost_per_token(- costs=llm_tokens_usage.unit_costs, cost_category=CostCategory.INPUT_NON_CACHED+ costs=llm_tokens_usage.unit_costs,+ cost_category=CostCategory.INPUT_NON_CACHED,)costs_input_cached = cost_report.costs_by_token_category.get(CostCategory.INPUT_CACHED, 0)cost_report.costs_by_token_category[CostCategory.INPUT_CACHED] = costs_input_cacheddiff --git a/pipelex/cogt/usage/costs_per_token.py b/pipelex/cogt/usage/costs_per_token.pyindex 6486e5a51..05027958d 100644--- a/pipelex/cogt/usage/costs_per_token.py+++ b/pipelex/cogt/usage/costs_per_token.py@@ -4,17 +4,28 @@ from pipelex.cogt.usage.cost_category import CostCategory, CostsByCategoryDictdef model_cost_per_token(costs: CostsByCategoryDict, cost_category: CostCategory) -> float:# cost_per_million_tokens_usd should be missing only for models that we run on our own GPUs# all token types are not used for all models- if cost_category == CostCategory.INPUT_CACHED:- if cost_per_million_tokens := costs.get(CostCategory.INPUT_CACHED):- return cost_per_million_tokens / 1000000- elif cost_per_million_tokens := costs.get(CostCategory.INPUT):- # according to openai docs, cached input tokens are discounted 50%- return 0.5 * cost_per_million_tokens / 1000000- else:- return 0.0- elif cost_category == CostCategory.INPUT_NON_CACHED:- return model_cost_per_token(costs=costs, cost_category=CostCategory.INPUT)- elif cost_per_million_tokens := costs.get(cost_category):- return cost_per_million_tokens / 1000000- else:- return 0.0+ match cost_category:+ case CostCategory.INPUT_CACHED:+ if cost_per_million_tokens := costs.get(CostCategory.INPUT_CACHED):+ return cost_per_million_tokens / 1000000+ elif cost_per_million_tokens := costs.get(CostCategory.INPUT):+ # according to openai docs, cached input tokens are discounted 50%+ return 0.5 * cost_per_million_tokens / 1000000+ else:+ return 0.0+ case CostCategory.INPUT_NON_CACHED:+ return model_cost_per_token(costs=costs, cost_category=CostCategory.INPUT)+ case (+ CostCategory.INPUT+ | CostCategory.INPUT_JOINED+ | CostCategory.INPUT_AUDIO+ | CostCategory.OUTPUT+ | CostCategory.OUTPUT_AUDIO+ | CostCategory.OUTPUT_REASONING+ | CostCategory.OUTPUT_ACCEPTED_PREDICTION+ | CostCategory.OUTPUT_REJECTED_PREDICTION+ ):+ if cost_per_million_tokens := costs.get(cost_category):+ return cost_per_million_tokens / 1000000+ else:+ return 0.0diff --git a/pipelex/cogt/usage/token_category.py b/pipelex/cogt/usage/token_category.pyindex 7e93b9082..f31e8d0ad 100644--- a/pipelex/cogt/usage/token_category.py+++ b/pipelex/cogt/usage/token_category.py@@ -1,5 +1,3 @@-from typing import Dict-from pipelex.cogt.usage.cost_category import CostCategoryfrom pipelex.types import StrEnum@@ -21,4 +19,4 @@ class TokenCategory(StrEnum):return CostCategory(self)-NbTokensByCategoryDict = Dict[TokenCategory, int]+NbTokensByCategoryDict = dict[TokenCategory, int]diff --git a/pipelex/config.py b/pipelex/config.pyindex 7bfc7cbfc..a41a7def0 100644--- a/pipelex/config.py+++ b/pipelex/config.py@@ -1,4 +1,4 @@-from typing import Dict, List, Optional, cast+from typing import castimport shortuuidfrom pydantic import Field, field_validator@@ -25,19 +25,19 @@ class StaticValidationReaction(StrEnum):class StaticValidationConfig(ConfigModel):default_reaction: StaticValidationReaction = Field(strict=False)- reactions: Dict[StaticValidationErrorType, StaticValidationReaction]+ reactions: dict[StaticValidationErrorType, StaticValidationReaction]@field_validator("reactions", mode="before")- def validate_reactions(cls, value: Dict[str, str]) -> Dict[StaticValidationErrorType, StaticValidationReaction]:- the_dict = cast(- Dict[StaticValidationErrorType, StaticValidationReaction],+ @staticmethod+ def validate_reactions(value: dict[str, str]) -> dict[StaticValidationErrorType, StaticValidationReaction]:+ return cast(+ "dict[StaticValidationErrorType, StaticValidationReaction]",ConfigModel.transform_dict_str_to_enum(input_dict=value,key_enum_cls=StaticValidationErrorType,value_enum_cls=StaticValidationReaction,),)- return the_dictclass PipeRunConfig(ConfigModel):@@ -49,13 +49,15 @@ class DryRunConfig(ConfigModel):text_gen_truncate_length: intnb_list_items: intnb_ocr_pages: int- image_urls: List[str]- allowed_to_fail_pipes: List[str] = Field(default_factory=list)+ image_urls: list[str]+ allowed_to_fail_pipes: list[str] = Field(default_factory=list)@field_validator("image_urls", mode="before")- def validate_image_urls(cls, value: List[str]) -> List[str]:+ @staticmethod+ def validate_image_urls(value: list[str]) -> list[str]:if not value:- raise PipelexConfigError("dry_run_config.image_urls must be a non-empty list")+ msg = "dry_run_config.image_urls must be a non-empty list"+ raise PipelexConfigError(msg)return value@@ -70,13 +72,12 @@ class StructureConfig(ConfigModel):class PromptingConfig(ConfigModel):default_prompting_style: PromptingStyle- prompting_styles: Dict[str, PromptingStyle]+ prompting_styles: dict[str, PromptingStyle]- def get_prompting_style(self, prompting_target: Optional[PromptingTarget] = None) -> Optional[PromptingStyle]:+ def get_prompting_style(self, prompting_target: PromptingTarget | None = None) -> PromptingStyle | None:if prompting_target:return self.prompting_styles.get(prompting_target, self.default_prompting_style)- else:- return None+ return Noneclass FeatureConfig(ConfigModel):@@ -125,5 +126,6 @@ class PipelexConfig(ConfigRoot):def get_config() -> PipelexConfig:singleton_config = get_required_config()if not isinstance(singleton_config, PipelexConfig):- raise RuntimeError(f"Expected {PipelexConfig}, but got {type(singleton_config)}")+ msg = f"Expected {PipelexConfig}, but got {type(singleton_config)}"+ raise TypeError(msg)return singleton_configdiff --git a/pipelex/core/bundles/__init__.py b/pipelex/core/bundles/__init__.pynew file mode 100644index 000000000..e69de29bbdiff --git a/pipelex/core/bundles/pipelex_bundle_blueprint.py b/pipelex/core/bundles/pipelex_bundle_blueprint.pyindex d649b3826..66ce37b84 100644--- a/pipelex/core/bundles/pipelex_bundle_blueprint.py+++ b/pipelex/core/bundles/pipelex_bundle_blueprint.py@@ -1,4 +1,4 @@-from typing import Annotated, Dict, Optional, Union+from typing import Annotatedfrom pydantic import BaseModel, ConfigDict, Field, field_validator@@ -15,19 +15,15 @@ from pipelex.pipe_operators.llm.pipe_llm_blueprint import PipeLLMBlueprintfrom pipelex.pipe_operators.ocr.pipe_ocr_blueprint import PipeOcrBlueprintPipeBlueprintUnion = Annotated[- Union[- # Pipe operators- PipeFuncBlueprint,- PipeImgGenBlueprint,- PipeJinja2Blueprint,- PipeLLMBlueprint,- PipeOcrBlueprint,- # Pipe controllers- PipeBatchBlueprint,- PipeConditionBlueprint,- PipeParallelBlueprint,- PipeSequenceBlueprint,- ],+ PipeFuncBlueprint+ | PipeImgGenBlueprint+ | PipeJinja2Blueprint+ | PipeLLMBlueprint+ | PipeOcrBlueprint+ | PipeBatchBlueprint+ | PipeConditionBlueprint+ | PipeParallelBlueprint+ | PipeSequenceBlueprint,Field(discriminator="type"),]@@ -36,14 +32,14 @@ class PipelexBundleBlueprint(BaseModel):model_config = ConfigDict(extra="forbid")domain: str- definition: Optional[str] = None- system_prompt: Optional[str] = None- system_prompt_to_structure: Optional[str] = None- prompt_template_to_structure: Optional[str] = None+ definition: str | None = None+ system_prompt: str | None = None+ system_prompt_to_structure: str | None = None+ prompt_template_to_structure: str | None = None- concept: Optional[Dict[str, Union[ConceptBlueprint, str]]] = Field(default_factory=dict)+ concept: dict[str, ConceptBlueprint | str] | None = Field(default_factory=dict)- pipe: Optional[Dict[str, PipeBlueprintUnion]] = Field(default_factory=dict)+ pipe: dict[str, PipeBlueprintUnion] | None = Field(default_factory=dict)@field_validator("domain", mode="before")@classmethoddiff --git a/pipelex/core/concepts/concept.py b/pipelex/core/concepts/concept.pyindex e88f925be..e8630ed25 100644--- a/pipelex/core/concepts/concept.py+++ b/pipelex/core/concepts/concept.py@@ -1,5 +1,3 @@-from typing import Optional-from kajson.kajson_manager import KajsonManagerfrom pydantic import BaseModel, ConfigDict, field_validator@@ -20,7 +18,7 @@ class Concept(BaseModel):domain: strdefinition: strstructure_class_name: str- refines: Optional[str] = None+ refines: str | None = None@propertydef concept_string(self) -> str:@@ -32,17 +30,20 @@ class Concept(BaseModel):return concept_string.startswith(SpecialDomain.IMPLICIT)@field_validator("code")- def validate_code(cls, code: str) -> str:+ @staticmethod+ def validate_code(code: str) -> str:ConceptBlueprint.validate_concept_code(concept_code=code)return code@field_validator("domain")- def validate_domain(cls, domain: str) -> str:+ @staticmethod+ def validate_domain(domain: str) -> str:DomainBlueprint.validate_domain_code(code=domain)return domain@field_validator("refines", mode="before")- def validate_refines(cls, refines: Optional[str]) -> Optional[str]:+ @staticmethod+ def validate_refines(refines: str | None) -> str | None:if refines is None:return NoneConceptBlueprint.validate_concept_string(concept_string=refines)@@ -72,16 +73,15 @@ class Concept(BaseModel):if strict:# Check if classes are equivalent (same fields, types, descriptions)return ClassRegistryUtils.are_classes_equivalent(concept_1_class, concept_2_class)- else:- # Check if concept_1 is a subclass of concept_2- try:- if issubclass(concept_1_class, concept_2_class):- return True- except TypeError:- pass-- # Check if concept_1 has a field that is of type concept_2_class- return ClassRegistryUtils.has_compatible_field(concept_1_class, concept_2_class)+ # Check if concept_1 is a subclass of concept_2+ try:+ if issubclass(concept_1_class, concept_2_class):+ return True+ except TypeError:+ pass++ # Check if concept_1 has a field that is of type concept_2_class+ return ClassRegistryUtils.has_compatible_field(concept_1_class, concept_2_class)return False@classmethod@@ -89,8 +89,7 @@ class Concept(BaseModel):# We get_class_registry directly from KajsonManager instead of pipelex hub to avoid circular importif KajsonManager.get_class_registry().has_subclass(name=structure_class_name, base_class=StuffContent):return True- else:- # We get_class_registry directly from KajsonManager instead of pipelex hub to avoid circular import- if KajsonManager.get_class_registry().has_class(name=structure_class_name):- log.warning(f"Concept class '{structure_class_name}' is registered but it's not a subclass of StuffContent")- return False+ # We get_class_registry directly from KajsonManager instead of pipelex hub to avoid circular import+ if KajsonManager.get_class_registry().has_class(name=structure_class_name):+ log.warning(f"Concept class '{structure_class_name}' is registered but it's not a subclass of StuffContent")+ return Falsediff --git a/pipelex/core/concepts/concept_blueprint.py b/pipelex/core/concepts/concept_blueprint.pyindex 5ca8655ae..493d4c7b0 100644--- a/pipelex/core/concepts/concept_blueprint.py+++ b/pipelex/core/concepts/concept_blueprint.py@@ -1,15 +1,14 @@from datetime import datetime-from typing import Any, Dict, List, Optional, Union+from typing import Anyfrom pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator-from typing_extensions import Selffrom pipelex.core.concepts.concept_native import NativeConceptEnum, NativeConceptManagerfrom pipelex.core.concepts.exceptions import ConceptCodeError, ConceptStringError, ConceptStringOrConceptCodeErrorfrom pipelex.core.domains.domain import SpecialDomainfrom pipelex.core.domains.domain_blueprint import DomainBlueprintfrom pipelex.tools.misc.string_utils import is_pascal_case-from pipelex.types import StrEnum+from pipelex.types import Self, StrEnumclass ConceptBlueprintError(Exception):@@ -32,13 +31,13 @@ class ConceptStructureBlueprintFieldType(StrEnum):class ConceptStructureBlueprint(BaseModel):definition: str- type: Optional[ConceptStructureBlueprintFieldType] = None- item_type: Optional[str] = None- key_type: Optional[str] = None- value_type: Optional[str] = None- choices: Optional[List[str]] = Field(default_factory=list)- required: Optional[bool] = Field(default=True)- default_value: Optional[Any] = None+ type: ConceptStructureBlueprintFieldType | None = None+ item_type: str | None = None+ key_type: str | None = None+ value_type: str | None = None+ choices: list[str] | None = Field(default_factory=list)+ required: bool | None = Field(default=True)+ default_value: Any | None = None# TODO: date translator for default_value@@ -47,27 +46,25 @@ class ConceptStructureBlueprint(BaseModel):"""Validate the structure blueprint according to type rules."""# If type is None (array), choices must not be Noneif self.type is None and not self.choices:- raise ConceptStructureBlueprintError(- f"When type is None (array), choices must not be empty. Actual type: {self.type}, choices: {self.choices}"- )+ msg = f"When type is None (array), choices must not be empty. Actual type: {self.type}, choices: {self.choices}"+ raise ConceptStructureBlueprintError(msg)# If type is "dict", key_type and value_type must not be emptyif self.type == ConceptStructureBlueprintFieldType.DICT:if not self.key_type:- raise ConceptStructureBlueprintError(- f"When type is '{ConceptStructureBlueprintFieldType.DICT}', key_type must not be empty. Actual key_type: {self.key_type}"- )+ msg = f"When type is '{ConceptStructureBlueprintFieldType.DICT}', key_type must not be empty. Actual key_type: {self.key_type}"+ raise ConceptStructureBlueprintError(msg)if not self.value_type:- raise ConceptStructureBlueprintError(- f"When type is '{ConceptStructureBlueprintFieldType.DICT}', value_type must not be empty. Actual value_type: {self.value_type}"- )+ msg = f"When type is '{ConceptStructureBlueprintFieldType.DICT}', value_type must not be empty. Actual value_type: {self.value_type}"+ raise ConceptStructureBlueprintError(msg)# Check when default_value is not None, type is not None (except for choice fields)if self.default_value is not None and self.type is None and not self.choices:- raise ConceptStructureBlueprintError(- f"When default_value is not None, type must be specified (unless choices are provided). "- f"Actual type: {self.type}, default_value: {self.default_value}, choices: {self.choices}"+ msg = (+ f"When default_value is not None, type must be specified (unless choices are provided). Actual type: {self.type},"+ f"default_value: {self.default_value}, choices: {self.choices}")+ raise ConceptStructureBlueprintError(msg)# Check default_value type is the same as typeif self.default_value is not None and self.type is not None:@@ -76,9 +73,8 @@ class ConceptStructureBlueprint(BaseModel):# Check default_value is valid for choice fieldsif self.default_value is not None and self.type is None and self.choices:if self.default_value not in self.choices:- raise ConceptStructureBlueprintError(- f"default_value must be one of the valid choices. Got '{self.default_value}', valid choices: {self.choices}"- )+ msg = f"default_value must be one of the valid choices. Got '{self.default_value}', valid choices: {self.choices}"+ raise ConceptStructureBlueprintError(msg)return self@@ -111,10 +107,8 @@ class ConceptStructureBlueprint(BaseModel):self._raise_type_mismatch_error("date", type(self.default_value).__name__)def _raise_type_mismatch_error(self, expected_type_name: str, actual_type_name: str) -> None:- """Raise a type mismatch error with consistent formatting."""- raise ConceptStructureBlueprintError(- f"default_value type mismatch: expected {expected_type_name} for type '{self.type}', but got {actual_type_name}"- )+ msg = f"default_value type mismatch: expected {expected_type_name} for type '{self.type}', but got {actual_type_name}"+ raise ConceptStructureBlueprintError(msg)class ConceptBlueprint(BaseModel):@@ -122,32 +116,32 @@ class ConceptBlueprint(BaseModel):definition: str# TODO (non-blockiing): define a type for Union[str, ConceptStructureBlueprint] (ConceptChoice to be consistent with LLMChoice)- structure: Optional[Union[str, Dict[str, Union[str, ConceptStructureBlueprint]]]] = None+ structure: str | dict[str, str | ConceptStructureBlueprint] | None = None# TODO: restore possibility of multiple refiles- refines: Optional[str] = None+ refines: str | None = None@classmethoddef is_native_concept_code(cls, concept_code: str) -> bool:ConceptBlueprint.validate_concept_code(concept_code=concept_code)- return concept_code in [native_concept for native_concept in NativeConceptEnum]+ return concept_code in NativeConceptEnum.values_list()@classmethoddef validate_concept_code(cls, concept_code: str) -> None:if not is_pascal_case(concept_code):- raise ConceptCodeError(- f"Concept code '{concept_code}' must be PascalCase (letters and numbers only, starting with uppercase, without `.`)"- )+ msg = f"Concept code '{concept_code}' must be PascalCase (letters and numbers only, starting with uppercase, without `.`)"+ raise ConceptCodeError(msg)@classmethoddef validate_concept_string_or_code(cls, concept_string_or_code: str) -> None:if concept_string_or_code.count(".") > 1:- raise ConceptStringOrConceptCodeError(+ msg = (f"concept_string_or_code '{concept_string_or_code}' is invalid. ""It should either contain a domain in snake_case and a concept code in PascalCase separated by one dot, "- "or be a concept code in PascalCase."+ "or be a concept code in PascalCase.",)+ raise ConceptStringOrConceptCodeError(msg)- elif concept_string_or_code.count(".") == 1:+ if concept_string_or_code.count(".") == 1:domain, concept_code = concept_string_or_code.split(".")DomainBlueprint.validate_domain_code(code=domain)cls.validate_concept_code(concept_code=concept_code)@@ -157,59 +151,57 @@ class ConceptBlueprint(BaseModel):@staticmethoddef validate_concept_string(concept_string: str) -> None:"""Validate that a concept code follows PascalCase convention."""- if "." not in concept_string:- raise ConceptStringError(- f"Concept string '{concept_string}' is invalid. It should contain a domain in snake_case "- "and a concept code in PascalCase separated by one dot."- )- elif concept_string.count(".") > 1:- raise ConceptStringError(+ if "." not in concept_string or concept_string.count(".") > 1:+ msg = (f"Concept string '{concept_string}' is invalid. It should contain a domain in snake_case ""and a concept code in PascalCase separated by one dot.")- else:- domain, concept_code = concept_string.split(".", 1)+ raise ConceptStringError(msg)+ domain, concept_code = concept_string.split(".", 1)# Validate domainDomainBlueprint.validate_domain_code(domain)# Validate concept codeif not is_pascal_case(concept_code):- raise ConceptCodeError(- f"Concept code '{concept_code}' must be PascalCase (letters and numbers only, starting with uppercase, without `.`)"- )+ msg = f"Concept code '{concept_code}' must be PascalCase (letters and numbers only, starting with uppercase, without `.`)"+ raise ConceptCodeError(msg)# Validate that if the concept code is among the native concepts, the domain MUST be native.- if concept_code in [concept.value for concept in [native_concept for native_concept in NativeConceptEnum]]:+ if concept_code in NativeConceptEnum.values_list():if not SpecialDomain.is_native(domain=domain):- raise ConceptStringError(+ msg = (f"Concept string '{concept_string}' is invalid. "f"Concept code '{concept_code}' is a native concept, so the domain must be '{SpecialDomain.NATIVE}', "f"or nothing, but not '{domain}'")-+ raise ConceptStringError(msg)# Validate that if the domain is native, the concept code is a native conceptif SpecialDomain.is_native(domain=domain):- if concept_code not in [native_concept for native_concept in NativeConceptEnum]:- raise ConceptStringError(+ if concept_code not in NativeConceptEnum.values_list():+ msg = (f"Concept string '{concept_string}' is invalid. "f"Concept code '{concept_code}' is not a native concept, so the domain must not be '{SpecialDomain.NATIVE}'.")+ raise ConceptStringError(msg)@field_validator("refines", mode="before")@classmethod- def validate_refines(cls, refines: Optional[str] = None) -> Optional[str]:+ def validate_refines(cls, refines: str | None = None) -> str | None:if refines is not None:if not NativeConceptManager.is_native_concept(refines):- raise ConceptBlueprintError(f"Forbidden to refine a non-native concept: '{refines}'. Refining non-native concepts will come soon.")+ msg = f"Forbidden to refine a non-native concept: '{refines}'. Refining non-native concepts will come soon."+ raise ConceptBlueprintError(msg)cls.validate_concept_string_or_code(concept_string_or_code=refines)return refines@model_validator(mode="before")- def model_validate_blueprint(cls, values: Union[Dict[str, Any], str]) -> Union[Dict[str, Any], str]:+ @classmethod+ def model_validate_blueprint(cls, values: dict[str, Any] | str) -> dict[str, Any] | str:if isinstance(values, dict) and values.get("refines") and values.get("structure"):- raise ConceptBlueprintError(+ msg = (f"Forbidden to have refines and structure at the same time: `{values.get('refines')}` "f"and `{values.get('structure')}` for concept that has the definition `{values.get('definition')}`")+ raise ConceptBlueprintError(msg)return valuesdiff --git a/pipelex/core/concepts/concept_factory.py b/pipelex/core/concepts/concept_factory.pyindex 90f8643bc..f501974be 100644--- a/pipelex/core/concepts/concept_factory.py+++ b/pipelex/core/concepts/concept_factory.py@@ -1,4 +1,4 @@-from typing import Any, Dict, List, Optional, Union+from typing import Anyfrom kajson.kajson_manager import KajsonManagerfrom pydantic import BaseModel@@ -25,7 +25,7 @@ class DomainAndConceptCode(BaseModel):class ConceptFactory:@classmethod- def normalize_structure_blueprint(cls, structure_dict: Dict[str, Union[str, ConceptStructureBlueprint]]) -> Dict[str, ConceptStructureBlueprint]:+ def normalize_structure_blueprint(cls, structure_dict: dict[str, str | ConceptStructureBlueprint]) -> dict[str, ConceptStructureBlueprint]:"""Convert a mixed structure dictionary to a proper ConceptStructureBlueprint dictionary.Args:@@ -33,8 +33,9 @@ class ConceptFactory:Returns:Dictionary with all values as ConceptStructureBlueprint objects+"""- normalized: Dict[str, ConceptStructureBlueprint] = {}+ normalized: dict[str, ConceptStructureBlueprint] = {}for field_name, field_value in structure_dict.items():if isinstance(field_value, str):@@ -64,7 +65,7 @@ class ConceptFactory:return f"{domain}.{concept_code}"@classmethod- def make(cls, concept_code: str, domain: str, definition: str, structure_class_name: str, refines: Optional[str] = None) -> Concept:+ def make(cls, concept_code: str, domain: str, definition: str, structure_class_name: str, refines: str | None = None) -> Concept:return Concept(code=concept_code,domain=domain,@@ -84,32 +85,34 @@ class ConceptFactory:@classmethoddef make_domain_and_concept_code_from_concept_string_or_code(- cls, domain: str, concept_string_or_code: str, concept_codes_from_the_same_domain: Optional[List[str]] = None+ cls,+ domain: str,+ concept_string_or_code: str,+ concept_codes_from_the_same_domain: list[str] | None = None,) -> DomainAndConceptCode:# At this point, the concept_string_or_code is already validatedif "." in concept_string_or_code:# Is a concept string.parts = concept_string_or_code.rsplit(".")return DomainAndConceptCode(domain=parts[0], concept_code=parts[1])- else:- if NativeConceptManager.is_native_concept(concept_string_or_code=concept_string_or_code):- return DomainAndConceptCode(domain=SpecialDomain.NATIVE, concept_code=concept_string_or_code)+ if NativeConceptManager.is_native_concept(concept_string_or_code=concept_string_or_code):+ return DomainAndConceptCode(domain=SpecialDomain.NATIVE, concept_code=concept_string_or_code)- elif (- concept_codes_from_the_same_domain and concept_string_or_code in concept_codes_from_the_same_domain- ): # Is a concept code from the same domain- return DomainAndConceptCode(domain=domain, concept_code=concept_string_or_code)- else:- return DomainAndConceptCode(domain=SpecialDomain.IMPLICIT, concept_code=concept_string_or_code)+ if (+ concept_codes_from_the_same_domain and concept_string_or_code in concept_codes_from_the_same_domain+ ): # Is a concept code from the same domain+ return DomainAndConceptCode(domain=domain, concept_code=concept_string_or_code)+ return DomainAndConceptCode(domain=SpecialDomain.IMPLICIT, concept_code=concept_string_or_code)@classmethoddef make_refine(cls, refine: str) -> str:if NativeConceptManager.is_native_concept(concept_string_or_code=refine):return NativeConceptManager.get_native_concept_string(concept_string_or_code=refine)- raise ConceptFactoryError(f"Refine '{refine}' is not a native concept")+ msg = f"Refine '{refine}' is not a native concept"+ raise ConceptFactoryError(msg)@classmethod- def make_refines(cls, blueprint: ConceptBlueprint) -> Optional[str]:+ def make_refines(cls, blueprint: ConceptBlueprint) -> str | None:if blueprint.refines:return cls.make_refine(refine=blueprint.refines)return None@@ -120,21 +123,22 @@ class ConceptFactory:domain: str,concept_code: str,blueprint: ConceptBlueprint,- concept_codes_from_the_same_domain: Optional[List[str]] = None,+ concept_codes_from_the_same_domain: list[str] | None = None,) -> Concept:ConceptBlueprint.validate_concept_code(concept_code=concept_code)structure_class_name: str- current_refine: Optional[str] = None+ current_refine: str | None = None# Handle structure definitionif blueprint.structure:if isinstance(blueprint.structure, str):# Structure is defined as a string - check if the class is in the registry and is validif not Concept.is_valid_structure_class(structure_class_name=blueprint.structure):- raise StructureClassError(+ msg = (f"Structure class '{blueprint.structure}' set for concept '{concept_code}' in domain '{domain}' ""is not a registered subclass of StuffContent")+ raise StructureClassError(msg)structure_class_name = blueprint.structureelse:# Structure is defined as a ConceptStructureBlueprint - run the structure generator and put it in the class registry@@ -148,7 +152,7 @@ class ConceptFactory:)# Execute the generated Python code to register the class- exec_globals: Dict[str, Any] = {}+ exec_globals: dict[str, Any] = {}exec(python_code, exec_globals)# Register the generated class@@ -158,26 +162,27 @@ class ConceptFactory:structure_class_name = concept_codeexcept Exception as exc:- raise ConceptFactoryError(f"Error generating structure class for concept '{concept_code}' in domain '{domain}': {exc}") from exc+ msg = f"Error generating structure class for concept '{concept_code}' in domain '{domain}': {exc}"+ raise ConceptFactoryError(msg) from exc# Handle refines definitionelif blueprint.refines:# If we have refines, validate that there is no structure related to the concept code in the class registryif Concept.is_valid_structure_class(structure_class_name=concept_code):- raise ConceptFactoryError(+ msg = (f"Concept '{concept_code}' in domain '{domain}' has refines but also has a structure class registered. ""A concept cannot have both structure and refines.")+ raise ConceptFactoryError(msg)current_refine = cls.make_refines(blueprint=blueprint)structure_class_name = current_refine.split(".")[1] + "Content" if current_refine else TextContent.__name__# Handle neither structure nor refines - check the class registry+ # If there is a class, use it. structure_class_name is then the concept_code+ elif Concept.is_valid_structure_class(structure_class_name=concept_code):+ structure_class_name = concept_codeelse:- # If there is a class, use it. structure_class_name is then the concept_code- if Concept.is_valid_structure_class(structure_class_name=concept_code):- structure_class_name = concept_code- else:- # If there is NO class, the fallback class is TextContent.__name__- structure_class_name = TextContent.__name__+ # If there is NO class, the fallback class is TextContent.__name__+ structure_class_name = TextContent.__name__domain_and_concept_code = cls.make_domain_and_concept_code_from_concept_string_or_code(domain=domain,diff --git a/pipelex/core/concepts/concept_library.py b/pipelex/core/concepts/concept_library.pyindex 716c44adb..2904820f7 100644--- a/pipelex/core/concepts/concept_library.py+++ b/pipelex/core/concepts/concept_library.py@@ -1,7 +1,7 @@-from typing import Any, Dict, List, Optional, Type+from typing import Anyfrom pydantic import Field, RootModel-from typing_extensions import Self, override+from typing_extensions import overridefrom pipelex.core.concepts.concept import Conceptfrom pipelex.core.concepts.concept_blueprint import ConceptBlueprint@@ -12,8 +12,9 @@ from pipelex.core.domains.domain import SpecialDomainfrom pipelex.core.stuffs.stuff_content import ImageContentfrom pipelex.exceptions import ConceptLibraryConceptNotFoundError, ConceptLibraryErrorfrom pipelex.hub import get_class_registry+from pipelex.types import Self-ConceptLibraryRoot = Dict[str, Concept]+ConceptLibraryRoot = dict[str, Concept]class ConceptLibrary(RootModel[ConceptLibraryRoot], ConceptProviderAbstract):@@ -22,16 +23,15 @@ class ConceptLibrary(RootModel[ConceptLibraryRoot], ConceptProviderAbstract):def validate_with_libraries(self):"""Validates that the each refine concept code in the refines array of each concept in the library exists in the library"""for concept in self.root.values():- if concept.refines and concept.refines not in self.root.keys():- raise ConceptLibraryError(- f"Concept '{concept.code}' refines '{concept.refines}' but no concept with the code '{concept.refines}' exists"- )+ if concept.refines and concept.refines not in self.root:+ msg = f"Concept '{concept.code}' refines '{concept.refines}' but no concept with the code '{concept.refines}' exists"+ raise ConceptLibraryError(msg)@overridedef setup(self):native_concepts = [ConceptFactory.make_native_concept(native_concept_data=NATIVE_CONCEPTS_DATA[native_concept])- for native_concept in [native_concept for native_concept in NativeConceptEnum]+ for native_concept in NativeConceptEnum.values_list()]self.add_concepts(native_concepts)@@ -52,79 +52,78 @@ class ConceptLibrary(RootModel[ConceptLibraryRoot], ConceptProviderAbstract):def get_native_concept(self, native_concept: NativeConceptEnum) -> Concept:try:return self.root[f"{SpecialDomain.NATIVE}.{native_concept}"]- except KeyError:- raise ConceptLibraryConceptNotFoundError(f"Native concept '{native_concept}' not found in the library")+ except KeyError as key_error:+ msg = f"Native concept '{native_concept}' not found in the library"+ raise ConceptLibraryConceptNotFoundError(msg) from key_error- def get_native_concepts(self) -> List[Concept]:+ def get_native_concepts(self) -> list[Concept]:"""Create all native concepts from the hardcoded data"""- return [self.get_native_concept(native_concept=native_concept) for native_concept in [native_concept for native_concept in NativeConceptEnum]]+ return [self.get_native_concept(native_concept=native_concept) for native_concept in NativeConceptEnum.values_list()]@override- def list_concepts(self) -> List[Concept]:+ def list_concepts(self) -> list[Concept]:return list(self.root.values())@override- def list_concepts_by_domain(self, domain: str) -> List[Concept]:+ def list_concepts_by_domain(self, domain: str) -> list[Concept]:return [concept for key, concept in self.root.items() if key.startswith(f"{domain}.")]@overridedef add_new_concept(self, concept: Concept):if concept.concept_string in self.root:- raise ConceptLibraryError(f"Concept '{concept.concept_string}' already exists in the library")+ msg = f"Concept '{concept.concept_string}' already exists in the library"+ raise ConceptLibraryError(msg)self.root[concept.concept_string] = concept@override- def add_concepts(self, concepts: List[Concept]):+ def add_concepts(self, concepts: list[Concept]):for concept in concepts:self.add_new_concept(concept=concept)- def remove_concepts_by_codes(self, concept_codes: List[str]) -> None:+ def remove_concepts_by_codes(self, concept_codes: list[str]) -> None:for concept_code in concept_codes:if concept_code in self.root:del self.root[concept_code]@overridedef is_compatible(self, tested_concept: Concept, wanted_concept: Concept, strict: bool = False) -> bool:- if Concept.are_concept_compatible(concept_1=tested_concept, concept_2=wanted_concept, strict=strict):- return True- return False+ return Concept.are_concept_compatible(concept_1=tested_concept, concept_2=wanted_concept, strict=strict)@overridedef get_required_concept(self, concept_string: str) -> Concept:- """- `concept_string` can have the domain or not. If it doesn't have the domain, it is assumed to be native.+ """`concept_string` can have the domain or not. If it doesn't have the domain, it is assumed to be native.If it is not native and doesnt have a domain, it should raise an error"""if Concept.is_implicit_concept(concept_string=concept_string):return ConceptFactory.make_implicit_concept(concept_string=concept_string)ConceptBlueprint.validate_concept_string(concept_string=concept_string)- concept = self.root[concept_string]- return concept+ return self.root[concept_string]@override- def get_class(self, concept_code: str) -> Optional[Type[Any]]:+ def get_class(self, concept_code: str) -> type[Any] | None:return get_class_registry().get_class(concept_code)@overridedef is_image_concept(self, concept: Concept) -> bool:- """- Check if the concept is an image concept.+ """Check if the concept is an image concept.It is an image concept if its structure class is a subclass of ImageContentor if it refines the native Image concept."""pydantic_model = self.get_class(concept_code=concept.structure_class_name)is_image_class = bool(pydantic_model and issubclass(pydantic_model, ImageContent))refines_image = self.is_compatible(- tested_concept=concept, wanted_concept=self.get_native_concept(native_concept=NativeConceptEnum.IMAGE), strict=True+ tested_concept=concept,+ wanted_concept=self.get_native_concept(native_concept=NativeConceptEnum.IMAGE),+ strict=True,)return is_image_class or refines_image@override- def search_for_concept_in_domains(self, concept_code: str, search_domains: List[str]) -> Optional[Concept]:+ def search_for_concept_in_domains(self, concept_code: str, search_domains: list[str]) -> Concept | None:ConceptBlueprint.validate_concept_code(concept_code=concept_code)for domain in search_domains:if found_concept := self.get_required_concept(- concept_string=ConceptFactory.construct_concept_string_with_domain(domain=domain, concept_code=concept_code)+ concept_string=ConceptFactory.construct_concept_string_with_domain(domain=domain, concept_code=concept_code),):return found_conceptdiff --git a/pipelex/core/concepts/concept_native.py b/pipelex/core/concepts/concept_native.pyindex 7732bac45..4e38c24ec 100644--- a/pipelex/core/concepts/concept_native.py+++ b/pipelex/core/concepts/concept_native.py@@ -1,4 +1,4 @@-from typing import Dict, NamedTuple+from typing import NamedTuplefrom pipelex.core.domains.domain import SpecialDomainfrom pipelex.types import StrEnum@@ -25,6 +25,10 @@ class NativeConceptEnum(StrEnum):PAGE = "Page"ANYTHING = "Anything"+ @classmethod+ def values_list(cls) -> list["NativeConceptEnum"]:+ return list(cls)+@classmethoddef is_text(cls, concept_code: str) -> bool:try:@@ -48,27 +52,41 @@ class NativeConceptEnum(StrEnum):return False-NATIVE_CONCEPTS_DATA: Dict[NativeConceptEnum, NativeConceptEnumData] = {+NATIVE_CONCEPTS_DATA: dict[NativeConceptEnum, NativeConceptEnumData] = {NativeConceptEnum.DYNAMIC: NativeConceptEnumData(- code=NativeConceptEnum.DYNAMIC, content_class_name=f"{NativeConceptEnum.DYNAMIC}Content", definition="A dynamic concept"+ code=NativeConceptEnum.DYNAMIC,+ content_class_name=f"{NativeConceptEnum.DYNAMIC}Content",+ definition="A dynamic concept",),NativeConceptEnum.TEXT: NativeConceptEnumData(- code=NativeConceptEnum.TEXT, content_class_name=f"{NativeConceptEnum.TEXT}Content", definition="A text"+ code=NativeConceptEnum.TEXT,+ content_class_name=f"{NativeConceptEnum.TEXT}Content",+ definition="A text",),NativeConceptEnum.IMAGE: NativeConceptEnumData(- code=NativeConceptEnum.IMAGE, content_class_name=f"{NativeConceptEnum.IMAGE}Content", definition="An image"+ code=NativeConceptEnum.IMAGE,+ content_class_name=f"{NativeConceptEnum.IMAGE}Content",+ definition="An image",),NativeConceptEnum.PDF: NativeConceptEnumData(- code=NativeConceptEnum.PDF, content_class_name=f"{NativeConceptEnum.PDF}Content", definition="A PDF"+ code=NativeConceptEnum.PDF,+ content_class_name=f"{NativeConceptEnum.PDF}Content",+ definition="A PDF",),NativeConceptEnum.TEXT_AND_IMAGES: NativeConceptEnumData(- code=NativeConceptEnum.TEXT_AND_IMAGES, content_class_name=f"{NativeConceptEnum.TEXT_AND_IMAGES}Content", definition="A text and an image"+ code=NativeConceptEnum.TEXT_AND_IMAGES,+ content_class_name=f"{NativeConceptEnum.TEXT_AND_IMAGES}Content",+ definition="A text and an image",),NativeConceptEnum.NUMBER: NativeConceptEnumData(- code=NativeConceptEnum.NUMBER, content_class_name=f"{NativeConceptEnum.NUMBER}Content", definition="A number"+ code=NativeConceptEnum.NUMBER,+ content_class_name=f"{NativeConceptEnum.NUMBER}Content",+ definition="A number",),NativeConceptEnum.LLM_PROMPT: NativeConceptEnumData(- code=NativeConceptEnum.LLM_PROMPT, content_class_name=f"{NativeConceptEnum.LLM_PROMPT}Content", definition="A prompt for an LLM"+ code=NativeConceptEnum.LLM_PROMPT,+ content_class_name=f"{NativeConceptEnum.LLM_PROMPT}Content",+ definition="A prompt for an LLM",),NativeConceptEnum.PAGE: NativeConceptEnumData(code=NativeConceptEnum.PAGE,@@ -76,7 +94,9 @@ NATIVE_CONCEPTS_DATA: Dict[NativeConceptEnum, NativeConceptEnumData] = {definition="The content of a page of a document, comprising text and linked images and an optional page view image",),NativeConceptEnum.ANYTHING: NativeConceptEnumData(- code=NativeConceptEnum.ANYTHING, content_class_name=f"{NativeConceptEnum.ANYTHING}Content", definition="Anything"+ code=NativeConceptEnum.ANYTHING,+ content_class_name=f"{NativeConceptEnum.ANYTHING}Content",+ definition="Anything",),}@@ -84,17 +104,14 @@ NATIVE_CONCEPTS_DATA: Dict[NativeConceptEnum, NativeConceptEnumData] = {class NativeConceptManager:@classmethoddef is_native_concept(cls, concept_string_or_code: str) -> bool:- native_concept_values = [concept.value for concept in NativeConceptEnum]+ native_concept_values = NativeConceptEnum.values_list()if "." in concept_string_or_code:domain, concept_code = concept_string_or_code.split(".", 1)if SpecialDomain.is_native(domain=domain) and concept_code in native_concept_values:return True- if concept_string_or_code in native_concept_values:- return True-- return False+ return concept_string_or_code in native_concept_values@classmethoddef get_native_concept_string(cls, concept_string_or_code: str) -> str:diff --git a/pipelex/core/concepts/concept_provider_abstract.py b/pipelex/core/concepts/concept_provider_abstract.pyindex 00ab47841..cbfbb404c 100644--- a/pipelex/core/concepts/concept_provider_abstract.py+++ b/pipelex/core/concepts/concept_provider_abstract.py@@ -1,10 +1,10 @@from abc import ABC, abstractmethod-from typing import Any, Dict, List, Optional, Type+from typing import Anyfrom pipelex.core.concepts.concept import Conceptfrom pipelex.core.concepts.concept_native import NativeConceptEnum-ConceptLibraryRoot = Dict[str, Concept]+ConceptLibraryRoot = dict[str, Concept]class ConceptProviderAbstract(ABC):@@ -13,15 +13,15 @@ class ConceptProviderAbstract(ABC):pass@abstractmethod- def add_concepts(self, concepts: List[Concept]) -> None:+ def add_concepts(self, concepts: list[Concept]) -> None:pass@abstractmethod- def list_concepts_by_domain(self, domain: str) -> List[Concept]:+ def list_concepts_by_domain(self, domain: str) -> list[Concept]:pass@abstractmethod- def list_concepts(self) -> List[Concept]:+ def list_concepts(self) -> list[Concept]:pass@abstractmethod@@ -49,11 +49,11 @@ class ConceptProviderAbstract(ABC):pass@abstractmethod- def search_for_concept_in_domains(self, concept_code: str, search_domains: List[str]) -> Optional[Concept]:+ def search_for_concept_in_domains(self, concept_code: str, search_domains: list[str]) -> Concept | None:pass@abstractmethod- def get_class(self, concept_code: str) -> Optional[Type[Any]]:+ def get_class(self, concept_code: str) -> type[Any] | None:pass@abstractmethoddiff --git a/pipelex/core/concepts/structure_generator.py b/pipelex/core/concepts/structure_generator.pyindex 2c6479783..b740d0f2e 100644--- a/pipelex/core/concepts/structure_generator.py+++ b/pipelex/core/concepts/structure_generator.py@@ -1,12 +1,13 @@-"""Generate Pydantic BaseModel classes from concept structure blueprints for structured outputs."""--from __future__ import annotations-import ast-from typing import Any, Dict, List, Type, Union, cast+from datetime import datetime+from enum import Enum+from typing import Any, Literal, Optional, cast++from pydantic import Fieldfrom pipelex import logfrom pipelex.core.concepts.concept_blueprint import ConceptStructureBlueprint, ConceptStructureBlueprintFieldType+from pipelex.core.stuffs.stuff_content import StructuredContentclass StructureGenerator:@@ -21,16 +22,15 @@ class StructureGenerator:"from pipelex.core.stuffs.stuff_content import StructuredContent","from pydantic import Field",}- self.enum_definitions: Dict[str, Dict[str, Any]] = {} # Store enum definitions+ self.enum_definitions: dict[str, dict[str, Any]] = {} # Store enum definitionsdef _format_default_value(self, value: Any) -> str:"""Format default value for Python code, ensuring strings use double quotes."""if isinstance(value, str):return f'"{value}"'- else:- return repr(value)+ return repr(value)- def generate_from_structure_blueprint(self, class_name: str, structure_blueprint: Dict[str, ConceptStructureBlueprint]) -> str:+ def generate_from_structure_blueprint(self, class_name: str, structure_blueprint: dict[str, ConceptStructureBlueprint]) -> str:"""Generate Python module content from structure blueprint.Args:@@ -39,6 +39,7 @@ class StructureGenerator:Returns:Generated Python module content+"""# Generate the classclass_code = self._generate_class_from_blueprint(class_name, structure_blueprint)@@ -50,11 +51,12 @@ class StructureGenerator:# Validate the generated codeif not self.validate_generated_code(generated_code, class_name):- raise ValueError(f"Generated code for class '{class_name}' failed validation")+ msg = f"Generated code for class '{class_name}' failed validation"+ raise ValueError(msg)return generated_code- def _generate_class_from_blueprint(self, class_name: str, structure_blueprint: Dict[str, ConceptStructureBlueprint]) -> str:+ def _generate_class_from_blueprint(self, class_name: str, structure_blueprint: dict[str, ConceptStructureBlueprint]) -> str:"""Generate a class definition from ConceptStructureBlueprint.Args:@@ -63,12 +65,13 @@ class StructureGenerator:Returns:Generated class code+"""# Generate class headerclass_header = f'class {class_name}(StructuredContent):\n """Generated {class_name} class"""\n'# Generate fields- field_definitions: List[str] = []+ field_definitions: list[str] = []for field_name, field_blueprint in structure_blueprint.items():field_code = self._generate_field_from_blueprint(field_name, field_blueprint)field_definitions.append(field_code)@@ -89,6 +92,7 @@ class StructureGenerator:Returns:Generated field code+"""# Determine Python typeif field_blueprint.choices:@@ -110,11 +114,10 @@ class StructureGenerator:field_params.insert(0, f"default={self._format_default_value(field_blueprint.default_value)}")else:field_params.insert(0, "...")+ elif field_blueprint.default_value is not None:+ field_params.insert(0, f"default={self._format_default_value(field_blueprint.default_value)}")else:- if field_blueprint.default_value is not None:- field_params.insert(0, f"default={self._format_default_value(field_blueprint.default_value)}")- else:- field_params.insert(0, "default=None")+ field_params.insert(0, "default=None")field_call = f"Field({', '.join(field_params)})"@@ -128,6 +131,7 @@ class StructureGenerator:Returns:Python type annotation string+"""if field_blueprint.type is None:# This should not happen based on validation, but handle gracefully@@ -176,7 +180,7 @@ class StructureGenerator:passreturn f"Dict[{key_type}, {value_type}]"- def _generate_field(self, field_name: str, field_def: Union[Dict[str, Any], str]) -> str:+ def _generate_field(self, field_name: str, field_def: dict[str, Any] | str) -> str:"""Generate a single field definition.Args:@@ -185,6 +189,7 @@ class StructureGenerator:Returns:Generated field code+"""# Handle simple string definitions (just the definition text)if isinstance(field_def, str):@@ -216,17 +221,16 @@ class StructureGenerator:field_params.insert(0, f"default={self._format_default_value(default_value)}")else:field_params.insert(0, "...")+ elif default_value is not None:+ field_params.insert(0, f"default={self._format_default_value(default_value)}")else:- if default_value is not None:- field_params.insert(0, f"default={self._format_default_value(default_value)}")- else:- field_params.insert(0, "default=None")+ field_params.insert(0, "default=None")field_call = f"Field({', '.join(field_params)})"return f" {field_name}: {python_type} = {field_call}"- def _get_python_type(self, field_type: Any, field_def: Dict[str, Any]) -> str:+ def _get_python_type(self, field_type: Any, field_def: dict[str, Any]) -> str:"""Convert high-level type to Python type annotation.Args:@@ -235,6 +239,7 @@ class StructureGenerator:Returns:Python type annotation string+"""# Check if it's a reference to a defined enumif isinstance(field_type, str) and field_type in self.enum_definitions:@@ -306,6 +311,7 @@ class StructureGenerator:Returns:True if the code is valid, False otherwise+"""# Step 1: Syntax validationif not self._validate_syntax(python_code):@@ -320,10 +326,7 @@ class StructureGenerator:return False# Step 4: Class instantiation validation- if not self._validate_instantiation(python_code, expected_class_name):- return False-- return True+ return self._validate_instantiation(python_code, expected_class_name)def _validate_syntax(self, python_code: str) -> bool:"""Validate that the code has valid Python syntax."""@@ -347,13 +350,13 @@ class StructureGenerator:"""Validate that the code executes and creates the expected class."""try:# Import necessary modules for the execution context- from datetime import datetime- from enum import Enum- from typing import Any, Dict, List, Literal, Optional+ from datetime import datetime # noqa: PLC0415+ from enum import Enum # noqa: PLC0415+ from typing import Any, Dict, List, Literal, Optional # noqa: PLC0415,F401- from pydantic import Field+ from pydantic import Field # noqa: PLC0415- from pipelex.core.stuffs.stuff_content import StructuredContent+ from pipelex.core.stuffs.stuff_content import StructuredContent # noqa: PLC0415# Provide necessary imports in the execution contextexec_globals = {@@ -361,14 +364,14 @@ class StructureGenerator:"datetime": datetime,"Enum": Enum,"Optional": Optional,- "List": List,- "Dict": Dict,+ "List": list,+ "Dict": dict,"Any": Any,"Literal": Literal,"Field": Field,"StructuredContent": StructuredContent,}- exec_locals: Dict[str, Any] = {}+ exec_locals: dict[str, Any] = {}exec(python_code, exec_globals, exec_locals)# Verify the expected class was created@@ -391,34 +394,23 @@ class StructureGenerator:return Falsedef _validate_instantiation(self, python_code: str, expected_class_name: str) -> bool:- """Validate that the generated class can be instantiated."""try:- # Import necessary modules for the execution context- from datetime import datetime- from enum import Enum- from typing import Any, Dict, List, Literal, Optional-- from pydantic import Field-- from pipelex.core.stuffs.stuff_content import StructuredContent-- # Provide necessary imports in the execution contextexec_globals = {"__builtins__": __builtins__,"datetime": datetime,"Enum": Enum,"Optional": Optional,- "List": List,- "Dict": Dict,+ "List": list,+ "Dict": dict,"Any": Any,"Literal": Literal,"Field": Field,"StructuredContent": StructuredContent,}- exec_locals: Dict[str, Any] = {}+ exec_locals: dict[str, Any] = {}exec(python_code, exec_globals, exec_locals)- generated_class = cast(Type[Any], exec_locals[expected_class_name])+ generated_class = cast("type[Any]", exec_locals[expected_class_name])# Try to create an instance (this will catch Pydantic validation issues)# For validation purposes, we'll try to create an instance with minimal valid data@@ -429,7 +421,7 @@ class StructureGenerator:except Exception:# If that fails, try with empty dict (some models accept this)try:- instance = generated_class(**{})+ instance = generated_class()except Exception:# If that fails too, the class structure is probably fine but requires specific data# For validation purposes, we'll just check that it's a valid Pydantic model classdiff --git a/pipelex/core/domains/__init__.py b/pipelex/core/domains/__init__.pynew file mode 100644index 000000000..e69de29bbdiff --git a/pipelex/core/domains/domain.py b/pipelex/core/domains/domain.pyindex 95f15f13d..59d1d08a4 100644--- a/pipelex/core/domains/domain.py+++ b/pipelex/core/domains/domain.py@@ -1,9 +1,6 @@-from typing import Optional-from pydantic import BaseModel-from typing_extensions import Self-from pipelex.types import StrEnum+from pipelex.types import Self, StrEnumclass SpecialDomain(StrEnum):@@ -26,10 +23,10 @@ class SpecialDomain(StrEnum):class Domain(BaseModel):code: str- definition: Optional[str] = None- system_prompt: Optional[str] = None- system_prompt_to_structure: Optional[str] = None- prompt_template_to_structure: Optional[str] = None+ definition: str | None = None+ system_prompt: str | None = None+ system_prompt_to_structure: str | None = None+ prompt_template_to_structure: str | None = None@classmethoddef make_default(cls) -> Self:diff --git a/pipelex/core/domains/domain_blueprint.py b/pipelex/core/domains/domain_blueprint.pyindex e73e3e6ef..844f3b380 100644--- a/pipelex/core/domains/domain_blueprint.py+++ b/pipelex/core/domains/domain_blueprint.py@@ -1,5 +1,3 @@-from typing import Optional-from pydantic import BaseModelfrom pipelex.core.domains.exceptions import DomainError@@ -8,13 +6,14 @@ from pipelex.tools.misc.string_utils import is_snake_caseclass DomainBlueprint(BaseModel):code: str- definition: Optional[str] = None- system_prompt: Optional[str] = None- system_prompt_to_structure: Optional[str] = None- prompt_template_to_structure: Optional[str] = None+ definition: str | None = None+ system_prompt: str | None = None+ system_prompt_to_structure: str | None = None+ prompt_template_to_structure: str | None = None@staticmethoddef validate_domain_code(code: str) -> None:"""Validate that a domain code follows snake_case convention."""if not is_snake_case(code):- raise DomainError(f"Domain code '{code}' must be snake_case (lowercase letters, numbers, and underscores only)")+ msg = f"Domain code '{code}' must be snake_case (lowercase letters, numbers, and underscores only)"+ raise DomainError(msg)diff --git a/pipelex/core/domains/domain_library.py b/pipelex/core/domains/domain_library.pyindex 4e863de48..f9a0abe14 100644--- a/pipelex/core/domains/domain_library.py+++ b/pipelex/core/domains/domain_library.py@@ -1,13 +1,12 @@-from typing import Dict, Optional-from pydantic import RootModel-from typing_extensions import Self, override+from typing_extensions import overridefrom pipelex.core.domains.domain import Domainfrom pipelex.core.domains.domain_provider_abstract import DomainProviderAbstractfrom pipelex.exceptions import DomainLibraryError+from pipelex.types import Self-DomainLibraryRoot = Dict[str, Domain]+DomainLibraryRoot = dict[str, Domain]class DomainLibrary(RootModel[DomainLibraryRoot], DomainProviderAbstract):@@ -34,14 +33,15 @@ class DomainLibrary(RootModel[DomainLibraryRoot], DomainProviderAbstract):del self.root[domain_code]@override- def get_domain(self, domain: str) -> Optional[Domain]:+ def get_domain(self, domain: str) -> Domain | None:return self.root.get(domain)@overridedef get_required_domain(self, domain: str) -> Domain:the_domain = self.get_domain(domain=domain)if not the_domain:- raise DomainLibraryError(f"Domain '{domain}' not found. Check for typos and make sure it is declared in a pipeline library.")+ msg = f"Domain '{domain}' not found. Check for typos and make sure it is declared in a pipeline library."+ raise DomainLibraryError(msg)return the_domain@overridediff --git a/pipelex/core/domains/domain_provider_abstract.py b/pipelex/core/domains/domain_provider_abstract.pyindex bb2b7bfc8..0a9cc769c 100644--- a/pipelex/core/domains/domain_provider_abstract.py+++ b/pipelex/core/domains/domain_provider_abstract.py@@ -1,12 +1,11 @@from abc import ABC, abstractmethod-from typing import Optionalfrom pipelex.core.domains.domain import Domainclass DomainProviderAbstract(ABC):@abstractmethod- def get_domain(self, domain: str) -> Optional[Domain]:+ def get_domain(self, domain: str) -> Domain | None:pass@abstractmethoddiff --git a/pipelex/core/exceptions.py b/pipelex/core/exceptions.pyindex 62f41fe28..2d2f5ccae 100644--- a/pipelex/core/exceptions.py+++ b/pipelex/core/exceptions.py@@ -4,16 +4,10 @@class PipelexInterpreterError(Exception):"""Base exception class for PipelexInterpreter errors."""- pass-class PipelexConfigurationError(PipelexInterpreterError):"""Raised when there are configuration issues with the PipelexInterpreter."""- pass-class PipelexUnknownPipeError(PipelexInterpreterError):"""Raised when encountering an unknown pipe blueprint type."""-- passdiff --git a/pipelex/core/interpreter.py b/pipelex/core/interpreter.pyindex 9413cd7a3..0d6d9509b 100644--- a/pipelex/core/interpreter.py+++ b/pipelex/core/interpreter.py@@ -1,12 +1,12 @@+from collections.abc import Mappingfrom pathlib import Path-from typing import Any, Dict, Mapping, Optional, Union, cast+from typing import Any, castimport tomlfrom pydantic import BaseModel, model_validator-from typing_extensions import Selffrom pipelex.core.bundles.pipelex_bundle_blueprint import PipelexBundleBlueprint-from pipelex.core.concepts.concept_blueprint import ConceptBlueprint+from pipelex.core.concepts.concept_blueprint import ConceptBlueprint, ConceptStructureBlueprintfrom pipelex.core.exceptions import (PipelexConfigurationError,PipelexUnknownPipeError,@@ -32,22 +32,21 @@ from pipelex.tools.misc.toml_utils import (validate_toml_content,validate_toml_file,)+from pipelex.types import Selfclass PLXDecodeError(toml.TomlDecodeError):"""Raised when PLX decoding fails."""- pass-class PipelexInterpreter(BaseModel):"""plx -> PipelexBundleBlueprint"""- file_path: Optional[Path] = None- file_content: Optional[str] = None+ file_path: Path | None = None+ file_content: str | None = None@staticmethod- def escape_plx_string(value: Optional[str]) -> str:+ def escape_plx_string(value: str | None) -> str:"""Escape a string for plx serialization."""if value is None:return ""@@ -58,14 +57,14 @@ class PipelexInterpreter(BaseModel):# Replace actual newlines with escaped newlinesvalue = value.replace("\n", "\\n")value = value.replace("\r", "\\r")- value = value.replace("\t", "\\t")- return value+ return value.replace("\t", "\\t")@model_validator(mode="after")def check_file_path_or_file_content(self) -> Self:"""Need to check if there is at least one of file_path or file_content"""if self.file_path is None and self.file_content is None:- raise PipelexConfigurationError("Either file_path or file_content must be provided")+ msg = "Either file_path or file_content must be provided"+ raise PipelexConfigurationError(msg)return self@model_validator(mode="after")@@ -79,7 +78,7 @@ class PipelexInterpreter(BaseModel):def _read_and_reformat(self) -> str:"""Load PLX content from file_path or use file_content directly."""if self.file_path:- with open(self.file_path, "r", encoding="utf-8") as file:+ with open(self.file_path, encoding="utf-8") as file:file_content = file.read()# Clean trailing whitespace and write back if needed@@ -89,8 +88,9 @@ class PipelexInterpreter(BaseModel):file.write(cleaned_content)return cleaned_contentreturn file_content- elif self.file_content is None:- raise PipelexConfigurationError("file_content must be provided if file_path is not provided")+ if self.file_content is None:+ msg = "file_content must be provided if file_path is not provided"+ raise PipelexConfigurationError(msg)return self.file_content@staticmethod@@ -106,6 +106,7 @@ class PipelexInterpreter(BaseModel):Criteria:- Has .plx extension- Starts with "domain =" (ignoring leading whitespace)+"""# Check if it has .toml extensionif file_path.suffix != ".plx":@@ -117,7 +118,7 @@ class PipelexInterpreter(BaseModel):try:# Read the first few lines to check for "domain ="- with open(file_path, "r", encoding="utf-8") as f:+ with open(file_path, encoding="utf-8") as f:# Read first 100 characters (should be enough to find domain)content = f.read(100)# Remove leading whitespace and check if it starts with "domain ="@@ -127,13 +128,14 @@ class PipelexInterpreter(BaseModel):# If we can't read the file, it's not a valid Pipelex filereturn False- def _parse_plx_content(self, content: str) -> Dict[str, Any]:+ def _parse_plx_content(self, content: str) -> dict[str, Any]:"""Parse PLX content and return the dictionary."""try:return toml.loads(content)except toml.TomlDecodeError as exc:file_path_str = str(self.file_path) if self.file_path else "content"- raise PLXDecodeError(f"PLX parsing error in '{file_path_str}': {exc}", exc.doc, exc.pos) from exc+ msg = f"PLX parsing error in '{file_path_str}': {exc}"+ raise PLXDecodeError(msg, exc.doc, exc.pos) from excdef make_pipelex_bundle_blueprint(self) -> PipelexBundleBlueprint:"""Make a PipelexBundleBlueprint from the file_path or file_content"""@@ -162,13 +164,13 @@ class PipelexInterpreter(BaseModel):# Concepts sectionif blueprint.concept:- concept_plx = PipelexInterpreter.concepts_to_plx_string(blueprint.concept, blueprint.domain)+ concept_plx = PipelexInterpreter.concepts_to_plx_string(blueprint.concept)if concept_plx: # Only add if not emptyplx_parts.append(concept_plx)# Pipes sectionif blueprint.pipe:- pipes_plx = PipelexInterpreter.pipes_to_plx_string(blueprint.pipe, blueprint.domain)+ pipes_plx = PipelexInterpreter.pipes_to_plx_string(blueprint.pipe)if pipes_plx: # Only add if not emptyplx_parts.append(pipes_plx)@@ -178,7 +180,7 @@ class PipelexInterpreter(BaseModel):return result@staticmethod- def concepts_to_plx_string(concepts: Dict[str, ConceptBlueprint | str], domain: str) -> str:+ def concepts_to_plx_string(concepts: dict[str, ConceptBlueprint | str]) -> str:"""Convert concepts dict to PLX string."""if not concepts:return ""@@ -190,15 +192,14 @@ class PipelexInterpreter(BaseModel):for concept_name, concept_blueprint in concepts.items():if isinstance(concept_blueprint, str):simple_concepts.append(f'{concept_name} = "{PipelexInterpreter.escape_plx_string(concept_blueprint)}"')+ # Handle ConceptBlueprint objects+ elif concept_blueprint.structure is None and concept_blueprint.refines is None:+ # Simple concept with just definition+ simple_concepts.append(f'{concept_name} = "{PipelexInterpreter.escape_plx_string(concept_blueprint.definition)}"')else:- # Handle ConceptBlueprint objects- if concept_blueprint.structure is None and concept_blueprint.refines is None:- # Simple concept with just definition- simple_concepts.append(f'{concept_name} = "{PipelexInterpreter.escape_plx_string(concept_blueprint.definition)}"')- else:- # Complex concept needs its own section- complex_concept_plx = PipelexInterpreter.complex_concept_to_plx_string(concept_name, concept_blueprint)- complex_concepts.append(complex_concept_plx)+ # Complex concept needs its own section+ complex_concept_plx = PipelexInterpreter.complex_concept_to_plx_string(concept_name, concept_blueprint)+ complex_concepts.append(complex_concept_plx)# Add simple concepts section if we have anyif simple_concepts:@@ -247,59 +248,58 @@ class PipelexInterpreter(BaseModel):"""Convert a structure field to PLX string."""if isinstance(field_def, str):return f'{field_name} = "{PipelexInterpreter.escape_plx_string(field_def)}"'- else:- # Complex field with type, definition, required, etc.- field_parts: list[str] = []+ # Complex field with type, definition, required, etc.+ field_parts: list[str] = []- if hasattr(field_def, "type") and field_def.type:- type_value = field_def.type.value if hasattr(field_def.type, "value") else field_def.type- field_parts.append(f'type = "{PipelexInterpreter.escape_plx_string(str(type_value))}"')+ if hasattr(field_def, "type") and field_def.type:+ type_value = field_def.type.value if hasattr(field_def.type, "value") else field_def.type+ field_parts.append(f'type = "{PipelexInterpreter.escape_plx_string(str(type_value))}"')- if hasattr(field_def, "definition") and field_def.definition:- field_parts.append(f'definition = "{PipelexInterpreter.escape_plx_string(field_def.definition)}"')+ if hasattr(field_def, "definition") and field_def.definition:+ field_parts.append(f'definition = "{PipelexInterpreter.escape_plx_string(field_def.definition)}"')- if hasattr(field_def, "required") and field_def.required is not None:- field_parts.append(f"required = {str(field_def.required).lower()}")+ if hasattr(field_def, "required") and field_def.required is not None:+ field_parts.append(f"required = {str(field_def.required).lower()}")- return f"{field_name} = {{ {', '.join(field_parts)} }}"+ return f"{field_name} = {{ {', '.join(field_parts)} }}"@staticmethod- def pipes_to_plx_string(pipes: Dict[str, Any], domain: str) -> str:+ def pipes_to_plx_string(pipes: dict[str, Any]) -> str:"""Convert pipes dict to PLX string."""plx_parts: list[str] = []for pipe_name, blueprint in pipes.items():- pipe_plx = PipelexInterpreter.pipe_to_plx_string(pipe_name, blueprint, domain)+ pipe_plx = PipelexInterpreter.pipe_to_plx_string(pipe_name, blueprint)plx_parts.append(pipe_plx)return "\n\n".join(plx_parts)@staticmethod- def pipe_to_plx_string(pipe_name: str, blueprint: Any, domain: str) -> str:+ def pipe_to_plx_string(pipe_name: str, blueprint: Any) -> str:"""Convert a single pipe blueprint to PLX string."""if isinstance(blueprint, PipeLLMBlueprint):- return PipelexInterpreter.llm_pipe_to_plx_string(pipe_name, blueprint, domain)+ return PipelexInterpreter.llm_pipe_to_plx_string(pipe_name, blueprint)elif isinstance(blueprint, PipeSequenceBlueprint):- return PipelexInterpreter.sequence_pipe_to_plx_string(pipe_name, blueprint, domain)+ return PipelexInterpreter.sequence_pipe_to_plx_string(pipe_name, blueprint)elif isinstance(blueprint, PipeOcrBlueprint):- return PipelexInterpreter.ocr_pipe_to_plx_string(pipe_name, blueprint, domain)+ return PipelexInterpreter.ocr_pipe_to_plx_string(pipe_name, blueprint)elif isinstance(blueprint, PipeFuncBlueprint):- return PipelexInterpreter.func_pipe_to_plx_string(pipe_name, blueprint, domain)+ return PipelexInterpreter.func_pipe_to_plx_string(pipe_name, blueprint)elif isinstance(blueprint, PipeImgGenBlueprint):- return PipelexInterpreter.img_gen_pipe_to_plx_string(pipe_name, blueprint, domain)+ return PipelexInterpreter.img_gen_pipe_to_plx_string(pipe_name, blueprint)elif isinstance(blueprint, PipeJinja2Blueprint):- return PipelexInterpreter.jinja2_pipe_to_plx_string(pipe_name, blueprint, domain)+ return PipelexInterpreter.jinja2_pipe_to_plx_string(pipe_name, blueprint)elif isinstance(blueprint, PipeConditionBlueprint):- return PipelexInterpreter.condition_pipe_to_plx_string(pipe_name, blueprint, domain)+ return PipelexInterpreter.condition_pipe_to_plx_string(pipe_name, blueprint)elif isinstance(blueprint, PipeParallelBlueprint):- return PipelexInterpreter.parallel_pipe_to_plx_string(pipe_name, blueprint, domain)+ return PipelexInterpreter.parallel_pipe_to_plx_string(pipe_name, blueprint)elif isinstance(blueprint, PipeBatchBlueprint):- return PipelexInterpreter.batch_pipe_to_plx_string(pipe_name, blueprint, domain)+ return PipelexInterpreter.batch_pipe_to_plx_string(pipe_name, blueprint)else:# Fallback to old dict approach for unknown pipe types- pipe_dict = PipelexInterpreter.serialize_pipe(blueprint, domain)+ pipe_dict = PipelexInterpreter.serialize_pipe(blueprint)return f"[pipe.{pipe_name}]\n" + "\n".join([f'{k} = "{v}"' if isinstance(v, str) else f"{k} = {v}" for k, v in pipe_dict.items()])@staticmethod- def llm_pipe_to_plx_string(pipe_name: str, pipe: PipeLLMBlueprint, domain: str) -> str:+ def llm_pipe_to_plx_string(pipe_name: str, pipe: PipeLLMBlueprint) -> str:"""Convert a PipeLLM blueprint directly to PLX section string."""lines: list[str] = [f"[pipe.{pipe_name}]",@@ -332,7 +332,7 @@ class PipelexInterpreter(BaseModel):return "\n".join(lines)@staticmethod- def ocr_pipe_to_plx_string(pipe_name: str, pipe: PipeOcrBlueprint, domain: str) -> str:+ def ocr_pipe_to_plx_string(pipe_name: str, pipe: PipeOcrBlueprint) -> str:"""Convert a PipeOcr blueprint directly to PLX section string."""lines: list[str] = [f"[pipe.{pipe_name}]",@@ -349,7 +349,7 @@ class PipelexInterpreter(BaseModel):return "\n".join(lines)@staticmethod- def func_pipe_to_plx_string(pipe_name: str, pipe: PipeFuncBlueprint, domain: str) -> str:+ def func_pipe_to_plx_string(pipe_name: str, pipe: PipeFuncBlueprint) -> str:"""Convert a PipeFunc blueprint directly to PLX section string."""lines: list[str] = [f"[pipe.{pipe_name}]",@@ -366,7 +366,7 @@ class PipelexInterpreter(BaseModel):return "\n".join(lines)@staticmethod- def img_gen_pipe_to_plx_string(pipe_name: str, pipe: PipeImgGenBlueprint, domain: str) -> str:+ def img_gen_pipe_to_plx_string(pipe_name: str, pipe: PipeImgGenBlueprint) -> str:"""Convert a PipeImgGen blueprint directly to PLX section string."""lines: list[str] = [f"[pipe.{pipe_name}]",@@ -394,7 +394,7 @@ class PipelexInterpreter(BaseModel):return "\n".join(lines)@staticmethod- def jinja2_pipe_to_plx_string(pipe_name: str, pipe: PipeJinja2Blueprint, domain: str) -> str:+ def jinja2_pipe_to_plx_string(pipe_name: str, pipe: PipeJinja2Blueprint) -> str:"""Convert a PipeJinja2 blueprint directly to PLX section string."""lines: list[str] = [f"[pipe.{pipe_name}]",@@ -416,7 +416,7 @@ class PipelexInterpreter(BaseModel):return "\n".join(lines)@staticmethod- def condition_pipe_to_plx_string(pipe_name: str, pipe: PipeConditionBlueprint, domain: str) -> str:+ def condition_pipe_to_plx_string(pipe_name: str, pipe: PipeConditionBlueprint) -> str:"""Convert a PipeCondition blueprint directly to PLX section string."""lines: list[str] = [f"[pipe.{pipe_name}]",@@ -450,7 +450,7 @@ class PipelexInterpreter(BaseModel):return "\n".join(lines)@staticmethod- def parallel_pipe_to_plx_string(pipe_name: str, pipe: PipeParallelBlueprint, domain: str) -> str:+ def parallel_pipe_to_plx_string(pipe_name: str, pipe: PipeParallelBlueprint) -> str:"""Convert a PipeParallel blueprint directly to PLX section string."""lines: list[str] = [f"[pipe.{pipe_name}]",@@ -480,7 +480,7 @@ class PipelexInterpreter(BaseModel):return "\n".join(lines)@staticmethod- def batch_pipe_to_plx_string(pipe_name: str, pipe: PipeBatchBlueprint, domain: str) -> str:+ def batch_pipe_to_plx_string(pipe_name: str, pipe: PipeBatchBlueprint) -> str:"""Convert a PipeBatch blueprint directly to PLX section string."""lines: list[str] = [f"[pipe.{pipe_name}]",@@ -505,12 +505,10 @@ class PipelexInterpreter(BaseModel):@staticmethoddef serialize_concept_structure_field(field_value: Any) -> Any:"""Serialize a single concept structure field."""- from pipelex.core.concepts.concept_blueprint import ConceptStructureBlueprint-if isinstance(field_value, str):return field_value- elif isinstance(field_value, ConceptStructureBlueprint):- field_data: Dict[str, Any] = {+ if isinstance(field_value, ConceptStructureBlueprint):+ field_data: dict[str, Any] = {"definition": field_value.definition,"required": field_value.required,}@@ -527,26 +525,24 @@ class PipelexInterpreter(BaseModel):if field_value.default_value is not None:field_data["default_value"] = field_value.default_valuereturn field_data- else:- return field_value+ return field_value@staticmethoddef serialize_concept_structure(- structure: Union[str, Dict[str, Any]],- ) -> Union[str, Dict[str, Any]]:+ structure: str | dict[str, Any],+ ) -> str | dict[str, Any]:"""Serialize concept structure field."""if isinstance(structure, str):return structure- else:- result: Dict[str, Any] = {}- for field_name, field_value in structure.items():- result[field_name] = PipelexInterpreter.serialize_concept_structure_field(field_value)- return result+ result: dict[str, Any] = {}+ for field_name, field_value in structure.items():+ result[field_name] = PipelexInterpreter.serialize_concept_structure_field(field_value)+ return result@staticmethoddef serialize_single_concept(- concept_blueprint: Union[ConceptBlueprint, str],- ) -> Union[str, Dict[str, Any]]:+ concept_blueprint: ConceptBlueprint | str,+ ) -> str | dict[str, Any]:"""Serialize a single concept blueprint."""if isinstance(concept_blueprint, str):return concept_blueprint@@ -554,70 +550,66 @@ class PipelexInterpreter(BaseModel):# Handle ConceptBlueprint objectif concept_blueprint.structure is not None:# Structured concept- concept_data = {+ return {"definition": concept_blueprint.definition,"structure": PipelexInterpreter.serialize_concept_structure(concept_blueprint.structure),}- return concept_data- elif concept_blueprint.refines is not None:+ if concept_blueprint.refines is not None:# Concept with refines- concept_data = {+ return {"definition": concept_blueprint.definition,"refines": concept_blueprint.refines,}- return concept_data- else:- # Simple concept with just definition- return concept_blueprint.definition+ # Simple concept with just definition+ return concept_blueprint.definition@staticmethod- def serialize_concepts(concepts: Optional[Dict[str, ConceptBlueprint | str]], domain: str) -> Dict[str, Any]:+ def serialize_concepts(concepts: dict[str, ConceptBlueprint | str] | None) -> dict[str, Any]:"""Serialize concepts section with domain context."""if concepts is None:return {}- result: Dict[str, Any] = {}+ result: dict[str, Any] = {}for concept_name, concept_blueprint in concepts.items():result[concept_name] = PipelexInterpreter.serialize_single_concept(concept_blueprint)return result@staticmethod- def serialize_pipes(pipes: Dict[str, Any], domain: str) -> Dict[str, Any]:+ def serialize_pipes(pipes: dict[str, Any]) -> dict[str, Any]:"""Serialize pipes section with domain context."""- result: Dict[str, Any] = {}+ result: dict[str, Any] = {}for pipe_name, blueprint in pipes.items():- result[pipe_name] = PipelexInterpreter.serialize_pipe(blueprint, domain)+ result[pipe_name] = PipelexInterpreter.serialize_pipe(blueprint)return result@staticmethod- def serialize_pipe(blueprint: Any, domain: str) -> Dict[str, Any]:- """Serialize a single pipe blueprint with domain context."""-+ def serialize_pipe(blueprint: Any) -> dict[str, Any]:if isinstance(blueprint, PipeLLMBlueprint):- return PipelexInterpreter.serialize_llm_pipe(blueprint, domain)+ return PipelexInterpreter.serialize_llm_pipe(blueprint)elif isinstance(blueprint, PipeOcrBlueprint):- return PipelexInterpreter.serialize_ocr_pipe(blueprint, domain)+ return PipelexInterpreter.serialize_ocr_pipe(blueprint)elif isinstance(blueprint, PipeFuncBlueprint):- return PipelexInterpreter._serialize_func_pipe(blueprint, domain)+ return PipelexInterpreter._serialize_func_pipe(blueprint)elif isinstance(blueprint, PipeImgGenBlueprint):- return PipelexInterpreter._serialize_img_gen_pipe(blueprint, domain)+ return PipelexInterpreter._serialize_img_gen_pipe(blueprint)elif isinstance(blueprint, PipeJinja2Blueprint):- return PipelexInterpreter.serialize_jinja2_pipe(blueprint, domain)+ return PipelexInterpreter.serialize_jinja2_pipe(blueprint)elif isinstance(blueprint, PipeSequenceBlueprint):- return PipelexInterpreter.serialize_sequence_pipe(blueprint, domain)+ return PipelexInterpreter.serialize_sequence_pipe(blueprint)elif isinstance(blueprint, PipeConditionBlueprint):- return PipelexInterpreter._serialize_condition_pipe(blueprint, domain)+ return PipelexInterpreter._serialize_condition_pipe(blueprint)elif isinstance(blueprint, PipeParallelBlueprint):- return PipelexInterpreter._serialize_parallel_pipe(blueprint, domain)+ return PipelexInterpreter._serialize_parallel_pipe(blueprint)elif isinstance(blueprint, PipeBatchBlueprint):- return PipelexInterpreter._serialize_batch_pipe(blueprint, domain)+ return PipelexInterpreter._serialize_batch_pipe(blueprint)else:- raise PipelexUnknownPipeError(f"Unknown pipe blueprint type: {type(blueprint)}")+ msg = f"Unknown pipe blueprint type: {type(blueprint)}"+ raise PipelexUnknownPipeError(msg)@staticmethod- def serialize_llm_pipe(pipe: PipeLLMBlueprint, domain: str) -> Dict[str, Any]:+ def serialize_llm_pipe(pipe: PipeLLMBlueprint) -> dict[str, Any]:"""Serialize PipeLLM blueprint."""- result: Dict[str, Any] = {+ result: dict[str, Any] = {"type": pipe.type,"definition": pipe.definition,}@@ -668,15 +660,15 @@ class PipelexInterpreter(BaseModel):return result@staticmethod- def add_common_pipe_fields(result: Dict[str, Any], pipe: Any) -> None:+ def add_common_pipe_fields(result: dict[str, Any], pipe: Any) -> None:"""Add common pipe fields (inputs) to result dict if they exist."""if hasattr(pipe, "inputs") and pipe.inputs:result["inputs"] = PipelexInterpreter.serialize_inputs(pipe.inputs)@staticmethod- def _serialize_ocr_pipe(pipe: PipeOcrBlueprint, domain: str) -> Dict[str, Any]:+ def _serialize_ocr_pipe(pipe: PipeOcrBlueprint) -> dict[str, Any]:"""Serialize PipeOcr blueprint."""- result: Dict[str, Any] = {+ result: dict[str, Any] = {"type": pipe.type,"definition": pipe.definition,"output": pipe.output,@@ -686,9 +678,9 @@ class PipelexInterpreter(BaseModel):return result@staticmethod- def _serialize_func_pipe(pipe: PipeFuncBlueprint, domain: str) -> Dict[str, Any]:+ def _serialize_func_pipe(pipe: PipeFuncBlueprint) -> dict[str, Any]:"""Serialize PipeFunc blueprint."""- result: Dict[str, Any] = {+ result: dict[str, Any] = {"type": pipe.type,"definition": pipe.definition,"output": pipe.output,@@ -699,9 +691,9 @@ class PipelexInterpreter(BaseModel):return result@staticmethod- def _serialize_img_gen_pipe(pipe: PipeImgGenBlueprint, domain: str) -> Dict[str, Any]:+ def _serialize_img_gen_pipe(pipe: PipeImgGenBlueprint) -> dict[str, Any]:"""Serialize PipeImgGen blueprint."""- result: Dict[str, Any] = {+ result: dict[str, Any] = {"type": pipe.type,"definition": pipe.definition,"output": pipe.output,@@ -724,9 +716,9 @@ class PipelexInterpreter(BaseModel):return result@staticmethod- def serialize_ocr_pipe(pipe: PipeOcrBlueprint, domain: str) -> Dict[str, Any]:+ def serialize_ocr_pipe(pipe: PipeOcrBlueprint) -> dict[str, Any]:"""Serialize a PipeOcr blueprint."""- result: Dict[str, Any] = {+ result: dict[str, Any] = {"type": pipe.type,"definition": pipe.definition,}@@ -753,9 +745,9 @@ class PipelexInterpreter(BaseModel):return result@staticmethod- def serialize_sequence_pipe(pipe: PipeSequenceBlueprint, domain: str) -> Dict[str, Any]:+ def serialize_sequence_pipe(pipe: PipeSequenceBlueprint) -> dict[str, Any]:"""Serialize a PipeSequence blueprint."""- result: Dict[str, Any] = {+ result: dict[str, Any] = {"type": pipe.type,"definition": pipe.definition,}@@ -773,9 +765,9 @@ class PipelexInterpreter(BaseModel):return result@staticmethod- def serialize_jinja2_pipe(pipe: PipeJinja2Blueprint, domain: str) -> Dict[str, Any]:+ def serialize_jinja2_pipe(pipe: PipeJinja2Blueprint) -> dict[str, Any]:"""Serialize PipeJinja2 blueprint."""- result: Dict[str, Any] = {+ result: dict[str, Any] = {"type": pipe.type,"definition": pipe.definition,"output": pipe.output,@@ -795,9 +787,9 @@ class PipelexInterpreter(BaseModel):return result@staticmethod- def serialize_sub_pipe(sub_pipe: Any) -> Dict[str, Any]:+ def serialize_sub_pipe(sub_pipe: Any) -> dict[str, Any]:"""Serialize a sub pipe (step or parallel) blueprint."""- step_data: Dict[str, Any] = {+ step_data: dict[str, Any] = {"pipe": sub_pipe.pipe,"result": sub_pipe.result,}@@ -833,9 +825,9 @@ class PipelexInterpreter(BaseModel):return "{ " + ", ".join(parts) + " }"@staticmethod- def _serialize_sequence_pipe(pipe: PipeSequenceBlueprint, domain: str) -> Dict[str, Any]:+ def _serialize_sequence_pipe(pipe: PipeSequenceBlueprint) -> dict[str, Any]:"""Serialize PipeSequence blueprint."""- result: Dict[str, Any] = {+ result: dict[str, Any] = {"type": pipe.type,"definition": pipe.definition,"output": pipe.output,@@ -849,7 +841,7 @@ class PipelexInterpreter(BaseModel):return result@staticmethod- def sequence_pipe_to_plx_string(pipe_name: str, pipe: PipeSequenceBlueprint, domain: str) -> str:+ def sequence_pipe_to_plx_string(pipe_name: str, pipe: PipeSequenceBlueprint) -> str:"""Convert a PipeSequence blueprint directly to PLX section string."""lines: list[str] = [f"[pipe.{pipe_name}]",@@ -875,9 +867,9 @@ class PipelexInterpreter(BaseModel):return "\n".join(lines)@staticmethod- def _serialize_condition_pipe(pipe: PipeConditionBlueprint, domain: str) -> Dict[str, Any]:+ def _serialize_condition_pipe(pipe: PipeConditionBlueprint) -> dict[str, Any]:"""Serialize PipeCondition blueprint."""- result: Dict[str, Any] = {+ result: dict[str, Any] = {"type": pipe.type,"definition": pipe.definition,"output": pipe.output,@@ -899,9 +891,9 @@ class PipelexInterpreter(BaseModel):return result@staticmethod- def _serialize_parallel_pipe(pipe: PipeParallelBlueprint, domain: str) -> Dict[str, Any]:+ def _serialize_parallel_pipe(pipe: PipeParallelBlueprint) -> dict[str, Any]:"""Serialize PipeParallel blueprint."""- result: Dict[str, Any] = {+ result: dict[str, Any] = {"type": pipe.type,"definition": pipe.definition,"output": pipe.output,@@ -919,9 +911,9 @@ class PipelexInterpreter(BaseModel):return result@staticmethod- def _serialize_batch_pipe(pipe: PipeBatchBlueprint, domain: str) -> Dict[str, Any]:+ def _serialize_batch_pipe(pipe: PipeBatchBlueprint) -> dict[str, Any]:"""Serialize PipeBatch blueprint."""- result: Dict[str, Any] = {+ result: dict[str, Any] = {"type": pipe.type,"definition": pipe.definition,"output": pipe.output,@@ -940,19 +932,19 @@ class PipelexInterpreter(BaseModel):@staticmethoddef serialize_input_requirement(input_req: InputRequirementBlueprint,- ) -> Dict[str, Any]:+ ) -> dict[str, Any]:"""Serialize a single InputRequirementBlueprint to PLX format."""- result: Dict[str, Any] = {"concept": input_req.concept}+ result: dict[str, Any] = {"concept": input_req.concept}if input_req.multiplicity is not None:result["multiplicity"] = input_req.multiplicityreturn result@staticmethoddef serialize_inputs(- inputs: Mapping[str, Union[str, InputRequirementBlueprint]],- ) -> Dict[str, Any]:+ inputs: Mapping[str, str | InputRequirementBlueprint],+ ) -> dict[str, Any]:"""Convert InputRequirementBlueprint objects to proper PLX format for serialization."""- result: Dict[str, Any] = {}+ result: dict[str, Any] = {}for key, value in inputs.items():if isinstance(value, InputRequirementBlueprint):result[key] = PipelexInterpreter.serialize_input_requirement(value)@@ -963,7 +955,7 @@ class PipelexInterpreter(BaseModel):@staticmethoddef inputs_to_plx_string(- inputs: Mapping[str, Union[str, InputRequirementBlueprint]],+ inputs: Mapping[str, str | InputRequirementBlueprint],) -> str:"""Convert inputs dictionary to PLX string format."""inputs_dict = PipelexInterpreter.serialize_inputs(inputs)@@ -973,27 +965,26 @@ class PipelexInterpreter(BaseModel):if isinstance(value, dict):# Nested dict (like InputRequirementBlueprint)nested_parts: list[str] = []- nested_dict: Dict[str, Any] = cast(Dict[str, Any], value)+ nested_dict: dict[str, Any] = cast("dict[str, Any]", value)for nested_key, nested_value in nested_dict.items():if isinstance(nested_value, str):nested_parts.append(f'{nested_key} = "{PipelexInterpreter.escape_plx_string(nested_value)}"')else:nested_parts.append(f"{nested_key} = {str(nested_value).lower()}")inputs_parts.append(f"{key} = {{ {', '.join(nested_parts)} }}")+ # Simple string value+ elif isinstance(value, str):+ inputs_parts.append(f'{key} = "{PipelexInterpreter.escape_plx_string(value)}"')else:- # Simple string value- if isinstance(value, str):- inputs_parts.append(f'{key} = "{PipelexInterpreter.escape_plx_string(value)}"')- else:- # Fallback for any other type- inputs_parts.append(f'{key} = "{PipelexInterpreter.escape_plx_string(str(value))}"')+ # Fallback for any other type+ inputs_parts.append(f'{key} = "{PipelexInterpreter.escape_plx_string(str(value))}"')return f"{{ {', '.join(inputs_parts)} }}"@staticmethoddef add_inputs_to_lines_if_exist(lines: list[str],- pipe_inputs: Optional[Mapping[str, Union[str, InputRequirementBlueprint]]],+ pipe_inputs: Mapping[str, str | InputRequirementBlueprint] | None,) -> None:"""Add inputs line to PLX lines if inputs exist."""if pipe_inputs:@@ -1002,16 +993,15 @@ class PipelexInterpreter(BaseModel):for key, value in inputs_dict.items():if isinstance(value, dict):nested_parts: list[str] = []- nested_dict: Dict[str, Any] = cast(Dict[str, Any], value)+ nested_dict: dict[str, Any] = cast("dict[str, Any]", value)for nested_key, nested_value in nested_dict.items():if isinstance(nested_value, str):nested_parts.append(f'{nested_key} = "{PipelexInterpreter.escape_plx_string(nested_value)}"')else:nested_parts.append(f"{nested_key} = {str(nested_value).lower()}")inputs_parts.append(f"{key} = {{ {', '.join(nested_parts)} }}")+ elif isinstance(value, str):+ inputs_parts.append(f'{key} = "{PipelexInterpreter.escape_plx_string(value)}"')else:- if isinstance(value, str):- inputs_parts.append(f'{key} = "{PipelexInterpreter.escape_plx_string(value)}"')- else:- inputs_parts.append(f'{key} = "{PipelexInterpreter.escape_plx_string(str(value))}"')+ inputs_parts.append(f'{key} = "{PipelexInterpreter.escape_plx_string(str(value))}"')lines.append(f"inputs = {{ {', '.join(inputs_parts)} }}")diff --git a/pipelex/core/memory/__init__.py b/pipelex/core/memory/__init__.pynew file mode 100644index 000000000..e69de29bbdiff --git a/pipelex/core/memory/working_memory.py b/pipelex/core/memory/working_memory.pyindex a430f6855..349a125cc 100644--- a/pipelex/core/memory/working_memory.py+++ b/pipelex/core/memory/working_memory.py@@ -1,8 +1,8 @@from operator import attrgetter-from typing import Any, Dict, List, Optional, Set, Type+from typing import Anyfrom pydantic import BaseModel, Field, model_validator-from typing_extensions import Self, override+from typing_extensions import overridefrom pipelex import log, pretty_printfrom pipelex.core.stuffs.stuff import Stuff@@ -25,28 +25,29 @@ from pipelex.exceptions import (WorkingMemoryTypeError,)from pipelex.tools.misc.context_provider_abstract import ContextProviderAbstract+from pipelex.types import SelfMAIN_STUFF_NAME = "main_stuff"BATCH_ITEM_STUFF_NAME = "BATCH_ITEM"PRETTY_PRINT_MAX_LENGTH = 1000-StuffDict = Dict[str, Stuff]-StuffArtefactDict = Dict[str, StuffArtefact]+StuffDict = dict[str, Stuff]+StuffArtefactDict = dict[str, StuffArtefact]class WorkingMemory(BaseModel, ContextProviderAbstract):root: StuffDict = Field(default_factory=dict)- aliases: Dict[str, str] = Field(default_factory=dict)+ aliases: dict[str, str] = Field(default_factory=dict)@model_validator(mode="after")def validate_stuff_names(self) -> Self:for key, stuff in self.root.items():- if key.startswith("_") and not key == BATCH_ITEM_STUFF_NAME:+ if key.startswith("_") and key != BATCH_ITEM_STUFF_NAME:log.warning(f"Stuff key '{key}' starts with '_', which is reserved for params")if not stuff.stuff_name:self.root[key].stuff_name = key- elif key != MAIN_STUFF_NAME and stuff.stuff_name != key:+ elif key not in (MAIN_STUFF_NAME, stuff.stuff_name):log.warning(f"Stuff name '{stuff.stuff_name}' does not match the key '{key}'")elif stuff.stuff_name.startswith("_") and stuff.stuff_name != BATCH_ITEM_STUFF_NAME:log.warning(f"Stuff name '{stuff.stuff_name}' starts with '_', which is reserved for params")@@ -63,14 +64,14 @@ class WorkingMemory(BaseModel, ContextProviderAbstract):def make_deep_copy(self) -> Self:return self.model_copy(deep=True)- def get_optional_stuff(self, name: str) -> Optional[Stuff]:+ def get_optional_stuff(self, name: str) -> Stuff | None:if named_stuff := self.root.get(name):return named_stuffif alias := self.aliases.get(name):return self.root.get(alias)return None- def get_optional_main_stuff(self) -> Optional[Stuff]:+ def get_optional_main_stuff(self) -> Stuff | None:return self.get_optional_stuff(name=MAIN_STUFF_NAME)# TODO: all calls to get_stuff should catch WorkingMemoryStuffNotFoundError in order to indicate what pipe is missing a required stuff@@ -90,24 +91,21 @@ class WorkingMemory(BaseModel, ContextProviderAbstract):message=f"Stuff '{name}' not found in working memory, valid keys are: {self.list_keys()}",)- def get_stuffs(self, names: Set[str]) -> List[Stuff]:- the_stuffs: List[Stuff] = []+ def get_stuffs(self, names: set[str]) -> list[Stuff]:+ the_stuffs: list[Stuff] = []for name in names:the_stuffs.append(self.get_stuff(name=name))return the_stuffs- def get_existing_stuffs(self, names: Set[str]) -> List[Stuff]:- the_stuffs: List[Stuff] = []+ def get_existing_stuffs(self, names: set[str]) -> list[Stuff]:+ the_stuffs: list[Stuff] = []for name in names:if stuff := self.get_optional_stuff(name=name):the_stuffs.append(stuff)return the_stuffsdef is_stuff_code_used(self, stuff_code: str) -> bool:- for stuff in self.root.values():- if stuff.concept.code == stuff_code:- return True- return False+ return any(stuff.concept.code == stuff_code for stuff in self.root.values())def get_main_stuff(self) -> Stuff:return self.get_stuff(name=MAIN_STUFF_NAME)@@ -122,20 +120,20 @@ class WorkingMemory(BaseModel, ContextProviderAbstract):def set_stuff(self, name: str, stuff: Stuff):self.root[name] = stuff- def add_new_stuff(self, name: str, stuff: Stuff, aliases: Optional[List[str]] = None):+ def add_new_stuff(self, name: str, stuff: Stuff, aliases: list[str] | None = None):# TODO: Add unit tests for this methodlog.debug(f"Adding new stuff '{name}' to WorkingMemory with aliases: {aliases}")if self.is_stuff_code_used(stuff_code=stuff.stuff_code):- raise WorkingMemoryConsistencyError(f"Stuff code '{stuff.stuff_code}' is already used by another stuff")+ msg = f"Stuff code '{stuff.stuff_code}' is already used by another stuff"+ raise WorkingMemoryConsistencyError(msg)if name in self.root or name in self.aliases:existing_stuff = self.get_stuff(name=name)if existing_stuff == stuff:log.warning(f"Key '{name}' already exists in WorkingMemory with the same stuff")return- else:- log.warning(f"Key '{name}' already exists in WorkingMemory and will be replaced by something different")- log.verbose(f"Existing stuff: {existing_stuff}")- log.verbose(f"New stuff: {stuff}")+ log.warning(f"Key '{name}' already exists in WorkingMemory and will be replaced by something different")+ log.verbose(f"Existing stuff: {existing_stuff}")+ log.verbose(f"New stuff: {stuff}")# it's a new stuffself.set_stuff(name=name, stuff=stuff)@@ -143,7 +141,7 @@ class WorkingMemory(BaseModel, ContextProviderAbstract):for alias in aliases:self.set_alias(alias, name)- def set_new_main_stuff(self, stuff: Stuff, name: Optional[str] = None):+ def set_new_main_stuff(self, stuff: Stuff, name: str | None = None):# TODO: Add unit tests for this methodif name:self.remove_main_stuff()@@ -157,16 +155,19 @@ class WorkingMemory(BaseModel, ContextProviderAbstract):def set_alias(self, alias: str, target: str) -> None:"""Add an alias pointing to a target name."""if alias == target:- raise WorkingMemoryConsistencyError(f"Cannot create alias '{alias}' pointing to itself")+ msg = f"Cannot create alias '{alias}' pointing to itself"+ raise WorkingMemoryConsistencyError(msg)if target not in self.root:- raise WorkingMemoryConsistencyError(f"Cannot create alias to non-existent target '{target}'")+ msg = f"Cannot create alias to non-existent target '{target}'"+ raise WorkingMemoryConsistencyError(msg)log.debug(f"Setting alias '{alias}' pointing to target '{target}'")self.aliases[alias] = targetdef add_alias(self, alias: str, target: str) -> None:"""Add an alias pointing to a target name."""if alias in self.root:- raise WorkingMemoryConsistencyError(f"Cannot add alias '{alias}' as it already exists")+ msg = f"Cannot add alias '{alias}' as it already exists"+ raise WorkingMemoryConsistencyError(msg)self.set_alias(alias=alias, target=target)log.debug(f"Added alias '{alias}' pointing to target '{target}'")@@ -179,11 +180,11 @@ class WorkingMemory(BaseModel, ContextProviderAbstract):"""Remove the alias pointing to the main stuff if it exists."""self.remove_alias(alias=MAIN_STUFF_NAME)- def get_aliases_for(self, target: str) -> List[str]:+ def get_aliases_for(self, target: str) -> list[str]:"""Get all aliases pointing to a target name."""return [alias for alias, t in self.aliases.items() if t == target]- def list_keys(self) -> List[str]:+ def list_keys(self) -> list[str]:return list(self.root.keys()) + list(self.aliases.keys())def pretty_print(self):@@ -195,7 +196,7 @@ class WorkingMemory(BaseModel, ContextProviderAbstract):################################################################################################@override- def generate_context(self) -> Dict[str, Any]:+ def generate_context(self) -> dict[str, Any]:# TODO: Add unit tests for this methodartefact_dict: StuffArtefactDict = {}for name, stuff in self.root.items():@@ -206,7 +207,7 @@ class WorkingMemory(BaseModel, ContextProviderAbstract):return artefact_dict@override- def get_typed_object_or_attribute(self, name: str, wanted_type: Optional[Type[Any]] = None) -> Any:+ def get_typed_object_or_attribute(self, name: str, wanted_type: type[Any] | None = None) -> Any:# TODO: Add unit tests for this method# TODO: Refactor this method. In the python paradigm, we should not have those ".", but arrays with field names.if "." in name:@@ -232,33 +233,31 @@ class WorkingMemory(BaseModel, ContextProviderAbstract):)return stuff_content- else:- content = self.get_stuff(name).content+ content = self.get_stuff(name).content- if wanted_type is not None and not isinstance(content, wanted_type):- raise WorkingMemoryTypeError(- variable_name=name,- message=f"Content of '{name}' is of type '{type(content).__name__}', it should be '{wanted_type.__name__}'",- )+ if wanted_type is not None and not isinstance(content, wanted_type):+ raise WorkingMemoryTypeError(+ variable_name=name,+ message=f"Content of '{name}' is of type '{type(content).__name__}', it should be '{wanted_type.__name__}'",+ )- return content+ return content################################################################################################# Stuff accessors################################################################################################- def get_stuff_as(self, name: str, content_type: Type[StuffContentType]) -> StuffContentType:+ def get_stuff_as(self, name: str, content_type: type[StuffContentType]) -> StuffContentType:"""Get stuff content as StuffContentType."""return self.get_stuff(name=name).content_as(content_type=content_type)- def get_stuff_as_list(self, name: str, item_type: Type[StuffContentType]) -> ListContent[StuffContentType]:- """- Get stuff content as ListContent with items of type StuffContentType.+ def get_stuff_as_list(self, name: str, item_type: type[StuffContentType]) -> ListContent[StuffContentType]:+ """Get stuff content as ListContent with items of type StuffContentType.If the items are of possibly various types, use item_type=StuffContent."""return self.get_stuff(name=name).as_list_of_fixed_content_type(item_type=item_type)- def get_list_stuff_first_item_as(self, name: str, item_type: Type[StuffContentType]) -> StuffContentType:+ def get_list_stuff_first_item_as(self, name: str, item_type: type[StuffContentType]) -> StuffContentType:"""Get stuff content as ListContent with items of type StuffContentType then return the first item."""return self.get_stuff_as_list(name=name, item_type=item_type).items[0]@@ -298,18 +297,17 @@ class WorkingMemory(BaseModel, ContextProviderAbstract):# Main stuff accessors################################################################################################- def main_stuff_as(self, content_type: Type[StuffContentType]) -> StuffContentType:+ def main_stuff_as(self, content_type: type[StuffContentType]) -> StuffContentType:"""Get main stuff content as StuffContentType."""return self.get_stuff_as(name=MAIN_STUFF_NAME, content_type=content_type)- def main_stuff_as_list(self, item_type: Type[StuffContentType]) -> ListContent[StuffContentType]:- """- Get main stuff content as ListContent with items of type StuffContentType.+ def main_stuff_as_list(self, item_type: type[StuffContentType]) -> ListContent[StuffContentType]:+ """Get main stuff content as ListContent with items of type StuffContentType.If the items are of possibly various types, use item_type=StuffContent."""return self.get_stuff_as_list(name=MAIN_STUFF_NAME, item_type=item_type)- def main_list_stuff_first_item_as(self, item_type: Type[StuffContentType]) -> StuffContentType:+ def main_list_stuff_first_item_as(self, item_type: type[StuffContentType]) -> StuffContentType:"""Get main stuff content as first list item of type StuffContentType."""return self.get_list_stuff_first_item_as(name=MAIN_STUFF_NAME, item_type=item_type)diff --git a/pipelex/core/memory/working_memory_factory.py b/pipelex/core/memory/working_memory_factory.pyindex a0f8f80f2..67f93973e 100644--- a/pipelex/core/memory/working_memory_factory.py+++ b/pipelex/core/memory/working_memory_factory.py@@ -1,4 +1,4 @@-from typing import Any, Dict, List, Optional, cast+from typing import Any, castimport shortuuidfrom polyfactory.factories.pydantic_factory import ModelFactory@@ -23,7 +23,7 @@ class WorkingMemoryFactory(BaseModel):cls,text: str,concept_string: str = SpecialDomain.NATIVE + "." + NativeConceptEnum.TEXT,- name: Optional[str] = "text",+ name: str | None = "text",) -> WorkingMemory:ConceptBlueprint.validate_concept_string(concept_string=concept_string)return cls.make_from_single_stuff(@@ -31,7 +31,7 @@ class WorkingMemoryFactory(BaseModel):concept=get_required_concept(concept_string=concept_string),content=TextContent(text=text),name=name,- )+ ),)@classmethod@@ -39,7 +39,7 @@ class WorkingMemoryFactory(BaseModel):cls,image_url: str,concept_string: str = SpecialDomain.NATIVE + "." + NativeConceptEnum.IMAGE,- name: Optional[str] = "image",+ name: str | None = "image",) -> WorkingMemory:# TODO: validate that the concept is compatible with an image conceptConceptBlueprint.validate_concept_string(concept_string=concept_string)@@ -55,7 +55,7 @@ class WorkingMemoryFactory(BaseModel):cls,pdf_url: str,concept_string: str = SpecialDomain.NATIVE + "." + NativeConceptEnum.PDF,- name: Optional[str] = "pdf",+ name: str | None = "pdf",) -> WorkingMemory:ConceptBlueprint.validate_concept_string(concept_string=concept_string)return cls.make_from_single_stuff(@@ -63,21 +63,22 @@ class WorkingMemoryFactory(BaseModel):concept=get_required_concept(concept_string=concept_string),content=PDFContent(url=pdf_url),name=name,- )+ ),)@classmethoddef make_from_single_stuff(cls, stuff: Stuff) -> WorkingMemory:if not stuff.stuff_name:- raise WorkingMemoryFactoryError(f"Cannot make_from_single_stuff because stuff has no name: {stuff}")+ msg = f"Cannot make_from_single_stuff because stuff has no name: {stuff}"+ raise WorkingMemoryFactoryError(msg)stuff_dict: StuffDict = {stuff.stuff_name: stuff}return WorkingMemory(root=stuff_dict, aliases={MAIN_STUFF_NAME: stuff.stuff_name})@classmethoddef make_from_multiple_stuffs(cls,- stuff_list: List[Stuff],- main_name: Optional[str] = None,+ stuff_list: list[Stuff],+ main_name: str | None = None,is_ignore_unnamed: bool = False,) -> WorkingMemory:stuff_dict: StuffDict = {}@@ -86,19 +87,19 @@ class WorkingMemoryFactory(BaseModel):if not name:if is_ignore_unnamed:continue- else:- raise WorkingMemoryFactoryError(f"Stuff {stuff} has no name")+ msg = f"Stuff {stuff} has no name"+ raise WorkingMemoryFactoryError(msg)stuff_dict[name] = stuff- aliases: Dict[str, str] = {}+ aliases: dict[str, str] = {}if stuff_dict:if main_name:aliases[MAIN_STUFF_NAME] = main_nameelse:- aliases[MAIN_STUFF_NAME] = list(stuff_dict.keys())[0]+ aliases[MAIN_STUFF_NAME] = next(iter(stuff_dict.keys()))return WorkingMemory(root=stuff_dict, aliases=aliases)@classmethod- def make_from_strings_from_dict(cls, input_dict: Dict[str, Any]) -> WorkingMemory:+ def make_from_strings_from_dict(cls, input_dict: dict[str, Any]) -> WorkingMemory:# TODO: Add unit tests for this methodstuff_dict: StuffDict = {}for name, content in input_dict.items():@@ -121,9 +122,9 @@ class WorkingMemoryFactory(BaseModel):def make_from_compact_memory(cls,compact_memory: CompactMemory,- search_domains: Optional[List[str]] = None,+ search_domains: list[str] | None = None,) -> WorkingMemory:- implicit_memory = cast(ImplicitMemory, compact_memory)+ implicit_memory = cast("ImplicitMemory", compact_memory)return cls.make_from_implicit_memory(implicit_memory=implicit_memory,search_domains=search_domains,@@ -133,16 +134,17 @@ class WorkingMemoryFactory(BaseModel):def make_from_implicit_memory(cls,implicit_memory: ImplicitMemory,- search_domains: Optional[List[str]] = None,+ search_domains: list[str] | None = None,) -> WorkingMemory:- """- Create a WorkingMemory from a compact memory dictionary.+ """Create a WorkingMemory from a implicit memory dictionary.Args:- compact_memory: Dictionary in the format from API serialization+ implicit_memory: Dictionary in the format from API serialization+ search_domains: List of search domains to use when making stuffReturns:- WorkingMemory object reconstructed from the compact format+ WorkingMemory object reconstructed from the implicit format+"""working_memory = cls.make_empty()@@ -162,34 +164,34 @@ class WorkingMemoryFactory(BaseModel):"""Helper method to create mock content for a requirement."""if requirement.structure_class:# Create mock object using polyfactory- class MockFactory(ModelFactory[requirement.structure_class]): # type: ignore+ class MockFactory(ModelFactory[requirement.structure_class]): # type: ignore[name-defined]__model__ = requirement.structure_class__check_model__ = True__use_examples__ = True__allow_none_optionals__ = False # Ensure Optional fields always get values- return MockFactory.build(factory_use_construct=True) # type: ignore- else:- # Fallback to text content- return TextContent(text=f"DRY RUN: Mock content for '{requirement.variable_name}' ({requirement.concept.code})")+ return MockFactory.build(factory_use_construct=True) # type: ignore[no-any-return]++ # Fallback to text content+ return TextContent(text=f"DRY RUN: Mock content for '{requirement.variable_name}' ({requirement.concept.code})")@classmethod- def make_for_dry_run(cls, needed_inputs: List[TypedNamedInputRequirement]) -> "WorkingMemory":- """- Create a WorkingMemory with mock objects for dry run mode.+ def make_for_dry_run(cls, needed_inputs: list[TypedNamedInputRequirement]) -> "WorkingMemory":+ """Create a WorkingMemory with mock objects for dry run mode.Args:needed_inputs: List of tuples (stuff_name, concept_code, structure_class)Returns:WorkingMemory with mock objects for each needed input+"""working_memory = cls.make_empty()for requirement in needed_inputs:log.debug(f"Creating dry run mock for '{requirement.variable_name}' with concept "- f"'{requirement.concept.code}' and class '{requirement.structure_class.__name__}'"+ f"'{requirement.concept.code}' and class '{requirement.structure_class.__name__}'",)try:@@ -214,7 +216,7 @@ class WorkingMemoryFactory(BaseModel):else:nb_stuffs = requirement.multiplicity- items: List[StuffContent] = []+ items: list[StuffContent] = []for _ in range(nb_stuffs):item_mock_content = cls.create_mock_content(requirement)items.append(item_mock_content)@@ -233,7 +235,7 @@ class WorkingMemoryFactory(BaseModel):except Exception as exc:log.warning(- f"Failed to create mock for '{requirement.variable_name}' ({requirement.concept.code}): {exc}. Using fallback text content."+ f"Failed to create mock for '{requirement.variable_name}' ({requirement.concept.code}): {exc}. Using fallback text content.",)# Create fallback text contentfallback_content = TextContent(text=f"DRY RUN: Fallback mock for '{requirement.variable_name}' ({requirement.concept.code})")diff --git a/pipelex/core/pipes/__init__.py b/pipelex/core/pipes/__init__.pynew file mode 100644index 000000000..e69de29bbdiff --git a/pipelex/core/pipes/exceptions.py b/pipelex/core/pipes/exceptions.pyindex baf2346d9..0fae11bce 100644--- a/pipelex/core/pipes/exceptions.py+++ b/pipelex/core/pipes/exceptions.py@@ -1,4 +1,2 @@class PipeBlueprintError(Exception):"""Exception raised for errors in the PipeBlueprint class."""-- passdiff --git a/pipelex/core/pipes/pipe_abstract.py b/pipelex/core/pipes/pipe_abstract.pyindex 3a7948bea..68705a742 100644--- a/pipelex/core/pipes/pipe_abstract.py+++ b/pipelex/core/pipes/pipe_abstract.py@@ -1,5 +1,5 @@from abc import ABC, abstractmethod-from typing import Any, List, Optional, Set, Type+from typing import Anyfrom pydantic import BaseModel, ConfigDict, Field, field_validator@@ -20,36 +20,30 @@ class PipeAbstract(ABC, BaseModel):type: Any # Any so that subclasses can put a Literalcode: strdomain: str- definition: Optional[str] = None+ definition: str | None = Noneinputs: PipeInputSpec = Field(default_factory=PipeInputSpec)output: Concept@field_validator("code", mode="before")- def validate_pipe_code_syntax(cls, code: str) -> str:+ @staticmethod+ def validate_pipe_code_syntax(code: str) -> str:PipeBlueprint.validate_pipe_code_syntax(pipe_code=code)return code@abstractmethoddef validate_output(self):- """- Validate the output for the pipe.- """- pass+ """Validate the output for the pipe."""@propertydef class_name(self) -> str:return self.__class__.__name__def validate_with_libraries(self):- """- Validate the pipe with the libraries, after the static validation- """- pass+ """Validate the pipe with the libraries, after the static validation"""@abstractmethod- def required_variables(self) -> Set[str]:- """- Return the variables that are required for the pipe to run.+ def required_variables(self) -> set[str]:+ """Return the variables that are required for the pipe to run.The required variables are only the list:# 1 - The inputs of dependency pipes# 2 - The variables in the pipe definition@@ -57,12 +51,10 @@ class PipeAbstract(ABC, BaseModel):- PipeBatch : Variables in the batch_params- PipeLLM : Variables in the prompt"""- pass@abstractmethod- def needed_inputs(self, visited_pipes: Optional[Set[str]] = None) -> PipeInputSpec:- """- Return the inputs that are needed for the pipe to run.+ def needed_inputs(self, visited_pipes: set[str] | None = None) -> PipeInputSpec:+ """Return the inputs that are needed for the pipe to run.Args:visited_pipes: Set of pipe codes currently being processed to prevent infinite recursion.@@ -70,22 +62,20 @@ class PipeAbstract(ABC, BaseModel):Returns:PipeInputSpec containing all needed inputs for this pipe- """- pass- def pipe_dependencies(self) -> Set[str]:"""- Return the pipes that are dependencies of the pipe.- - PipeBatch : The pipe that is being batched- - PipeCondition : The pipes in the pipe_map- - PipeSequence : The pipes in the steps++ def pipe_dependencies(self) -> set[str]:+ """Return the pipes that are dependencies of the pipe.+ - PipeBatch : The pipe that is being batched+ - PipeCondition : The pipes in the pipe_map+ - PipeSequence : The pipes in the steps"""return set()- def concept_dependencies(self) -> List[Concept]:- required_concepts: List[Concept] = [self.output]- for concept in self.inputs.concepts:- required_concepts.append(concept)+ def concept_dependencies(self) -> list[Concept]:+ required_concepts: list[Concept] = [self.output]+ required_concepts.extend(self.inputs.concepts)required_concepts.append(self.output)return required_concepts@@ -95,8 +85,8 @@ class PipeAbstract(ABC, BaseModel):job_metadata: JobMetadata,working_memory: WorkingMemory,pipe_run_params: PipeRunParams,- output_name: Optional[str] = None,- print_intermediate_outputs: Optional[bool] = False,+ output_name: str | None = None,+ print_intermediate_outputs: bool | None = False,) -> PipeOutput:pass@@ -104,7 +94,8 @@ class PipeAbstract(ABC, BaseModel):pipe_stack = pipe_run_params.pipe_stacklimit = pipe_run_params.pipe_stack_limitif len(pipe_stack) > limit:- raise PipeStackOverflowError(f"Exceeded pipe stack limit of {limit}. You can raise that limit in the config. Stack:\n{pipe_stack}")+ msg = f"Exceeded pipe stack limit of {limit}. You can raise that limit in the config. Stack:\n{pipe_stack}"+ raise PipeStackOverflowError(msg)-PipeAbstractType = Type[PipeAbstract]+PipeAbstractType = type[PipeAbstract]diff --git a/pipelex/core/pipes/pipe_blueprint.py b/pipelex/core/pipes/pipe_blueprint.pyindex b776a3b52..d684c82ed 100644--- a/pipelex/core/pipes/pipe_blueprint.py+++ b/pipelex/core/pipes/pipe_blueprint.py@@ -1,4 +1,4 @@-from typing import Any, Dict, List, Optional, Union+from typing import Anyfrom pydantic import BaseModel, field_validator@@ -14,8 +14,8 @@ class AllowedPipeCategories(StrEnum):PIPE_CONTROLLER = "PipeController"@classmethod- def value_list(cls) -> List[str]:- return [value for value in cls]+ def value_list(cls) -> list[str]:+ return list(cls)class AllowedPipeTypes(StrEnum):@@ -32,38 +32,44 @@ class AllowedPipeTypes(StrEnum):PIPE_SEQUENCE = "PipeSequence"@classmethod- def value_list(cls) -> List[str]:- return [value for value in cls]+ def value_list(cls) -> list[str]:+ return list(cls)class PipeBlueprint(BaseModel):category: Anytype: Any # TODO: Find a better way to handle this.- definition: Optional[str] = None- inputs: Optional[Dict[str, Union[str, InputRequirementBlueprint]]] = None+ definition: str | None = None+ inputs: dict[str, str | InputRequirementBlueprint] | None = Noneoutput: str@field_validator("type", mode="after")- def validate_pipe_type(cls, value: Any) -> Any:+ @staticmethod+ def validate_pipe_type(value: Any) -> Any:"""Validate that the pipe type is one of the allowed values."""if value not in AllowedPipeTypes.value_list():- raise PipeBlueprintError(f"Invalid pipe type '{value}'. Must be one of: {AllowedPipeTypes.value_list()}")+ msg = f"Invalid pipe type '{value}'. Must be one of: {AllowedPipeTypes.value_list()}"+ raise PipeBlueprintError(msg)return value@field_validator("category", mode="after")- def validate_pipe_category(cls, value: Any) -> Any:+ @staticmethod+ def validate_pipe_category(value: Any) -> Any:"""Validate that the pipe category is one of the allowed values."""if value not in AllowedPipeCategories.value_list():- raise PipeBlueprintError(f"Invalid pipe category '{value}'. Must be one of: {AllowedPipeCategories.value_list()}")+ msg = f"Invalid pipe category '{value}'. Must be one of: {AllowedPipeCategories.value_list()}"+ raise PipeBlueprintError(msg)return value@field_validator("output", mode="before")- def validate_concept_string_or_code(cls, output: str) -> str:+ @staticmethod+ def validate_concept_string_or_code(output: str) -> str:ConceptBlueprint.validate_concept_string_or_code(concept_string_or_code=output)return output@classmethoddef validate_pipe_code_syntax(cls, pipe_code: str) -> str:if not is_snake_case(pipe_code):- raise PipeBlueprintError(f"Invalid pipe code syntax '{pipe_code}'. Must be in snake_case.")+ msg = f"Invalid pipe code syntax '{pipe_code}'. Must be in snake_case."+ raise PipeBlueprintError(msg)return pipe_codediff --git a/pipelex/core/pipes/pipe_factory.py b/pipelex/core/pipes/pipe_factory.pyindex d4a3ba12b..1b9bfc37d 100644--- a/pipelex/core/pipes/pipe_factory.py+++ b/pipelex/core/pipes/pipe_factory.py@@ -1,4 +1,4 @@-from typing import Any, List, Optional, Protocol, Type, TypeVar+from typing import Any, Protocol, TypeVarfrom kajson.exceptions import ClassRegistryInheritanceError, ClassRegistryNotFoundErrorfrom kajson.kajson_manager import KajsonManager@@ -20,7 +20,7 @@ class PipeFactoryProtocol(Protocol[PipeBlueprintType, PipeType]):domain: str,pipe_code: str,blueprint: PipeBlueprintType,- concept_codes_from_the_same_domain: Optional[List[str]] = None,+ concept_codes_from_the_same_domain: list[str] | None = None,) -> PipeType: ...@@ -32,23 +32,21 @@ class PipeFactory(PipeFactoryProtocol[PipeBlueprint, PipeAbstract]):domain: str,pipe_code: str,blueprint: PipeBlueprint,- concept_codes_from_the_same_domain: Optional[List[str]] = None,+ concept_codes_from_the_same_domain: list[str] | None = None,) -> PipeAbstract:# The factory class name for that specific type of Pipe is the pipe class name with "Factory" suffixfactory_class_name = f"{blueprint.type}Factory"try:- pipe_factory: Type[PipeFactoryProtocol[Any, Any]] = KajsonManager.get_class_registry().get_required_subclass(+ pipe_factory: type[PipeFactoryProtocol[Any, Any]] = KajsonManager.get_class_registry().get_required_subclass(name=factory_class_name,base_class=PipeFactoryProtocol,)except ClassRegistryNotFoundError as factory_not_found_error:- raise PipeFactoryError(- f"Pipe '{pipe_code}' couldn't be created: factory '{factory_class_name}' not found: {factory_not_found_error}"- ) from factory_not_found_error+ msg = f"Pipe '{pipe_code}' couldn't be created: factory '{factory_class_name}' not found: {factory_not_found_error}"+ raise PipeFactoryError(msg) from factory_not_found_errorexcept ClassRegistryInheritanceError as factory_inheritance_error:- raise PipeFactoryError(- f"Pipe '{pipe_code}' couldn't be created: factory '{factory_class_name}' is not a subclass of {type(PipeFactoryProtocol)}."- ) from factory_inheritance_error+ msg = f"Pipe '{pipe_code}' couldn't be created: factory '{factory_class_name}' is not a subclass of {type(PipeFactoryProtocol)}."+ raise PipeFactoryError(msg) from factory_inheritance_errorpipe_from_blueprint: PipeAbstract = pipe_factory.make_from_blueprint(domain=domain,diff --git a/pipelex/core/pipes/pipe_input.py b/pipelex/core/pipes/pipe_input.pyindex db2ca2071..1aa6b86fe 100644--- a/pipelex/core/pipes/pipe_input.py+++ b/pipelex/core/pipes/pipe_input.py@@ -1,4 +1,4 @@-from typing import Callable, Dict, List, Optional, Tuple, Type+from collections.abc import Callablefrom pydantic import BaseModel, Field, RootModel, field_validator@@ -11,27 +11,27 @@ from pipelex.exceptions import PipeInputNotFoundErrorclass InputRequirement(BaseModel):concept: Concept- multiplicity: Optional[PipeOutputMultiplicity] = None+ multiplicity: PipeOutputMultiplicity | None = Noneclass NamedInputRequirement(InputRequirement):variable_name: str- requirement_expression: Optional[str] = None+ requirement_expression: str | None = Noneclass TypedNamedInputRequirement(NamedInputRequirement):- structure_class: Type[StuffContent]+ structure_class: type[StuffContent]@classmethoddef make_from_named(cls,named: NamedInputRequirement,- structure_class: Type[StuffContent],+ structure_class: type[StuffContent],) -> "TypedNamedInputRequirement":return cls(**named.model_dump(), structure_class=structure_class)-PipeInputSpecRoot = Dict[str, InputRequirement]+PipeInputSpecRoot = dict[str, InputRequirement]class PipeInputSpec(RootModel[PipeInputSpecRoot]):@@ -41,14 +41,14 @@ class PipeInputSpec(RootModel[PipeInputSpecRoot]):@classmethoddef validate_concept_codes(cls,- input_value: Dict[str, InputRequirement],- handler: Callable[[Dict[str, InputRequirement]], Dict[str, InputRequirement]],- ) -> Dict[str, InputRequirement]:+ input_value: dict[str, InputRequirement],+ handler: Callable[[dict[str, InputRequirement]], dict[str, InputRequirement]],+ ) -> dict[str, InputRequirement]:# First let Pydantic handle the basic type validation- validated_dict: Dict[str, InputRequirement] = handler(input_value)+ validated_dict: dict[str, InputRequirement] = handler(input_value)# Now we can transform and validate the keys and values- transformed_dict: Dict[str, InputRequirement] = {}+ transformed_dict: dict[str, InputRequirement] = {}for required_input, requirement in validated_dict.items():# in case of sub-attribute, the variable name is the object name, before the 1st dottransformed_key: str = required_input.split(".", 1)[0]@@ -57,7 +57,7 @@ class PipeInputSpec(RootModel[PipeInputSpecRoot]):if transformed_key in transformed_dict and transformed_dict[transformed_key] != requirement:log.verbose(- f"Variable {transformed_key} already exists with a different concept code: {transformed_dict[transformed_key]} -> {requirement}"+ f"Variable {transformed_key} already exists with a different concept code: {transformed_dict[transformed_key]} -> {requirement}",)transformed_dict[transformed_key] = InputRequirement(concept=requirement.concept, multiplicity=requirement.multiplicity)@@ -73,39 +73,40 @@ class PipeInputSpec(RootModel[PipeInputSpecRoot]):def get_required_input_requirement(self, variable_name: str) -> InputRequirement:requirement = self.root.get(variable_name)if not requirement:- raise PipeInputNotFoundError(f"Variable '{variable_name}' not found in input spec")+ msg = f"Variable '{variable_name}' not found in input spec"+ raise PipeInputNotFoundError(msg)return requirement- def add_requirement(self, variable_name: str, concept: Concept, multiplicity: Optional[PipeOutputMultiplicity] = None):+ def add_requirement(self, variable_name: str, concept: Concept, multiplicity: PipeOutputMultiplicity | None = None):self.root[variable_name] = InputRequirement(concept=concept, multiplicity=multiplicity)@property- def items(self) -> List[Tuple[str, InputRequirement]]:+ def items(self) -> list[tuple[str, InputRequirement]]:return list(self.root.items())@property- def concepts(self) -> List[Concept]:- all_concepts: List[Concept] = []+ def concepts(self) -> list[Concept]:+ all_concepts: list[Concept] = []for requirement in self.root.values():if requirement.concept.concept_string not in [c.concept_string for c in all_concepts]:all_concepts.append(requirement.concept)return all_concepts@property- def variables(self) -> List[str]:+ def variables(self) -> list[str]:return list(self.root.keys())@property- def required_names(self) -> List[str]:- the_required_names: List[str] = []- for requirement_expression in self.root.keys():+ def required_names(self) -> list[str]:+ the_required_names: list[str] = []+ for requirement_expression in self.root:required_variable_name = requirement_expression.split(".", 1)[0]the_required_names.append(required_variable_name)return the_required_names@property- def named_input_requirements(self) -> List[NamedInputRequirement]:- the_requirements: List[NamedInputRequirement] = []+ def named_input_requirements(self) -> list[NamedInputRequirement]:+ the_requirements: list[NamedInputRequirement] = []for requirement_expression, requirement in self.root.items():required_variable_name = requirement_expression.split(".", 1)[0]# TODO: refactor this with a proper class like InputRequirement@@ -115,6 +116,6 @@ class PipeInputSpec(RootModel[PipeInputSpecRoot]):requirement_expression=requirement_expression,concept=requirement.concept,multiplicity=requirement.multiplicity,- )+ ),)return the_requirementsdiff --git a/pipelex/core/pipes/pipe_input_blueprint.py b/pipelex/core/pipes/pipe_input_blueprint.pyindex 13da21156..4af759768 100644--- a/pipelex/core/pipes/pipe_input_blueprint.py+++ b/pipelex/core/pipes/pipe_input_blueprint.py@@ -1,5 +1,3 @@-from typing import Optional-from pydantic import BaseModel, field_validatorfrom pipelex.core.concepts.concept_blueprint import ConceptBlueprint@@ -8,7 +6,7 @@ from pipelex.core.pipes.pipe_run_params import PipeOutputMultiplicityclass InputRequirementBlueprint(BaseModel):concept: str- multiplicity: Optional[PipeOutputMultiplicity] = None+ multiplicity: PipeOutputMultiplicity | None = None@field_validator("concept", mode="before")@classmethoddiff --git a/pipelex/core/pipes/pipe_input_factory.py b/pipelex/core/pipes/pipe_input_factory.pyindex e5ce3bf8b..351f4006f 100644--- a/pipelex/core/pipes/pipe_input_factory.py+++ b/pipelex/core/pipes/pipe_input_factory.py@@ -1,5 +1,3 @@-from typing import Dict, List, Optional, Union-from pipelex.core.concepts.concept_blueprint import ConceptBlueprintfrom pipelex.core.concepts.concept_factory import ConceptFactoryfrom pipelex.core.pipes.pipe_input import InputRequirement, PipeInputSpec@@ -14,9 +12,12 @@ class PipeInputSpecFactory:@classmethoddef make_from_blueprint(- cls, domain: str, blueprint: Dict[str, Union[str, InputRequirementBlueprint]], concept_codes_from_the_same_domain: Optional[List[str]] = None+ cls,+ domain: str,+ blueprint: dict[str, str | InputRequirementBlueprint],+ concept_codes_from_the_same_domain: list[str] | None = None,) -> PipeInputSpec:- inputs: Dict[str, InputRequirement] = {}+ inputs: dict[str, InputRequirement] = {}for var_name, input_requirement_blueprint in blueprint.items():if isinstance(input_requirement_blueprint, str):input_requirement_blueprint = InputRequirementBlueprint(concept=input_requirement_blueprint)@@ -32,8 +33,9 @@ class PipeInputSpecFactory:inputs[var_name] = InputRequirement(concept=get_concept_provider().get_required_concept(concept_string=ConceptFactory.construct_concept_string_with_domain(- domain=input_domain_and_code.domain, concept_code=input_domain_and_code.concept_code- )+ domain=input_domain_and_code.domain,+ concept_code=input_domain_and_code.concept_code,+ ),),multiplicity=input_requirement_blueprint.multiplicity,)diff --git a/pipelex/core/pipes/pipe_library.py b/pipelex/core/pipes/pipe_library.pyindex 9ece942fb..ebccc3279 100644--- a/pipelex/core/pipes/pipe_library.py+++ b/pipelex/core/pipes/pipe_library.py@@ -1,18 +1,18 @@from itertools import groupby-from typing import Dict, List, Optionalfrom pydantic import RootModelfrom rich import boxfrom rich.table import Table-from typing_extensions import Self, override+from typing_extensions import overridefrom pipelex import pretty_printfrom pipelex.core.pipes.pipe_abstract import PipeAbstractfrom pipelex.core.pipes.pipe_provider_abstract import PipeProviderAbstractfrom pipelex.exceptions import ConceptError, ConceptLibraryConceptNotFoundError, PipeLibraryError, PipeLibraryPipeNotFoundErrorfrom pipelex.hub import get_concept_provider+from pipelex.types import Self-PipeLibraryRoot = Dict[str, PipeAbstract]+PipeLibraryRoot = dict[str, PipeAbstract]class PipeLibrary(RootModel[PipeLibraryRoot], PipeProviderAbstract):@@ -26,14 +26,14 @@ class PipeLibrary(RootModel[PipeLibraryRoot], PipeProviderAbstract):try:concept_provider.get_required_concept(concept_string=concept.concept_string)except ConceptError as concept_error:- raise PipeLibraryError(- f"Error validating pipe '{pipe.code}' dependency concept '{concept.concept_string}' because of: {concept_error}"- ) from concept_error+ msg = f"Error validating pipe '{pipe.code}' dependency concept '{concept.concept_string}' because of: {concept_error}"+ raise PipeLibraryError(msg) from concept_errorfor pipe_code in pipe.pipe_dependencies():self.get_required_pipe(pipe_code=pipe_code)pipe.validate_with_libraries()except (ConceptLibraryConceptNotFoundError, PipeLibraryPipeNotFoundError) as not_found_error:- raise PipeLibraryError(f"Missing dependency for pipe '{pipe.code}': {not_found_error}") from not_found_error+ msg = f"Missing dependency for pipe '{pipe.code}': {not_found_error}"+ raise PipeLibraryError(msg) from not_found_error@classmethoddef make_empty(cls) -> Self:@@ -42,11 +42,12 @@ class PipeLibrary(RootModel[PipeLibraryRoot], PipeProviderAbstract):@overridedef add_new_pipe(self, pipe: PipeAbstract):if pipe.code in self.root:- raise PipeLibraryError(f"Pipe '{pipe.code}' already exists in the library")+ msg = f"Pipe '{pipe.code}' already exists in the library"+ raise PipeLibraryError(msg)self.root[pipe.code] = pipe@override- def add_pipes(self, pipes: List[PipeAbstract]):+ def add_pipes(self, pipes: list[PipeAbstract]):for pipe in pipes:self.add_new_pipe(pipe=pipe)@@ -58,27 +59,26 @@ class PipeLibrary(RootModel[PipeLibraryRoot], PipeProviderAbstract):self.root[name] = pipe@override- def get_optional_pipe(self, pipe_code: str) -> Optional[PipeAbstract]:+ def get_optional_pipe(self, pipe_code: str) -> PipeAbstract | None:return self.root.get(pipe_code)@overridedef get_required_pipe(self, pipe_code: str) -> PipeAbstract:the_pipe = self.get_optional_pipe(pipe_code=pipe_code)if not the_pipe:- raise PipeLibraryPipeNotFoundError(- f"Pipe '{pipe_code}' not found. Check for typos and make sure it is declared in a library listed in the config."- )+ msg = f"Pipe '{pipe_code}' not found. Check for typos and make sure it is declared in a library listed in the config."+ raise PipeLibraryPipeNotFoundError(msg)return the_pipe@override- def get_pipes(self) -> List[PipeAbstract]:+ def get_pipes(self) -> list[PipeAbstract]:return list(self.root.values())@override- def get_pipes_dict(self) -> Dict[str, PipeAbstract]:+ def get_pipes_dict(self) -> dict[str, PipeAbstract]:return self.root- def remove_pipes_by_codes(self, pipe_codes: List[str]) -> None:+ def remove_pipes_by_codes(self, pipe_codes: list[str]) -> None:# TODO: We should create a separate library, that copies the original one, and then removes the pipes from it# Then run the dry run + validation to see if removing those pipe has not broken any other pipe.# If validated, it should update the real library.@@ -92,7 +92,7 @@ class PipeLibrary(RootModel[PipeLibraryRoot], PipeProviderAbstract):@overridedef pretty_list_pipes(self) -> None:- def _format_concept_code(concept_code: Optional[str], current_domain: str) -> str:+ def _format_concept_code(concept_code: str | None, current_domain: str) -> str:"""Format concept code by removing domain prefix if it matches current domain."""if not concept_code:return ""@@ -107,7 +107,7 @@ class PipeLibrary(RootModel[PipeLibraryRoot], PipeProviderAbstract):ordered_items = sorted(pipes, key=lambda pipe: (pipe.domain or "", pipe.code or ""))# Create dictionary for return value- pipes_dict: Dict[str, Dict[str, Dict[str, str]]] = {}+ pipes_dict: dict[str, dict[str, dict[str, str]]] = {}# Group by domain and create separate tablesfor domain, domain_pipes in groupby(ordered_items, key=lambda pipe: pipe.domain):diff --git a/pipelex/core/pipes/pipe_output.py b/pipelex/core/pipes/pipe_output.pyindex 1a5143d52..73e8669bb 100644--- a/pipelex/core/pipes/pipe_output.py+++ b/pipelex/core/pipes/pipe_output.py@@ -1,4 +1,4 @@-from typing import List, Type, TypeVar+from typing import TypeVarfrom pydantic import BaseModel, Field@@ -25,23 +25,20 @@ class PipeOutput(BaseModel):def main_stuff(self) -> Stuff:return self.working_memory.get_main_stuff()- def main_stuff_as_list(self, item_type: Type[StuffContentType]) -> ListContent[StuffContentType]:- """- Get main stuff content as ListContent with items of type StuffContentType.+ def main_stuff_as_list(self, item_type: type[StuffContentType]) -> ListContent[StuffContentType]:+ """Get main stuff content as ListContent with items of type StuffContentType.If the items are of possibly various types, use item_type=StuffContent."""return self.working_memory.main_stuff_as_list(item_type=item_type)- def main_stuff_as_items(self, item_type: Type[StuffContentType]) -> List[StuffContentType]:- """- Get main stuff content as ListContent with items of type StuffContentType.+ def main_stuff_as_items(self, item_type: type[StuffContentType]) -> list[StuffContentType]:+ """Get main stuff content as ListContent with items of type StuffContentType.Return the actual items"""return self.working_memory.main_stuff_as_list(item_type=item_type).items- def main_stuff_as(self, content_type: Type[StuffContentType]) -> StuffContentType:- """- Get main stuff content as StuffContentType.+ def main_stuff_as(self, content_type: type[StuffContentType]) -> StuffContentType:+ """Get main stuff content as StuffContentType.If the items are of possibly various types, use item_type=StuffContent."""return self.working_memory.main_stuff_as(content_type=content_type)diff --git a/pipelex/core/pipes/pipe_provider_abstract.py b/pipelex/core/pipes/pipe_provider_abstract.pyindex 48fca9e43..89456a6f8 100644--- a/pipelex/core/pipes/pipe_provider_abstract.py+++ b/pipelex/core/pipes/pipe_provider_abstract.py@@ -1,9 +1,8 @@from abc import ABC, abstractmethod-from typing import Dict, List, Optionalfrom pipelex.core.pipes.pipe_abstract import PipeAbstract-PipeLibraryRoot = Dict[str, PipeAbstract]+PipeLibraryRoot = dict[str, PipeAbstract]class PipeProviderAbstract(ABC):@@ -16,15 +15,15 @@ class PipeProviderAbstract(ABC):pass@abstractmethod- def get_optional_pipe(self, pipe_code: str) -> Optional[PipeAbstract]:+ def get_optional_pipe(self, pipe_code: str) -> PipeAbstract | None:pass@abstractmethod- def get_pipes(self) -> List[PipeAbstract]:+ def get_pipes(self) -> list[PipeAbstract]:pass@abstractmethod- def get_pipes_dict(self) -> Dict[str, PipeAbstract]:+ def get_pipes_dict(self) -> dict[str, PipeAbstract]:pass@abstractmethod@@ -40,5 +39,5 @@ class PipeProviderAbstract(ABC):pass@abstractmethod- def add_pipes(self, pipes: List[PipeAbstract]) -> None:+ def add_pipes(self, pipes: list[PipeAbstract]) -> None:passdiff --git a/pipelex/core/pipes/pipe_run_params.py b/pipelex/core/pipes/pipe_run_params.pyindex d4d424e6d..2a71d40d4 100644--- a/pipelex/core/pipes/pipe_run_params.py+++ b/pipelex/core/pipes/pipe_run_params.py@@ -1,11 +1,12 @@-from typing import Any, Dict, List, Optional, Union+from __future__ import annotations++from typing import Any, Unionfrom pydantic import BaseModel, Field, field_validator-from typing_extensions import Selffrom pipelex import logfrom pipelex.core.memory.working_memory import BATCH_ITEM_STUFF_NAME, MAIN_STUFF_NAME-from pipelex.types import StrEnum+from pipelex.types import Self, StrEnumclass PipeRunParamKey(StrEnum):@@ -24,23 +25,21 @@ PipeOutputMultiplicity = Union[bool, int]class OutputMultiplicityResolution(BaseModel):- """- Result of resolving output multiplicity settings between base and override values.+ """Result of resolving output multiplicity settings between base and override values.This model provides a clear, structured representation of how output multiplicityshould be applied, replacing the previous tuple-based return format."""- resolved_multiplicity: Optional[PipeOutputMultiplicity] = Field(description="The final multiplicity value to use after resolution")+ resolved_multiplicity: PipeOutputMultiplicity | None = Field(description="The final multiplicity value to use after resolution")enable_multiple_outputs: bool = Field(description="Whether multiple outputs should be generated")- specific_output_count: Optional[int] = Field(default=None, description="Exact number of outputs to generate, if specified")+ specific_output_count: int | None = Field(default=None, description="Exact number of outputs to generate, if specified")-def make_output_multiplicity(nb_output: Optional[int], multiple_output: Optional[bool]) -> Optional[PipeOutputMultiplicity]:- """- This function takes two mutually exclusive parameters that control how many outputs+def make_output_multiplicity(nb_output: int | None, multiple_output: bool | None) -> PipeOutputMultiplicity | None:+ """This function takes two mutually exclusive parameters that control how many outputsa pipe should generate and converts them into a single PipeOutputMultiplicity type.Args:@@ -58,8 +57,9 @@ def make_output_multiplicity(nb_output: Optional[int], multiple_output: OptionalNone>>> make_output_multiplicity(nb_output=0, multiple_output=True)True+"""- output_multiplicity: Optional[PipeOutputMultiplicity]+ output_multiplicity: PipeOutputMultiplicity | Noneif nb_output:output_multiplicity = nb_outputelif multiple_output:@@ -70,11 +70,10 @@ def make_output_multiplicity(nb_output: Optional[int], multiple_output: Optionaldef output_multiplicity_to_apply(- base_multiplicity: Optional[PipeOutputMultiplicity],- override_multiplicity: Optional[PipeOutputMultiplicity],+ base_multiplicity: PipeOutputMultiplicity | None,+ override_multiplicity: PipeOutputMultiplicity | None,) -> OutputMultiplicityResolution:- """- Resolve output multiplicity settings by combining base configuration with override.+ """Resolve output multiplicity settings by combining base configuration with override.This function implements a priority system where override values take precedence overbase values, with clear logic to handle different type combinations and return@@ -113,6 +112,7 @@ def output_multiplicity_to_apply(>>> result = output_multiplicity_to_apply(3, None)>>> (result.resolved_multiplicity, result.enable_multiple_outputs, result.specific_output_count)(3, True, 3)+"""log.debug(f"base_multiplicity = {base_multiplicity}")log.debug(f"override_multiplicity = {override_multiplicity}")@@ -124,12 +124,16 @@ def output_multiplicity_to_apply(if isinstance(base_multiplicity, bool):log.debug("base_multiplicity is bool")return OutputMultiplicityResolution(- resolved_multiplicity=base_multiplicity, enable_multiple_outputs=base_multiplicity, specific_output_count=None+ resolved_multiplicity=base_multiplicity,+ enable_multiple_outputs=base_multiplicity,+ specific_output_count=None,)elif isinstance(base_multiplicity, int):log.debug("base_multiplicity is int")return OutputMultiplicityResolution(- resolved_multiplicity=base_multiplicity, enable_multiple_outputs=True, specific_output_count=base_multiplicity+ resolved_multiplicity=base_multiplicity,+ enable_multiple_outputs=True,+ specific_output_count=base_multiplicity,)else:log.debug("base_multiplicity is None")@@ -156,11 +160,13 @@ def output_multiplicity_to_apply(log.debug("override_multiplicity is False - forcing single output")return OutputMultiplicityResolution(resolved_multiplicity=False, enable_multiple_outputs=False, specific_output_count=None)- # Case 3: Override is an integerelse:+ # Case 3: Override is an integerlog.debug("override_multiplicity is int - using override value")return OutputMultiplicityResolution(- resolved_multiplicity=override_multiplicity, enable_multiple_outputs=True, specific_output_count=override_multiplicity+ resolved_multiplicity=override_multiplicity,+ enable_multiple_outputs=True,+ specific_output_count=override_multiplicity,)@@ -171,10 +177,10 @@ class BatchParams(BaseModel):@classmethoddef make_optional_batch_params(cls,- input_list_name: Union[bool, str],- input_item_name: Optional[str] = None,- ) -> Optional["BatchParams"]:- the_batch_params: Optional[BatchParams] = None+ input_list_name: bool | str,+ input_item_name: str | None = None,+ ) -> BatchParams | None:+ the_batch_params: BatchParams | None = Noneif input_list_name or input_item_name:input_list_stuff_name: strif isinstance(input_list_name, str):@@ -189,7 +195,7 @@ class BatchParams(BaseModel):return the_batch_params@classmethod- def make_default(cls) -> "BatchParams":+ def make_default(cls) -> BatchParams:return BatchParams(input_list_stuff_name=MAIN_STUFF_NAME,input_item_stuff_name=BATCH_ITEM_STUFF_NAME,@@ -198,26 +204,27 @@ class BatchParams(BaseModel):class PipeRunParams(BaseModel):run_mode: PipeRunMode = PipeRunMode.LIVE- final_stuff_code: Optional[str] = None- is_with_preliminary_text: Optional[bool] = None- output_multiplicity: Optional[PipeOutputMultiplicity] = None- dynamic_output_concept_code: Optional[str] = None- batch_params: Optional[BatchParams] = None- params: Dict[str, Any] = Field(default_factory=dict)+ final_stuff_code: str | None = None+ is_with_preliminary_text: bool | None = None+ output_multiplicity: PipeOutputMultiplicity | None = None+ dynamic_output_concept_code: str | None = None+ batch_params: BatchParams | None = None+ params: dict[str, Any] = Field(default_factory=dict)pipe_stack_limit: int- pipe_stack: List[str] = Field(default_factory=list)- pipe_layers: List[str] = Field(default_factory=list)+ pipe_stack: list[str] = Field(default_factory=list)+ pipe_layers: list[str] = Field(default_factory=list)@field_validator("params")@classmethod- def validate_param_keys(cls, value: Dict[str, Any]) -> Dict[str, Any]:+ def validate_param_keys(cls, value: dict[str, Any]) -> dict[str, Any]:for key in value:if key == PipeRunParamKey.DYNAMIC_OUTPUT_CONCEPT:# TODO: validate the concept codepassif not key.startswith("_"):- raise ValueError(f"Parameter key '{key}' must start with an underscore '_'")+ msg = f"Parameter key '{key}' must start with an underscore '_'"+ raise ValueError(msg)return valuedef make_deep_copy(self) -> Self:@@ -230,11 +237,10 @@ class PipeRunParams(BaseModel):def copy_by_injecting_multiplicity(cls,pipe_run_params: Self,- applied_output_multiplicity: Optional[PipeOutputMultiplicity],- is_with_preliminary_text: Optional[bool] = None,+ applied_output_multiplicity: PipeOutputMultiplicity | None,+ is_with_preliminary_text: bool | None = None,) -> Self:- """- Copy the run params the nb_output into the params, and remove the attribute.+ """Copy the run params the nb_output into the params, and remove the attribute.This is useful to make a single prompt with multiple outputs."""new_run_params = pipe_run_params.model_copy()diff --git a/pipelex/core/pipes/pipe_run_params_factory.py b/pipelex/core/pipes/pipe_run_params_factory.pyindex 65b006529..75665c7ed 100644--- a/pipelex/core/pipes/pipe_run_params_factory.py+++ b/pipelex/core/pipes/pipe_run_params_factory.py@@ -1,4 +1,4 @@-from typing import Any, Dict, Optional+from typing import Anyfrom pipelex.config import get_configfrom pipelex.core.pipes.pipe_run_params import BatchParams, PipeOutputMultiplicity, PipeRunMode, PipeRunParams@@ -9,11 +9,11 @@ class PipeRunParamsFactory:def make_run_params(cls,pipe_run_mode: PipeRunMode = PipeRunMode.LIVE,- pipe_stack_limit: Optional[int] = None,- output_multiplicity: Optional[PipeOutputMultiplicity] = None,- dynamic_output_concept_code: Optional[str] = None,- batch_params: Optional[BatchParams] = None,- params: Optional[Dict[str, Any]] = None,+ pipe_stack_limit: int | None = None,+ output_multiplicity: PipeOutputMultiplicity | None = None,+ dynamic_output_concept_code: str | None = None,+ batch_params: BatchParams | None = None,+ params: dict[str, Any] | None = None,) -> PipeRunParams:pipe_stack_limit = pipe_stack_limit or get_config().pipelex.pipe_run_config.pipe_stack_limitreturn PipeRunParams(diff --git a/pipelex/core/registry_models.py b/pipelex/core/registry_models.pyindex 413e79663..644b8795b 100644--- a/pipelex/core/registry_models.py+++ b/pipelex/core/registry_models.py@@ -1,4 +1,4 @@-from typing import Any, ClassVar, List+from typing import Any, ClassVarfrom pipelex.core.pipes.pipe_abstract import PipeAbstractTypefrom pipelex.core.pipes.pipe_factory import PipeFactoryProtocol@@ -38,9 +38,9 @@ from pipelex.tools.registry_models import ModelType, RegistryModelsclass PipelexRegistryModels(RegistryModels):- FIELD_EXTRACTION: ClassVar[List[ModelType]] = []+ FIELD_EXTRACTION: ClassVar[list[ModelType]] = []- PIPE_OPERATORS: ClassVar[List[PipeAbstractType]] = [+ PIPE_OPERATORS: ClassVar[list[PipeAbstractType]] = [PipeFunc,PipeImgGen,PipeJinja2,@@ -48,7 +48,7 @@ class PipelexRegistryModels(RegistryModels):PipeOcr,]- PIPE_OPERATORS_FACTORY: ClassVar[List[PipeFactoryProtocol[Any, Any]]] = [+ PIPE_OPERATORS_FACTORY: ClassVar[list[PipeFactoryProtocol[Any, Any]]] = [PipeFuncFactory,PipeImgGenFactory,PipeJinja2Factory,@@ -56,21 +56,21 @@ class PipelexRegistryModels(RegistryModels):PipeOcrFactory,]- PIPE_CONTROLLERS: ClassVar[List[PipeAbstractType]] = [+ PIPE_CONTROLLERS: ClassVar[list[PipeAbstractType]] = [PipeBatch,PipeCondition,PipeParallel,PipeSequence,]- PIPE_CONTROLLERS_FACTORY: ClassVar[List[PipeFactoryProtocol[Any, Any]]] = [+ PIPE_CONTROLLERS_FACTORY: ClassVar[list[PipeFactoryProtocol[Any, Any]]] = [PipeBatchFactory,PipeConditionFactory,PipeParallelFactory,PipeSequenceFactory,]- STUFF: ClassVar[List[ModelType]] = [+ STUFF: ClassVar[list[ModelType]] = [TextContent,NumberContent,ImageContent,@@ -84,6 +84,6 @@ class PipelexRegistryModels(RegistryModels):PageContent,]- EXPERIMENTAL: ClassVar[List[ModelType]] = [+ EXPERIMENTAL: ClassVar[list[ModelType]] = [DynamicContent,]diff --git a/pipelex/core/stuffs/__init__.py b/pipelex/core/stuffs/__init__.pynew file mode 100644index 000000000..e69de29bbdiff --git a/pipelex/core/stuffs/stuff.py b/pipelex/core/stuffs/stuff.pyindex 354ed7235..29abdce1c 100644--- a/pipelex/core/stuffs/stuff.py+++ b/pipelex/core/stuffs/stuff.py@@ -1,4 +1,4 @@-from typing import Any, Dict, Optional, Type, Union+from typing import Any, castfrom pydantic import ConfigDict, ValidationErrorfrom typing_extensions import override@@ -27,23 +27,22 @@ class Stuff(CustomBaseModel):model_config = ConfigDict(extra="ignore", strict=True)stuff_code: str- stuff_name: Optional[str] = None+ stuff_name: str | None = Noneconcept: Conceptcontent: StuffContentdef make_artefact(self) -> StuffArtefact:- artefact_dict: Dict[str, Any] = self.content.model_dump(serialize_as_any=True)+ artefact_dict: dict[str, Any] = self.content.model_dump(serialize_as_any=True)- def set_artefact_field(key: str, value: Optional[Union[str, StuffContent]]):+ def set_artefact_field(key: str, value: str | StuffContent | None):if value is None:returnif key in artefact_dict:stuff_name = self.stuff_name or f"unnamed using concept code {self.concept.code}"- raise StuffError(- f"""Cannot create stuff artefact for stuff '{stuff_name}' of concept '{self.concept.code}' because reserved field '{key}'+ msg = f"""Cannot create stuff artefact for stuff '{stuff_name}' of concept '{self.concept.code}' because reserved field '{key}'in the structured output '{self.content.__class__.__name__}' already exists in the stuff content.Forbidden fields are: 'stuff_name', 'content_class', 'concept_code', 'stuff_code', 'content'."""- )+ raise StuffError(msg)artefact_dict[key] = valueset_artefact_field("stuff_name", self.stuff_name)@@ -101,12 +100,12 @@ Forbidden fields are: 'stuff_name', 'content_class', 'concept_code', 'stuff_codedef is_number(self) -> bool:return isinstance(self.content, NumberContent)- def content_as(self, content_type: Type[StuffContentType]) -> StuffContentType:+ def content_as(self, content_type: type[StuffContentType]) -> StuffContentType:"""Get content with proper typing if it's of the expected type."""return self.verify_content_type(self.content, content_type)@classmethod- def verify_content_type(cls, content: StuffContent, content_type: Type[StuffContentType]) -> StuffContentType:+ def verify_content_type(cls, content: StuffContent, content_type: type[StuffContentType]) -> StuffContentType:"""Verify and convert content to the expected type."""# First try the direct isinstance check for performanceif isinstance(content, content_type):@@ -123,18 +122,20 @@ Forbidden fields are: 'stuff_name', 'content_class', 'concept_code', 'stuff_codeexcept ValidationError as exc:formatted_error = format_pydantic_validation_error(exc)raise StuffContentValidationError(- original_type=type(content).__name__, target_type=content_type.__name__, validation_error=formatted_error+ original_type=type(content).__name__,+ target_type=content_type.__name__,+ validation_error=formatted_error,) from exc- raise TypeError(f"Content is of type '{type(content)}', instead of the expected '{content_type}'")+ msg = f"Content is of type '{type(content)}', instead of the expected '{content_type}'"+ raise TypeError(msg)def as_list_content(self) -> ListContent: # pyright: ignore[reportMissingTypeArgument, reportUnknownParameterType]"""Get content as ListContent with items of any type."""return self.content_as(content_type=ListContent) # pyright: ignore[reportUnknownVariableType]- def as_list_of_fixed_content_type(self, item_type: Type[StuffContentType]) -> ListContent[StuffContentType]:- """- Get content as ListContent with items of type T.+ def as_list_of_fixed_content_type(self, item_type: type[StuffContentType]) -> ListContent[StuffContentType]:+ """Get content as ListContent with items of type T.Args:item_type: The expected type of items in the list.@@ -144,8 +145,9 @@ Forbidden fields are: 'stuff_name', 'content_class', 'concept_code', 'stuff_codeRaises:TypeError: If content is not ListContent or items don't match expected type+"""- list_content: ListContent[StuffContentType] = self.content_as(content_type=ListContent)+ list_content = cast("ListContent[StuffContentType]", self.content_as(content_type=ListContent))# Validate all items are of the expected typefor item in list_content.items:diff --git a/pipelex/core/stuffs/stuff_artefact.py b/pipelex/core/stuffs/stuff_artefact.pyindex e1cc2e771..cf494b212 100644--- a/pipelex/core/stuffs/stuff_artefact.py+++ b/pipelex/core/stuffs/stuff_artefact.py@@ -1,4 +1,4 @@-from typing import Any, Dict, Optional, Tuple+from typing import Anyfrom jinja2.runtime import Contextfrom pydantic import RootModel@@ -10,7 +10,7 @@ from pipelex.tools.templating.jinja2_models import Jinja2ContextKey, Jinja2Taggafrom pipelex.tools.templating.templating_models import TextFormat-class StuffArtefact(RootModel[Dict[str, Any]], Jinja2TaggableAbstract):+class StuffArtefact(RootModel[dict[str, Any]], Jinja2TaggableAbstract):"""A flattened representation of Stuff and its content as a dictionary.This RootModel implementation allows for subscript access to the underlying dictionary@@ -45,12 +45,12 @@ class StuffArtefact(RootModel[Dict[str, Any]], Jinja2TaggableAbstract):def rendered_str(self, text_format: TextFormat) -> str:content = self.root["content"]if not isinstance(content, StuffContent):- raise StuffArtefactError(f"StuffArtefact has no StuffContent, content: {self}")- rendered_str = content.rendered_str(text_format=text_format)- return rendered_str+ msg = f"StuffArtefact has no StuffContent, content: {self}"+ raise StuffArtefactError(msg)+ return content.rendered_str(text_format=text_format)@override- def render_tagged_for_jinja2(self, context: Context, tag_name: Optional[str] = None) -> Tuple[Any, Optional[str]]:+ def render_tagged_for_jinja2(self, context: Context, tag_name: str | None = None) -> tuple[Any, str | None]:# TODO: factorize the text formatting with the jinja2 "text_format" filtertext_format = context.get(Jinja2ContextKey.TEXT_FORMAT, default=TextFormat.PLAIN)rendered_str = self.rendered_str(text_format=text_format)diff --git a/pipelex/core/stuffs/stuff_content.py b/pipelex/core/stuffs/stuff_content.pyindex 43f54ada9..0da0eea6a 100644--- a/pipelex/core/stuffs/stuff_content.py+++ b/pipelex/core/stuffs/stuff_content.py@@ -2,14 +2,14 @@ import base64import jsonfrom abc import ABC, abstractmethodfrom io import BytesIO-from typing import Any, Dict, Generic, List, Optional, Type, TypeVar, Union+from typing import Any, Generic, TypeVarimport markdownfrom json2html import json2htmlfrom kajson import kajsonfrom PIL import Imagefrom pydantic import BaseModel-from typing_extensions import Self, override+from typing_extensions import overridefrom yattag import Docfrom pipelex.cogt.ocr.ocr_output import ExtractedImage@@ -20,6 +20,7 @@ from pipelex.tools.misc.markdown_utils import convert_to_markdownfrom pipelex.tools.misc.path_utils import InterpretedPathOrUrl, interpret_path_or_urlfrom pipelex.tools.templating.templating_models import TextFormatfrom pipelex.tools.typing.pydantic_utils import CustomBaseModel, clean_model_to_dict+from pipelex.types import SelfObjectContentType = TypeVar("ObjectContentType", bound=BaseModel)StuffContentType = TypeVar("StuffContentType", bound="StuffContent")@@ -32,7 +33,7 @@ class StuffContent(ABC, CustomBaseModel):def short_desc(self) -> str:return f"some {self.__class__.__name__}"- def smart_dump(self) -> Union[str, Dict[str, Any], List[str], List[Dict[str, Any]]]:+ def smart_dump(self) -> str | dict[str, Any] | list[str] | list[dict[str, Any]]:return self.model_dump(serialize_as_any=True)@override@@ -81,7 +82,7 @@ class TextContent(StuffContentInitableFromStr):text: str@override- def smart_dump(self) -> Union[str, Dict[str, Any], List[str], List[Dict[str, Any]]]:+ def smart_dump(self) -> str | dict[str, Any] | list[str] | list[dict[str, Any]]:return self.text@property@@ -105,8 +106,7 @@ class TextContent(StuffContentInitableFromStr):@overridedef rendered_html(self) -> str:# Convert a markdown string to HTML and return HTML as a Unicode string.- html = markdown.markdown(self.text)- return html+ return markdown.markdown(self.text)@overridedef rendered_markdown(self, level: int = 1, is_pretty: bool = False) -> str:@@ -138,10 +138,10 @@ class DynamicContent(StuffContent):class NumberContent(StuffContentInitableFromStr):- number: Union[int, float]+ number: int | float@override- def smart_dump(self) -> Union[str, Dict[str, Any], List[str], List[Dict[str, Any]]]:+ def smart_dump(self) -> str | dict[str, Any] | list[str] | list[dict[str, Any]]:return str(self.number)@property@@ -182,9 +182,9 @@ class NumberContent(StuffContentInitableFromStr):class ImageContent(StuffContentInitableFromStr):url: str- source_prompt: Optional[str] = None- caption: Optional[str] = None- base_64: Optional[str] = None+ source_prompt: str | None = None+ caption: str | None = None+ base_64: str | None = None@property@override@@ -234,27 +234,26 @@ class ImageContent(StuffContentInitableFromStr):base_64=base_64,)- def save_to_directory(self, directory: str, base_name: Optional[str] = None, extension: Optional[str] = None):+ def save_to_directory(self, directory: str, base_name: str | None = None, extension: str | None = None):ensure_directory_exists(directory)base_name = base_name or "img"- if base_64 := self.base_64:- if not extension:- match interpret_path_or_url(path_or_uri=self.url):- case InterpretedPathOrUrl.FILE_NAME:- parts = self.url.rsplit(".", 1)- base_name = parts[0]- extension = parts[1]- case _:- file_type = detect_file_type_from_base64(b64=base_64)- base_name = base_name or "img"- extension = file_type.extension- file_path = get_incremental_file_path(- base_path=directory,- base_name=base_name,- extension=extension,- avoid_suffix_if_possible=True,- )- save_base64_to_binary_file(b64=base_64, file_path=file_path)+ if (base_64 := self.base_64) and not extension:+ match interpret_path_or_url(path_or_uri=self.url):+ case InterpretedPathOrUrl.FILE_NAME:+ parts = self.url.rsplit(".", 1)+ base_name = parts[0]+ extension = parts[1]+ case InterpretedPathOrUrl.FILE_PATH | InterpretedPathOrUrl.FILE_URI | InterpretedPathOrUrl.URL | InterpretedPathOrUrl.BASE_64:+ file_type = detect_file_type_from_base64(b64=base_64)+ base_name = base_name or "img"+ extension = file_type.extension+ file_path = get_incremental_file_path(+ base_path=directory,+ base_name=base_name,+ extension=extension,+ avoid_suffix_if_possible=True,+ )+ save_base64_to_binary_file(b64=base_64, file_path=file_path)if caption := self.caption:caption_file_path = get_incremental_file_path(@@ -399,13 +398,13 @@ class StructuredContent(StuffContent):class ListContent(StuffContent, Generic[StuffContentType]):- items: List[StuffContentType]+ items: list[StuffContentType]@propertydef nb_items(self) -> int:return len(self.items)- def get_items(self, item_type: Type[StuffContent]) -> List[StuffContent]:+ def get_items(self, item_type: type[StuffContent]) -> list[StuffContent]:return [item for item in self.items if isinstance(item, item_type)]@property@@ -414,22 +413,21 @@ class ListContent(StuffContent, Generic[StuffContentType]):nb_items = len(self.items)if nb_items == 0:return "empty list"- elif nb_items == 1:+ if nb_items == 1:return f"list of 1 {self.items[0].__class__.__name__}"+ item_classes: list[str] = [item.__class__.__name__ for item in self.items]+ item_classes_set = set(item_classes)+ nb_classes = len(item_classes_set)+ if nb_classes == 1:+ return f"list of {len(self.items)} {item_classes[0]}s"+ elif nb_items == nb_classes:+ return f"list of {len(self.items)} items of different types"else:- item_classes: List[str] = [item.__class__.__name__ for item in self.items]- item_classes_set = set(item_classes)- nb_classes = len(item_classes_set)- if nb_classes == 1:- return f"list of {len(self.items)} {item_classes[0]}s"- elif nb_items == nb_classes:- return f"list of {len(self.items)} items of different types"- else:- return f"list of {len(self.items)} items of {nb_classes} different types"+ return f"list of {len(self.items)} items of {nb_classes} different types"@property- def _single_class_name(self) -> Optional[str]:- item_classes: List[str] = [item.__class__.__name__ for item in self.items]+ def _single_class_name(self) -> str | None:+ item_classes: list[str] = [item.__class__.__name__ for item in self.items]item_classes_set = set(item_classes)nb_classes = len(item_classes_set)if nb_classes == 1:@@ -473,8 +471,8 @@ class ListContent(StuffContent, Generic[StuffContentType]):class TextAndImagesContent(StuffContent):- text: Optional[TextContent]- images: Optional[List[ImageContent]]+ text: TextContent | None+ images: list[ImageContent] | None@property@override@@ -510,7 +508,7 @@ class TextAndImagesContent(StuffContent):class PageContent(StructuredContent):text_and_images: TextAndImagesContent- page_view: Optional[ImageContent] = None+ page_view: ImageContent | None = Nonedef save_to_directory(self, directory: str):ensure_directory_exists(directory)diff --git a/pipelex/core/stuffs/stuff_factory.py b/pipelex/core/stuffs/stuff_factory.pyindex b2d202367..18a33be98 100644--- a/pipelex/core/stuffs/stuff_factory.py+++ b/pipelex/core/stuffs/stuff_factory.py@@ -1,4 +1,4 @@-from typing import Any, Dict, List, Optional, Type, cast+from typing import Any, castimport shortuuidfrom pydantic import BaseModel, ValidationError, field_validator@@ -26,10 +26,11 @@ class StuffFactoryError(PipelexError):class StuffBlueprint(BaseModel):stuff_name: strconcept_string: str- content: Dict[str, Any] | str+ content: dict[str, Any] | str@field_validator("concept_string")- def validate_concept_string(cls, concept_string: str) -> str:+ @staticmethod+ def validate_concept_string(concept_string: str) -> str:ConceptBlueprint.validate_concept_string(concept_string)return concept_string@@ -62,8 +63,8 @@ class StuffFactory:cls,concept: Concept,content: StuffContent,- name: Optional[str] = None,- code: Optional[str] = None,+ name: str | None = None,+ code: str | None = None,) -> Stuff:if not name:name = cls.make_stuff_name(concept=concept)@@ -78,10 +79,10 @@ class StuffFactory:def make_stuff_using_concept_name_and_search_domains(cls,concept_name: str,- search_domains: List[str],+ search_domains: list[str],content: StuffContent,- name: Optional[str] = None,- code: Optional[str] = None,+ name: str | None = None,+ code: str | None = None,) -> Stuff:# TODO: Add unit tests for this methodconcept_provider = get_concept_provider()@@ -90,7 +91,8 @@ class StuffFactory:search_domains=search_domains,)if not concept:- raise StuffFactoryError(f"Could not find a concept named '{concept_name}' in domains {search_domains}")+ msg = f"Could not find a concept named '{concept_name}' in domains {search_domains}"+ raise StuffFactoryError(msg)return cls.make_stuff(concept=concept, content=content, name=name, code=code)@classmethod@@ -121,20 +123,17 @@ class StuffFactory:def combine_stuffs(cls,concept: Concept,- stuff_contents: Dict[str, StuffContent],- name: Optional[str] = None,+ stuff_contents: dict[str, StuffContent],+ name: str | None = None,) -> Stuff:# TODO: Add unit tests for this method- """- Combine a dictionary of stuffs into a single stuff.- """+ """Combine a dictionary of stuffs into a single stuff."""the_subclass = get_class_registry().get_required_subclass(name=concept.structure_class_name, base_class=StuffContent)try:the_stuff_content = the_subclass.model_validate(obj=stuff_contents)except ValidationError as exc:- raise StuffFactoryError(- f"Error combining stuffs for concept {concept.code}, stuff named `{name}`: {format_pydantic_validation_error(exc=exc)}"- ) from exc+ msg = f"Error combining stuffs for concept {concept.code}, stuff named `{name}`: {format_pydantic_validation_error(exc=exc)}"+ raise StuffFactoryError(msg) from excreturn cls.make_stuff(concept=concept,content=the_stuff_content,@@ -146,16 +145,17 @@ class StuffFactory:cls,name: str,stuff_content_or_data: StuffContentOrData,- search_domains: List[str],- stuff_code: Optional[str] = None,+ search_domains: list[str],+ stuff_code: str | None = None,) -> Stuff:# TODO: Add unit tests for this methodcontent: StuffContentconcept_name: strif isinstance(stuff_content_or_data, ListContent):- content = cast(ListContent[Any], stuff_content_or_data)+ content = cast("ListContent[Any]", stuff_content_or_data)if len(content.items) == 0:- raise StuffFactoryError("ListContent in compact memory has no items")+ msg = "ListContent in compact memory has no items"+ raise StuffFactoryError(msg)concept_name = type(content.items[0]).__name__try:return cls.make_stuff_using_concept_name_and_search_domains(@@ -166,7 +166,8 @@ class StuffFactory:code=stuff_code,)except StuffFactoryError as exc:- raise StuffFactoryError(f"Could not make stuff for ListContent '{name}': {exc}") from exc+ msg = f"Could not make stuff for ListContent '{name}': {exc}"+ raise StuffFactoryError(msg) from excelif isinstance(stuff_content_or_data, StuffContent):content = stuff_content_or_dataconcept_class_name = type(content).__name__@@ -191,11 +192,13 @@ class StuffFactory:code=stuff_code,)except StuffFactoryError as exc:- raise StuffFactoryError(f"Could not make stuff for StuffContent '{name}': {exc}") from exc+ msg = f"Could not make stuff for StuffContent '{name}': {exc}"+ raise StuffFactoryError(msg) from excelif isinstance(stuff_content_or_data, list):items = stuff_content_or_dataif len(items) == 0:- raise StuffFactoryError("List in compact memory has no items")+ msg = "List in compact memory has no items"+ raise StuffFactoryError(msg)first_item = items[0]concept_name = type(first_item).__name__content = ListContent[Any](items=items)@@ -208,7 +211,8 @@ class StuffFactory:code=stuff_code,)except StuffFactoryError as exc:- raise StuffFactoryError(f"Could not make stuff for list of StuffContent '{name}': {exc}") from exc+ msg = f"Could not make stuff for list of StuffContent '{name}': {exc}"+ raise StuffFactoryError(msg) from excelif isinstance(stuff_content_or_data, str):str_stuff: str = stuff_content_or_datareturn StuffFactory.make_stuff(@@ -217,11 +221,12 @@ class StuffFactory:name=name,)else:- stuff_content_dict: Dict[str, Any] = stuff_content_or_data+ stuff_content_dict: dict[str, Any] = stuff_content_or_datatry:concept_code = stuff_content_dict.get("concept") or stuff_content_dict.get("concept_code")if not concept_code:- raise StuffFactoryError("Stuff content data dict is badly formed: no concept code")+ msg = "Stuff content data dict is badly formed: no concept code"+ raise StuffFactoryError(msg)content_value = stuff_content_dict["content"]if NativeConceptManager.is_native_concept(concept_string_or_code=concept_code):concept = ConceptFactory.make_native_concept(native_concept_data=NATIVE_CONCEPTS_DATA[NativeConceptEnum(concept_code)])@@ -236,7 +241,8 @@ class StuffFactory:code=stuff_code,)except KeyError as exc:- raise StuffFactoryError(f"Stuff content data dict is badly formed: {exc}") from exc+ msg = f"Stuff content data dict is badly formed: {exc}"+ raise StuffFactoryError(msg) from excif isinstance(content_value, StuffContent):return StuffFactory.make_stuff(@@ -245,17 +251,16 @@ class StuffFactory:content=content_value,code=stuff_code,)- else:- content = StuffContentFactory.make_stuff_content_from_concept_with_fallback(- concept=get_concept_provider().get_required_concept(concept_string=concept_code),- value=content_value,- )- return StuffFactory.make_stuff(- concept=get_concept_provider().get_required_concept(concept_string=concept_code),- name=name,- content=content,- code=stuff_code,- )+ content = StuffContentFactory.make_stuff_content_from_concept_with_fallback(+ concept=get_concept_provider().get_required_concept(concept_string=concept_code),+ value=content_value,+ )+ return StuffFactory.make_stuff(+ concept=get_concept_provider().get_required_concept(concept_string=concept_code),+ name=name,+ content=content,+ code=stuff_code,+ )class StuffContentFactoryError(PipelexError):@@ -264,15 +269,14 @@ class StuffContentFactoryError(PipelexError):class StuffContentFactory:@classmethod- def make_content_from_value(cls, stuff_content_subclass: Type[StuffContent], value: Dict[str, Any] | str) -> StuffContent:+ def make_content_from_value(cls, stuff_content_subclass: type[StuffContent], value: dict[str, Any] | str) -> StuffContent:if isinstance(value, str) and stuff_content_subclass == TextContent:return TextContent(text=value)return stuff_content_subclass.model_validate(obj=value)@classmethod- def make_stuff_content_from_concept_required(cls, concept: Concept, value: Dict[str, Any] | str) -> StuffContent:- """- Create StuffContent from concept code, requiring the concept to be linked to a class in the registry.+ def make_stuff_content_from_concept_required(cls, concept: Concept, value: dict[str, Any] | str) -> StuffContent:+ """Create StuffContent from concept code, requiring the concept to be linked to a class in the registry.Raises StuffContentFactoryError if no registry class is found."""the_subclass_name = concept.structure_class_name@@ -280,16 +284,15 @@ class StuffContentFactory:return cls.make_content_from_value(stuff_content_subclass=the_subclass, value=value)@classmethod- def make_stuff_content_from_concept_with_fallback(cls, concept: Concept, value: Dict[str, Any] | str) -> StuffContent:- """- Create StuffContent from concept code, falling back to TextContent if no registry class is found.- """+ def make_stuff_content_from_concept_with_fallback(cls, concept: Concept, value: dict[str, Any] | str) -> StuffContent:+ """Create StuffContent from concept code, falling back to TextContent if no registry class is found."""the_structure_class = get_class_registry().get_class(name=concept.structure_class_name)if the_structure_class is None:return cls.make_content_from_value(stuff_content_subclass=TextContent, value=value)if not issubclass(the_structure_class, StuffContent):- raise StuffContentFactoryError(f"Concept '{concept.code}', subclass '{the_structure_class}' is not a subclass of StuffContent")+ msg = f"Concept '{concept.code}', subclass '{the_structure_class}' is not a subclass of StuffContent"+ raise StuffContentFactoryError(msg)return cls.make_content_from_value(stuff_content_subclass=the_structure_class, value=value)diff --git a/pipelex/exceptions.py b/pipelex/exceptions.pyindex 320aa600f..1b83b4d3e 100644--- a/pipelex/exceptions.py+++ b/pipelex/exceptions.py@@ -1,5 +1,3 @@-from typing import List, Optional-from click import ClickExceptionfrom typing_extensions import override@@ -24,12 +22,12 @@ class StaticValidationError(Exception):self,error_type: StaticValidationErrorType,domain: str,- pipe_code: Optional[str] = None,- variable_names: Optional[List[str]] = None,- required_concept_codes: Optional[List[str]] = None,- provided_concept_code: Optional[str] = None,- file_path: Optional[str] = None,- explanation: Optional[str] = None,+ pipe_code: str | None = None,+ variable_names: list[str] | None = None,+ required_concept_codes: list[str] | None = None,+ provided_concept_code: str | None = None,+ file_path: str | None = None,+ explanation: str | None = None,):self.error_type = error_typeself.domain = domain@@ -93,8 +91,6 @@ class WorkingMemoryStuffNotFoundError(WorkingMemoryVariableError):class PipelexCLIError(PipelexError, ClickException):"""Raised when there's an error in CLI usage or operation."""- pass-class PipelexConfigError(PipelexError):pass@@ -185,7 +181,7 @@ class PipeStackOverflowError(PipeExecutionError):class DryRunError(PipeExecutionError):"""Raised when a dry run fails due to missing inputs or other validation issues."""- def __init__(self, message: str, missing_inputs: Optional[List[str]] = None, pipe_code: Optional[str] = None):+ def __init__(self, message: str, missing_inputs: list[str] | None = None, pipe_code: str | None = None):self.missing_inputs = missing_inputs or []self.pipe_code = pipe_codesuper().__init__(message)@@ -206,32 +202,22 @@ class PipeRunParamsError(PipelexError):class PipeBatchError(PipelexError):"""Base class for all PipeBatch-related errors."""- pass-class PipeBatchRecursionError(PipeBatchError):"""Raised when a PipeBatch attempts to run itself recursively."""- pass-class PipeBatchInputError(PipeBatchError):"""Raised when the input to a PipeBatch is not a ListContent or is invalid."""- pass-class PipeBatchOutputError(PipeBatchError):"""Raised when there's an error with the output structure of a PipeBatch operation."""- pass-class PipeBatchBranchError(PipeBatchError):"""Raised when there's an error with a branch pipe execution in PipeBatch."""- pass-class JobHistoryError(PipelexError):pass@@ -272,10 +258,8 @@ class PipeInputDetailsError(PipelexError):class ApiSerializationError(Exception):"""Exception raised when API serialization fails."""- pass--class StartPipelineException(Exception):+class StartPipelineError(Exception):passdiff --git a/pipelex/hub.py b/pipelex/hub.pyindex d6348390d..8f1364d1e 100644--- a/pipelex/hub.py+++ b/pipelex/hub.py@@ -1,4 +1,4 @@-from typing import ClassVar, Optional, Type+from typing import ClassVar, Optionalfrom kajson.class_registry_abstract import ClassRegistryAbstract@@ -35,8 +35,7 @@ from pipelex.tools.templating.template_provider_abstract import TemplateProviderclass PipelexHub:- """- PipelexHub serves as a central dependency manager to break cyclic imports between components.+ """PipelexHub serves as a central dependency manager to break cyclic imports between components.It provides access to core providers and factories through a singleton instance,allowing components to retrieve dependencies based on protocols without direct imports that could create cycles."""@@ -45,30 +44,30 @@ class PipelexHub:def __init__(self):# tools- self._config: Optional[ConfigRoot] = None- self._secrets_provider: Optional[SecretsProviderAbstract] = None- self._template_provider: Optional[TemplateProviderAbstract] = None- self._class_registry: Optional[ClassRegistryAbstract] = None- self._storage_provider: Optional[StorageProviderAbstract] = None+ self._config: ConfigRoot | None = None+ self._secrets_provider: SecretsProviderAbstract | None = None+ self._template_provider: TemplateProviderAbstract | None = None+ self._class_registry: ClassRegistryAbstract | None = None+ self._storage_provider: StorageProviderAbstract | None = None# cogt- self._models_manager: Optional[ModelManagerAbstract] = None- self._plugin_manager: Optional[PluginManager] = None+ self._models_manager: ModelManagerAbstract | None = None+ self._plugin_manager: PluginManager | None = Noneself._inference_manager: InferenceManagerProtocolself._report_delegate: ReportingProtocol- self._content_generator: Optional[ContentGeneratorProtocol] = None+ self._content_generator: ContentGeneratorProtocol | None = None# pipelex- self._domain_provider: Optional[DomainProviderAbstract] = None- self._concept_provider: Optional[ConceptProviderAbstract] = None- self._pipe_provider: Optional[PipeProviderAbstract] = None- self._pipe_router: Optional[PipeRouterProtocol] = None- self._library_manager: Optional[LibraryManagerAbstract] = None+ self._domain_provider: DomainProviderAbstract | None = None+ self._concept_provider: ConceptProviderAbstract | None = None+ self._pipe_provider: PipeProviderAbstract | None = None+ self._pipe_router: PipeRouterProtocol | None = None+ self._library_manager: LibraryManagerAbstract | None = None# pipeline- self._pipeline_tracker: Optional[PipelineTrackerProtocol] = None- self._pipeline_manager: Optional[PipelineManagerAbstract] = None- self._activity_manager: Optional[ActivityManagerProtocol] = None- self._observer_provider: Optional[ObserverProtocol] = None+ self._pipeline_tracker: PipelineTrackerProtocol | None = None+ self._pipeline_manager: PipelineManagerAbstract | None = None+ self._activity_manager: ActivityManagerProtocol | None = None+ self._observer_provider: ObserverProtocol | None = None############################################################# Class methods for singleton management@@ -77,7 +76,8 @@ class PipelexHub:@classmethoddef get_instance(cls) -> "PipelexHub":if cls._instance is None:- raise RuntimeError("PipelexHub is not initialized")+ msg = "PipelexHub is not initialized"+ raise RuntimeError(msg)return cls._instance@classmethod@@ -90,9 +90,8 @@ class PipelexHub:# tools- def setup_config(self, config_cls: Type[ConfigRoot], specific_config_path: Optional[str] = None):- """- Set the global configuration instance.+ def setup_config(self, config_cls: type[ConfigRoot], specific_config_path: str | None = None):+ """Set the global configuration instance.# Args:# config (Config): The configuration instance to set.@@ -108,9 +107,7 @@ class PipelexHub:self._config = configdef reset_config(self) -> None:- """- Reset the global configuration instance and the config manager.- """+ """Reset the global configuration instance and the config manager."""self._config = Nonelog.reset()@@ -179,8 +176,7 @@ class PipelexHub:# toolsdef get_required_config(self) -> ConfigRoot:- """- Get the current configuration instance as an instance of a particular subclass of ConfigRoot. This should be used only from pipelex.tools.+ """Get the current configuration instance as an instance of a particular subclass of ConfigRoot. This should be used only from pipelex.tools.when getting the config from other projects, use their own project.get_config() method to get the Configwith the proper subclass which is required for proper type checking.@@ -189,41 +185,49 @@ class PipelexHub:Raises:RuntimeError: If the configuration has not been set.+"""if self._config is None:- raise RuntimeError("Config instance is not set. You must initialize Pipelex first.")+ msg = "Config instance is not set. You must initialize Pipelex first."+ raise RuntimeError(msg)return self._configdef get_required_secrets_provider(self) -> SecretsProviderAbstract:if self._secrets_provider is None:- raise RuntimeError("Secrets provider is not set. You must initialize Pipelex first.")+ msg = "Secrets provider is not set. You must initialize Pipelex first."+ raise RuntimeError(msg)return self._secrets_providerdef get_required_template_provider(self) -> TemplateProviderAbstract:if self._template_provider is None:- raise RuntimeError("Template provider is not set. You must initialize Pipelex first.")+ msg = "Template provider is not set. You must initialize Pipelex first."+ raise RuntimeError(msg)return self._template_providerdef get_required_class_registry(self) -> ClassRegistryAbstract:if self._class_registry is None:- raise RuntimeError("ClassRegistry is not initialized")+ msg = "ClassRegistry is not initialized"+ raise RuntimeError(msg)return self._class_registrydef get_storage_provider(self) -> StorageProviderAbstract:if self._storage_provider is None:- raise RuntimeError("StorageProvider is not initialized")+ msg = "StorageProvider is not initialized"+ raise RuntimeError(msg)return self._storage_provider# cogtdef get_required_models_manager(self) -> ModelManagerAbstract:if self._models_manager is None:- raise RuntimeError("ModelsManager is not initialized")+ msg = "ModelsManager is not initialized"+ raise RuntimeError(msg)return self._models_managerdef get_plugin_manager(self) -> PluginManager:if self._plugin_manager is None:- raise RuntimeError("PluginManager2 is not initialized")+ msg = "PluginManager2 is not initialized"+ raise RuntimeError(msg)return self._plugin_managerdef get_inference_manager(self) -> InferenceManagerProtocol:@@ -234,63 +238,73 @@ class PipelexHub:def get_required_content_generator(self) -> ContentGeneratorProtocol:if self._content_generator is None:- raise RuntimeError("ContentGenerator is not initialized")+ msg = "ContentGenerator is not initialized"+ raise RuntimeError(msg)return self._content_generator# pipelexdef get_required_domain_provider(self) -> DomainProviderAbstract:if self._domain_provider is None:- raise RuntimeError("DomainProvider is not initialized")+ msg = "DomainProvider is not initialized"+ raise RuntimeError(msg)return self._domain_provider- def get_optional_domain_provider(self) -> Optional[DomainProviderAbstract]:+ def get_optional_domain_provider(self) -> DomainProviderAbstract | None:return self._domain_providerdef get_required_concept_provider(self) -> ConceptProviderAbstract:if self._concept_provider is None:- raise RuntimeError("ConceptProvider is not initialized")+ msg = "ConceptProvider is not initialized"+ raise RuntimeError(msg)return self._concept_provider- def get_optional_concept_provider(self) -> Optional[ConceptProviderAbstract]:+ def get_optional_concept_provider(self) -> ConceptProviderAbstract | None:return self._concept_providerdef get_required_pipe_provider(self) -> PipeProviderAbstract:if self._pipe_provider is None:- raise RuntimeError("PipeProvider is not initialized")+ msg = "PipeProvider is not initialized"+ raise RuntimeError(msg)return self._pipe_providerdef get_required_pipe_router(self) -> PipeRouterProtocol:if self._pipe_router is None:- raise RuntimeError("PipeRouter is not initialized")+ msg = "PipeRouter is not initialized"+ raise RuntimeError(msg)return self._pipe_routerdef get_pipeline_tracker(self) -> PipelineTrackerProtocol:if self._pipeline_tracker is None:- raise RuntimeError("PipelineTracker is not initialized")+ msg = "PipelineTracker is not initialized"+ raise RuntimeError(msg)return self._pipeline_trackerdef get_required_pipeline_manager(self) -> PipelineManagerAbstract:if self._pipeline_manager is None:- raise RuntimeError("PipelineManager is not initialized")+ msg = "PipelineManager is not initialized"+ raise RuntimeError(msg)return self._pipeline_managerdef get_activity_manager(self) -> ActivityManagerProtocol:if self._activity_manager is None:- raise RuntimeError("Activity manager is not set. You must initialize Pipelex first.")+ msg = "Activity manager is not set. You must initialize Pipelex first."+ raise RuntimeError(msg)return self._activity_managerdef get_required_library_manager(self) -> LibraryManagerAbstract:if self._library_manager is None:- raise RuntimeError("Library manager is not set. You must initialize Pipelex first.")+ msg = "Library manager is not set. You must initialize Pipelex first."+ raise RuntimeError(msg)return self._library_manager- def get_optional_library_manager(self) -> Optional[LibraryManagerAbstract]:+ def get_optional_library_manager(self) -> LibraryManagerAbstract | None:return self._library_managerdef get_observer_provider(self) -> ObserverProtocol:if self._observer_provider is None:- raise RuntimeError("Observer is not set. You must initialize Pipelex first.")+ msg = "Observer is not set. You must initialize Pipelex first."+ raise RuntimeError(msg)return self._observer_provider@@ -390,11 +404,10 @@ def get_required_domain(domain: str) -> Domain:return get_pipelex_hub().get_required_domain_provider().get_required_domain(domain=domain)-def get_optional_domain(domain: str) -> Optional[Domain]:+def get_optional_domain(domain: str) -> Domain | None:if domain_provider := get_pipelex_hub().get_optional_domain_provider():return domain_provider.get_domain(domain=domain)- else:- return None+ return Nonedef get_pipe_provider() -> PipeProviderAbstract:@@ -405,7 +418,7 @@ def get_required_pipe(pipe_code: str) -> PipeAbstract:return get_pipelex_hub().get_required_pipe_provider().get_required_pipe(pipe_code=pipe_code)-def get_optional_pipe(pipe_code: str) -> Optional[PipeAbstract]:+def get_optional_pipe(pipe_code: str) -> PipeAbstract | None:return get_pipelex_hub().get_required_pipe_provider().get_optional_pipe(pipe_code=pipe_code)@@ -413,7 +426,7 @@ def get_concept_provider() -> ConceptProviderAbstract:return get_pipelex_hub().get_required_concept_provider()-def get_optional_concept_provider() -> Optional[ConceptProviderAbstract]:+def get_optional_concept_provider() -> ConceptProviderAbstract | None:return get_pipelex_hub().get_optional_concept_provider()diff --git a/pipelex/libraries/library_config.py b/pipelex/libraries/library_config.pyindex 3bc048d19..bfb5650b4 100644--- a/pipelex/libraries/library_config.py+++ b/pipelex/libraries/library_config.py@@ -1,4 +1,4 @@-from typing import ClassVar, List+from typing import ClassVarfrom pipelex.tools.config.config_model import ConfigModelfrom pipelex.tools.misc.file_utils import copy_file_from_package, copy_folder_from_package, find_files_in_dir@@ -27,10 +27,10 @@ class LibraryConfig(ConfigModel):return "tests/test_pipelines"@property- def failing_pipelines_file_paths(self) -> List[str]:+ def failing_pipelines_file_paths(self) -> list[str]:return ["tests/test_pipelines/failing_pipelines.plx"]- def get_templates_paths(self) -> List[str]:+ def get_templates_paths(self) -> list[str]:return [str(path) for path in find_files_in_dir(dir_path=self.templates_dir_path, pattern="*.toml", is_recursive=True)]def export_libraries(self, overwrite: bool = False) -> None:diff --git a/pipelex/libraries/library_manager.py b/pipelex/libraries/library_manager.pyindex 6a0ad2922..164fb76b5 100644--- a/pipelex/libraries/library_manager.py+++ b/pipelex/libraries/library_manager.py@@ -1,6 +1,6 @@import osfrom pathlib import Path-from typing import ClassVar, List, Optional, Type+from typing import ClassVarfrom pydantic import ValidationErrorfrom typing_extensions import override@@ -42,7 +42,7 @@ class LibraryComponent(StrEnum):PIPE = "pipe"@property- def error_class(self) -> Type[LibraryError]:+ def error_class(self) -> type[LibraryError]:match self:case LibraryComponent.CONCEPT:return ConceptLibraryError@@ -51,7 +51,7 @@ class LibraryComponent(StrEnum):class LibraryManager(LibraryManagerAbstract):- allowed_root_attributes: ClassVar[List[str]] = [+ allowed_root_attributes: ClassVar[list[str]] = ["domain","definition","system_prompt",@@ -90,8 +90,9 @@ class LibraryManager(LibraryManagerAbstract):try:validate_toml_file(template_path)except TOMLValidationError as exc:- log.error(f"PLX formatting issues in template file '{template_path}': {exc}")- raise LibraryError(f"PLX validation failed for template file '{template_path}': {exc}") from exc+ msg = f"PLX validation failed for template file '{template_path}': {exc}"+ log.error(msg)+ raise LibraryError(msg) from exc@overridedef setup(self) -> None:@@ -108,20 +109,20 @@ class LibraryManager(LibraryManagerAbstract):self.teardown()self.setup()- def _get_pipeline_library_dirs(self) -> List[Path]:+ def _get_pipeline_library_dirs(self) -> list[Path]:library_dirs = [Path(self.library_config.pipelines_dir_path)]if runtime_manager.is_unit_testing:log.debug("Registering test pipeline structures for unit testing")library_dirs += [Path(self.library_config.test_pipelines_dir_path)]return library_dirs- def _get_pipelex_plx_files_from_dirs(self, dirs: List[Path]) -> List[Path]:+ def _get_pipelex_plx_files_from_dirs(self, dirs: list[Path]) -> list[Path]:"""Get all valid Pipelex PLX files from the given directories."""-- all_plx_paths: List[Path] = []+ all_plx_paths: list[Path] = []for dir_path in dirs:if not dir_path.exists():- raise LibraryError(f"Directory does not exist: {dir_path}")+ msg = f"Directory does not exist: {dir_path}"+ raise LibraryError(msg)# Find all TOML files in the directoryplx_files = find_files_in_dir(@@ -143,13 +144,14 @@ class LibraryManager(LibraryManagerAbstract):def load_from_file(self, plx_path: Path) -> None:"""Load a single file - this method is kept for compatibility."""if not PipelexInterpreter.is_pipelex_file(plx_path):- raise LibraryError(f"File is not a valid Pipelex PLX file: {plx_path}")+ msg = f"File is not a valid Pipelex PLX file: {plx_path}"+ raise LibraryError(msg)blueprint = PipelexInterpreter(file_path=plx_path).make_pipelex_bundle_blueprint()self.load_from_blueprint(blueprint)@override- def load_from_blueprint(self, blueprint: PipelexBundleBlueprint) -> List[PipeAbstract]:+ def load_from_blueprint(self, blueprint: PipelexBundleBlueprint) -> list[PipeAbstract]:"""Load a blueprint."""# Create and load domaindomain = self._load_domain_from_blueprint(blueprint)@@ -174,7 +176,7 @@ class LibraryManager(LibraryManagerAbstract):if blueprint.concept is not None:concept_codes_to_remove = [ConceptFactory.construct_concept_string_with_domain(domain=blueprint.domain, concept_code=concept_code)- for concept_code in blueprint.concept.keys()+ for concept_code in blueprint.concept]self.concept_library.remove_concepts_by_codes(concept_codes=concept_codes_to_remove)@@ -188,11 +190,11 @@ class LibraryManager(LibraryManagerAbstract):system_prompt=blueprint.system_prompt,system_prompt_to_structure=blueprint.system_prompt_to_structure,prompt_template_to_structure=blueprint.prompt_template_to_structure,- )+ ),)- def _load_concepts_from_blueprint(self, blueprint: PipelexBundleBlueprint) -> List[Concept]:- concepts: List[Concept] = []+ def _load_concepts_from_blueprint(self, blueprint: PipelexBundleBlueprint) -> list[Concept]:+ concepts: list[Concept] = []if blueprint.concept is not None:for concept_code, concept_blueprint_or_str in blueprint.concept.items():@@ -207,8 +209,8 @@ class LibraryManager(LibraryManagerAbstract):concepts.append(concept)return concepts- def _load_pipes_from_blueprint(self, blueprint: PipelexBundleBlueprint) -> List[PipeAbstract]:- pipes: List[PipeAbstract] = []+ def _load_pipes_from_blueprint(self, blueprint: PipelexBundleBlueprint) -> list[PipeAbstract]:+ pipes: list[PipeAbstract] = []if blueprint.pipe is not None:for pipe_name, pipe_blueprint in blueprint.pipe.items():pipe = PipeFactory.make_from_blueprint(@@ -223,11 +225,11 @@ class LibraryManager(LibraryManagerAbstract):@overridedef load_libraries(self,- library_dirs: Optional[List[Path]] = None,- library_file_paths: Optional[List[Path]] = None,+ library_dirs: list[Path] | None = None,+ library_file_paths: list[Path] | None = None,) -> None:- dirs_to_use: List[Path] = self._get_pipeline_library_dirs()- all_plx_paths: List[Path] = self._get_pipelex_plx_files_from_dirs(dirs_to_use)+ dirs_to_use: list[Path] = self._get_pipeline_library_dirs()+ all_plx_paths: list[Path] = self._get_pipelex_plx_files_from_dirs(dirs_to_use)# Remove failing pipelines from the listfailing_pipelines_file_paths = get_config().pipelex.library_config.failing_pipelines_file_paths@@ -246,19 +248,21 @@ class LibraryManager(LibraryManagerAbstract):all_plx_paths = library_file_paths# Parse all blueprints first- blueprints: List[PipelexBundleBlueprint] = []+ blueprints: list[PipelexBundleBlueprint] = []for plx_file_path in all_plx_paths:try:blueprint = PipelexInterpreter(file_path=plx_file_path).make_pipelex_bundle_blueprint()except FileNotFoundError as exc:- raise PipelexSetupError(f"Could not find PLX blueprint at '{plx_file_path}'") from exc+ msg = f"Could not find PLX blueprint at '{plx_file_path}'"+ raise PipelexSetupError(msg) from excexcept ValidationError as exc:- error_msg = format_pydantic_validation_error(exc)- raise PipelexSetupError(f"Could not load PLX blueprint from '{plx_file_path}' because of: {error_msg}") from exc+ formatted_error_msg = format_pydantic_validation_error(exc)+ msg = f"Could not load PLX blueprint from '{plx_file_path}' because of: {formatted_error_msg}"+ raise PipelexSetupError(msg) from excblueprints.append(blueprint)# Load all domains first- all_domains: List[Domain] = []+ all_domains: list[Domain] = []for blueprint in blueprints:domain = self._load_domain_from_blueprint(blueprint)all_domains.append(domain)@@ -266,14 +270,14 @@ class LibraryManager(LibraryManagerAbstract):self.domain_library.add_domain(domain=domain)# Load all concepts second- all_concepts: List[Concept] = []+ all_concepts: list[Concept] = []for blueprint in blueprints:concepts = self._load_concepts_from_blueprint(blueprint)all_concepts.extend(concepts)self.concept_library.add_concepts(concepts=all_concepts)# Load all pipes third- all_pipes: List[PipeAbstract] = []+ all_pipes: list[PipeAbstract] = []for blueprint in blueprints:pipes = self._load_pipes_from_blueprint(blueprint)all_pipes.extend(pipes)diff --git a/pipelex/libraries/library_manager_abstract.py b/pipelex/libraries/library_manager_abstract.pyindex 37e77c21b..3f93d87ed 100644--- a/pipelex/libraries/library_manager_abstract.py+++ b/pipelex/libraries/library_manager_abstract.py@@ -1,6 +1,5 @@from abc import ABC, abstractmethodfrom pathlib import Path-from typing import List, Optionalfrom pipelex.core.bundles.pipelex_bundle_blueprint import PipelexBundleBlueprintfrom pipelex.core.pipes.pipe_abstract import PipeAbstract@@ -24,7 +23,7 @@ class LibraryManagerAbstract(ABC):pass@abstractmethod- def load_libraries(self, library_dirs: Optional[List[Path]] = None, library_file_paths: Optional[List[Path]] = None) -> None:+ def load_libraries(self, library_dirs: list[Path] | None = None, library_file_paths: list[Path] | None = None) -> None:pass@abstractmethod@@ -32,7 +31,7 @@ class LibraryManagerAbstract(ABC):pass@abstractmethod- def load_from_blueprint(self, blueprint: PipelexBundleBlueprint) -> List[PipeAbstract]:+ def load_from_blueprint(self, blueprint: PipelexBundleBlueprint) -> list[PipeAbstract]:pass@abstractmethoddiff --git a/pipelex/libraries/library_manager_factory.py b/pipelex/libraries/library_manager_factory.pyindex 686b31f29..c63ff4412 100644--- a/pipelex/libraries/library_manager_factory.py+++ b/pipelex/libraries/library_manager_factory.py@@ -24,10 +24,13 @@ class LibraryManagerFactory:@classmethoddef make(- cls, domain_library: DomainLibrary, concept_library: ConceptLibrary, pipe_library: PipeLibrary, config_dir_path: str+ cls,+ domain_library: DomainLibrary,+ concept_library: ConceptLibrary,+ pipe_library: PipeLibrary,+ config_dir_path: str,) -> "LibraryManager":"""Create a LibraryManager with provided libraries."""-library_config = LibraryConfig(config_dir_path=config_dir_path)return LibraryManager(diff --git a/pipelex/libraries/pipelines/builder/__init__.py b/pipelex/libraries/pipelines/builder/__init__.pynew file mode 100644index 000000000..e69de29bbdiff --git a/pipelex/libraries/pipelines/builder/builder.py b/pipelex/libraries/pipelines/builder/builder.pyindex 89b7bc6da..e3ad8d3fe 100644--- a/pipelex/libraries/pipelines/builder/builder.py+++ b/pipelex/libraries/pipelines/builder/builder.py@@ -1,4 +1,4 @@-from typing import Annotated, Dict, List, Optional, Union, cast+from typing import Annotated, castfrom pydantic import ConfigDict, Field, ValidationError, field_validator@@ -20,7 +20,7 @@ from pipelex.libraries.pipelines.builder.pipe.pipe_parallel_spec import PipeParafrom pipelex.libraries.pipelines.builder.pipe.pipe_sequence_spec import PipeSequenceSpecfrom pipelex.libraries.pipelines.builder.pipe.pipe_signature import PipeSignaturefrom pipelex.pipe_works.pipe_dry import dry_run_pipes-from pipelex.tools.typing.pydantic_utils import format_pydantic_validation_error+from pipelex.tools.typing.pydantic_utils import empty_list_factory_of, format_pydantic_validation_errorfrom pipelex.types import StrEnum@@ -39,25 +39,21 @@ class PipelexBundleSpecDraft(StructuredContent):domain: str = Field(description="The domain of the pipeline library.")definition: str = Field(description="The definition of the pipeline library.")- concept: Dict[str, ConceptSpec] = Field(default_factory=dict, description="The concepts of the pipeline library.")+ concept: dict[str, ConceptSpec] = Field(default_factory=dict, description="The concepts of the pipeline library.")- pipe: Dict[str, PipeSignature] = Field(default_factory=dict, description="The pipes of the pipeline library.")+ pipe: dict[str, PipeSignature] = Field(default_factory=dict, description="The pipes of the pipeline library.")PipeSpecUnion = Annotated[- Union[- # Pipe operators- PipeFuncSpec,- PipeImgGenSpec,- PipeJinja2Spec,- PipeLLMSpec,- PipeOcrSpec,- # Pipe controllers- PipeBatchSpec,- PipeConditionSpec,- PipeParallelSpec,- PipeSequenceSpec,- ],+ PipeFuncSpec+ | PipeImgGenSpec+ | PipeJinja2Spec+ | PipeLLMSpec+ | PipeOcrSpec+ | PipeBatchSpec+ | PipeConditionSpec+ | PipeParallelSpec+ | PipeSequenceSpec,Field(discriminator="type"),]@@ -92,19 +88,20 @@ class PipelexBundleSpec(StructuredContent):3. Pipe keys must be in snake_case format.4. Extra fields are forbidden (strict mode).5. Pipe types must match their blueprint discriminator.+"""model_config = ConfigDict(extra="forbid")domain: str- definition: Optional[str] = None- system_prompt: Optional[str] = None- system_prompt_to_structure: Optional[str] = None- prompt_template_to_structure: Optional[str] = None+ definition: str | None = None+ system_prompt: str | None = None+ system_prompt_to_structure: str | None = None+ prompt_template_to_structure: str | None = None- concept: Optional[Dict[str, Union[ConceptSpec, str]]] = Field(default_factory=dict)+ concept: dict[str, ConceptSpec | str] | None = Field(default_factory=dict)- pipe: Optional[Dict[str, PipeSpecUnion]] = Field(default_factory=dict)+ pipe: dict[str, PipeSpecUnion] | None = Field(default_factory=dict)@field_validator("domain", mode="before")@classmethod@@ -113,7 +110,7 @@ class PipelexBundleSpec(StructuredContent):return domaindef to_blueprint(self) -> PipelexBundleBlueprint:- concept: Optional[Dict[str, Union[ConceptBlueprint, str]]] = None+ concept: dict[str, ConceptBlueprint | str] | None = Noneif self.concept:concept = {}@@ -123,7 +120,7 @@ class PipelexBundleSpec(StructuredContent):else:concept[concept_code] = ConceptBlueprint(definition=concept_code, structure=concept_blueprint)- pipe: Optional[Dict[str, PipeBlueprintUnion]] = None+ pipe: dict[str, PipeBlueprintUnion] | None = Noneif self.pipe:pipe = {}for pipe_code, pipe_blueprint in self.pipe.items():@@ -142,7 +139,7 @@ class PipelexBundleSpec(StructuredContent):# TODO: Put this in a factory. Investigate why it is necessary.def _convert_pipe_spec(pipe_spec: PipeSpecUnion) -> PipeSpecUnion:- pipe_type_to_class: Dict[str, type] = {+ pipe_type_to_class: dict[str, type] = {"PipeFunc": PipeFuncSpec,"PipeImgGen": PipeImgGenSpec,"PipeJinja2": PipeJinja2Spec,@@ -158,7 +155,7 @@ def _convert_pipe_spec(pipe_spec: PipeSpecUnion) -> PipeSpecUnion:if pipe_class is None:msg = f"Unknown pipe type: {pipe_spec.type}"raise PipeBuilderError(msg)- return cast(PipeSpecUnion, pipe_class(**pipe_spec.model_dump(serialize_as_any=True)))+ return cast("PipeSpecUnion", pipe_class(**pipe_spec.model_dump(serialize_as_any=True)))async def compile_in_pipelex_bundle_spec(working_memory: WorkingMemory) -> PipelexBundleSpec:@@ -169,6 +166,7 @@ async def compile_in_pipelex_bundle_spec(working_memory: WorkingMemory) -> PipelReturns:PipelexBundleSpec: The constructed pipeline spec.+"""# The working memory actually contains ConceptSpec objects (not ConceptSpecDraft)# but they may have been deserialized incorrectly@@ -177,11 +175,11 @@ async def compile_in_pipelex_bundle_spec(working_memory: WorkingMemory) -> Pipelitem_type=ConceptSpec,)- pipe_specs = cast(ListContent[PipeSpecUnion], working_memory.get_stuff(name="pipe_specs").content)+ pipe_specs = cast("ListContent[PipeSpecUnion]", working_memory.get_stuff(name="pipe_specs").content)domain_information = working_memory.get_stuff_as(name="domain_information", content_type=DomainInformation)# Properly validate and reconstruct concept specs to ensure proper Pydantic validation- validated_concepts: Dict[str, Union[ConceptSpec, str]] = {}+ validated_concepts: dict[str, ConceptSpec | str] = {}for concept_spec in concept_specs.items:try:# Re-create the ConceptSpec to ensure proper Pydantic validation@@ -200,6 +198,7 @@ async def compile_in_pipelex_bundle_spec(working_memory: WorkingMemory) -> Pipel)+# TODO: rename or merge with PipeDry.DryRunStatusclass DryRunStatus(StrEnum):SUCCESS = "SUCCESS"FAILURE = "FAILURE"@@ -208,8 +207,6 @@ class DryRunStatus(StrEnum):class ValidateDryRunError(Exception):"""Raised when validating the dry run of a pipelex bundle blueprint."""- pass-class PipeFailure(StructuredContent):"""Details of a single pipe failure during dry run."""@@ -222,7 +219,10 @@ class DryRunResult(StructuredContent):"""A result of a dry run of a pipelex bundle blueprint."""status: DryRunStatus- failed_pipes: List[PipeFailure] = Field(default_factory=list, description="List of pipes that failed during dry run")+ failed_pipes: list[PipeFailure] = Field(+ default_factory=empty_list_factory_of(PipeFailure),+ description="List of pipes that failed during dry run",+ )async def validate_dry_run(working_memory: WorkingMemory) -> ListContent[PipeFailure]:@@ -247,29 +247,28 @@ async def validate_dry_run(working_memory: WorkingMemory) -> ListContent[PipeFai"PipeSequence": PipeSequenceSpec,}- failed_pipes: List[PipeFailure] = []+ failed_pipes: list[PipeFailure] = []for pipe_code, dry_run_output in dry_run_result.items():- if dry_run_output.status == DryRunStatus.FAILURE:- if pipelex_bundle_spec.pipe and pipe_code in pipelex_bundle_spec.pipe:- pipe_spec = pipelex_bundle_spec.pipe[pipe_code]- spec_class = pipe_type_to_spec_class.get(pipe_spec.type)- if not spec_class:- msg = f"Unknown pipe type: {pipe_spec.type}"- raise ValidateDryRunError(msg)- pipe_spec = spec_class(**pipe_spec.model_dump(serialize_as_any=True))- failed_pipes.append(- PipeFailure(- pipe=pipe_spec,- error_message=dry_run_output.error_message or "",- )- )+ if dry_run_output.status.is_failure and pipelex_bundle_spec.pipe and pipe_code in pipelex_bundle_spec.pipe:+ pipe_spec = pipelex_bundle_spec.pipe[pipe_code]+ spec_class = pipe_type_to_spec_class.get(pipe_spec.type)+ if not spec_class:+ msg = f"Unknown pipe type: {pipe_spec.type}"+ raise ValidateDryRunError(msg)+ pipe_spec = spec_class(**pipe_spec.model_dump(serialize_as_any=True))+ failed_pipes.append(+ PipeFailure(+ pipe=pipe_spec,+ error_message=dry_run_output.error_message or "",+ ),+ )return ListContent[PipeFailure](items=failed_pipes)async def reconstruct_bundle_with_all_fixes(working_memory: WorkingMemory) -> PipelexBundleSpec:pipelex_bundle_spec = working_memory.get_stuff_as(name="pipelex_bundle_spec", content_type=PipelexBundleSpec)- fixed_pipes_list = cast(ListContent[PipeSpecUnion], working_memory.get_stuff(name="fixed_pipes").content)+ fixed_pipes_list = cast("ListContent[PipeSpecUnion]", working_memory.get_stuff(name="fixed_pipes").content)if not pipelex_bundle_spec.pipe:msg = "No pipes section found in bundle spec"diff --git a/pipelex/libraries/pipelines/builder/concept/__init__.py b/pipelex/libraries/pipelines/builder/concept/__init__.pynew file mode 100644index 000000000..e69de29bbdiff --git a/pipelex/libraries/pipelines/builder/concept/concept_spec.py b/pipelex/libraries/pipelines/builder/concept/concept_spec.pyindex 78e22f008..7e0751fc9 100644--- a/pipelex/libraries/pipelines/builder/concept/concept_spec.py+++ b/pipelex/libraries/pipelines/builder/concept/concept_spec.py@@ -1,8 +1,7 @@from datetime import datetime-from typing import Any, Dict, Optional, Union, cast+from typing import Any, castfrom pydantic import ConfigDict, Field, field_validator, model_validator-from typing_extensions import Selffrom pipelex.core.concepts.concept_blueprint import (ConceptBlueprint,@@ -18,7 +17,7 @@ from pipelex.core.domains.domain_blueprint import DomainBlueprintfrom pipelex.core.memory.working_memory import WorkingMemoryfrom pipelex.core.stuffs.stuff_content import StructuredContentfrom pipelex.tools.misc.string_utils import is_pascal_case-from pipelex.types import StrEnum+from pipelex.types import Self, StrEnumclass ConceptStructureSpecFieldType(StrEnum):@@ -54,13 +53,14 @@ class ConceptStructureSpec(StructuredContent):- For typed fields: type must be specified and default_value must match that type- Type validation includes: text (str), integer (int), boolean (bool),number (int/float), dict (dict)+"""the_field_name: str = Field(description="Field name. Must be snake_case.")definition: str- type: Optional[ConceptStructureSpecFieldType] = Field(default=None, description="The type of the field.")- required: Optional[bool] = True- default_value: Optional[Any] = None+ type: ConceptStructureSpecFieldType | None = Field(default=None, description="The type of the field.")+ required: bool | None = True+ default_value: Any | None = None@model_validator(mode="after")def validate_structure_blueprint(self) -> Self:@@ -95,13 +95,10 @@ class ConceptStructureSpec(StructuredContent):self._raise_type_mismatch_error("date", type(self.default_value).__name__)def _raise_type_mismatch_error(self, expected_type_name: str, actual_type_name: str) -> None:- """Raise a type mismatch error with consistent formatting."""- raise ConceptStructureBlueprintError(- f"default_value type mismatch: expected {expected_type_name} for type '{self.type}', but got {actual_type_name}"- )+ msg = f"default_value type mismatch: expected {expected_type_name} for type '{self.type}', but got {actual_type_name}"+ raise ConceptStructureBlueprintError(msg)def to_blueprint(self) -> ConceptStructureBlueprint:- """Convert this ConceptStructureBlueprint to the core ConceptStructureBlueprint."""# Convert the type enum value - self.type is already a ConceptStructureBlueprintFieldType enum# We need to get the corresponding value in the core enumcore_type = None@@ -121,9 +118,9 @@ class ConceptSpecDraft(StructuredContent):the_concept_code: str = Field(description="Concept code. Must be PascalCase.")definition: str = Field(description="Description of the concept, in natural language.")structure: str = Field(- description="A description of a dict with fieldnames as keys, and values being a dict with: definition, type, required, default_value"+ description="A description of a dict with fieldnames as keys, and values being a dict with: definition, type, required, default_value",)- refines: Optional[str] = Field(+ refines: str | None = Field(default=None,description="The native concept this concept extends (Text, Image, PDF, TextAndImages, Number, Page) ""in PascalCase format. Cannot be used together with 'structure'.",@@ -158,37 +155,38 @@ class ConceptSpec(StructuredContent):5. Native concepts: When refining, must be one of the valid native concepts.6. Structure values: In structure dict, values must be either valid concept stringsor ConceptStructureBlueprint instances.+"""model_config = ConfigDict(extra="forbid")the_concept_code: str = Field(description="Concept code. Must be PascalCase.")definition: str- structure: Optional[Union[str, Dict[str, Union[str, ConceptStructureSpec]]]] = None- refines: Optional[str] = None+ structure: str | dict[str, str | ConceptStructureSpec] | None = None+ refines: str | None = None@classmethoddef is_native_concept_code(cls, concept_code: str) -> bool:ConceptSpec.validate_concept_code(concept_code=concept_code)- return concept_code in [native_concept for native_concept in [native_concept for native_concept in NativeConceptEnum]]+ return concept_code in NativeConceptEnum.values_list()@classmethoddef validate_concept_code(cls, concept_code: str) -> None:if not is_pascal_case(concept_code):- raise ConceptCodeError(- f"Concept code '{concept_code}' must be PascalCase (letters and numbers only, starting with uppercase, without `.`)"- )+ msg = f"Concept code '{concept_code}' must be PascalCase (letters and numbers only, starting with uppercase, without `.`)"+ raise ConceptCodeError(msg)@classmethoddef validate_concept_string_or_code(cls, concept_string_or_code: str) -> None:if concept_string_or_code.count(".") > 1:- raise ConceptStringOrConceptCodeError(+ msg = (f"concept_string_or_code '{concept_string_or_code}' is invalid. ""It should either contain a domain in snake_case and a concept code in PascalCase separated by one dot, ""or be a concept code in PascalCase.")+ raise ConceptStringOrConceptCodeError(msg)- elif concept_string_or_code.count(".") == 1:+ if concept_string_or_code.count(".") == 1:domain, concept_code = concept_string_or_code.split(".")DomainBlueprint.validate_domain_code(code=domain)cls.validate_concept_code(concept_code=concept_code)@@ -198,70 +196,68 @@ class ConceptSpec(StructuredContent):@staticmethoddef validate_concept_string(concept_string: str) -> None:"""Validate that a concept code follows PascalCase convention."""- if "." not in concept_string:- raise ConceptStringError(- f"Concept string '{concept_string}' is invalid. It should contain a domain in snake_case "- "and a concept code in PascalCase separated by one dot."- )- elif concept_string.count(".") > 1:- raise ConceptStringError(+ if "." not in concept_string or concept_string.count(".") > 1:+ msg = (f"Concept string '{concept_string}' is invalid. It should contain a domain in snake_case ""and a concept code in PascalCase separated by one dot.")- else:- domain, concept_code = concept_string.split(".", 1)+ raise ConceptStringError(msg)+ domain, concept_code = concept_string.split(".", 1)DomainBlueprint.validate_domain_code(domain)if not is_pascal_case(concept_code):- raise ConceptCodeError(- f"Concept code '{concept_code}' must be PascalCase (letters and numbers only, starting with uppercase, without `.`)"- )+ msg = f"Concept code '{concept_code}' must be PascalCase (letters and numbers only, starting with uppercase, without `.`)"+ raise ConceptCodeError(msg)# Validate that if the concept code is among the native concepts, the domain MUST be native.- if concept_code in [concept.value for concept in [native_concept for native_concept in NativeConceptEnum]]:- if domain != SpecialDomain.NATIVE:- raise ConceptStringError(- f"Concept string '{concept_string}' is invalid. "- f"Concept code '{concept_code}' is a native concept, so the domain must be '{SpecialDomain.NATIVE}', "- f"or nothing, but not '{domain}'"- )+ if not SpecialDomain.is_native(domain=domain) and concept_code in NativeConceptEnum.values_list():+ msg = (+ f"Concept string '{concept_string}' is invalid. "+ f"Concept code '{concept_code}' is a native concept, so the domain must be '{SpecialDomain.NATIVE}', "+ f"or nothing, but not '{domain}'"+ )+ raise ConceptStringError(msg)# Validate that if the domain is native, the concept code is a native conceptif SpecialDomain.is_native(domain=domain):- if concept_code not in [native_concept for native_concept in NativeConceptEnum]:- raise ConceptStringError(+ if concept_code not in NativeConceptEnum.values_list():+ msg = (f"Concept string '{concept_string}' is invalid. "f"Concept code '{concept_code}' is not a native concept, so the domain must not be '{SpecialDomain.NATIVE}'.")+ raise ConceptStringError(msg)@field_validator("refines", mode="before")@classmethod- def validate_refines(cls, refines: Optional[str] = None) -> Optional[str]:+ def validate_refines(cls, refines: str | None = None) -> str | None:if refines is not None:if not NativeConceptManager.is_native_concept(refines):- raise ConceptBlueprintError(f"Forbidden to refine a non-native concept: '{refines}'. Refining non-native concepts will come soon.")+ msg = f"Forbidden to refine a non-native concept: '{refines}'. Refining non-native concepts will come soon."+ raise ConceptBlueprintError(msg)cls.validate_concept_string_or_code(concept_string_or_code=refines)return refines@model_validator(mode="before")- def model_validate_spec(cls, values: Dict[str, Any]) -> Dict[str, Any]:+ @classmethod+ def model_validate_spec(cls, values: dict[str, Any]) -> dict[str, Any]:if values.get("refines") and values.get("structure"):- raise ConceptSpecError(+ msg = (f"Forbidden to have refines and structure at the same time: `{values.get('refines')}` "- f"and `{values.get('structure')}` for concept that has the definition `{values.get('definition')}`"+ f"and `{values.get('structure')}` for concept that has the definition `{values.get('definition')}`",)+ raise ConceptSpecError(msg)return valuesdef to_blueprint(self) -> ConceptBlueprint:"""Convert this ConceptBlueprint to the original core ConceptBlueprint."""# TODO: Clarify concept structure blueprint- converted_structure: Optional[Union[str, Dict[str, Union[str, ConceptStructureBlueprint]]]] = None+ converted_structure: str | dict[str, str | ConceptStructureBlueprint] | None = Noneif self.structure:converted_structure = {}if isinstance(self.structure, str):converted_structure = self.structureelse:- for field_name, field_spec in cast(Dict[str, ConceptStructureSpec], self.structure).items():+ for field_name, field_spec in cast("dict[str, ConceptStructureSpec]", self.structure).items():converted_structure[field_name] = field_spec.to_blueprint()return ConceptBlueprint(definition=self.definition, structure=converted_structure, refines=self.refines)@@ -271,7 +267,7 @@ async def create_concept_spec(working_memory: WorkingMemory) -> ConceptSpec:concept_spec_draft = working_memory.get_stuff_as(name="concept_spec_draft", content_type=ConceptSpecDraft)concept_spec_structures_stuff = working_memory.get_stuff_as_list(name="concept_spec_structures", item_type=ConceptStructureSpec)- structure_dict: Dict[str, Union[str, ConceptStructureSpec]] = {}+ structure_dict: dict[str, str | ConceptStructureSpec] = {}for structure_item in concept_spec_structures_stuff.items:structure_spec = ConceptStructureSpec(the_field_name=structure_item.the_field_name,diff --git a/pipelex/libraries/pipelines/builder/pipe/__init__.py b/pipelex/libraries/pipelines/builder/pipe/__init__.pynew file mode 100644index 000000000..e69de29bbdiff --git a/pipelex/libraries/pipelines/builder/pipe/inputs_spec.py b/pipelex/libraries/pipelines/builder/pipe/inputs_spec.pyindex 43c6f0b64..159ad2d6b 100644--- a/pipelex/libraries/pipelines/builder/pipe/inputs_spec.py+++ b/pipelex/libraries/pipelines/builder/pipe/inputs_spec.py@@ -1,5 +1,3 @@-from typing import Optional-from pydantic import Field, field_validatorfrom pipelex.core.concepts.concept_blueprint import ConceptBlueprint@@ -25,10 +23,11 @@ class InputRequirementSpec(StructuredContent):1. Concept must be a valid concept code (PascalCase) or concept string (domain.ConceptCode).2. Domain and concept code are separated by a dot when using full concept strings.3. Concept validation is performed using ConceptBlueprint.validate_concept_string_or_code.+"""concept: str = Field(description="Concept code or concept string in PascalCase format")- multiplicity: Optional[PipeOutputMultiplicity] = None+ multiplicity: PipeOutputMultiplicity | None = None@field_validator("concept", mode="before")@classmethoddiff --git a/pipelex/libraries/pipelines/builder/pipe/pipe_batch_spec.py b/pipelex/libraries/pipelines/builder/pipe/pipe_batch_spec.pyindex 558ba5aec..46741432b 100644--- a/pipelex/libraries/pipelines/builder/pipe/pipe_batch_spec.py+++ b/pipelex/libraries/pipelines/builder/pipe/pipe_batch_spec.py@@ -1,4 +1,4 @@-from typing import Literal, Optional+from typing import Literalfrom pydantic import Fieldfrom typing_extensions import override@@ -32,14 +32,15 @@ class PipeBatchSpec(PipeSpec):1. branch_pipe_code must reference an existing pipe in the pipeline.2. When input_list_name is specified, it must reference a list in context.3. The branch pipe should be designed to process single items.+"""type: Literal["PipeBatch"] = "PipeBatch"category: Literal["PipeController"] = "PipeController"the_pipe_code: str = Field(description="Pipe code. Must be snake_case.")branch_pipe_code: str- input_list_name: Optional[str] = None- input_item_name: Optional[str] = None+ input_list_name: str | None = None+ input_item_name: str | None = None@overridedef to_blueprint(self) -> PipeBatchBlueprint:diff --git a/pipelex/libraries/pipelines/builder/pipe/pipe_condition_spec.py b/pipelex/libraries/pipelines/builder/pipe/pipe_condition_spec.pyindex 0127a4719..65d745fba 100644--- a/pipelex/libraries/pipelines/builder/pipe/pipe_condition_spec.py+++ b/pipelex/libraries/pipelines/builder/pipe/pipe_condition_spec.py@@ -1,4 +1,4 @@-from typing import Dict, Literal, Optional+from typing import Literalfrom pydantic import Field, RootModelfrom typing_extensions import override@@ -6,7 +6,7 @@ from typing_extensions import overridefrom pipelex.libraries.pipelines.builder.pipe.pipe_signature import PipeSpecfrom pipelex.pipe_controllers.condition.pipe_condition_blueprint import PipeConditionBlueprint, PipeConditionPipeMapBlueprint-PipeConditionPipeMapRoot = Dict[str, str]+PipeConditionPipeMapRoot = dict[str, str]class PipeConditionPipeMapSpec(RootModel[PipeConditionPipeMapRoot]):@@ -18,6 +18,7 @@ class PipeConditionPipeMapSpec(RootModel[PipeConditionPipeMapRoot]):root: Dictionary mapping condition results (keys) to pipe codes (values).Each key represents a possible condition outcome, and its valueis the pipe code to execute when that condition is met.+"""root: PipeConditionPipeMapRoot = Field(default_factory=dict)@@ -48,16 +49,17 @@ class PipeConditionSpec(PipeSpec):1. Either expression or expression_template should be provided, not both.2. pipe_map keys must be strings representing possible condition outcomes.3. All pipe codes in pipe_map and default_pipe_code must be valid pipe references.+"""type: Literal["PipeCondition"] = "PipeCondition"category: Literal["PipeController"] = "PipeController"the_pipe_code: str = Field(description="Pipe code. Must be snake_case.")- expression_template: Optional[str] = None- expression: Optional[str] = None+ expression_template: str | None = None+ expression: str | None = Nonepipe_map: PipeConditionPipeMapSpec = Field(default_factory=PipeConditionPipeMapSpec)- default_pipe_code: Optional[str] = None- add_alias_from_expression_to: Optional[str] = None+ default_pipe_code: str | None = None+ add_alias_from_expression_to: str | None = None@overridedef to_blueprint(self) -> PipeConditionBlueprint:diff --git a/pipelex/libraries/pipelines/builder/pipe/pipe_func_spec.py b/pipelex/libraries/pipelines/builder/pipe/pipe_func_spec.pyindex 48773f9d4..08e39c20e 100644--- a/pipelex/libraries/pipelines/builder/pipe/pipe_func_spec.py+++ b/pipelex/libraries/pipelines/builder/pipe/pipe_func_spec.py@@ -8,14 +8,14 @@ from pipelex.pipe_operators.func.pipe_func_blueprint import PipeFuncBlueprintclass PipeFuncSpec(PipeSpec):- """- PipeFunc enables calling custom functions in the Pipelex framework.+ """PipeFunc enables calling custom functions in the Pipelex framework.Attributes:the_pipe_code: Pipe code. Must be snake_case.type: Fixed to "PipeFunc" for this pipe type.category: Fixed to "PipeOperator" for this pipe category.function_name: The name of the function to call.+"""type: Literal["PipeFunc"] = "PipeFunc"diff --git a/pipelex/libraries/pipelines/builder/pipe/pipe_img_spec.py b/pipelex/libraries/pipelines/builder/pipe/pipe_img_spec.pyindex 331b3dbae..a0c4f83b3 100644--- a/pipelex/libraries/pipelines/builder/pipe/pipe_img_spec.py+++ b/pipelex/libraries/pipelines/builder/pipe/pipe_img_spec.py@@ -1,4 +1,4 @@-from typing import Literal, Optional, Union+from typing import Literalfrom pydantic import Fieldfrom typing_extensions import override@@ -44,24 +44,25 @@ class PipeImgGenSpec(PipeSpec):Raises:PipeDefinitionError: When validation rules are violated or mutually exclusivefields are set simultaneously.+"""type: Literal["PipeImgGen"] = "PipeImgGen"category: Literal["PipeOperator"] = "PipeOperator"the_pipe_code: str = Field(description="Pipe code. Must be snake_case.")- img_gen_prompt: Optional[str] = None- img_gen_prompt_var_name: Optional[str] = None+ img_gen_prompt: str | None = None+ img_gen_prompt_var_name: str | None = None# New ImgGenChoice pattern (like LLM)- img_gen: Optional[ImgGenChoice] = None+ img_gen: ImgGenChoice | None = None# One-time settings (not in ImgGenSetting)- aspect_ratio: Optional[AspectRatio] = Field(default=None, strict=False)- is_raw: Optional[bool] = None- seed: Optional[Union[int, Literal["auto"]]] = None- nb_output: Optional[int] = Field(default=None, ge=1)- background: Optional[Background] = Field(default=None, strict=False)- output_format: Optional[OutputFormat] = Field(default=None, strict=False)+ aspect_ratio: AspectRatio | None = Field(default=None, strict=False)+ is_raw: bool | None = None+ seed: int | Literal["auto"] | None = None+ nb_output: int | None = Field(default=None, ge=1)+ background: Background | None = Field(default=None, strict=False)+ output_format: OutputFormat | None = Field(default=None, strict=False)@overridedef to_blueprint(self) -> PipeImgGenBlueprint:diff --git a/pipelex/libraries/pipelines/builder/pipe/pipe_jinja2_spec.py b/pipelex/libraries/pipelines/builder/pipe/pipe_jinja2_spec.pyindex 4b6fc56dd..83a9d1cc8 100644--- a/pipelex/libraries/pipelines/builder/pipe/pipe_jinja2_spec.py+++ b/pipelex/libraries/pipelines/builder/pipe/pipe_jinja2_spec.py@@ -1,4 +1,4 @@-from typing import Any, Dict, Literal, Optional+from typing import Any, Literalfrom pydantic import Fieldfrom typing_extensions import override@@ -16,14 +16,14 @@ class PromptingStyleSpec(StructuredContent):class Jinja2Spec(StructuredContent):- jinja2_name: Optional[str] = Field(default=None, description="Name of the Jinja2 template to use")- jinja2: Optional[str] = Field(default=None, description="Raw Jinja2 template string")- prompting_style: Optional[PromptingStyleSpec] = Field(default=None, description="Style of prompting to use (typically for different LLMs)")+ jinja2_name: str | None = Field(default=None, description="Name of the Jinja2 template to use")+ jinja2: str | None = Field(default=None, description="Raw Jinja2 template string")+ prompting_style: PromptingStyleSpec | None = Field(default=None, description="Style of prompting to use (typically for different LLMs)")template_category: Jinja2TemplateCategory = Field(default=Jinja2TemplateCategory.LLM_PROMPT,description="Category of the template (could also be HTML, MARKDOWN, MERMAID, etc.), influences Jinja2 rendering environment config",)- extra_context: Optional[Dict[str, Any]] = Field(default=None, description="Additional context variables for template rendering")+ extra_context: dict[str, Any] | None = Field(default=None, description="Additional context variables for template rendering")class PipeJinja2Spec(PipeSpec, Jinja2Spec):diff --git a/pipelex/libraries/pipelines/builder/pipe/pipe_llm_spec.py b/pipelex/libraries/pipelines/builder/pipe/pipe_llm_spec.pyindex b3d8d8ce9..c193f884f 100644--- a/pipelex/libraries/pipelines/builder/pipe/pipe_llm_spec.py+++ b/pipelex/libraries/pipelines/builder/pipe/pipe_llm_spec.py@@ -1,7 +1,7 @@-from typing import Literal, Optional, Union+from typing import Literal, Unionfrom pydantic import Field, field_validator, model_validator-from typing_extensions import Self, override+from typing_extensions import overridefrom pipelex.cogt.llm.llm_setting import LLMChoice, LLMSettingfrom pipelex.core.stuffs.stuff_content import StructuredContent@@ -9,21 +9,20 @@ from pipelex.exceptions import PipeDefinitionErrorfrom pipelex.libraries.pipelines.builder.pipe.pipe_signature import PipeSpecfrom pipelex.pipe_operators.llm.pipe_llm_blueprint import PipeLLMBlueprint, StructuringMethodfrom pipelex.tools.typing.validation_utils import has_more_than_one_among_attributes_from_lists+from pipelex.types import Selfclass LLMSettingSpec(StructuredContent):llm_handle: strtemperature: float = Field(..., ge=0, le=1)- max_tokens: Optional[int] = None+ max_tokens: int | None = None@field_validator("max_tokens", mode="before")@classmethod- def validate_max_tokens(cls, value: Union[int, Literal["auto"], None]) -> Optional[int]:- if value is None:+ def validate_max_tokens(cls, value: int | Literal["auto"] | None) -> int | None:+ if value is None or (isinstance(value, str) and value == "auto"):return None- elif isinstance(value, str) and value == "auto":- return None- elif isinstance(value, int): # pyright: ignore[reportUnnecessaryIsInstance]+ if isinstance(value, int): # pyright: ignore[reportUnnecessaryIsInstance]return value@@ -73,35 +72,38 @@ class PipeLLMSpec(PipeSpec):3. Output cardinality: nb_output and multiple_output are mutually exclusive.4. nb_output must be greater than 0 when specified.5. Structuring method must be 'direct' or 'preliminary_text' when specified.+"""type: Literal["PipeLLM"] = "PipeLLM"category: Literal["PipeOperator"] = "PipeOperator"the_pipe_code: str = Field(description="Pipe code. Must be snake_case.")- system_prompt_template: Optional[str] = None- system_prompt_template_name: Optional[str] = None- system_prompt_name: Optional[str] = None- system_prompt: Optional[str] = None+ system_prompt_template: str | None = None+ system_prompt_template_name: str | None = None+ system_prompt_name: str | None = None+ system_prompt: str | None = None- prompt_template: Optional[str] = None- template_name: Optional[str] = None- prompt_name: Optional[str] = None- prompt: Optional[str] = None+ prompt_template: str | None = None+ template_name: str | None = None+ prompt_name: str | None = None+ prompt: str | None = None- llm: Optional[LLMChoiceSpec] = None- llm_to_structure: Optional[LLMChoiceSpec] = None+ llm: LLMChoiceSpec | None = None+ llm_to_structure: LLMChoiceSpec | None = None- structuring_method: Optional[StructuringMethod] = None- prompt_template_to_structure: Optional[str] = None- system_prompt_to_structure: Optional[str] = None+ structuring_method: StructuringMethod | None = None+ prompt_template_to_structure: str | None = None+ system_prompt_to_structure: str | None = None- nb_output: Optional[int] = None- multiple_output: Optional[bool] = None+ nb_output: int | None = None+ multiple_output: bool | None = None@field_validator("nb_output", mode="after")- def validate_nb_output(cls, value: Optional[int] = None) -> Optional[int]:+ @staticmethod+ def validate_nb_output(value: int | None = None) -> int | None:if value and value < 1:- raise PipeDefinitionError("PipeLLMBlueprint nb_output must be greater than 0")+ msg = "PipeLLMBlueprint nb_output must be greater than 0"+ raise PipeDefinitionError(msg)return value@model_validator(mode="after")@@ -114,19 +116,20 @@ class PipeLLMSpec(PipeSpec):["prompt", "prompt_name", "prompt_template", "template_name"],],):- raise PipeDefinitionError(f"PipeLLMBlueprint should have no more than one of {excess_attributes_list} among them")+ msg = f"PipeLLMBlueprint should have no more than one of {excess_attributes_list} among them"+ raise PipeDefinitionError(msg)return self@overridedef to_blueprint(self) -> PipeLLMBlueprint:base_blueprint = super().to_blueprint()- llm: Optional[LLMChoice] = None+ llm: LLMChoice | None = Noneif isinstance(self.llm, LLMSettingSpec):llm = LLMSetting(llm_handle=self.llm.llm_handle, temperature=self.llm.temperature, max_tokens=self.llm.max_tokens)elif isinstance(self.llm, str):llm = self.llm- llm_to_structure: Optional[LLMChoice] = None+ llm_to_structure: LLMChoice | None = Noneif isinstance(self.llm_to_structure, LLMSettingSpec):llm_to_structure = LLMSetting(llm_handle=self.llm_to_structure.llm_handle,diff --git a/pipelex/libraries/pipelines/builder/pipe/pipe_ocr_spec.py b/pipelex/libraries/pipelines/builder/pipe/pipe_ocr_spec.pyindex 299197280..2a377d064 100644--- a/pipelex/libraries/pipelines/builder/pipe/pipe_ocr_spec.py+++ b/pipelex/libraries/pipelines/builder/pipe/pipe_ocr_spec.py@@ -1,4 +1,4 @@-from typing import Literal, Optional+from typing import Literalfrom pydantic import Fieldfrom typing_extensions import override@@ -34,16 +34,17 @@ class PipeOcrSpec(PipeSpec):1. OCR model must be "base_ocr_mistral".2. Boolean flags (page_images, page_image_captions, page_views) are optional.3. page_views_dpi should be a positive integer when specified.+"""type: Literal["PipeOcr"] = "PipeOcr"category: Literal["PipeOperator"] = "PipeOperator"the_pipe_code: str = Field(description="Pipe code. Must be snake_case.")ocr: str = "base_ocr_mistral"- page_images: Optional[bool] = None- page_image_captions: Optional[bool] = None- page_views: Optional[bool] = None- page_views_dpi: Optional[int] = None+ page_images: bool | None = None+ page_image_captions: bool | None = None+ page_views: bool | None = None+ page_views_dpi: int | None = None@overridedef to_blueprint(self) -> PipeOcrBlueprint:diff --git a/pipelex/libraries/pipelines/builder/pipe/pipe_parallel_spec.py b/pipelex/libraries/pipelines/builder/pipe/pipe_parallel_spec.pyindex ec6180e59..2afdf8b6e 100644--- a/pipelex/libraries/pipelines/builder/pipe/pipe_parallel_spec.py+++ b/pipelex/libraries/pipelines/builder/pipe/pipe_parallel_spec.py@@ -1,4 +1,4 @@-from typing import List, Literal, Optional+from typing import Literalfrom pydantic import Field, field_validatorfrom typing_extensions import override@@ -31,17 +31,19 @@ class PipeParallelSpec(PipeSpec):2. Each parallel step must be a valid SubPipeSpec.3. combined_output, when specified, must be a valid concept string or code.4. Pipe codes in parallels must reference existing pipes.+"""type: Literal["PipeParallel"] = "PipeParallel"category: Literal["PipeController"] = "PipeController"the_pipe_code: str = Field(description="Pipe code. Must be snake_case.")- parallels: List[SubPipeSpec]+ parallels: list[SubPipeSpec]add_each_output: bool = True- combined_output: Optional[str] = None+ combined_output: str | None = None@field_validator("combined_output", mode="before")- def validate_combined_output(cls, combined_output: str) -> str:+ @staticmethod+ def validate_combined_output(combined_output: str) -> str:if combined_output:ConceptBlueprint.validate_concept_string_or_code(concept_string_or_code=combined_output)return combined_outputdiff --git a/pipelex/libraries/pipelines/builder/pipe/pipe_sequence_spec.py b/pipelex/libraries/pipelines/builder/pipe/pipe_sequence_spec.pyindex bda5a78a4..2921c0a2e 100644--- a/pipelex/libraries/pipelines/builder/pipe/pipe_sequence_spec.py+++ b/pipelex/libraries/pipelines/builder/pipe/pipe_sequence_spec.py@@ -1,4 +1,4 @@-from typing import List, Literal+from typing import Literalfrom pydantic import Fieldfrom typing_extensions import override@@ -26,12 +26,13 @@ class PipeSequenceSpec(PipeSpec):1. Steps list must not be empty.2. Each step must be a valid SubPipeBlueprint instance.3. Pipe codes referenced in steps must exist in the pipeline.+"""type: Literal["PipeSequence"] = "PipeSequence"category: Literal["PipeController"] = "PipeController"the_pipe_code: str = Field(description="Pipe code. Must be snake_case.")- steps: List[SubPipeSpec]+ steps: list[SubPipeSpec]@overridedef to_blueprint(self) -> PipeSequenceBlueprint:diff --git a/pipelex/libraries/pipelines/builder/pipe/pipe_signature.py b/pipelex/libraries/pipelines/builder/pipe/pipe_signature.pyindex f8424f81a..90a5132ca 100644--- a/pipelex/libraries/pipelines/builder/pipe/pipe_signature.py+++ b/pipelex/libraries/pipelines/builder/pipe/pipe_signature.py@@ -1,4 +1,4 @@-from typing import Any, Dict, Optional, Union+from typing import Anyfrom pydantic import Field, field_validator@@ -16,10 +16,10 @@ class PipeSignature(StructuredContent):type: AllowedPipeTypes = Field(description="Pipe type.")category: AllowedPipeCategories = Field(description="Pipe category.")definition: str = Field(description="What the pipe does")- inputs: Dict[str, ConceptSpecDraft] = Field(description="Pipe inputs: key is the concept code in pascal Case.")+ inputs: dict[str, ConceptSpecDraft] = Field(description="Pipe inputs: key is the concept code in pascal Case.")result: str = Field(description="The name of the result of the pipe. Must be snake_case. It will be used in the inputs of the next pipes.")output: ConceptSpecDraft = Field(description="Concept as output")- important_features: Optional[Dict[str, Any]] = Field(+ important_features: dict[str, Any] | None = Field(default=None,description="Important features specific to this pipe type ""(e.g., referenced pipe codes for controllers, specific configuration for operators)",@@ -51,40 +51,44 @@ class PipeSpec(StructuredContent):2. Output concept: Must be valid concept string or code in PascalCase.3. Input concepts: When provided, must use PascalCase for concept references.4. Pipe codes: When validating pipe codes, must be in snake_case format.+"""type: Any = Field(description=f"Pipe type. Must be one of: {AllowedPipeTypes}")category: Any = Field(description=f"Pipe category. Must be one of: {AllowedPipeCategories}")- definition: Optional[str] = Field(description="Natural language description of what the pipe does.")- inputs: Optional[Dict[str, Union[str, InputRequirementSpec]]] = Field(+ definition: str | None = Field(description="Natural language description of what the pipe does.")+ inputs: dict[str, str | InputRequirementSpec] | None = Field(description=("Input concept specifications. Can be either: ""InputRequirementSpec with additional constraints""Dictionary keys are input names, values are concept specifications. If Its the concept itself, use the concept code in PascalCase."- )+ ),)output: str = Field(description="Output concept code in PascalCase format!! Very important")@field_validator("type", mode="after")- def validate_pipe_type(cls, value: Any) -> Any:- allowed_types = [_type.value for _type in AllowedPipeTypes]- if value not in allowed_types:- raise PipeBlueprintError(f"Invalid pipe type '{value}'. Must be one of: {allowed_types}")+ @staticmethod+ def validate_pipe_type(value: Any) -> Any:+ if value not in AllowedPipeTypes.value_list():+ msg = f"Invalid pipe type '{value}'. Must be one of: {AllowedPipeTypes.value_list()}"+ raise PipeBlueprintError(msg)return value@field_validator("output", mode="before")- def validate_concept_string_or_code(cls, output: str) -> str:+ @staticmethod+ def validate_concept_string_or_code(output: str) -> str:ConceptBlueprint.validate_concept_string_or_code(concept_string_or_code=output)return output@classmethoddef validate_pipe_code_syntax(cls, pipe_code: str) -> str:if not is_snake_case(pipe_code):- raise PipeBlueprintError(f"Invalid pipe code syntax '{pipe_code}'. Must be in snake_case.")+ msg = f"Invalid pipe code syntax '{pipe_code}'. Must be in snake_case."+ raise PipeBlueprintError(msg)return pipe_codedef to_blueprint(self) -> PipeBlueprint:- converted_inputs: Optional[Dict[str, Union[str, InputRequirementBlueprint]]] = None+ converted_inputs: dict[str, str | InputRequirementBlueprint] | None = Noneif self.inputs is not None:converted_inputs = {}for input_name, input_spec in self.inputs.items():diff --git a/pipelex/libraries/pipelines/builder/pipe/sub_pipe_spec.py b/pipelex/libraries/pipelines/builder/pipe/sub_pipe_spec.pyindex aafc03094..42423bd3b 100644--- a/pipelex/libraries/pipelines/builder/pipe/sub_pipe_spec.py+++ b/pipelex/libraries/pipelines/builder/pipe/sub_pipe_spec.py@@ -1,12 +1,10 @@-from typing import Optional, Union-from pydantic import ConfigDict, model_validator-from typing_extensions import Selffrom pipelex.core.stuffs.stuff_content import StructuredContentfrom pipelex.exceptions import PipeDefinitionErrorfrom pipelex.pipe_controllers.sub_pipe_blueprint import SubPipeBlueprintfrom pipelex.tools.typing.validation_utils import has_more_than_one_among_attributes_from_list+from pipelex.types import Selfclass SubPipeSpec(StructuredContent):@@ -35,21 +33,23 @@ class SubPipeSpec(StructuredContent):2. batch_over and batch_as must be specified together (both or neither).3. pipe must reference a valid pipe code.4. result, when specified, should follow naming conventions.+"""model_config = ConfigDict(extra="forbid")the_pipe_code: str- result: Optional[str] = None- nb_output: Optional[int] = None- multiple_output: Optional[bool] = None- batch_over: Union[bool, str] = False- batch_as: Optional[str] = None+ result: str | None = None+ nb_output: int | None = None+ multiple_output: bool | None = None+ batch_over: bool | str = False+ batch_as: str | None = None@model_validator(mode="after")def validate_multiple_output(self) -> Self:if has_more_than_one_among_attributes_from_list(self, attributes_list=["nb_output", "multiple_output"]):- raise PipeDefinitionError("PipeStepBlueprint should have no more than '1' of nb_output or multiple_output")+ msg = "PipeStepBlueprint should have no more than '1' of nb_output or multiple_output"+ raise PipeDefinitionError(msg)return self@model_validator(mode="after")@@ -58,10 +58,12 @@ class SubPipeSpec(StructuredContent):batch_as_is_specified = self.batch_as is not None and self.batch_as != ""if batch_over_is_specified and not batch_as_is_specified:- raise PipeDefinitionError(f"In pipe '{self.the_pipe_code}': When 'batch_over' is specified, 'batch_as' must also be provided")+ msg = f"In pipe '{self.the_pipe_code}': When 'batch_over' is specified, 'batch_as' must also be provided"+ raise PipeDefinitionError(msg)if batch_as_is_specified and not batch_over_is_specified:- raise PipeDefinitionError(f"In pipe '{self.the_pipe_code}': When 'batch_as' is specified, 'batch_over' must also be provided")+ msg = f"In pipe '{self.the_pipe_code}': When 'batch_as' is specified, 'batch_over' must also be provided"+ raise PipeDefinitionError(msg)return selfdiff --git a/pipelex/migration/migrate_v0_1_0_to_v0_2_0.py b/pipelex/migration/migrate_v0_1_0_to_v0_2_0.pyindex 0d536e3d4..e9ec77e70 100644--- a/pipelex/migration/migrate_v0_1_0_to_v0_2_0.py+++ b/pipelex/migration/migrate_v0_1_0_to_v0_2_0.py@@ -3,7 +3,8 @@import reimport shutilfrom pathlib import Path-from typing import Any, ClassVar, Dict, List, Match+from re import Match+from typing import Any, ClassVarfrom pipelex.migration.migration_result import MigrationResult@@ -36,10 +37,11 @@ class TOMLMigrator:re.MULTILINE,)- def find_toml_files(self, directory: Path) -> List[Path]:+ def find_toml_files(self, directory: Path) -> list[Path]:"""Find all TOML files in directory and subdirectories."""if not directory.exists():- raise FileNotFoundError(f"Directory not found: {directory}")+ msg = f"Directory not found: {directory}"+ raise FileNotFoundError(msg)return list(directory.glob("**/*.toml"))@@ -65,15 +67,11 @@ class TOMLMigrator:# Check pipe patternspipe_matches = list(self.pipe_pattern.finditer(content))- for match in pipe_matches:- if not self._is_line_inside_multiline_string(content, match.start()):- return True-- return False+ return any(not self._is_line_inside_multiline_string(content, match.start()) for match in pipe_matches)- def get_migration_preview(self, content: str) -> List[Dict[str, Any]]:+ def get_migration_preview(self, content: str) -> list[dict[str, Any]]:"""Get preview of changes that would be made."""- changes: List[Dict[str, Any]] = []+ changes: list[dict[str, Any]] = []# Handle concept migrationsconcept_matches = list(self.concept_pattern.finditer(content))@@ -117,46 +115,43 @@ class TOMLMigrator:if self._is_line_inside_multiline_string(content, match.start()):# Return the original text unchangedreturn match.group(0)- else:- # Apply the normal replacement: Concept = -> definition =- return f"{match.group(1)}definition{match.group(3)}{match.group(4)}"+ # Apply the normal replacement: Concept = -> definition =+ return f"{match.group(1)}definition{match.group(3)}{match.group(4)}"def pipe_replacement_function(match: Match[str]) -> str:# Check if this match is inside a multiline stringif self._is_line_inside_multiline_string(content, match.start()):# Return the original text unchangedreturn match.group(0)- else:- # Apply pipe replacement: PipeClassName = "desc" -> type = "PipeClassName"\ndefinition = "desc"- leading_whitespace = match.group(1)- pipe_class_name = match.group(2)- definition_value = match.group(4)- trailing_whitespace = match.group(5)+ # Apply pipe replacement: PipeClassName = "desc" -> type = "PipeClassName"\ndefinition = "desc"+ leading_whitespace = match.group(1)+ pipe_class_name = match.group(2)+ definition_value = match.group(4)+ trailing_whitespace = match.group(5)- type_line = f'{leading_whitespace}type = "{pipe_class_name}"'- definition_line = f"{leading_whitespace}definition = {definition_value}"+ type_line = f'{leading_whitespace}type = "{pipe_class_name}"'+ definition_line = f"{leading_whitespace}definition = {definition_value}"- return f"{type_line}\n{definition_line}{trailing_whitespace}"+ return f"{type_line}\n{definition_line}{trailing_whitespace}"# Apply concept migrations firstmigrated_content = self.concept_pattern.sub(concept_replacement_function, content)# Then apply pipe migrations- migrated_content = self.pipe_pattern.sub(pipe_replacement_function, migrated_content)-- return migrated_content+ return self.pipe_pattern.sub(pipe_replacement_function, migrated_content)def migrate_file(self, file_path: Path, create_backup: bool = True) -> int:- """- Migrate a single file from old to new syntax.+ """Migrate a single file from old to new syntax.Returns:Number of changes made in the file+"""try:- with open(file_path, "r", encoding="utf-8") as f:+ with open(file_path, encoding="utf-8") as f:original_content = f.read()except Exception as e:- raise IOError(f"Failed to read file {file_path}: {e}")+ msg = f"Failed to read file {file_path}: {e}"+ raise OSError(msg) from eif not self.needs_migration(original_content):return 0@@ -175,19 +170,20 @@ class TOMLMigrator:try:shutil.copy2(file_path, backup_path)except Exception as e:- raise IOError(f"Failed to create backup for {file_path}: {e}")+ msg = f"Failed to create backup for {file_path}: {e}"+ raise OSError(msg) from etry:with open(file_path, "w", encoding="utf-8") as f:f.write(migrated_content)except Exception as e:- raise IOError(f"Failed to write migrated content to {file_path}: {e}")+ msg = f"Failed to write migrated content to {file_path}: {e}"+ raise OSError(msg) from ereturn changes_countdef migrate_directory(self, directory: Path, create_backups: bool = True, dry_run: bool = False) -> MigrationResult:- """- Migrate all TOML files in a directory.+ """Migrate all TOML files in a directory.Args:directory: Directory containing TOML files@@ -196,17 +192,18 @@ class TOMLMigrator:Returns:MigrationResult with statistics and details+"""toml_files = self.find_toml_files(directory)files_processed = len(toml_files)files_modified = 0total_changes = 0- modified_files: List[Path] = []- errors: List[str] = []+ modified_files: list[Path] = []+ errors: list[str] = []for toml_file in toml_files:try:- with open(toml_file, "r", encoding="utf-8") as f:+ with open(toml_file, encoding="utf-8") as f:content = f.read()if not self.needs_migration(content):@@ -244,8 +241,7 @@ class TOMLMigrator:def migrate_concept_syntax(directory: Path, create_backups: bool = True, dry_run: bool = False) -> MigrationResult:- """- Convenience function to migrate TOML files from Concept = to definition = syntax.+ """Convenience function to migrate TOML files from Concept = to definition = syntax.This function accepts either a directory (the historical behavior) or a path to a single TOML file.@@ -256,6 +252,7 @@ def migrate_concept_syntax(directory: Path, create_backups: bool = True, dry_runReturns:MigrationResult with migration statistics+"""migrator = TOMLMigrator()@@ -270,7 +267,7 @@ def migrate_concept_syntax(directory: Path, create_backups: bool = True, dry_runreturn resulttry:- with open(input_path, "r", encoding="utf-8") as f:+ with open(input_path, encoding="utf-8") as f:content = f.read()except Exception as e:result.errors.append(f"Failed to read file {input_path}: {e}")diff --git a/pipelex/migration/migration_result.py b/pipelex/migration/migration_result.pyindex 935f82806..184000741 100644--- a/pipelex/migration/migration_result.py+++ b/pipelex/migration/migration_result.py@@ -1,10 +1,11 @@"""Shared migration result types."""from pathlib import Path-from typing import Listfrom pydantic import BaseModel, Field+from pipelex.tools.typing.pydantic_utils import empty_list_factory_of+class MigrationResult(BaseModel):"""Result of migration operation."""@@ -12,5 +13,5 @@ class MigrationResult(BaseModel):files_processed: int = 0files_modified: int = 0total_changes: int = 0- modified_files: List[Path] = Field(default_factory=list)- errors: List[str] = Field(default_factory=list)+ modified_files: list[Path] = Field(default_factory=empty_list_factory_of(Path))+ errors: list[str] = Field(default_factory=list)diff --git a/pipelex/observer/local_observer.py b/pipelex/observer/local_observer.pyindex 831ce50aa..506b77922 100644--- a/pipelex/observer/local_observer.py+++ b/pipelex/observer/local_observer.py@@ -1,5 +1,4 @@import os-from typing import Optionalimport kajsonfrom typing_extensions import override@@ -9,7 +8,7 @@ from pipelex.observer.observer_protocol import ObserverProtocol, PayloadTypeclass LocalObserver(ObserverProtocol):- def __init__(self, storage_dir: Optional[str] = None) -> None:+ def __init__(self, storage_dir: str | None = None) -> None:self.storage_dir = storage_dir or get_config().pipelex.observer_config.observer_diros.makedirs(self.storage_dir, exist_ok=True)diff --git a/pipelex/observer/observer_protocol.py b/pipelex/observer/observer_protocol.pyindex b0bd33c85..293b4daea 100644--- a/pipelex/observer/observer_protocol.py+++ b/pipelex/observer/observer_protocol.py@@ -1,6 +1,6 @@-from typing import Any, Dict, Protocol+from typing import Any, Protocol-PayloadType = Dict[str, Any]+PayloadType = dict[str, Any]class ObserverProtocol(Protocol):diff --git a/pipelex/pipe_controllers/batch/__init__.py b/pipelex/pipe_controllers/batch/__init__.pynew file mode 100644index 000000000..e69de29bbdiff --git a/pipelex/pipe_controllers/batch/pipe_batch.py b/pipelex/pipe_controllers/batch/pipe_batch.pyindex 4ca87ad49..5bdb437c5 100644--- a/pipelex/pipe_controllers/batch/pipe_batch.py+++ b/pipelex/pipe_controllers/batch/pipe_batch.py@@ -1,9 +1,9 @@import asyncio-from typing import Any, Coroutine, List, Literal, Optional, Set, cast+from typing import TYPE_CHECKING, Any, Literal, castimport shortuuidfrom pydantic import model_validator-from typing_extensions import Self, override+from typing_extensions import overridefrom pipelex import logfrom pipelex.config import get_config@@ -11,7 +11,6 @@ from pipelex.core.memory.working_memory import MAIN_STUFF_NAME, WorkingMemoryfrom pipelex.core.pipes.pipe_input import PipeInputSpecfrom pipelex.core.pipes.pipe_output import PipeOutputfrom pipelex.core.pipes.pipe_run_params import BatchParams, PipeRunMode, PipeRunParams-from pipelex.core.stuffs.stuff import Stufffrom pipelex.core.stuffs.stuff_content import ListContent, StuffContentfrom pipelex.core.stuffs.stuff_factory import StuffFactoryfrom pipelex.exceptions import (@@ -22,17 +21,23 @@ from pipelex.exceptions import (from pipelex.hub import get_pipeline_tracker, get_required_pipefrom pipelex.pipe_controllers.pipe_controller import PipeControllerfrom pipelex.pipeline.job_metadata import JobMetadata+from pipelex.types import Self++if TYPE_CHECKING:+ from collections.abc import Coroutine++ from pipelex.core.stuffs.stuff import Stuffclass PipeBatch(PipeController):type: Literal["PipeBatch"] = "PipeBatch"branch_pipe_code: str- batch_params: Optional[BatchParams] = None+ batch_params: BatchParams | None = None@override- def pipe_dependencies(self) -> Set[str]:- return set([self.branch_pipe_code])+ def pipe_dependencies(self) -> set[str]:+ return {self.branch_pipe_code}@overridedef validate_output(self):@@ -51,13 +56,14 @@ class PipeBatch(PipeController):# Now check that the required vairables ARE in the inputs of the piperequired_variables = self.required_variables()for variable_name in required_variables:- if variable_name not in self.inputs.root.keys():- raise PipeInputError(f"Input '{variable_name}' of pipe '{self.code}' is not in the inputs of the pipe '{self.branch_pipe_code}'")+ if variable_name not in self.inputs.root:+ msg = f"Input '{variable_name}' of pipe '{self.code}' is not in the inputs of the pipe '{self.branch_pipe_code}'"+ raise PipeInputError(msg)return self@override- def required_variables(self) -> Set[str]:- required_variables: Set[str] = set()+ def required_variables(self) -> set[str]:+ required_variables: set[str] = set()# 1. Check that the inputs of the pipe branch_pipe_code are in the inputs of the pipepipe = get_required_pipe(pipe_code=self.branch_pipe_code)for variable_name, _ in pipe.inputs.items:@@ -68,7 +74,7 @@ class PipeBatch(PipeController):return required_variables@override- def needed_inputs(self, visited_pipes: Optional[Set[str]] = None) -> PipeInputSpec:+ def needed_inputs(self, visited_pipes: set[str] | None = None) -> PipeInputSpec:return self.inputsasync def _run_batch_pipe(@@ -76,7 +82,7 @@ class PipeBatch(PipeController):job_metadata: JobMetadata,working_memory: WorkingMemory,pipe_run_params: PipeRunParams,- output_name: Optional[str] = None,+ output_name: str | None = None,) -> PipeOutput:"""Common logic for running or dry-running a pipe in batch mode."""batch_params = pipe_run_params.batch_params or self.batch_params or BatchParams.make_default()@@ -84,9 +90,8 @@ class PipeBatch(PipeController):try:input_item_concept_code = self.inputs.get_required_input_requirement(input_item_stuff_name)except PipeInputNotFoundError as exc:- raise PipeInputError(- f"Batch input item stuff named '{input_item_stuff_name}' is not in this PipeBatch '{self.code}' input spec: {self.inputs}"- ) from exc+ msg = f"Batch input item stuff named '{input_item_stuff_name}' is not in this PipeBatch '{self.code}' input spec: {self.inputs}"+ raise PipeInputError(msg) from excif pipe_run_params.final_stuff_code:method_name = "dry_run_pipe" if pipe_run_params.run_mode == PipeRunMode.DRY else "_run_controller_pipe"@@ -97,26 +102,27 @@ class PipeBatch(PipeController):try:input_stuff = working_memory.get_stuff(batch_params.input_list_stuff_name)except WorkingMemoryStuffNotFoundError as exc:- raise PipeInputError(+ msg = (f"Input list stuff '{batch_params.input_list_stuff_name}' required by this PipeBatch '{self.code}' not found in working memory: {exc}"- ) from exc+ )+ raise PipeInputError(msg) from exc+input_stuff_code = input_stuff.stuff_codeinput_content = input_stuff.contentif not isinstance(input_content, ListContent):- raise PipeInputError(- f"Input of PipeBatch must be ListContent, got {input_stuff.stuff_name or 'unnamed'} = {type(input_content)}. stuff: {input_stuff}"- )- input_content = cast(ListContent[StuffContent], input_content)+ msg = f"Input of PipeBatch must be ListContent, got {input_stuff.stuff_name or 'unnamed'} = {type(input_content)}. stuff: {input_stuff}"+ raise PipeInputError(msg)+ input_content = cast("ListContent[StuffContent]", input_content)# TODO: Make commented code work when inputing images named "a.b.c"sub_pipe = get_required_pipe(pipe_code=self.branch_pipe_code)nb_history_items_limit = get_config().pipelex.tracker_config.applied_nb_items_limitbatch_output_stuff_code = shortuuid.uuid()- tasks: List[Coroutine[Any, Any, PipeOutput]] = []- item_stuffs: List[Stuff] = []- required_stuff_lists: List[List[Stuff]] = []- branch_output_item_codes: List[str] = []+ tasks: list[Coroutine[Any, Any, PipeOutput]] = []+ item_stuffs: list[Stuff] = []+ required_stuff_lists: list[list[Stuff]] = []+ branch_output_item_codes: list[str] = []for branch_index, item in enumerate(input_content.items):branch_output_item_code = f"{batch_output_stuff_code}-branch-{branch_index}"@@ -160,10 +166,10 @@ class PipeBatch(PipeController):pipe_outputs = await asyncio.gather(*tasks)- output_items: List[StuffContent] = []- output_stuffs: List[Stuff] = []+ output_items: list[StuffContent] = []+ output_stuffs: list[Stuff] = []output_stuff_code = shortuuid.uuid()[:5]- for branch_index, pipe_output in enumerate(pipe_outputs):+ for pipe_output in pipe_outputs:branch_output_stuff = pipe_output.main_stuffoutput_stuffs.append(branch_output_stuff)output_items.append(branch_output_stuff.content)@@ -181,7 +187,7 @@ class PipeBatch(PipeController):required_stuff_list,item_input_stuff,item_output_stuff,- ) in enumerate(zip(required_stuff_lists, item_stuffs, output_stuffs)):+ ) in enumerate(zip(required_stuff_lists, item_stuffs, output_stuffs, strict=False)):get_pipeline_tracker().add_batch_step(from_stuff=input_stuff,to_stuff=item_input_stuff,@@ -225,7 +231,7 @@ class PipeBatch(PipeController):job_metadata: JobMetadata,working_memory: WorkingMemory,pipe_run_params: PipeRunParams,- output_name: Optional[str] = None,+ output_name: str | None = None,) -> PipeOutput:"""Run a pipe in batch mode for each item in the input list."""return await self._run_batch_pipe(@@ -241,7 +247,7 @@ class PipeBatch(PipeController):job_metadata: JobMetadata,working_memory: WorkingMemory,pipe_run_params: PipeRunParams,- output_name: Optional[str] = None,+ output_name: str | None = None,) -> PipeOutput:"""Dry run a pipe in batch mode for each item in the input list."""return await self._run_batch_pipe(diff --git a/pipelex/pipe_controllers/batch/pipe_batch_blueprint.py b/pipelex/pipe_controllers/batch/pipe_batch_blueprint.pyindex 80707fdef..7f0655c0f 100644--- a/pipelex/pipe_controllers/batch/pipe_batch_blueprint.py+++ b/pipelex/pipe_controllers/batch/pipe_batch_blueprint.py@@ -1,4 +1,4 @@-from typing import Literal, Optional+from typing import Literalfrom pipelex.core.pipes.pipe_blueprint import PipeBlueprint@@ -7,5 +7,5 @@ class PipeBatchBlueprint(PipeBlueprint):type: Literal["PipeBatch"] = "PipeBatch"category: Literal["PipeController"] = "PipeController"branch_pipe_code: str- input_list_name: Optional[str] = None- input_item_name: Optional[str] = None+ input_list_name: str | None = None+ input_item_name: str | None = Nonediff --git a/pipelex/pipe_controllers/batch/pipe_batch_factory.py b/pipelex/pipe_controllers/batch/pipe_batch_factory.pyindex 91a2f9e6e..3eeb04bb2 100644--- a/pipelex/pipe_controllers/batch/pipe_batch_factory.py+++ b/pipelex/pipe_controllers/batch/pipe_batch_factory.py@@ -1,5 +1,3 @@-from typing import List, Optional-from typing_extensions import overridefrom pipelex.core.concepts.concept_factory import ConceptFactory@@ -19,7 +17,7 @@ class PipeBatchFactory(PipeFactoryProtocol[PipeBatchBlueprint, PipeBatch]):domain: str,pipe_code: str,blueprint: PipeBatchBlueprint,- concept_codes_from_the_same_domain: Optional[List[str]] = None,+ concept_codes_from_the_same_domain: list[str] | None = None,) -> PipeBatch:output_domain_and_code = ConceptFactory.make_domain_and_concept_code_from_concept_string_or_code(domain=domain,@@ -31,12 +29,15 @@ class PipeBatchFactory(PipeFactoryProtocol[PipeBatchBlueprint, PipeBatch]):code=pipe_code,definition=blueprint.definition,inputs=PipeInputSpecFactory.make_from_blueprint(- domain=domain, blueprint=blueprint.inputs or {}, concept_codes_from_the_same_domain=concept_codes_from_the_same_domain+ domain=domain,+ blueprint=blueprint.inputs or {},+ concept_codes_from_the_same_domain=concept_codes_from_the_same_domain,),output=get_concept_provider().get_required_concept(concept_string=ConceptFactory.construct_concept_string_with_domain(- domain=output_domain_and_code.domain, concept_code=output_domain_and_code.concept_code- )+ domain=output_domain_and_code.domain,+ concept_code=output_domain_and_code.concept_code,+ ),),branch_pipe_code=blueprint.branch_pipe_code,batch_params=BatchParams.make_optional_batch_params(diff --git a/pipelex/pipe_controllers/condition/__init__.py b/pipelex/pipe_controllers/condition/__init__.pynew file mode 100644index 000000000..e69de29bbdiff --git a/pipelex/pipe_controllers/condition/pipe_condition.py b/pipelex/pipe_controllers/condition/pipe_condition.pyindex f53a8940e..793a84c16 100644--- a/pipelex/pipe_controllers/condition/pipe_condition.py+++ b/pipelex/pipe_controllers/condition/pipe_condition.py@@ -1,8 +1,8 @@-from typing import Dict, List, Literal, Optional, Set, Union, cast+from typing import TYPE_CHECKING, Literal, castimport shortuuidfrom pydantic import field_validator, model_validator-from typing_extensions import Self, override+from typing_extensions import overridefrom pipelex import logfrom pipelex.config import StaticValidationReaction, get_config@@ -34,24 +34,27 @@ from pipelex.pipe_operators.jinja2.pipe_jinja2_factory import PipeJinja2Factoryfrom pipelex.pipe_works.pipe_job_factory import PipeJobFactoryfrom pipelex.pipeline.job_metadata import JobCategory, JobMetadatafrom pipelex.tools.typing.validation_utils import has_exactly_one_among_attributes_from_list+from pipelex.types import Self++if TYPE_CHECKING:+ from pipelex.pipe_operators.jinja2.pipe_jinja2 import PipeJinja2Outputclass PipeCondition(PipeController):type: Literal["PipeCondition"] = "PipeCondition"- expression_template: Optional[str] = None- expression: Optional[str] = None+ expression_template: str | None = None+ expression: str | None = None# TODO: rething this pipe_map.- pipe_map: List[PipeConditionPipeMap]- default_pipe_code: Optional[str] = None- add_alias_from_expression_to: Optional[str] = None+ pipe_map: list[PipeConditionPipeMap]+ default_pipe_code: str | None = None+ add_alias_from_expression_to: str | None = None########################################################################################## Validation#########################################################################################@overridedef validate_output(self):- """- Validate the output for the pipe condition.+ """Validate the output for the pipe condition.The output of the pipe condition should match the output of all the conditional pipes, and the default pipe."""# This pipe CONTINUE enables to leave a PipeCondition and continue the sequence.@@ -82,24 +85,23 @@ class PipeCondition(PipeController):@field_validator("pipe_map")@classmethod- def validate_pipe_map(cls, pipe_map: List[PipeConditionPipeMap]) -> List[PipeConditionPipeMap]:+ def validate_pipe_map(cls, pipe_map: list[PipeConditionPipeMap]) -> list[PipeConditionPipeMap]:# Validate that the expressions and pipe_code are UNIQUEexpression_results = [pipe_condition_pipe_map.expression_result for pipe_condition_pipe_map in pipe_map]pipe_codes = [pipe_condition_pipe_map.pipe_code for pipe_condition_pipe_map in pipe_map]if len(expression_results) != len(set(expression_results)):- raise PipeDefinitionError(- f"PipeCondition '{cls.code}' must have a unique expression result for each pipe in pipe_map in pipe_map: {pipe_map}"- )+ msg = f"PipeCondition '{cls.code}' must have a unique expression result for each pipe in pipe_map in pipe_map: {pipe_map}"+ raise PipeDefinitionError(msg)if len(pipe_codes) != len(set(pipe_codes)):- raise PipeDefinitionError(- f"PipeCondition '{cls.code}' must have a unique pipe code for each expression result in pipe_map in pipe_map: {pipe_map}"- )+ msg = f"PipeCondition '{cls.code}' must have a unique pipe code for each expression result in pipe_map in pipe_map: {pipe_map}"+ raise PipeDefinitionError(msg)return pipe_map@model_validator(mode="after")def validate_expression(self) -> Self:if not has_exactly_one_among_attributes_from_list(self, attributes_list=["expression_template", "expression"]):- raise PipeDefinitionError("PipeCondition should have exactly one of expression_template or expression")+ msg = "PipeCondition should have exactly one of expression_template or expression"+ raise PipeDefinitionError(msg)return selfdef _make_pipe_condition_details(self, evaluated_expression: str, chosen_pipe_code: str) -> PipeConditionDetails:@@ -116,18 +118,18 @@ class PipeCondition(PipeController):def applied_expression_template(self) -> str:if self.expression_template:return self.expression_template- elif self.expression:+ if self.expression:return "{{ " + self.expression + " }}"- else:- raise PipeExecutionError("No expression or expression_template provided")+ msg = "No expression or expression_template provided"+ raise PipeExecutionError(msg)########################################################################################## Inputs#########################################################################################@override- def required_variables(self) -> Set[str]:- required_variables: Set[str] = set()+ def required_variables(self) -> set[str]:+ required_variables: set[str] = set()# Variables from the expression/expression_templatepipe_jinja2 = PipeJinja2Factory.make_pipe_jinja2_from_template_str(domain=self.domain,@@ -144,11 +146,12 @@ class PipeCondition(PipeController):def _validate_required_variables(self) -> Self:for required_variable_name in self.required_variables():if required_variable_name not in self.inputs.variables:- raise PipeDefinitionError(f"Required variable '{required_variable_name}' is not in the inputs of pipe {self.code}")+ msg = f"Required variable '{required_variable_name}' is not in the inputs of pipe {self.code}"+ raise PipeDefinitionError(msg)return self@override- def needed_inputs(self, visited_pipes: Optional[Set[str]] = None) -> PipeInputSpec:+ def needed_inputs(self, visited_pipes: set[str] | None = None) -> PipeInputSpec:if visited_pipes is None:visited_pipes = set()@@ -194,15 +197,14 @@ class PipeCondition(PipeController):@model_validator(mode="after")def validate_inputs(self) -> Self:if not self.pipe_map:- raise ValueError(f"Pipe'{self.code}'(PipeCondition) must have at least one mapping in pipe_map")+ msg = f"Pipe'{self.code}'(PipeCondition) must have at least one mapping in pipe_map"+ raise ValueError(msg)# Skip validation during model creation - it will be done in validate_with_libraries()return selfdef _validate_inputs(self):- """- Validate that the inputs declared for this PipeCondition match what is actually needed.- """+ """Validate that the inputs declared for this PipeCondition match what is actually needed."""static_validation_config = get_config().pipelex.static_validation_configdefault_reaction = static_validation_config.default_reactionreactions = static_validation_config.reactions@@ -245,15 +247,14 @@ class PipeCondition(PipeController):@overridedef validate_with_libraries(self):- """- Perform full validation after all libraries are loaded.+ """Perform full validation after all libraries are loaded.This is called after all pipes and concepts are available."""self._validate_inputs()self._validate_required_variables()@override- def pipe_dependencies(self) -> Set[str]:+ def pipe_dependencies(self) -> set[str]:pipe_codes = [pipe_condition_pipe_map.pipe_codefor pipe_condition_pipe_map in self.pipe_map@@ -269,7 +270,7 @@ class PipeCondition(PipeController):job_metadata: JobMetadata,working_memory: WorkingMemory,pipe_run_params: PipeRunParams,- output_name: Optional[str] = None,+ output_name: str | None = None,) -> PipeOutput:log.dev(f"{self.class_name} generating a '{self.output.code}'")@@ -277,7 +278,7 @@ class PipeCondition(PipeController):# pipe_run_params.push_pipe_code(pipe_code=pipe_code)# Convert PipeInputSpec to blueprint format- inputs_blueprint: Dict[str, Union[str, InputRequirementBlueprint]] = {}+ inputs_blueprint: dict[str, str | InputRequirementBlueprint] = {}for var_name, requirement in self.inputs.root.items():inputs_blueprint[var_name] = InputRequirementBlueprint(concept=requirement.concept.concept_string,@@ -300,7 +301,7 @@ class PipeCondition(PipeController):jinja2_job_metadata = job_metadata.copy_with_update(updated_metadata=JobMetadata(job_category=JobCategory.JINJA2_JOB,- )+ ),)log.debug(f"Jinja2 expression: {self.applied_expression_template}")# evaluated_expression = (@@ -312,7 +313,7 @@ class PipeCondition(PipeController):# ).rendered_text.strip()# TODO: restore the possibility above, without need to explicitly cast the outputpipe_jinja2_output = cast(- PipeJinja2Output,+ "PipeJinja2Output",await pipe_jinja2.run_pipe(job_metadata=jinja2_job_metadata,working_memory=working_memory,@@ -357,14 +358,15 @@ class PipeCondition(PipeController):)required_variables = pipe_jinja2.required_variables()log.debug(required_variables, title=f"Required variables for PipeCondition '{self.code}'")- required_stuff_names = set([required_variable for required_variable in required_variables if not required_variable.startswith("_")])+ required_stuff_names = {required_variable for required_variable in required_variables if not required_variable.startswith("_")}try:required_stuffs = working_memory.get_stuffs(names=required_stuff_names)except WorkingMemoryStuffNotFoundError as exc:- pipe_condition_path = pipe_run_params.pipe_layers + [self.code]+ pipe_condition_path = [*pipe_run_params.pipe_layers, self.code]pipe_condition_path_str = ".".join(pipe_condition_path)error_details = f"PipeCondition '{pipe_condition_path_str}', required_variables: {required_variables}, missing: '{exc.variable_name}'"- raise PipeInputError(f"Some required stuff(s) not found: {error_details}") from exc+ msg = f"Some required stuff(s) not found: {error_details}"+ raise PipeInputError(msg) from excfor required_stuff in required_stuffs:get_pipeline_tracker().add_condition_step(@@ -399,17 +401,16 @@ class PipeCondition(PipeController):job_metadata: JobMetadata,working_memory: WorkingMemory,pipe_run_params: PipeRunParams,- output_name: Optional[str] = None,+ output_name: str | None = None,) -> PipeOutput:- """- Dry run implementation for PipeCondition.+ """Dry run implementation for PipeCondition.Validates that all required inputs are present, expression is valid, and target pipes exist."""log.debug(f"PipeCondition: dry run controller pipe: {self.code}")# 1. Validate that all required inputs are present in the working memoryneeded_inputs = self.needed_inputs()- missing_input_names: List[str] = []+ missing_input_names: list[str] = []for named_input_requirement in needed_inputs.named_input_requirements:if not working_memory.get_optional_stuff(named_input_requirement.variable_name):@@ -442,14 +443,14 @@ class PipeCondition(PipeController):message=error_msg,missing_inputs=[],pipe_code=self.code,- )+ ) from exc# 3. Validate that all pipes in the pipe_map exist- all_pipe_codes = set([pipe_condition_pipe_map.pipe_code for pipe_condition_pipe_map in self.pipe_map])+ all_pipe_codes = {pipe_condition_pipe_map.pipe_code for pipe_condition_pipe_map in self.pipe_map}if self.default_pipe_code:all_pipe_codes.add(self.default_pipe_code)- missing_pipes: List[str] = []+ missing_pipes: list[str] = []for pipe_code in all_pipe_codes:try:get_required_pipe(pipe_code=pipe_code)diff --git a/pipelex/pipe_controllers/condition/pipe_condition_blueprint.py b/pipelex/pipe_controllers/condition/pipe_condition_blueprint.pyindex 6898cb51c..fa5ed48a4 100644--- a/pipelex/pipe_controllers/condition/pipe_condition_blueprint.py+++ b/pipelex/pipe_controllers/condition/pipe_condition_blueprint.py@@ -1,10 +1,10 @@-from typing import Dict, Literal, Optional+from typing import Literalfrom pydantic import Field, RootModelfrom pipelex.core.pipes.pipe_blueprint import PipeBlueprint-PipeConditionPipeMapRoot = Dict[str, str]+PipeConditionPipeMapRoot = dict[str, str]class PipeConditionPipeMapBlueprint(RootModel[PipeConditionPipeMapRoot]):@@ -14,8 +14,8 @@ class PipeConditionPipeMapBlueprint(RootModel[PipeConditionPipeMapRoot]):class PipeConditionBlueprint(PipeBlueprint):type: Literal["PipeCondition"] = "PipeCondition"category: Literal["PipeController"] = "PipeController"- expression_template: Optional[str] = None- expression: Optional[str] = None+ expression_template: str | None = None+ expression: str | None = Nonepipe_map: PipeConditionPipeMapBlueprint = Field(default_factory=PipeConditionPipeMapBlueprint)- default_pipe_code: Optional[str] = None- add_alias_from_expression_to: Optional[str] = None+ default_pipe_code: str | None = None+ add_alias_from_expression_to: str | None = Nonediff --git a/pipelex/pipe_controllers/condition/pipe_condition_details.py b/pipelex/pipe_controllers/condition/pipe_condition_details.pyindex f88084956..c97f513b2 100644--- a/pipelex/pipe_controllers/condition/pipe_condition_details.py+++ b/pipelex/pipe_controllers/condition/pipe_condition_details.py@@ -1,5 +1,3 @@-from typing import List, Optional-from pydantic import BaseModel@@ -11,7 +9,7 @@ class PipeConditionPipeMap(BaseModel):class PipeConditionDetails(BaseModel):code: strtest_expression: str- pipe_map: List[PipeConditionPipeMap]- default_pipe_code: Optional[str] = None+ pipe_map: list[PipeConditionPipeMap]+ default_pipe_code: str | None = Noneevaluated_expression: strchosen_pipe_code: strdiff --git a/pipelex/pipe_controllers/condition/pipe_condition_factory.py b/pipelex/pipe_controllers/condition/pipe_condition_factory.pyindex fb7a8c65f..215c7c978 100644--- a/pipelex/pipe_controllers/condition/pipe_condition_factory.py+++ b/pipelex/pipe_controllers/condition/pipe_condition_factory.py@@ -1,5 +1,3 @@-from typing import List, Optional-from typing_extensions import overridefrom pipelex.core.concepts.concept_factory import ConceptFactory@@ -13,7 +11,7 @@ from pipelex.pipe_controllers.condition.pipe_condition_details import PipeConditclass PipeConditionFactory(PipeFactoryProtocol[PipeConditionBlueprint, PipeCondition]):@classmethod- def make_pipe_condition_pipe_map(cls, pipe_map: PipeConditionPipeMapBlueprint) -> List[PipeConditionPipeMap]:+ def make_pipe_condition_pipe_map(cls, pipe_map: PipeConditionPipeMapBlueprint) -> list[PipeConditionPipeMap]:return [PipeConditionPipeMap(expression_result=expression_result, pipe_code=pipe_code) for expression_result, pipe_code in pipe_map.root.items()]@@ -25,7 +23,7 @@ class PipeConditionFactory(PipeFactoryProtocol[PipeConditionBlueprint, PipeCondidomain: str,pipe_code: str,blueprint: PipeConditionBlueprint,- concept_codes_from_the_same_domain: Optional[List[str]] = None,+ concept_codes_from_the_same_domain: list[str] | None = None,) -> PipeCondition:output_domain_and_code = ConceptFactory.make_domain_and_concept_code_from_concept_string_or_code(domain=domain,@@ -37,12 +35,15 @@ class PipeConditionFactory(PipeFactoryProtocol[PipeConditionBlueprint, PipeCondicode=pipe_code,definition=blueprint.definition,inputs=PipeInputSpecFactory.make_from_blueprint(- domain=domain, blueprint=blueprint.inputs or {}, concept_codes_from_the_same_domain=concept_codes_from_the_same_domain+ domain=domain,+ blueprint=blueprint.inputs or {},+ concept_codes_from_the_same_domain=concept_codes_from_the_same_domain,),output=get_concept_provider().get_required_concept(concept_string=ConceptFactory.construct_concept_string_with_domain(- domain=output_domain_and_code.domain, concept_code=output_domain_and_code.concept_code- )+ domain=output_domain_and_code.domain,+ concept_code=output_domain_and_code.concept_code,+ ),),expression_template=blueprint.expression_template,expression=blueprint.expression,diff --git a/pipelex/pipe_controllers/parallel/__init__.py b/pipelex/pipe_controllers/parallel/__init__.pynew file mode 100644index 000000000..e69de29bbdiff --git a/pipelex/pipe_controllers/parallel/pipe_parallel.py b/pipelex/pipe_controllers/parallel/pipe_parallel.pyindex 933152bcb..7afd7d605 100644--- a/pipelex/pipe_controllers/parallel/pipe_parallel.py+++ b/pipelex/pipe_controllers/parallel/pipe_parallel.py@@ -1,8 +1,8 @@import asyncio-from typing import Any, Coroutine, Dict, List, Literal, Optional, Set+from typing import TYPE_CHECKING, Any, Literalfrom pydantic import model_validator-from typing_extensions import Self, override+from typing_extensions import overridefrom pipelex import logfrom pipelex.config import StaticValidationReaction, get_config@@ -12,29 +12,34 @@ from pipelex.core.pipes.pipe_input import PipeInputSpecfrom pipelex.core.pipes.pipe_input_factory import PipeInputSpecFactoryfrom pipelex.core.pipes.pipe_output import PipeOutputfrom pipelex.core.pipes.pipe_run_params import PipeRunMode, PipeRunParams-from pipelex.core.stuffs.stuff import Stuff-from pipelex.core.stuffs.stuff_content import StuffContentfrom pipelex.core.stuffs.stuff_factory import StuffFactoryfrom pipelex.exceptions import DryRunError, PipeDefinitionError, PipeRunParamsError, StaticValidationError, StaticValidationErrorTypefrom pipelex.hub import get_pipeline_tracker, get_required_pipefrom pipelex.pipe_controllers.pipe_controller import PipeControllerfrom pipelex.pipe_controllers.sub_pipe import SubPipefrom pipelex.pipeline.job_metadata import JobMetadata+from pipelex.types import Self++if TYPE_CHECKING:+ from collections.abc import Coroutine++ from pipelex.core.stuffs.stuff import Stuff+ from pipelex.core.stuffs.stuff_content import StuffContentclass PipeParallel(PipeController):type: Literal["PipeParallel"] = "PipeParallel"- parallel_sub_pipes: List[SubPipe]+ parallel_sub_pipes: list[SubPipe]add_each_output: bool- combined_output: Optional[Concept]+ combined_output: Concept | None@override- def required_variables(self) -> Set[str]:+ def required_variables(self) -> set[str]:return set()@override- def needed_inputs(self, visited_pipes: Optional[Set[str]] = None) -> PipeInputSpec:+ def needed_inputs(self, visited_pipes: set[str] | None = None) -> PipeInputSpec:if visited_pipes is None:visited_pipes = set()@@ -69,7 +74,8 @@ class PipeParallel(PipeController):def validate_inputs(self) -> Self:# Validate that either add_each_output or combined_output is setif not self.add_each_output and not self.combined_output:- raise PipeDefinitionError(f"PipeParallel'{self.code}'requires either add_each_output or combined_output to be set")+ msg = f"PipeParallel'{self.code}'requires either add_each_output or combined_output to be set"+ raise PipeDefinitionError(msg)return self@@ -78,9 +84,7 @@ class PipeParallel(PipeController):passdef _validate_inputs(self):- """- Validate that the inputs declared for this PipeParallel match what is actually needed.- """+ """Validate that the inputs declared for this PipeParallel match what is actually needed."""static_validation_config = get_config().pipelex.static_validation_configdefault_reaction = static_validation_config.default_reactionreactions = static_validation_config.reactions@@ -123,15 +127,14 @@ class PipeParallel(PipeController):@overridedef validate_with_libraries(self):- """- Perform full validation after all libraries are loaded.+ """Perform full validation after all libraries are loaded.This is called after all pipes and concepts are available."""self._validate_inputs()@override- def pipe_dependencies(self) -> Set[str]:- return set(sub_pipe.pipe_code for sub_pipe in self.parallel_sub_pipes)+ def pipe_dependencies(self) -> set[str]:+ return {sub_pipe.pipe_code for sub_pipe in self.parallel_sub_pipes}@overrideasync def _run_controller_pipe(@@ -139,19 +142,17 @@ class PipeParallel(PipeController):job_metadata: JobMetadata,working_memory: WorkingMemory,pipe_run_params: PipeRunParams,- output_name: Optional[str] = None,+ output_name: str | None = None,) -> PipeOutput:- """- Run a list of pipes in parallel.- """-+ """Run a list of pipes in parallel."""if not self.add_each_output and not self.combined_output:- raise PipeDefinitionError("PipeParallel requires either add_each_output or combined_output to be set")+ msg = "PipeParallel requires either add_each_output or combined_output to be set"+ raise PipeDefinitionError(msg)if pipe_run_params.final_stuff_code:log.debug(f"PipeBatch.run_pipe() final_stuff_code: {pipe_run_params.final_stuff_code}")pipe_run_params.final_stuff_code = None- tasks: List[Coroutine[Any, Any, PipeOutput]] = []+ tasks: list[Coroutine[Any, Any, PipeOutput]] = []for sub_pipe in self.parallel_sub_pipes:tasks.append(@@ -160,35 +161,34 @@ class PipeParallel(PipeController):job_metadata=job_metadata,working_memory=working_memory.make_deep_copy(),sub_pipe_run_params=pipe_run_params.make_deep_copy(),- )+ ),)pipe_outputs = await asyncio.gather(*tasks)- output_stuff_content_items: List[StuffContent] = []- output_stuffs: Dict[str, Stuff] = {}- output_stuff_contents: Dict[str, StuffContent] = {}+ output_stuff_content_items: list[StuffContent] = []+ output_stuffs: dict[str, Stuff] = {}+ output_stuff_contents: dict[str, StuffContent] = {}# TODO: refactor this to use a specific function for this that can also be used in dry runfor output_index, pipe_output in enumerate(pipe_outputs):output_stuff = pipe_output.main_stuffsub_pipe_output_name = self.parallel_sub_pipes[output_index].output_nameif not sub_pipe_output_name:- raise PipeDefinitionError("PipeParallel requires a result specified for each parallel sub pipe")+ msg = "PipeParallel requires a result specified for each parallel sub pipe"+ raise PipeDefinitionError(msg)if self.add_each_output:working_memory.add_new_stuff(name=sub_pipe_output_name, stuff=output_stuff)output_stuff_content_items.append(output_stuff.content)if sub_pipe_output_name in output_stuffs:# TODO: check that at the blueprint / factory level- raise PipeDefinitionError(- f"PipeParallel requires unique output names for each parallel sub pipe, but {sub_pipe_output_name} is already used"- )+ msg = f"PipeParallel requires unique output names for each parallel sub pipe, but {sub_pipe_output_name} is already used"+ raise PipeDefinitionError(msg)output_stuffs[sub_pipe_output_name] = output_stuffif sub_pipe_output_name in output_stuff_contents:# TODO: check that at the blueprint / factory level- raise PipeDefinitionError(- f"PipeParallel requires unique output names for each parallel sub pipe, but {sub_pipe_output_name} is already used"- )+ msg = f"PipeParallel requires unique output names for each parallel sub pipe, but {sub_pipe_output_name} is already used"+ raise PipeDefinitionError(msg)output_stuff_contents[sub_pipe_output_name] = output_stuff.contentlog.debug(f"PipeParallel '{self.code}': output_stuff_contents[{sub_pipe_output_name}]: {output_stuff_contents[sub_pipe_output_name]}")@@ -220,28 +220,29 @@ class PipeParallel(PipeController):job_metadata: JobMetadata,working_memory: WorkingMemory,pipe_run_params: PipeRunParams,- output_name: Optional[str] = None,+ output_name: str | None = None,) -> PipeOutput:- """- Dry run implementation for PipeParallel.+ """Dry run implementation for PipeParallel.Validates that all required inputs are present and that all parallel sub-pipes can be dry run."""log.debug(f"PipeParallel: dry run controller pipe: {self.code}")if pipe_run_params.run_mode != PipeRunMode.DRY:- raise PipeRunParamsError(f"PipeSequence._dry_run_controller_pipe() called with run_mode = {pipe_run_params.run_mode} in pipe {self.code}")+ msg = f"PipeSequence._dry_run_controller_pipe() called with run_mode = {pipe_run_params.run_mode} in pipe {self.code}"+ raise PipeRunParamsError(msg)# 1. Validate that all required inputs are present in the working memoryneeded_inputs = self.needed_inputs()- missing_input_names: List[str] = []+ missing_input_names: list[str] = []for named_input_requirement in needed_inputs.named_input_requirements:if not working_memory.get_optional_stuff(named_input_requirement.variable_name):missing_input_names.append(named_input_requirement.variable_name)if missing_input_names:+ msg = f"Dry run failed: missing required inputs: {missing_input_names}"log.error(f"Dry run failed: missing required inputs: {missing_input_names}")raise DryRunError(- message=f"Dry run failed for pipe '{self.code}' (PipeParallel): missing required inputs: {', '.join(missing_input_names)}",+ message=msg,missing_inputs=missing_input_names,pipe_code=self.code,)@@ -251,10 +252,11 @@ class PipeParallel(PipeController):try:get_required_pipe(pipe_code=sub_pipe.pipe_code)except Exception as exc:- raise PipeDefinitionError(f"PipeParallel'{self.code}'sub-pipe '{sub_pipe.pipe_code}' not found") from exc+ msg = f"PipeParallel'{self.code}'sub-pipe '{sub_pipe.pipe_code}' not found"+ raise PipeDefinitionError(msg) from exc# 3. Run all sub-pipes in dry mode- tasks: List[Coroutine[Any, Any, PipeOutput]] = []+ tasks: list[Coroutine[Any, Any, PipeOutput]] = []for sub_pipe in self.parallel_sub_pipes:tasks.append(@@ -263,29 +265,31 @@ class PipeParallel(PipeController):job_metadata=job_metadata,working_memory=working_memory.make_deep_copy(),sub_pipe_run_params=pipe_run_params.make_deep_copy(),- )+ ),)try:pipe_outputs = await asyncio.gather(*tasks)except Exception as exc:- log.error(f"Dry run failed: parallel sub-pipe execution failed: {exc}")+ msg = f"Dry run failed: parallel sub-pipe execution failed: {exc}"+ log.error(msg)raise DryRunError(- message=f"Dry run failed for pipe '{self.code}' (PipeParallel): parallel sub-pipe execution failed: {exc}",+ message=msg,missing_inputs=[],pipe_code=self.code,- )+ ) from exc# 4. Process outputs as in the regular run- output_stuffs: Dict[str, Stuff] = {}- output_stuff_contents: Dict[str, StuffContent] = {}+ output_stuffs: dict[str, Stuff] = {}+ output_stuff_contents: dict[str, StuffContent] = {}for output_index, pipe_output in enumerate(pipe_outputs):output_stuff = pipe_output.main_stuffsub_pipe_output_name = self.parallel_sub_pipes[output_index].output_nameif not sub_pipe_output_name:+ msg = f"Dry run failed for pipe '{self.code}' (PipeParallel): sub-pipe output name not specified"raise DryRunError(- message=f"Dry run failed for pipe '{self.code}' (PipeParallel): sub-pipe output name not specified",+ message=msg,missing_inputs=[],pipe_code=self.code,)@@ -294,8 +298,9 @@ class PipeParallel(PipeController):working_memory.add_new_stuff(name=sub_pipe_output_name, stuff=output_stuff)if sub_pipe_output_name in output_stuffs:+ msg = f"Dry run failed for pipe '{self.code}' (PipeParallel): duplicate output name '{sub_pipe_output_name}'"raise DryRunError(- message=f"Dry run failed for pipe '{self.code}' (PipeParallel): duplicate output name '{sub_pipe_output_name}'",+ message=msg,missing_inputs=[],pipe_code=self.code,)diff --git a/pipelex/pipe_controllers/parallel/pipe_parallel_blueprint.py b/pipelex/pipe_controllers/parallel/pipe_parallel_blueprint.pyindex 1d810ddb4..f9f5599a0 100644--- a/pipelex/pipe_controllers/parallel/pipe_parallel_blueprint.py+++ b/pipelex/pipe_controllers/parallel/pipe_parallel_blueprint.py@@ -1,4 +1,4 @@-from typing import List, Literal, Optional+from typing import Literalfrom pydantic import field_validator@@ -10,12 +10,13 @@ from pipelex.pipe_controllers.sub_pipe_blueprint import SubPipeBlueprintclass PipeParallelBlueprint(PipeBlueprint):type: Literal["PipeParallel"] = "PipeParallel"category: Literal["PipeController"] = "PipeController"- parallels: List[SubPipeBlueprint]+ parallels: list[SubPipeBlueprint]add_each_output: bool = True- combined_output: Optional[str] = None+ combined_output: str | None = None@field_validator("combined_output", mode="before")- def validate_combined_output(cls, combined_output: str) -> str:+ @staticmethod+ def validate_combined_output(combined_output: str) -> str:if combined_output:ConceptBlueprint.validate_concept_string_or_code(concept_string_or_code=combined_output)return combined_outputdiff --git a/pipelex/pipe_controllers/parallel/pipe_parallel_factory.py b/pipelex/pipe_controllers/parallel/pipe_parallel_factory.pyindex 012556b05..af322d650 100644--- a/pipelex/pipe_controllers/parallel/pipe_parallel_factory.py+++ b/pipelex/pipe_controllers/parallel/pipe_parallel_factory.py@@ -1,4 +1,4 @@-from typing import List, Optional+from typing import TYPE_CHECKINGfrom typing_extensions import override@@ -9,9 +9,11 @@ from pipelex.exceptions import PipeDefinitionErrorfrom pipelex.hub import get_concept_providerfrom pipelex.pipe_controllers.parallel.pipe_parallel import PipeParallelfrom pipelex.pipe_controllers.parallel.pipe_parallel_blueprint import PipeParallelBlueprint-from pipelex.pipe_controllers.sub_pipe import SubPipefrom pipelex.pipe_controllers.sub_pipe_factory import SubPipeFactory+if TYPE_CHECKING:+ from pipelex.pipe_controllers.sub_pipe import SubPipe+class PipeParallelFactory(PipeFactoryProtocol[PipeParallelBlueprint, PipeParallel]):@classmethod@@ -21,16 +23,18 @@ class PipeParallelFactory(PipeFactoryProtocol[PipeParallelBlueprint, PipeParalledomain: str,pipe_code: str,blueprint: PipeParallelBlueprint,- concept_codes_from_the_same_domain: Optional[List[str]] = None,+ concept_codes_from_the_same_domain: list[str] | None = None,) -> PipeParallel:- parallel_sub_pipes: List[SubPipe] = []+ parallel_sub_pipes: list[SubPipe] = []for sub_pipe_blueprint in blueprint.parallels:if not sub_pipe_blueprint.result:- raise PipeDefinitionError("PipeParallel requires a result specified for each parallel sub pipe")+ msg = "PipeParallel requires a result specified for each parallel sub pipe"+ raise PipeDefinitionError(msg)sub_pipe = SubPipeFactory.make_from_blueprint(sub_pipe_blueprint, concept_codes_from_the_same_domain=concept_codes_from_the_same_domain)parallel_sub_pipes.append(sub_pipe)if not blueprint.add_each_output and not blueprint.combined_output:- raise PipeDefinitionError("PipeParallel requires either add_each_output or combined_output to be set")+ msg = "PipeParallel requires either add_each_output or combined_output to be set"+ raise PipeDefinitionError(msg)if blueprint.combined_output:combined_output_domain_and_code = ConceptFactory.make_domain_and_concept_code_from_concept_string_or_code(@@ -40,8 +44,9 @@ class PipeParallelFactory(PipeFactoryProtocol[PipeParallelBlueprint, PipeParalle)combined_output = get_concept_provider().get_required_concept(concept_string=ConceptFactory.construct_concept_string_with_domain(- domain=combined_output_domain_and_code.domain, concept_code=combined_output_domain_and_code.concept_code- )+ domain=combined_output_domain_and_code.domain,+ concept_code=combined_output_domain_and_code.concept_code,+ ),)else:combined_output = None@@ -56,12 +61,15 @@ class PipeParallelFactory(PipeFactoryProtocol[PipeParallelBlueprint, PipeParallecode=pipe_code,definition=blueprint.definition,inputs=PipeInputSpecFactory.make_from_blueprint(- domain=domain, blueprint=blueprint.inputs or {}, concept_codes_from_the_same_domain=concept_codes_from_the_same_domain+ domain=domain,+ blueprint=blueprint.inputs or {},+ concept_codes_from_the_same_domain=concept_codes_from_the_same_domain,),output=get_concept_provider().get_required_concept(concept_string=ConceptFactory.construct_concept_string_with_domain(- domain=output_domain_and_code.domain, concept_code=output_domain_and_code.concept_code- )+ domain=output_domain_and_code.domain,+ concept_code=output_domain_and_code.concept_code,+ ),),parallel_sub_pipes=parallel_sub_pipes,add_each_output=blueprint.add_each_output,diff --git a/pipelex/pipe_controllers/pipe_controller.py b/pipelex/pipe_controllers/pipe_controller.pyindex 5cb192290..8211f7449 100644--- a/pipelex/pipe_controllers/pipe_controller.py+++ b/pipelex/pipe_controllers/pipe_controller.py@@ -1,5 +1,5 @@from abc import abstractmethod-from typing import Literal, Optional+from typing import Literalfrom typing_extensions import override@@ -20,8 +20,8 @@ class PipeController(PipeAbstract):job_metadata: JobMetadata,working_memory: WorkingMemory,pipe_run_params: PipeRunParams,- output_name: Optional[str] = None,- print_intermediate_outputs: Optional[bool] = False,+ output_name: str | None = None,+ print_intermediate_outputs: bool | None = False,) -> PipeOutput:pipe_run_params.push_pipe_to_stack(pipe_code=self.code)self.monitor_pipe_stack(pipe_run_params=pipe_run_params)@@ -67,7 +67,7 @@ class PipeController(PipeAbstract):job_metadata: JobMetadata,working_memory: WorkingMemory,pipe_run_params: PipeRunParams,- output_name: Optional[str] = None,+ output_name: str | None = None,) -> PipeOutput:pass@@ -77,6 +77,6 @@ class PipeController(PipeAbstract):job_metadata: JobMetadata,working_memory: WorkingMemory,pipe_run_params: PipeRunParams,- output_name: Optional[str] = None,+ output_name: str | None = None,) -> PipeOutput:passdiff --git a/pipelex/pipe_controllers/sequence/__init__.py b/pipelex/pipe_controllers/sequence/__init__.pynew file mode 100644index 000000000..e69de29bbdiff --git a/pipelex/pipe_controllers/sequence/pipe_sequence.py b/pipelex/pipe_controllers/sequence/pipe_sequence.pyindex 38e008ca1..7b6e157c6 100644--- a/pipelex/pipe_controllers/sequence/pipe_sequence.py+++ b/pipelex/pipe_controllers/sequence/pipe_sequence.py@@ -1,7 +1,7 @@-from typing import List, Literal, Optional, Set+from typing import Literalfrom pydantic import model_validator-from typing_extensions import Self, override+from typing_extensions import overridefrom pipelex import logfrom pipelex.config import StaticValidationReaction, get_config@@ -21,14 +21,15 @@ from pipelex.pipe_controllers.pipe_controller import PipeControllerfrom pipelex.pipe_controllers.sequence.exceptions import PipeSequenceErrorfrom pipelex.pipe_controllers.sub_pipe import SubPipefrom pipelex.pipeline.job_metadata import JobMetadata+from pipelex.types import Selfclass PipeSequence(PipeController):type: Literal["PipeSequence"] = "PipeSequence"- sequential_sub_pipes: List[SubPipe]+ sequential_sub_pipes: list[SubPipe]@override- def needed_inputs(self, visited_pipes: Optional[Set[str]] = None) -> PipeInputSpec:+ def needed_inputs(self, visited_pipes: set[str] | None = None) -> PipeInputSpec:if visited_pipes is None:visited_pipes = set()@@ -40,33 +41,30 @@ class PipeSequence(PipeController):visited_pipes_with_current = visited_pipes | {self.code}needed_inputs = PipeInputSpecFactory.make_empty()- generated_outputs: Set[str] = set()+ generated_outputs: set[str] = set()for sequential_sub_pipe in self.sequential_sub_pipes:sub_pipe = get_required_pipe(pipe_code=sequential_sub_pipe.pipe_code)# Use the centralized recursion detectionsub_pipe_needed_inputs = sub_pipe.needed_inputs(visited_pipes_with_current)- if isinstance(sub_pipe, PipeParallel):- if sub_pipe.add_each_output:- for sub_parallel_pipe in sub_pipe.parallel_sub_pipes:- if sub_pipe.add_each_output and sub_parallel_pipe.output_name:- generated_outputs.add(sub_parallel_pipe.output_name)- elif sub_parallel_pipe.output_name:- generated_outputs.add(sub_parallel_pipe.output_name)+ if isinstance(sub_pipe, PipeParallel) and sub_pipe.add_each_output:+ for sub_parallel_pipe in sub_pipe.parallel_sub_pipes:+ if (sub_pipe.add_each_output and sub_parallel_pipe.output_name) or sub_parallel_pipe.output_name:+ generated_outputs.add(sub_parallel_pipe.output_name)if sequential_sub_pipe.batch_params:if sequential_sub_pipe.batch_params.input_list_stuff_name not in generated_outputs:needed_inputs.add_requirement(variable_name=sequential_sub_pipe.batch_params.input_list_stuff_name,concept=sub_pipe_needed_inputs.get_required_input_requirement(- variable_name=sequential_sub_pipe.batch_params.input_item_stuff_name+ variable_name=sequential_sub_pipe.batch_params.input_item_stuff_name,).concept,multiplicity=True,)- for input_name, requirement in sub_pipe_needed_inputs.items:- if input_name != sequential_sub_pipe.batch_params.input_item_stuff_name and input_name not in generated_outputs:- needed_inputs.add_requirement(input_name, requirement.concept, requirement.multiplicity)+ for input_name, requirement in sub_pipe_needed_inputs.items:+ if input_name != sequential_sub_pipe.batch_params.input_item_stuff_name and input_name not in generated_outputs:+ needed_inputs.add_requirement(input_name, requirement.concept, requirement.multiplicity)else:for input_name, requirement in sub_pipe_needed_inputs.items:if input_name not in generated_outputs:@@ -78,40 +76,38 @@ class PipeSequence(PipeController):return needed_inputs@override- def required_variables(self) -> Set[str]:+ def required_variables(self) -> set[str]:return set()@overridedef validate_output(self):- """- Validate the output for the pipe sequence.+ """Validate the output for the pipe sequence.The output of the pipe sequence should match the output of the last step."""last_step_output_pipe = get_required_pipe(pipe_code=self.sequential_sub_pipes[-1].pipe_code)if self.output.concept_string != last_step_output_pipe.output.concept_string:- raise PipeSequenceError(+ msg = (f"The output concept code '{self.output.concept_string}' of the pipe '{self.code}' is "f"not matching the output concept code '{last_step_output_pipe.output.concept_string}' "- f"of the last step '{self.sequential_sub_pipes[-1].pipe_code}'"+ f"of the last step '{self.sequential_sub_pipes[-1].pipe_code}'",)+ raise PipeSequenceError(msg)@model_validator(mode="after")def validate_inputs(self) -> Self:if len(self.sequential_sub_pipes) == 0:- raise ValueError(f"Pipe'{self.code}'(PipeSequence) must have at least 1 step")+ msg = f"Pipe'{self.code}'(PipeSequence) must have at least 1 step"+ raise ValueError(msg)return selfdef _validate_output_multiplicity_support(self, pipe_run_params: PipeRunParams) -> None:"""Validate that the pipe supports the requested output multiplicity."""if pipe_run_params.is_multiple_output_required:- raise PipeRunParamsError(- f"{self.__class__.__name__} does not support multiple outputs, got output_multiplicity = {pipe_run_params.output_multiplicity}"- )+ msg = f"{self.__class__.__name__} does not support multiple outputs, got output_multiplicity = {pipe_run_params.output_multiplicity}"+ raise PipeRunParamsError(msg)def _validate_inputs(self):- """- Validate that the inputs declared for this PipeSequence match what is actually needed.- """+ """Validate that the inputs declared for this PipeSequence match what is actually needed."""static_validation_config = get_config().pipelex.static_validation_configdefault_reaction = static_validation_config.default_reactionreactions = static_validation_config.reactions@@ -159,15 +155,14 @@ class PipeSequence(PipeController):@overridedef validate_with_libraries(self):- """- Perform full validation after all libraries are loaded.+ """Perform full validation after all libraries are loaded.This is called after all pipes and concepts are available."""self._validate_inputs()@override- def pipe_dependencies(self) -> Set[str]:- return set(sub_pipe.pipe_code for sub_pipe in self.sequential_sub_pipes)+ def pipe_dependencies(self) -> set[str]:+ return {sub_pipe.pipe_code for sub_pipe in self.sequential_sub_pipes}@overrideasync def _run_controller_pipe(@@ -175,7 +170,7 @@ class PipeSequence(PipeController):job_metadata: JobMetadata,working_memory: WorkingMemory,pipe_run_params: PipeRunParams,- output_name: Optional[str] = None,+ output_name: str | None = None,) -> PipeOutput:pipe_run_params.push_pipe_layer(pipe_code=self.code)self._validate_output_multiplicity_support(pipe_run_params)@@ -206,18 +201,20 @@ class PipeSequence(PipeController):job_metadata: JobMetadata,working_memory: WorkingMemory,pipe_run_params: PipeRunParams,- output_name: Optional[str] = None,+ output_name: str | None = None,) -> PipeOutput:if pipe_run_params.run_mode != PipeRunMode.DRY:- raise PipeRunParamsError(f"PipeSequence._dry_run_controller_pipe() called with run_mode = {pipe_run_params.run_mode} in pipe {self.code}")+ msg = f"PipeSequence._dry_run_controller_pipe() called with run_mode = {pipe_run_params.run_mode} in pipe {self.code}"+ raise PipeRunParamsError(msg)log.debug(f"PipeSequence._dry_run_controller_pipe() called with {self.code=} {pipe_run_params=}")# Verify the output of this pipe is matching the output of the last step.concept_string_of_last_step = get_required_pipe(pipe_code=self.sequential_sub_pipes[-1].pipe_code).output.concept_stringif self.output.concept_string != concept_string_of_last_step:- raise ValueError(+ msg = (f"The output concept code '{self.output.concept_string}' of the pipe '{self.code}' is "f"not matching the output concept code '{concept_string_of_last_step}' of the last step '{self.sequential_sub_pipes[-1].pipe_code}'")+ raise ValueError(msg)return await self._run_controller_pipe(job_metadata=job_metadata,working_memory=working_memory,diff --git a/pipelex/pipe_controllers/sequence/pipe_sequence_blueprint.py b/pipelex/pipe_controllers/sequence/pipe_sequence_blueprint.pyindex f20fe2582..47fe17dda 100644--- a/pipelex/pipe_controllers/sequence/pipe_sequence_blueprint.py+++ b/pipelex/pipe_controllers/sequence/pipe_sequence_blueprint.py@@ -1,4 +1,4 @@-from typing import List, Literal+from typing import Literalfrom pipelex.core.pipes.pipe_blueprint import PipeBlueprintfrom pipelex.pipe_controllers.sub_pipe_blueprint import SubPipeBlueprint@@ -7,4 +7,4 @@ from pipelex.pipe_controllers.sub_pipe_blueprint import SubPipeBlueprintclass PipeSequenceBlueprint(PipeBlueprint):type: Literal["PipeSequence"] = "PipeSequence"category: Literal["PipeController"] = "PipeController"- steps: List[SubPipeBlueprint]+ steps: list[SubPipeBlueprint]diff --git a/pipelex/pipe_controllers/sequence/pipe_sequence_factory.py b/pipelex/pipe_controllers/sequence/pipe_sequence_factory.pyindex 7ecf86457..a91d084cd 100644--- a/pipelex/pipe_controllers/sequence/pipe_sequence_factory.py+++ b/pipelex/pipe_controllers/sequence/pipe_sequence_factory.py@@ -1,5 +1,3 @@-from typing import List, Optional-from typing_extensions import overridefrom pipelex.core.concepts.concept_factory import ConceptFactory@@ -19,7 +17,7 @@ class PipeSequenceFactory(PipeFactoryProtocol[PipeSequenceBlueprint, PipeSequencdomain: str,pipe_code: str,blueprint: PipeSequenceBlueprint,- concept_codes_from_the_same_domain: Optional[List[str]] = None,+ concept_codes_from_the_same_domain: list[str] | None = None,) -> PipeSequence:output_domain_and_code = ConceptFactory.make_domain_and_concept_code_from_concept_string_or_code(domain=domain,@@ -31,12 +29,15 @@ class PipeSequenceFactory(PipeFactoryProtocol[PipeSequenceBlueprint, PipeSequenccode=pipe_code,definition=blueprint.definition,inputs=PipeInputSpecFactory.make_from_blueprint(- domain=domain, blueprint=blueprint.inputs or {}, concept_codes_from_the_same_domain=concept_codes_from_the_same_domain+ domain=domain,+ blueprint=blueprint.inputs or {},+ concept_codes_from_the_same_domain=concept_codes_from_the_same_domain,),output=get_concept_provider().get_required_concept(concept_string=ConceptFactory.construct_concept_string_with_domain(- domain=output_domain_and_code.domain, concept_code=output_domain_and_code.concept_code- )+ domain=output_domain_and_code.domain,+ concept_code=output_domain_and_code.concept_code,+ ),),sequential_sub_pipes=[SubPipeFactory.make_from_blueprint(blueprint=step, concept_codes_from_the_same_domain=concept_codes_from_the_same_domain)diff --git a/pipelex/pipe_controllers/sub_pipe.py b/pipelex/pipe_controllers/sub_pipe.pyindex eb18970f0..1ddff16f8 100644--- a/pipelex/pipe_controllers/sub_pipe.py+++ b/pipelex/pipe_controllers/sub_pipe.py@@ -1,5 +1,3 @@-from typing import List, Optional-from pydantic import BaseModelfrom pipelex import log@@ -19,10 +17,10 @@ from pipelex.pipeline.job_metadata import JobMetadataclass SubPipe(BaseModel):pipe_code: str- output_name: Optional[str] = None- output_multiplicity: Optional[PipeOutputMultiplicity] = None- batch_params: Optional[BatchParams] = None- concept_codes_from_the_same_domain: Optional[List[str]] = None+ output_name: str | None = None+ output_multiplicity: PipeOutputMultiplicity | None = None+ batch_params: BatchParams | None = None+ concept_codes_from_the_same_domain: list[str] | None = Noneasync def run_pipe(self,@@ -44,10 +42,11 @@ class SubPipe(BaseModel):try:working_memory.get_typed_object_or_attribute(name=batch_params.input_list_stuff_name, wanted_type=ListContent)except WorkingMemoryStuffNotFoundError as exc:- raise PipeInputError(+ msg = (f"Input list stuff named '{batch_params.input_list_stuff_name}' required by sub_pipe '{self.pipe_code}' "f"of pipe '{calling_pipe_code}' not found in working memory: {exc}"- ) from exc+ )+ raise PipeInputError(msg) from excpipe_batch_blueprint = PipeBatchBlueprint(definition=f"Batch processing for {self.pipe_code}",@@ -57,7 +56,7 @@ class SubPipe(BaseModel):input_item_name=batch_params.input_item_stuff_name,inputs={batch_params.input_item_stuff_name: InputRequirementBlueprint(- concept=sub_pipe.inputs.root[batch_params.input_item_stuff_name].concept.concept_string+ concept=sub_pipe.inputs.root[batch_params.input_item_stuff_name].concept.concept_string,),},)@@ -111,14 +110,15 @@ class SubPipe(BaseModel):# Case 3: Normal processingrequired_variables = sub_pipe.required_variables()log.debug(required_variables, title=f"Required variables for {self.pipe_code}")- required_stuff_names = set([required_variable for required_variable in required_variables if not required_variable.startswith("_")])+ required_stuff_names = {rv for rv in required_variables if not rv.startswith("_")}try:required_stuffs = working_memory.get_stuffs(names=required_stuff_names)except WorkingMemoryStuffNotFoundError as exc:- sub_pipe_path = sub_pipe_run_params.pipe_layers + [self.pipe_code]+ sub_pipe_path = [*sub_pipe_run_params.pipe_layers, self.pipe_code]sub_pipe_path_str = ".".join(sub_pipe_path)error_details = f"SubPipe '{sub_pipe_path_str}', required_variables: {required_variables}, missing: '{exc.variable_name}'"- raise PipeInputError(f"Some required stuff(s) not found: {error_details}") from exc+ msg = f"Some required stuff(s) not found: {error_details}"+ raise PipeInputError(msg) from exclog.verbose(required_stuffs, title=f"Required stuffs for {self.pipe_code}")# This is the only line that changes between run and dry_runif sub_pipe_run_params.run_mode == PipeRunMode.DRY:diff --git a/pipelex/pipe_controllers/sub_pipe_blueprint.py b/pipelex/pipe_controllers/sub_pipe_blueprint.pyindex 256251d78..73ee2d1ac 100644--- a/pipelex/pipe_controllers/sub_pipe_blueprint.py+++ b/pipelex/pipe_controllers/sub_pipe_blueprint.py@@ -1,26 +1,25 @@-from typing import Optional, Union-from pydantic import BaseModel, ConfigDict, model_validator-from typing_extensions import Selffrom pipelex.exceptions import PipeDefinitionErrorfrom pipelex.tools.typing.validation_utils import has_more_than_one_among_attributes_from_list+from pipelex.types import Selfclass SubPipeBlueprint(BaseModel):model_config = ConfigDict(extra="forbid")pipe: str- result: Optional[str] = None- nb_output: Optional[int] = None- multiple_output: Optional[bool] = None- batch_over: Union[bool, str] = False- batch_as: Optional[str] = None+ result: str | None = None+ nb_output: int | None = None+ multiple_output: bool | None = None+ batch_over: bool | str = False+ batch_as: str | None = None@model_validator(mode="after")def validate_multiple_output(self) -> Self:if has_more_than_one_among_attributes_from_list(self, attributes_list=["nb_output", "multiple_output"]):- raise PipeDefinitionError("PipeStepBlueprint should have no more than '1' of nb_output or multiple_output")+ msg = "PipeStepBlueprint should have no more than '1' of nb_output or multiple_output"+ raise PipeDefinitionError(msg)return self@model_validator(mode="after")@@ -29,9 +28,11 @@ class SubPipeBlueprint(BaseModel):batch_as_is_specified = self.batch_as is not None and self.batch_as != ""if batch_over_is_specified and not batch_as_is_specified:- raise PipeDefinitionError(f"In pipe '{self.pipe}': When 'batch_over' is specified, 'batch_as' must also be provided")+ msg = f"In pipe '{self.pipe}': When 'batch_over' is specified, 'batch_as' must also be provided"+ raise PipeDefinitionError(msg)if batch_as_is_specified and not batch_over_is_specified:- raise PipeDefinitionError(f"In pipe '{self.pipe}': When 'batch_as' is specified, 'batch_over' must also be provided")+ msg = f"In pipe '{self.pipe}': When 'batch_as' is specified, 'batch_over' must also be provided"+ raise PipeDefinitionError(msg)return selfdiff --git a/pipelex/pipe_controllers/sub_pipe_factory.py b/pipelex/pipe_controllers/sub_pipe_factory.pyindex 39987175d..aa7a33e84 100644--- a/pipelex/pipe_controllers/sub_pipe_factory.py+++ b/pipelex/pipe_controllers/sub_pipe_factory.py@@ -1,5 +1,3 @@-from typing import List, Optional-from pipelex.core.pipes.pipe_run_params import BatchParams, make_output_multiplicityfrom pipelex.pipe_controllers.sub_pipe import SubPipefrom pipelex.pipe_controllers.sub_pipe_blueprint import SubPipeBlueprint@@ -10,7 +8,7 @@ class SubPipeFactory:def make_from_blueprint(cls,blueprint: SubPipeBlueprint,- concept_codes_from_the_same_domain: Optional[List[str]] = None,+ concept_codes_from_the_same_domain: list[str] | None = None,) -> SubPipe:"""Create a SubPipe from a SubPipeBlueprint."""output_multiplicity = make_output_multiplicity(diff --git a/pipelex/pipe_operators/func/__init__.py b/pipelex/pipe_operators/func/__init__.pynew file mode 100644index 000000000..e69de29bbdiff --git a/pipelex/pipe_operators/func/pipe_func.py b/pipelex/pipe_operators/func/pipe_func.pyindex ac65f2e0d..932925e3e 100644--- a/pipelex/pipe_operators/func/pipe_func.py+++ b/pipelex/pipe_operators/func/pipe_func.py@@ -1,4 +1,4 @@-from typing import List, Literal, Optional, Set, cast, get_type_hints+from typing import Literal, cast, get_type_hintsfrom typing_extensions import override@@ -26,11 +26,11 @@ class PipeFunc(PipeOperator[PipeFuncOutput]):function_name: str@override- def required_variables(self) -> Set[str]:+ def required_variables(self) -> set[str]:return set()@override- def needed_inputs(self, visited_pipes: Optional[Set[str]] = None) -> PipeInputSpec:+ def needed_inputs(self, visited_pipes: set[str] | None = None) -> PipeInputSpec:return self.inputs@override@@ -43,25 +43,27 @@ class PipeFunc(PipeOperator[PipeFuncOutput]):job_metadata: JobMetadata,working_memory: WorkingMemory,pipe_run_params: PipeRunParams,- output_name: Optional[str] = None,+ output_name: str | None = None,) -> PipeFuncOutput:log.debug(f"Applying function '{self.function_name}'")function = func_registry.get_required_function(self.function_name)if not callable(function):- raise ValueError(f"Function '{self.function_name}' is not callable")+ msg = f"Function '{self.function_name}' is not callable"+ raise TypeError(msg)func_output_object = await function(working_memory=working_memory)the_content: StuffContentif isinstance(func_output_object, StuffContent):the_content = func_output_objectelif isinstance(func_output_object, list):- func_result_list = cast(List[StuffContent], func_output_object)+ func_result_list = cast("list[StuffContent]", func_output_object)the_content = ListContent(items=func_result_list)elif isinstance(func_output_object, str):the_content = TextContent(text=func_output_object)else:- raise ValueError(f"Function '{self.function_name}' must return a StuffContent or a list, got {type(func_output_object)}")+ msg = f"Function '{self.function_name}' must return a StuffContent or a list, got {type(func_output_object)}"+ raise TypeError(msg)output_stuff = StuffFactory.make_stuff(name=output_name,@@ -74,11 +76,10 @@ class PipeFunc(PipeOperator[PipeFuncOutput]):name=output_name,)- pipe_output = PipeFuncOutput(+ return PipeFuncOutput(working_memory=working_memory,pipeline_run_id=job_metadata.pipeline_run_id,)- return pipe_output@overrideasync def _dry_run_operator_pipe(@@ -86,46 +87,48 @@ class PipeFunc(PipeOperator[PipeFuncOutput]):job_metadata: JobMetadata,working_memory: WorkingMemory,pipe_run_params: PipeRunParams,- output_name: Optional[str] = None,+ output_name: str | None = None,) -> PipeFuncOutput:log.debug(f"Dry run for PipeFunc '{self.function_name}'")function = func_registry.get_required_function(self.function_name)if not callable(function):- raise ValueError(f"Function '{self.function_name}' is not callable")+ msg = f"Function '{self.function_name}' is not callable"+ raise TypeError(msg)# Check that all needed inputs are present in working memoryneeded_inputs = self.needed_inputs()for input_name, _ in needed_inputs.items:if input_name not in working_memory.root:- raise DryRunError(- f"Required input '{input_name}' not found in working memory for function '{self.function_name}' in pipe '{self.code}'"- )+ msg = f"Required input '{input_name}' not found in working memory for function '{self.function_name}' in pipe '{self.code}'"+ raise DryRunError(msg)try:return_type = get_type_hints(function).get("return")if return_type is None:- raise DryRunError(f"Function '{self.function_name}' has no return type annotation")- else:- if not issubclass(return_type, StuffContent):- raise DryRunError(f"Function '{self.function_name}' return type {return_type} is not a subclass of StuffContent")-- requirement = TypedNamedInputRequirement(- variable_name="mock_output",- concept=ConceptFactory.make(- concept_code=self.output.code,- domain="generic",- definition="Lorem Ipsum",- structure_class_name=self.output.structure_class_name,- ),- structure_class=return_type,- multiplicity=False,- )- mock_content = WorkingMemoryFactory.create_mock_content(requirement)+ msg = f"Function '{self.function_name}' has no return type annotation"+ raise DryRunError(msg)+ if not issubclass(return_type, StuffContent):+ msg = f"Function '{self.function_name}' return type {return_type} is not a subclass of StuffContent"+ raise DryRunError(msg)++ requirement = TypedNamedInputRequirement(+ variable_name="mock_output",+ concept=ConceptFactory.make(+ concept_code=self.output.code,+ domain="generic",+ definition="Lorem Ipsum",+ structure_class_name=self.output.structure_class_name,+ ),+ structure_class=return_type,+ multiplicity=False,+ )+ mock_content = WorkingMemoryFactory.create_mock_content(requirement)except Exception as exc:- raise DryRunError(f"Failed to get type hints for function '{self.function_name}' in pipe '{self.code}': {exc}")+ msg = f"Failed to get type hints for function '{self.function_name}' in pipe '{self.code}': {exc}"+ raise DryRunError(msg) from excoutput_stuff = StuffFactory.make_stuff(name=output_name,diff --git a/pipelex/pipe_operators/func/pipe_func_factory.py b/pipelex/pipe_operators/func/pipe_func_factory.pyindex 773ff4e64..685423dc6 100644--- a/pipelex/pipe_operators/func/pipe_func_factory.py+++ b/pipelex/pipe_operators/func/pipe_func_factory.py@@ -1,5 +1,3 @@-from typing import List, Optional-from typing_extensions import overridefrom pipelex.core.concepts.concept_factory import ConceptFactory@@ -18,7 +16,7 @@ class PipeFuncFactory(PipeFactoryProtocol[PipeFuncBlueprint, PipeFunc]):domain: str,pipe_code: str,blueprint: PipeFuncBlueprint,- concept_codes_from_the_same_domain: Optional[List[str]] = None,+ concept_codes_from_the_same_domain: list[str] | None = None,) -> PipeFunc:output_domain_and_code = ConceptFactory.make_domain_and_concept_code_from_concept_string_or_code(domain=domain,@@ -30,12 +28,15 @@ class PipeFuncFactory(PipeFactoryProtocol[PipeFuncBlueprint, PipeFunc]):code=pipe_code,definition=blueprint.definition,inputs=PipeInputSpecFactory.make_from_blueprint(- domain=domain, blueprint=blueprint.inputs or {}, concept_codes_from_the_same_domain=concept_codes_from_the_same_domain+ domain=domain,+ blueprint=blueprint.inputs or {},+ concept_codes_from_the_same_domain=concept_codes_from_the_same_domain,),output=get_concept_provider().get_required_concept(concept_string=ConceptFactory.construct_concept_string_with_domain(- domain=output_domain_and_code.domain, concept_code=output_domain_and_code.concept_code- )+ domain=output_domain_and_code.domain,+ concept_code=output_domain_and_code.concept_code,+ ),),function_name=blueprint.function_name,)diff --git a/pipelex/pipe_operators/img_gen/__init__.py b/pipelex/pipe_operators/img_gen/__init__.pynew file mode 100644index 000000000..e69de29bbdiff --git a/pipelex/pipe_operators/img_gen/pipe_img_gen.py b/pipelex/pipe_operators/img_gen/pipe_img_gen.pyindex 5ec4f3f1c..be9036289 100644--- a/pipelex/pipe_operators/img_gen/pipe_img_gen.py+++ b/pipelex/pipe_operators/img_gen/pipe_img_gen.py@@ -1,7 +1,7 @@-from typing import List, Literal, Optional, Set, Union+from typing import Literalfrom pydantic import Field, field_validator, model_validator-from typing_extensions import Self, override+from typing_extensions import overridefrom pipelex import logfrom pipelex.cogt.content_generation.content_generator_dry import ContentGeneratorDry@@ -33,12 +33,13 @@ from pipelex.exceptions import (from pipelex.hub import get_concept_provider, get_content_generator, get_model_deckfrom pipelex.pipe_operators.pipe_operator import PipeOperatorfrom pipelex.pipeline.job_metadata import JobMetadata+from pipelex.types import Selfclass PipeImgGenOutput(PipeOutput):@property- def image_urls(self) -> List[str]:- the_urls: List[str] = []+ def image_urls(self) -> list[str]:+ the_urls: list[str] = []content = self.main_stuff.contentif isinstance(content, ListContent):items = self.main_stuff_as_items(item_type=ImageContent)@@ -46,7 +47,8 @@ class PipeImgGenOutput(PipeOutput):elif isinstance(content, ImageContent):the_urls = [content.url]else:- raise PipeRunParamsError(f"PipeImgGen output should be a ListContent or an ImageContent, got {type(content)}")+ msg = f"PipeImgGen output should be a ListContent or an ImageContent, got {type(content)}"+ raise PipeRunParamsError(msg)return the_urls@@ -55,33 +57,34 @@ DEFAULT_PROMPT_VAR_NAME = "prompt"class PipeImgGen(PipeOperator[PipeImgGenOutput]):type: Literal["PipeImgGen"] = "PipeImgGen"- img_gen_prompt: Optional[str] = None- img_gen_prompt_var_name: Optional[str] = None+ img_gen_prompt: str | None = None+ img_gen_prompt_var_name: str | None = None# New ImgGenChoice pattern (like LLM)- img_gen: Optional[ImgGenChoice] = None+ img_gen: ImgGenChoice | None = None# Legacy individual settings (for backwards compatibility)- img_gen_handle: Optional[str] = None- quality: Optional[Quality] = Field(default=None, strict=False)- nb_steps: Optional[int] = Field(default=None, gt=0)- guidance_scale: Optional[float] = Field(default=None, gt=0)- is_moderated: Optional[bool] = None- safety_tolerance: Optional[int] = Field(default=None, ge=1, le=6)+ img_gen_handle: str | None = None+ quality: Quality | None = Field(default=None, strict=False)+ nb_steps: int | None = Field(default=None, gt=0)+ guidance_scale: float | None = Field(default=None, gt=0)+ is_moderated: bool | None = None+ safety_tolerance: int | None = Field(default=None, ge=1, le=6)# One-time settings (not in ImgGenSetting)- aspect_ratio: Optional[AspectRatio] = Field(default=None, strict=False)- is_raw: Optional[bool] = None- seed: Optional[Union[int, Literal["auto"]]] = None- background: Optional[Background] = Field(default=None, strict=False)- output_format: Optional[OutputFormat] = Field(default=None, strict=False)+ aspect_ratio: AspectRatio | None = Field(default=None, strict=False)+ is_raw: bool | None = None+ seed: int | Literal["auto"] | None = None+ background: Background | None = Field(default=None, strict=False)+ output_format: OutputFormat | None = Field(default=None, strict=False)output_multiplicity: PipeOutputMultiplicity@field_validator("img_gen_prompt_var_name")@classmethod- def validate_input_var_name_not_provided_as_attribute(cls, value: Optional[str]) -> Optional[str]:+ def validate_input_var_name_not_provided_as_attribute(cls, value: str | None) -> str | None:if value is not None:- raise PipeDefinitionError("img_gen_prompt_var_name must be None before input validation")+ msg = "img_gen_prompt_var_name must be None before input validation"+ raise PipeDefinitionError(msg)return value@model_validator(mode="after")@@ -102,13 +105,14 @@ class PipeImgGen(PipeOperator[PipeImgGenOutput]):wanted_concept=get_concept_provider().get_native_concept(native_concept=NativeConceptEnum.IMAGE),strict=True,):- raise PipeDefinitionError(+ msg = (f"The output of a ImgGen pipe must be compatible with the Image concept. "f"In the pipe '{self.code}' the output is '{self.output.concept_string}'")+ raise PipeDefinitionError(msg)@override- def needed_inputs(self, visited_pipes: Optional[Set[str]] = None) -> PipeInputSpec:+ def needed_inputs(self, visited_pipes: set[str] | None = None) -> PipeInputSpec:needed_inputs = PipeInputSpecFactory.make_empty()if self.img_gen_prompt:needed_inputs.add_requirement(@@ -123,11 +127,10 @@ class PipeImgGen(PipeOperator[PipeImgGenOutput]):return needed_inputs@override- def required_variables(self) -> Set[str]:+ def required_variables(self) -> set[str]:if self.img_gen_prompt_var_name:return {self.img_gen_prompt_var_name}- else:- return {DEFAULT_PROMPT_VAR_NAME}+ return {DEFAULT_PROMPT_VAR_NAME}def _validate_inputs(self):concept_provider = get_concept_provider()@@ -137,12 +140,12 @@ class PipeImgGen(PipeOperator[PipeImgGenOutput]):# check that we have either an img_gen_prompt passed as attribute or as a single text inputif self.img_gen_prompt:if self.inputs.items:- raise PipeDefinitionError("img_gen_prompt_var_name must be None if img_gen_prompt is provided")- else:- # we're good with the prompt provided as attribute- return+ msg = "img_gen_prompt_var_name must be None if img_gen_prompt is provided"+ raise PipeDefinitionError(msg)+ # we're good with the prompt provided as attribute+ return- candidate_prompt_var_names: List[str] = []+ candidate_prompt_var_names: list[str] = []for input_name, requirement in self.inputs.items:log.debug(f"Validating input '{input_name}' with concept code '{requirement.concept.code}'")if concept_provider.is_compatible(@@ -205,8 +208,8 @@ class PipeImgGen(PipeOperator[PipeImgGenOutput]):job_metadata: JobMetadata,working_memory: WorkingMemory,pipe_run_params: PipeRunParams,- output_name: Optional[str] = None,- content_generator: Optional[ContentGeneratorProtocol] = None,+ output_name: str | None = None,+ content_generator: ContentGeneratorProtocol | None = None,) -> PipeImgGenOutput:content_generator = content_generator or get_content_generator()@@ -223,9 +226,11 @@ class PipeImgGen(PipeOperator[PipeImgGenOutput]):try:img_gen_prompt_text = working_memory.get_stuff_as_str(stuff_name)except WorkingMemoryStuffNotFoundError as exc:- raise PipeInputError(f"Could not find a valid user image named '{stuff_name}' in the working_memory: {exc}") from exc+ msg = f"Could not find a valid user image named '{stuff_name}' in the working_memory: {exc}"+ raise PipeInputError(msg) from excelse:- raise UnexpectedPipeDefinitionError("You must provide an image gen prompt either as attribute of the pipe or as a single text input")+ msg = "You must provide an image gen prompt either as attribute of the pipe or as a single text input"+ raise UnexpectedPipeDefinitionError(msg)img_gen_config = get_config().cogt.img_gen_configimg_gen_param_defaults = img_gen_config.img_gen_param_defaults@@ -252,13 +257,13 @@ class PipeImgGen(PipeOperator[PipeImgGenOutput]):# Process one-time settingsseed_setting = self.seed or img_gen_param_defaults.seed- seed: Optional[int]+ seed: int | Noneif isinstance(seed_setting, str) and seed_setting == "auto":seed = Noneelse:seed = seed_setting- # Build ImggJobParams from ImgGenSetting + one-time settings+ # Build ImgGenJobParams from ImgGenSetting + one-time settingsimg_gen_job_params = ImgGenJobParams(aspect_ratio=self.aspect_ratio or img_gen_param_defaults.aspect_ratio,background=self.background or img_gen_param_defaults.background,@@ -276,7 +281,7 @@ class PipeImgGen(PipeOperator[PipeImgGenOutput]):log.debug(f"Using img_gen handle: {img_gen_handle}")the_content: StuffContent- image_urls: List[str] = []+ image_urls: list[str] = []nb_images: intif isinstance(applied_output_multiplicity, bool):if self.output_multiplicity:@@ -285,8 +290,7 @@ class PipeImgGen(PipeOperator[PipeImgGenOutput]):msg += f" and pipe_run_params.output_multiplicity = {pipe_run_params.output_multiplicity}."msg += f" Tried to apply applied_output_multiplicity = {applied_output_multiplicity}."raise PipeRunParamsError(msg)- else:- nb_images = 1+ nb_images = 1elif isinstance(applied_output_multiplicity, int):nb_images = applied_output_multiplicityelse:@@ -303,13 +307,13 @@ class PipeImgGen(PipeOperator[PipeImgGenOutput]):img_gen_job_params=img_gen_job_params,img_gen_job_config=img_gen_config.img_gen_job_config,)- image_content_items: List[StuffContent] = []+ image_content_items: list[StuffContent] = []for generated_image in generated_image_list:image_content_items.append(ImageContent(url=generated_image.url,source_prompt=img_gen_prompt_text,- )+ ),)image_urls.append(generated_image.url)the_content = ListContent(@@ -347,11 +351,10 @@ class PipeImgGen(PipeOperator[PipeImgGenOutput]):name=output_name,)- pipe_output = PipeImgGenOutput(+ return PipeImgGenOutput(working_memory=working_memory,pipeline_run_id=job_metadata.pipeline_run_id,)- return pipe_output@overrideasync def _dry_run_operator_pipe(@@ -359,15 +362,14 @@ class PipeImgGen(PipeOperator[PipeImgGenOutput]):job_metadata: JobMetadata,working_memory: WorkingMemory,pipe_run_params: PipeRunParams,- output_name: Optional[str] = None,+ output_name: str | None = None,) -> PipeImgGenOutput:log.debug(f"PipeImgGen: dry run operator pipe: {self.code}")content_generator_dry = ContentGeneratorDry()- pipe_output = await self._run_operator_pipe(+ return await self._run_operator_pipe(job_metadata=job_metadata,working_memory=working_memory,pipe_run_params=pipe_run_params or PipeRunParamsFactory.make_run_params(pipe_run_mode=PipeRunMode.DRY),output_name=output_name,content_generator=content_generator_dry,)- return pipe_outputdiff --git a/pipelex/pipe_operators/img_gen/pipe_img_gen_blueprint.py b/pipelex/pipe_operators/img_gen/pipe_img_gen_blueprint.pyindex 27e39ba0d..938ce84d4 100644--- a/pipelex/pipe_operators/img_gen/pipe_img_gen_blueprint.py+++ b/pipelex/pipe_operators/img_gen/pipe_img_gen_blueprint.py@@ -1,4 +1,4 @@-from typing import Literal, Optional, Union+from typing import Literalfrom pydantic import Field@@ -10,16 +10,16 @@ from pipelex.core.pipes.pipe_blueprint import PipeBlueprintclass PipeImgGenBlueprint(PipeBlueprint):type: Literal["PipeImgGen"] = "PipeImgGen"category: Literal["PipeOperator"] = "PipeOperator"- img_gen_prompt: Optional[str] = None- img_gen_prompt_var_name: Optional[str] = None+ img_gen_prompt: str | None = None+ img_gen_prompt_var_name: str | None = None# New ImgGenChoice pattern (like LLM)- img_gen: Optional[ImgGenChoice] = None+ img_gen: ImgGenChoice | None = None# One-time settings (not in ImgGenSetting)- aspect_ratio: Optional[AspectRatio] = Field(default=None, strict=False)- is_raw: Optional[bool] = None- seed: Optional[Union[int, Literal["auto"]]] = None- nb_output: Optional[int] = Field(default=None, ge=1)- background: Optional[Background] = Field(default=None, strict=False)- output_format: Optional[OutputFormat] = Field(default=None, strict=False)+ aspect_ratio: AspectRatio | None = Field(default=None, strict=False)+ is_raw: bool | None = None+ seed: int | Literal["auto"] | None = None+ nb_output: int | None = Field(default=None, ge=1)+ background: Background | None = Field(default=None, strict=False)+ output_format: OutputFormat | None = Field(default=None, strict=False)diff --git a/pipelex/pipe_operators/img_gen/pipe_img_gen_factory.py b/pipelex/pipe_operators/img_gen/pipe_img_gen_factory.pyindex ad09adc6a..de36dde65 100644--- a/pipelex/pipe_operators/img_gen/pipe_img_gen_factory.py+++ b/pipelex/pipe_operators/img_gen/pipe_img_gen_factory.py@@ -1,5 +1,3 @@-from typing import List, Optional-from typing_extensions import overridefrom pipelex.core.concepts.concept_factory import ConceptFactory@@ -18,7 +16,7 @@ class PipeImgGenFactory(PipeFactoryProtocol[PipeImgGenBlueprint, PipeImgGen]):domain: str,pipe_code: str,blueprint: PipeImgGenBlueprint,- concept_codes_from_the_same_domain: Optional[List[str]] = None,+ concept_codes_from_the_same_domain: list[str] | None = None,) -> PipeImgGen:output_domain_and_code = ConceptFactory.make_domain_and_concept_code_from_concept_string_or_code(domain=domain,@@ -30,12 +28,15 @@ class PipeImgGenFactory(PipeFactoryProtocol[PipeImgGenBlueprint, PipeImgGen]):code=pipe_code,definition=blueprint.definition,inputs=PipeInputSpecFactory.make_from_blueprint(- domain=domain, blueprint=blueprint.inputs or {}, concept_codes_from_the_same_domain=concept_codes_from_the_same_domain+ domain=domain,+ blueprint=blueprint.inputs or {},+ concept_codes_from_the_same_domain=concept_codes_from_the_same_domain,),output=get_concept_provider().get_required_concept(concept_string=ConceptFactory.construct_concept_string_with_domain(- domain=output_domain_and_code.domain, concept_code=output_domain_and_code.concept_code- )+ domain=output_domain_and_code.domain,+ concept_code=output_domain_and_code.concept_code,+ ),),output_multiplicity=blueprint.nb_output or 1,img_gen_prompt=blueprint.img_gen_prompt,diff --git a/pipelex/pipe_operators/jinja2/__init__.py b/pipelex/pipe_operators/jinja2/__init__.pynew file mode 100644index 000000000..e69de29bbdiff --git a/pipelex/pipe_operators/jinja2/pipe_jinja2.py b/pipelex/pipe_operators/jinja2/pipe_jinja2.pyindex a37ab4174..342b4a74d 100644--- a/pipelex/pipe_operators/jinja2/pipe_jinja2.py+++ b/pipelex/pipe_operators/jinja2/pipe_jinja2.py@@ -1,8 +1,8 @@-from typing import Any, ClassVar, Dict, Literal, Optional, Set+from typing import Any, ClassVar, Literalfrom jinja2 import TemplateSyntaxErrorfrom pydantic import ConfigDict, model_validator-from typing_extensions import Self, override+from typing_extensions import overridefrom pipelex import logfrom pipelex.cogt.content_generation.content_generator_dry import ContentGeneratorDry@@ -29,6 +29,7 @@ from pipelex.tools.templating.jinja2_required_variables import detect_jinja2_reqfrom pipelex.tools.templating.jinja2_template_category import Jinja2TemplateCategoryfrom pipelex.tools.templating.templating_models import PromptingStylefrom pipelex.tools.typing.validation_utils import has_exactly_one_among_attributes_from_list+from pipelex.types import Selfclass PipeJinja2Output(PipeOutput):@@ -46,21 +47,23 @@ class PipeJinja2(PipeOperator[PipeJinja2Output]):native_concept_data=NATIVE_CONCEPTS_DATA[NativeConceptEnum.TEXT],)- jinja2_name: Optional[str] = None- jinja2: Optional[str] = None- prompting_style: Optional[PromptingStyle] = None+ jinja2_name: str | None = None+ jinja2: str | None = None+ prompting_style: PromptingStyle | None = Nonetemplate_category: Jinja2TemplateCategory = Jinja2TemplateCategory.LLM_PROMPT- extra_context: Optional[Dict[str, Any]] = None+ extra_context: dict[str, Any] | None = None@model_validator(mode="after")def validate_jinja2(self) -> Self:if not has_exactly_one_among_attributes_from_list(self, attributes_list=["jinja2_name", "jinja2"]):- raise PipeDefinitionError("PipeJinja2 should have exactly one of jinja2_name or jinja2")+ msg = "PipeJinja2 should have exactly one of jinja2_name or jinja2"+ raise PipeDefinitionError(msg)if self.jinja2:try:check_jinja2_parsing(jinja2_template_source=self.jinja2, template_category=self.template_category)except TemplateSyntaxError as exc:- raise Jinja2TemplateError(f"Could not parse Jinja2 template included in PipeJinja2: {exc}") from exc+ msg = f"Could not parse Jinja2 template included in PipeJinja2: {exc}"+ raise Jinja2TemplateError(msg) from excreturn self@model_validator(mode="after")@@ -73,7 +76,8 @@ class PipeJinja2(PipeOperator[PipeJinja2Output]):required_variables = self.required_variables()for required_variable_name in required_variables:if required_variable_name not in self.inputs.variables:- raise PipeDefinitionError(f"Required variable '{required_variable_name}' is not in the inputs of pipe {self.code}")+ msg = f"Required variable '{required_variable_name}' is not in the inputs of pipe {self.code}"+ raise PipeDefinitionError(msg)return self@override@@ -87,7 +91,7 @@ class PipeJinja2(PipeOperator[PipeJinja2Output]):log.debug(f"Validated jinja2 template '{self.jinja2_name}':\n{the_template}")@override- def needed_inputs(self, visited_pipes: Optional[Set[str]] = None) -> PipeInputSpec:+ def needed_inputs(self, visited_pipes: set[str] | None = None) -> PipeInputSpec:needed_inputs = PipeInputSpecFactory.make_empty()for input_name, requirement in self.inputs.root.items():needed_inputs.add_requirement(variable_name=input_name, concept=requirement.concept)@@ -103,7 +107,7 @@ class PipeJinja2(PipeOperator[PipeJinja2Output]):return "Jinja2 template not defined"@override- def required_variables(self) -> Set[str]:+ def required_variables(self) -> set[str]:required_variables = detect_jinja2_required_variables(template_category=self.template_category,template_provider=get_template_provider(),@@ -113,7 +117,7 @@ class PipeJinja2(PipeOperator[PipeJinja2Output]):return {variable_namefor variable_name in required_variables- if not variable_name.startswith("_") and variable_name != "preliminary_text" and variable_name != "place_holder"+ if not variable_name.startswith("_") and variable_name not in ("preliminary_text", "place_holder")}@override@@ -122,16 +126,15 @@ class PipeJinja2(PipeOperator[PipeJinja2Output]):job_metadata: JobMetadata,working_memory: WorkingMemory,pipe_run_params: PipeRunParams,- output_name: Optional[str] = None,- content_generator: Optional[ContentGeneratorProtocol] = None,+ output_name: str | None = None,+ content_generator: ContentGeneratorProtocol | None = None,) -> PipeJinja2Output:content_generator = content_generator or get_content_generator()if pipe_run_params.is_multiple_output_required:- raise PipeRunParamsError(- f"PipeJinja2 does not suppport multiple outputs, got output_multiplicity = {pipe_run_params.output_multiplicity}"- )+ msg = f"PipeJinja2 does not suppport multiple outputs, got output_multiplicity = {pipe_run_params.output_multiplicity}"+ raise PipeRunParamsError(msg)- context: Dict[str, Any] = working_memory.generate_context()+ context: dict[str, Any] = working_memory.generate_context()if pipe_run_params:context.update(**pipe_run_params.params)if self.extra_context:@@ -155,20 +158,18 @@ class PipeJinja2(PipeOperator[PipeJinja2Output]):name=output_name,)- pipe_output = PipeJinja2Output(+ return PipeJinja2Output(working_memory=working_memory,pipeline_run_id=job_metadata.pipeline_run_id,)- return pipe_output-@overrideasync def _dry_run_operator_pipe(self,job_metadata: JobMetadata,working_memory: WorkingMemory,pipe_run_params: PipeRunParams,- output_name: Optional[str] = None,+ output_name: str | None = None,) -> PipeJinja2Output:content_generator_used: ContentGeneratorProtocolif get_config().pipelex.dry_run_config.apply_to_jinja2_rendering:@@ -178,11 +179,10 @@ class PipeJinja2(PipeOperator[PipeJinja2Output]):log.debug(f"PipeJinja2: using regular operator pipe for jinja2 rendering (dry run not applied to jinja2): {self.code}")content_generator_used = get_content_generator()- pipe_output = await self._run_operator_pipe(+ return await self._run_operator_pipe(job_metadata=job_metadata,working_memory=working_memory,pipe_run_params=pipe_run_params or PipeRunParamsFactory.make_run_params(pipe_run_mode=PipeRunMode.DRY),output_name=output_name,content_generator=content_generator_used,)- return pipe_outputdiff --git a/pipelex/pipe_operators/jinja2/pipe_jinja2_factory.py b/pipelex/pipe_operators/jinja2/pipe_jinja2_factory.pyindex e0b26a6e9..2ce9d7436 100644--- a/pipelex/pipe_operators/jinja2/pipe_jinja2_factory.py+++ b/pipelex/pipe_operators/jinja2/pipe_jinja2_factory.py@@ -1,5 +1,3 @@-from typing import List, Optional-from typing_extensions import overridefrom pipelex.core.concepts.concept_factory import ConceptFactory@@ -23,9 +21,9 @@ class PipeJinja2Factory(PipeFactoryProtocol[PipeJinja2Blueprint, PipeJinja2]):domain: str,pipe_code: str,blueprint: PipeJinja2Blueprint,- concept_codes_from_the_same_domain: Optional[List[str]] = None,+ concept_codes_from_the_same_domain: list[str] | None = None,) -> PipeJinja2:- preprocessed_template: Optional[str] = None+ preprocessed_template: str | None = Noneif blueprint.jinja2:preprocessed_template = preprocess_template(blueprint.jinja2)check_jinja2_parsing(@@ -45,12 +43,15 @@ class PipeJinja2Factory(PipeFactoryProtocol[PipeJinja2Blueprint, PipeJinja2]):code=pipe_code,definition=blueprint.definition,inputs=PipeInputSpecFactory.make_from_blueprint(- domain=domain, blueprint=blueprint.inputs or {}, concept_codes_from_the_same_domain=concept_codes_from_the_same_domain+ domain=domain,+ blueprint=blueprint.inputs or {},+ concept_codes_from_the_same_domain=concept_codes_from_the_same_domain,),output=get_concept_provider().get_required_concept(concept_string=ConceptFactory.construct_concept_string_with_domain(- domain=output_domain_and_code.domain, concept_code=output_domain_and_code.concept_code- )+ domain=output_domain_and_code.domain,+ concept_code=output_domain_and_code.concept_code,+ ),),jinja2_name=blueprint.jinja2_name,jinja2=preprocessed_template,@@ -63,9 +64,9 @@ class PipeJinja2Factory(PipeFactoryProtocol[PipeJinja2Blueprint, PipeJinja2]):def make_pipe_jinja2_from_template_str(cls,domain: str,- inputs: Optional[PipeInputSpec] = None,- template_str: Optional[str] = None,- template_name: Optional[str] = None,+ inputs: PipeInputSpec | None = None,+ template_str: str | None = None,+ template_name: str | None = None,) -> PipeJinja2:if template_str:preprocessed_template = preprocess_template(template_str)@@ -79,12 +80,12 @@ class PipeJinja2Factory(PipeFactoryProtocol[PipeJinja2Blueprint, PipeJinja2]):jinja2=preprocessed_template,inputs=inputs or PipeInputSpecFactory.make_empty(),)- elif template_name:+ if template_name:return PipeJinja2(domain=domain,code="adhoc_pipe_jinja2_from_template_name",jinja2_name=template_name,inputs=inputs or PipeInputSpecFactory.make_empty(),)- else:- raise PipeDefinitionError("Either template_str or template_name must be provided to make_pipe_jinja2_from_template_str")+ msg = "Either template_str or template_name must be provided to make_pipe_jinja2_from_template_str"+ raise PipeDefinitionError(msg)diff --git a/pipelex/pipe_operators/llm/__init__.py b/pipelex/pipe_operators/llm/__init__.pynew file mode 100644index 000000000..e69de29bbdiff --git a/pipelex/pipe_operators/llm/pipe_llm.py b/pipelex/pipe_operators/llm/pipe_llm.pyindex 64c892878..3204c4503 100644--- a/pipelex/pipe_operators/llm/pipe_llm.py+++ b/pipelex/pipe_operators/llm/pipe_llm.py@@ -1,7 +1,7 @@-from typing import List, Literal, Optional, Set, Type, cast+from typing import Literal, castfrom pydantic import model_validator-from typing_extensions import Self, override+from typing_extensions import overridefrom pipelex import logfrom pipelex.cogt.content_generation.content_generator_dry import ContentGeneratorDry@@ -50,6 +50,7 @@ from pipelex.pipe_operators.llm.pipe_llm_blueprint import StructuringMethodfrom pipelex.pipe_operators.pipe_operator import PipeOperatorfrom pipelex.pipeline.job_metadata import JobMetadatafrom pipelex.tools.typing.structure_printer import StructurePrinter+from pipelex.types import Selfclass PipeLLMOutput(PipeOutput):@@ -59,11 +60,11 @@ class PipeLLMOutput(PipeOutput):class PipeLLM(PipeOperator[PipeLLMOutput]):type: Literal["PipeLLM"] = "PipeLLM"llm_prompt_spec: LLMPromptSpec- llm_choices: Optional[LLMSettingChoices] = None- structuring_method: Optional[StructuringMethod] = None- prompt_template_to_structure: Optional[str] = None- system_prompt_to_structure: Optional[str] = None- output_multiplicity: Optional[PipeOutputMultiplicity] = None+ llm_choices: LLMSettingChoices | None = None+ structuring_method: StructuringMethod | None = None+ prompt_template_to_structure: str | None = None+ system_prompt_to_structure: str | None = None+ output_multiplicity: PipeOutputMultiplicity | None = None@model_validator(mode="after")def validate_inputs(self) -> Self:@@ -73,12 +74,12 @@ class PipeLLM(PipeOperator[PipeLLMOutput]):@model_validator(mode="after")def validate_output_concept_consistency(self) -> Self:- if self.structuring_method is not None:- if self.output.structure_class_name == NativeConceptEnum.TEXT:- raise PipeDefinitionError(- f"Output concept '{self.output.code}' is considered a Text concept, "- f"so it cannot be structured. Maybe you forgot to add '{NativeConceptEnum.TEXT}' to the class registry?"- )+ if self.structuring_method is not None and self.output.structure_class_name == NativeConceptEnum.TEXT:+ msg = (+ f"Output concept '{self.output.code}' is considered a Text concept, "+ f"so it cannot be structured. Maybe you forgot to add '{NativeConceptEnum.TEXT}' to the class registry?"+ )+ raise PipeDefinitionError(msg)return self@override@@ -99,13 +100,14 @@ class PipeLLM(PipeOperator[PipeLLMOutput]):tested_concept=self.output,wanted_concept=get_concept_provider().get_native_concept(native_concept=NativeConceptEnum.IMAGE),):- raise PipeDefinitionError(+ msg = (f"The output of a LLM pipe cannot be compatible with the Image concept. In the "f"pipe '{self.code}' the output is '{self.output.concept_string}'")+ raise PipeDefinitionError(msg)@override- def needed_inputs(self, visited_pipes: Optional[Set[str]] = None) -> PipeInputSpec:+ def needed_inputs(self, visited_pipes: set[str] | None = None) -> PipeInputSpec:"""Needed inputs are the inputs needed to run the pipe, specified in the inputs attribute of the pipe"""# The images are not tagged in the prompt_template.# Therefore if an image is provided in the inputs, it becomes a needed input.@@ -115,7 +117,8 @@ class PipeLLM(PipeOperator[PipeLLMOutput]):for input_name, requirement in self.inputs.items:if concept_provider.is_image_concept(concept=requirement.concept):needed_inputs.add_requirement(- variable_name=input_name, concept=concept_provider.get_native_concept(native_concept=NativeConceptEnum.IMAGE)+ variable_name=input_name,+ concept=concept_provider.get_native_concept(native_concept=NativeConceptEnum.IMAGE),)else:needed_inputs.add_requirement(variable_name=input_name, concept=requirement.concept)@@ -123,19 +126,19 @@ class PipeLLM(PipeOperator[PipeLLMOutput]):return needed_inputs@override- def required_variables(self) -> Set[str]:+ def required_variables(self) -> set[str]:"""Required variables are the variables that are used in the current prompt template or system prompt"""- required_variables: Set[str] = set()+ required_variables: set[str] = set()required_variables.update(self.llm_prompt_spec.required_variables())- required_variables = {variable_name for variable_name in required_variables if not variable_name.startswith("_")}- return required_variables+ return {variable_name for variable_name in required_variables if not variable_name.startswith("_")}def _validate_required_variables(self) -> Self:"""This method checks that all required variables are in the inputs"""required_variables = self.required_variables()for required_variable_name in required_variables:if required_variable_name not in self.inputs.variables:- raise PipeDefinitionError(f"Required variable '{required_variable_name}' is not in the inputs of pipe {self.code}")+ msg = f"Required variable '{required_variable_name}' is not in the inputs of pipe {self.code}"+ raise PipeDefinitionError(msg)return selfdef _validate_inputs(self):@@ -166,12 +169,11 @@ class PipeLLM(PipeOperator[PipeLLMOutput]):if named_input_requirement.concept == concept_provider.get_native_concept(native_concept=NativeConceptEnum.IMAGE):try:input_requirement_of_declared_input = self.inputs.get_required_input_requirement(- variable_name=named_input_requirement.variable_name+ variable_name=named_input_requirement.variable_name,)except PipeInputNotFoundError as exc:- raise PipeInputError(- f"Input variable '{named_input_requirement.variable_name}' is not in this PipeLLM '{self.code}' input spec: {self.inputs}"- ) from exc+ msg = f"Input variable '{named_input_requirement.variable_name}' is not in this PipeLLM '{self.code}' input spec: {self.inputs}"+ raise PipeInputError(msg) from excif not concept_provider.is_compatible(tested_concept=input_requirement_of_declared_input.concept,wanted_concept=named_input_requirement.concept,@@ -230,8 +232,8 @@ class PipeLLM(PipeOperator[PipeLLMOutput]):job_metadata: JobMetadata,working_memory: WorkingMemory,pipe_run_params: PipeRunParams,- output_name: Optional[str] = None,- content_generator: Optional[ContentGeneratorProtocol] = None,+ output_name: str | None = None,+ content_generator: ContentGeneratorProtocol | None = None,) -> PipeLLMOutput:content_generator = content_generator or get_content_generator()# interpret / unwrap the arguments@@ -246,7 +248,7 @@ class PipeLLM(PipeOperator[PipeLLMOutput]):output_concept_code = SpecialDomain.NATIVE + "." + NativeConceptEnum.TEXTelse:output_concept = get_concept_provider().get_required_concept(- concept_string=ConceptFactory.construct_concept_string_with_domain(domain=self.domain, concept_code=output_concept_code)+ concept_string=ConceptFactory.construct_concept_string_with_domain(domain=self.domain, concept_code=output_concept_code),)multiplicity_resolution = output_multiplicity_to_apply(@@ -258,8 +260,8 @@ class PipeLLM(PipeOperator[PipeLLMOutput]):fixed_nb_output = multiplicity_resolution.specific_output_count# Collect what LLM settings we have for this particular PipeLLM- llm_for_text_choice: Optional[LLMChoice] = None- llm_for_object_choice: Optional[LLMChoice] = None+ llm_for_text_choice: LLMChoice | None = None+ llm_for_object_choice: LLMChoice | None = Noneif self.llm_choices:llm_for_text_choice = self.llm_choices.for_textllm_for_object_choice = self.llm_choices.for_object@@ -295,10 +297,10 @@ class PipeLLM(PipeOperator[PipeLLMOutput]):self.structuring_method == StructuringMethod.PRELIMINARY_TEXT) or get_config().pipelex.structure_config.is_default_text_then_structurelog.verbose(- f"is_with_preliminary_text: {is_with_preliminary_text} for pipe {self.code} because the structuring_method is {self.structuring_method}"+ f"is_with_preliminary_text: {is_with_preliminary_text} for pipe {self.code} because the structuring_method is {self.structuring_method}",)# Append output structure prompt if needed- output_structure_prompt: Optional[str] = PipeLLM.get_output_structure_prompt(+ output_structure_prompt: str | None = PipeLLM.get_output_structure_prompt(concept_string=pipe_run_params.dynamic_output_concept_code or output_concept.concept_string,is_with_preliminary_text=is_with_preliminary_text,)@@ -337,9 +339,9 @@ class PipeLLM(PipeOperator[PipeLLMOutput]):# TODO: we need a better solution for structuring_method (text then object), meanwhile,# we acknowledge the code here with llm_prompt_1 and llm_prompt_2 is overly complex and should be refactored.- llm_prompt_2_factory: Optional[LLMPromptFactoryAbstract]+ llm_prompt_2_factory: LLMPromptFactoryAbstract | Noneif self.structuring_method:- structuring_method = cast(StructuringMethod, self.structuring_method)+ structuring_method = cast("StructuringMethod", self.structuring_method)log.debug(f"PipeLLM pipe_code is '{self.code}' and structuring_method is '{structuring_method}'")match structuring_method:case StructuringMethod.DIRECT:@@ -409,25 +411,24 @@ class PipeLLM(PipeOperator[PipeLLMOutput]):name=output_name,)- pipe_output = PipeLLMOutput(+ return PipeLLMOutput(working_memory=working_memory,pipeline_run_id=job_metadata.pipeline_run_id,)- return pipe_outputasync def _llm_gen_object_stuff_content(self,job_metadata: JobMetadata,is_multiple_output: bool,- fixed_nb_output: Optional[int],+ fixed_nb_output: int | None,output_class_name: str,llm_setting_main: LLMSetting,llm_setting_for_object: LLMSetting,llm_prompt_1: LLMPrompt,- llm_prompt_2_factory: Optional[LLMPromptFactoryAbstract],+ llm_prompt_2_factory: LLMPromptFactoryAbstract | None,content_generator: ContentGeneratorProtocol,) -> StuffContent:- content_class: Type[StuffContent] = get_class_registry().get_required_subclass(name=output_class_name, base_class=StuffContent)+ content_class: type[StuffContent] = get_class_registry().get_required_subclass(name=output_class_name, base_class=StuffContent)task_desc: strthe_content: StuffContent@@ -438,7 +439,7 @@ class PipeLLM(PipeOperator[PipeLLMOutput]):else:task_desc = f"{self.class_name}_gen_list_{content_class.__class__.__name__}"log.dev(task_desc)- generated_objects: List[StuffContent]+ generated_objects: list[StuffContent]if llm_prompt_2_factory is not None:# We're generating a list of objects using preliminary textmethod_desc = "text_then_object"@@ -504,17 +505,16 @@ class PipeLLM(PipeOperator[PipeLLMOutput]):job_metadata: JobMetadata,working_memory: WorkingMemory,pipe_run_params: PipeRunParams,- output_name: Optional[str] = None,+ output_name: str | None = None,) -> PipeLLMOutput:content_generator_dry = ContentGeneratorDry()- pipe_output = await self._run_operator_pipe(+ return await self._run_operator_pipe(job_metadata=job_metadata,working_memory=working_memory,pipe_run_params=pipe_run_params,output_name=output_name,content_generator=content_generator_dry,)- return pipe_output@staticmethoddef get_output_structure_prompt(concept_string: str, is_with_preliminary_text: bool) -> str:diff --git a/pipelex/pipe_operators/llm/pipe_llm_blueprint.py b/pipelex/pipe_operators/llm/pipe_llm_blueprint.pyindex 84605a5b3..a7c15ee59 100644--- a/pipelex/pipe_operators/llm/pipe_llm_blueprint.py+++ b/pipelex/pipe_operators/llm/pipe_llm_blueprint.py@@ -1,13 +1,12 @@-from typing import Literal, Optional+from typing import Literalfrom pydantic import field_validator, model_validator-from typing_extensions import Selffrom pipelex.cogt.llm.llm_setting import LLMChoicefrom pipelex.core.pipes.pipe_blueprint import PipeBlueprintfrom pipelex.exceptions import PipeDefinitionErrorfrom pipelex.tools.typing.validation_utils import has_more_than_one_among_attributes_from_lists-from pipelex.types import StrEnum+from pipelex.types import Self, StrEnumclass StructuringMethod(StrEnum):@@ -18,30 +17,32 @@ class StructuringMethod(StrEnum):class PipeLLMBlueprint(PipeBlueprint):type: Literal["PipeLLM"] = "PipeLLM"category: Literal["PipeOperator"] = "PipeOperator"- system_prompt_template: Optional[str] = None- system_prompt_template_name: Optional[str] = None- system_prompt_name: Optional[str] = None- system_prompt: Optional[str] = None+ system_prompt_template: str | None = None+ system_prompt_template_name: str | None = None+ system_prompt_name: str | None = None+ system_prompt: str | None = None- prompt_template: Optional[str] = None- template_name: Optional[str] = None- prompt_name: Optional[str] = None- prompt: Optional[str] = None+ prompt_template: str | None = None+ template_name: str | None = None+ prompt_name: str | None = None+ prompt: str | None = None- llm: Optional[LLMChoice] = None- llm_to_structure: Optional[LLMChoice] = None+ llm: LLMChoice | None = None+ llm_to_structure: LLMChoice | None = None- structuring_method: Optional[StructuringMethod] = None- prompt_template_to_structure: Optional[str] = None- system_prompt_to_structure: Optional[str] = None+ structuring_method: StructuringMethod | None = None+ prompt_template_to_structure: str | None = None+ system_prompt_to_structure: str | None = None- nb_output: Optional[int] = None- multiple_output: Optional[bool] = None+ nb_output: int | None = None+ multiple_output: bool | None = None@field_validator("nb_output", mode="after")- def validate_nb_output(cls, value: Optional[int] = None) -> Optional[int]:+ @staticmethod+ def validate_nb_output(value: int | None = None) -> int | None:if value and value < 1:- raise PipeDefinitionError("PipeLLMBlueprint nb_output must be greater than 0")+ msg = "PipeLLMBlueprint nb_output must be greater than 0"+ raise PipeDefinitionError(msg)return value@model_validator(mode="after")@@ -54,5 +55,6 @@ class PipeLLMBlueprint(PipeBlueprint):["prompt", "prompt_name", "prompt_template", "template_name"],],):- raise PipeDefinitionError(f"PipeLLMBlueprint should have no more than one of {excess_attributes_list} among them")+ msg = f"PipeLLMBlueprint should have no more than one of {excess_attributes_list} among them"+ raise PipeDefinitionError(msg)return selfdiff --git a/pipelex/pipe_operators/llm/pipe_llm_factory.py b/pipelex/pipe_operators/llm/pipe_llm_factory.pyindex e1479f74e..60410525b 100644--- a/pipelex/pipe_operators/llm/pipe_llm_factory.py+++ b/pipelex/pipe_operators/llm/pipe_llm_factory.py@@ -1,5 +1,3 @@-from typing import List, Optional-from typing_extensions import overridefrom pipelex.cogt.llm.llm_prompt_spec import LLMPromptSpec@@ -26,10 +24,10 @@ class PipeLLMFactory(PipeFactoryProtocol[PipeLLMBlueprint, PipeLLM]):domain: str,pipe_code: str,blueprint: PipeLLMBlueprint,- concept_codes_from_the_same_domain: Optional[List[str]] = None,+ concept_codes_from_the_same_domain: list[str] | None = None,) -> PipeLLM:- system_prompt_jinja2_blueprint: Optional[Jinja2Blueprint] = None- system_prompt: Optional[str] = None+ system_prompt_jinja2_blueprint: Jinja2Blueprint | None = None+ system_prompt: str | None = Noneif blueprint.system_prompt_template or blueprint.system_prompt_template_name:try:system_prompt_jinja2_blueprint = Jinja2Blueprint(@@ -48,7 +46,7 @@ class PipeLLMFactory(PipeFactoryProtocol[PipeLLMBlueprint, PipeLLM]):if domain_obj := get_optional_domain(domain=domain):system_prompt = domain_obj.system_prompt- user_text_jinja2_blueprint: Optional[Jinja2Blueprint] = None+ user_text_jinja2_blueprint: Jinja2Blueprint | None = Noneif blueprint.prompt_template or blueprint.template_name:try:user_text_jinja2_blueprint = Jinja2Blueprint(@@ -72,12 +70,15 @@ class PipeLLMFactory(PipeFactoryProtocol[PipeLLMBlueprint, PipeLLM]):error_msg = f"Jinja2 template not found for pipe '{pipe_code}' in domain '{domain}': {exc}."raise PipeDefinitionError(error_msg) from exc- user_images: List[str] = []+ user_images: list[str] = []if blueprint.inputs:for stuff_name, requirement in blueprint.inputs.items():if isinstance(requirement, str):- requirement = InputRequirementBlueprint(concept=requirement)- concept_string = requirement.concept+ input_requirement_blueprint = InputRequirementBlueprint(concept=requirement)+ else:+ input_requirement_blueprint = requirement++ concept_string = input_requirement_blueprint.conceptdomain_and_code = ConceptFactory.make_domain_and_concept_code_from_concept_string_or_code(domain=domain,concept_string_or_code=concept_string,@@ -85,8 +86,9 @@ class PipeLLMFactory(PipeFactoryProtocol[PipeLLMBlueprint, PipeLLM]):)concept = get_concept_provider().get_required_concept(concept_string=ConceptFactory.construct_concept_string_with_domain(- domain=domain_and_code.domain, concept_code=domain_and_code.concept_code- )+ domain=domain_and_code.domain,+ concept_code=domain_and_code.concept_code,+ ),)if get_concept_provider().is_image_concept(concept=concept):@@ -125,10 +127,12 @@ class PipeLLMFactory(PipeFactoryProtocol[PipeLLMBlueprint, PipeLLM]):code=pipe_code,definition=blueprint.definition,inputs=PipeInputSpecFactory.make_from_blueprint(- domain=domain, blueprint=blueprint.inputs or {}, concept_codes_from_the_same_domain=concept_codes_from_the_same_domain+ domain=domain,+ blueprint=blueprint.inputs or {},+ concept_codes_from_the_same_domain=concept_codes_from_the_same_domain,),output=get_concept_provider().get_required_concept(- concept_string=ConceptFactory.construct_concept_string_with_domain(domain=output_concept_domain, concept_code=output_concept_code)+ concept_string=ConceptFactory.construct_concept_string_with_domain(domain=output_concept_domain, concept_code=output_concept_code),),llm_prompt_spec=llm_prompt_spec,llm_choices=llm_choices,diff --git a/pipelex/pipe_operators/ocr/__init__.py b/pipelex/pipe_operators/ocr/__init__.pynew file mode 100644index 000000000..e69de29bbdiff --git a/pipelex/pipe_operators/ocr/pipe_ocr.py b/pipelex/pipe_operators/ocr/pipe_ocr.pyindex 6530fb618..34af1cf67 100644--- a/pipelex/pipe_operators/ocr/pipe_ocr.py+++ b/pipelex/pipe_operators/ocr/pipe_ocr.py@@ -1,7 +1,7 @@-from typing import List, Literal, Optional, Set+from typing import Literalfrom pydantic import model_validator-from typing_extensions import Self, override+from typing_extensions import overridefrom pipelex import logfrom pipelex.cogt.content_generation.content_generator_dry import ContentGeneratorDry@@ -33,6 +33,7 @@ from pipelex.hub import (from pipelex.pipe_operators.pipe_operator import PipeOperatorfrom pipelex.pipeline.job_metadata import JobMetadatafrom pipelex.tools.pdf.pypdfium2_renderer import pypdfium2_renderer+from pipelex.types import Selfclass PipeOcrOutput(PipeOutput):@@ -44,14 +45,14 @@ PIPE_OCR_INPUT_NAME = "ocr_input"class PipeOcr(PipeOperator[PipeOcrOutput]):type: Literal["PipeOcr"] = "PipeOcr"- ocr_choice: Optional[OcrChoice]+ ocr_choice: OcrChoice | Noneshould_caption_images: boolshould_include_images: boolshould_include_page_views: boolpage_views_dpi: int- image_stuff_name: Optional[str] = None- pdf_stuff_name: Optional[str] = None+ image_stuff_name: str | None = None+ pdf_stuff_name: str | None = None@model_validator(mode="after")def validate_inputs(self) -> Self:@@ -65,13 +66,14 @@ class PipeOcr(PipeOperator[PipeOcrOutput]):check_ocr_choice_with_deck(ocr_choice=self.ocr_choice)@override- def required_variables(self) -> Set[str]:+ def required_variables(self) -> set[str]:return {PIPE_OCR_INPUT_NAME}@overridedef validate_output(self):if self.output != get_concept_provider().get_native_concept(native_concept=NativeConceptEnum.PAGE):- raise PipeDefinitionError(f"PipeOcr output should be a Page concept, but is {self.output.concept_string}")+ msg = f"PipeOcr output should be a Page concept, but is {self.output.concept_string}"+ raise PipeDefinitionError(msg)def _validate_inputs(self):concept_provider = get_concept_provider()@@ -80,7 +82,7 @@ class PipeOcr(PipeOperator[PipeOcrOutput]):reactions = static_validation_config.reactions# check that we have either an image or a pdf in inputs, at most one of them and nothing else- candidate_prompt_var_names: List[str] = []+ candidate_prompt_var_names: list[str] = []for input_name, requirement in self.inputs.items:log.debug(f"{input_name=}")log.debug(f"{requirement=}")@@ -147,7 +149,7 @@ class PipeOcr(PipeOperator[PipeOcrOutput]):raise missing_input_var_error@override- def needed_inputs(self, visited_pipes: Optional[Set[str]] = None) -> PipeInputSpec:+ def needed_inputs(self, visited_pipes: set[str] | None = None) -> PipeInputSpec:return PipeInputSpecFactory.make_from_blueprint(domain=self.domain,blueprint={PIPE_OCR_INPUT_NAME: InputRequirementBlueprint(concept=self.inputs.root[PIPE_OCR_INPUT_NAME].concept.concept_string)},@@ -159,13 +161,13 @@ class PipeOcr(PipeOperator[PipeOcrOutput]):job_metadata: JobMetadata,working_memory: WorkingMemory,pipe_run_params: PipeRunParams,- output_name: Optional[str] = None,- content_generator: Optional[ContentGeneratorProtocol] = None,+ output_name: str | None = None,+ content_generator: ContentGeneratorProtocol | None = None,) -> PipeOcrOutput:content_generator = content_generator or get_content_generator()- image_uri: Optional[str] = None- pdf_uri: Optional[str] = None+ image_uri: str | None = None+ pdf_uri: str | None = Noneif self.image_stuff_name:image_stuff = working_memory.get_stuff_as_image(name=self.image_stuff_name)image_uri = image_stuff.url@@ -173,7 +175,8 @@ class PipeOcr(PipeOperator[PipeOcrOutput]):pdf_stuff = working_memory.get_stuff_as_pdf(name=self.pdf_stuff_name)pdf_uri = pdf_stuff.urlelse:- raise PipeDefinitionError("PipeOcr should have a non-None image_stuff_name or pdf_stuff_name")+ msg = "PipeOcr should have a non-None image_stuff_name or pdf_stuff_name"+ raise PipeDefinitionError(msg)ocr_choice: OcrChoice = self.ocr_choice or get_model_deck().ocr_choice_defaultocr_setting: OcrSetting = get_model_deck().get_ocr_setting(ocr_choice=ocr_choice)@@ -199,13 +202,13 @@ class PipeOcr(PipeOperator[PipeOcrOutput]):)# Build the output stuff, which is a list of page contents- page_view_contents: List[ImageContent] = []+ page_view_contents: list[ImageContent] = []if self.should_include_page_views:log.debug(f"should_include_page_views: {self.should_include_page_views}, pdf_uri: {pdf_uri}, image_uri: {image_uri}")if pdf_uri:- for page in ocr_output.pages.values():- if page.page_view:- page_view_contents.append(ImageContent.make_from_extracted_image(extracted_image=page.page_view))+ page_view_contents.extend(+ ImageContent.make_from_extracted_image(extracted_image=page.page_view) for page in ocr_output.pages.values() if page.page_view+ )log.debug(f"page_view_contents: {page_view_contents}")needs_to_generate_page_views: boolif len(page_view_contents) == 0:@@ -224,7 +227,7 @@ class PipeOcr(PipeOperator[PipeOcrOutput]):elif image_uri:page_view_contents = [ImageContent.make_from_str(str_value=image_uri)]- page_contents: List[PageContent] = []+ page_contents: list[PageContent] = []for page_index, page in ocr_output.pages.items():images = [ImageContent.make_from_extracted_image(extracted_image=img) for img in page.extracted_images]log.debug(f"images: {images}, page_view_contents: {page_view_contents}, index: {page_index}")@@ -236,7 +239,7 @@ class PipeOcr(PipeOperator[PipeOcrOutput]):images=images,),page_view=page_view,- )+ ),)content: ListContent[PageContent] = ListContent(items=page_contents)@@ -252,11 +255,10 @@ class PipeOcr(PipeOperator[PipeOcrOutput]):name=output_name,)- pipe_output = PipeOcrOutput(+ return PipeOcrOutput(working_memory=working_memory,pipeline_run_id=job_metadata.pipeline_run_id,)- return pipe_output@overrideasync def _dry_run_operator_pipe(@@ -264,18 +266,18 @@ class PipeOcr(PipeOperator[PipeOcrOutput]):job_metadata: JobMetadata,working_memory: WorkingMemory,pipe_run_params: PipeRunParams,- output_name: Optional[str] = None,+ output_name: str | None = None,) -> PipeOcrOutput:log.debug(f"PipeOcr: dry run operator pipe: {self.code}")if pipe_run_params.run_mode != PipeRunMode.DRY:- raise PipeDefinitionError(f"Running pipe '{self.code}' (PipeOcr) _dry_run_operator_pipe() in non-dry mode is not allowed.")+ msg = f"Running pipe '{self.code}' (PipeOcr) _dry_run_operator_pipe() in non-dry mode is not allowed."+ raise PipeDefinitionError(msg)content_generator_dry = ContentGeneratorDry()- pipe_output = await self._run_operator_pipe(+ return await self._run_operator_pipe(job_metadata=job_metadata,working_memory=working_memory,pipe_run_params=pipe_run_params,output_name=output_name,content_generator=content_generator_dry,)- return pipe_outputdiff --git a/pipelex/pipe_operators/ocr/pipe_ocr_blueprint.py b/pipelex/pipe_operators/ocr/pipe_ocr_blueprint.pyindex 610eb5e96..b09179f33 100644--- a/pipelex/pipe_operators/ocr/pipe_ocr_blueprint.py+++ b/pipelex/pipe_operators/ocr/pipe_ocr_blueprint.py@@ -1,4 +1,4 @@-from typing import Literal, Optional+from typing import Literalfrom pipelex.cogt.ocr.ocr_setting import OcrChoicefrom pipelex.core.pipes.pipe_blueprint import PipeBlueprint@@ -7,8 +7,8 @@ from pipelex.core.pipes.pipe_blueprint import PipeBlueprintclass PipeOcrBlueprint(PipeBlueprint):type: Literal["PipeOcr"] = "PipeOcr"category: Literal["PipeOperator"] = "PipeOperator"- ocr: Optional[OcrChoice] = None- page_images: Optional[bool] = None- page_image_captions: Optional[bool] = None- page_views: Optional[bool] = None- page_views_dpi: Optional[int] = None+ ocr: OcrChoice | None = None+ page_images: bool | None = None+ page_image_captions: bool | None = None+ page_views: bool | None = None+ page_views_dpi: int | None = Nonediff --git a/pipelex/pipe_operators/ocr/pipe_ocr_factory.py b/pipelex/pipe_operators/ocr/pipe_ocr_factory.pyindex 80251a98a..65fd09980 100644--- a/pipelex/pipe_operators/ocr/pipe_ocr_factory.py+++ b/pipelex/pipe_operators/ocr/pipe_ocr_factory.py@@ -1,5 +1,3 @@-from typing import List, Optional-from typing_extensions import overridefrom pipelex.config import get_config@@ -19,7 +17,7 @@ class PipeOcrFactory(PipeFactoryProtocol[PipeOcrBlueprint, PipeOcr]):domain: str,pipe_code: str,blueprint: PipeOcrBlueprint,- concept_codes_from_the_same_domain: Optional[List[str]] = None,+ concept_codes_from_the_same_domain: list[str] | None = None,) -> PipeOcr:output_domain_and_code = ConceptFactory.make_domain_and_concept_code_from_concept_string_or_code(domain=domain,@@ -33,11 +31,14 @@ class PipeOcrFactory(PipeFactoryProtocol[PipeOcrBlueprint, PipeOcr]):definition=blueprint.definition,output=get_concept_provider().get_required_concept(concept_string=ConceptFactory.construct_concept_string_with_domain(- domain=output_domain_and_code.domain, concept_code=output_domain_and_code.concept_code- )+ domain=output_domain_and_code.domain,+ concept_code=output_domain_and_code.concept_code,+ ),),inputs=PipeInputSpecFactory.make_from_blueprint(- domain=domain, blueprint=blueprint.inputs or {}, concept_codes_from_the_same_domain=concept_codes_from_the_same_domain+ domain=domain,+ blueprint=blueprint.inputs or {},+ concept_codes_from_the_same_domain=concept_codes_from_the_same_domain,),ocr_choice=blueprint.ocr,should_include_images=blueprint.page_images or False,diff --git a/pipelex/pipe_operators/pipe_operator.py b/pipelex/pipe_operators/pipe_operator.pyindex b0f650534..75ad892f0 100644--- a/pipelex/pipe_operators/pipe_operator.py+++ b/pipelex/pipe_operators/pipe_operator.py@@ -1,5 +1,5 @@from abc import abstractmethod-from typing import Generic, Literal, Optional, TypeVar+from typing import Generic, Literal, TypeVarfrom typing_extensions import override@@ -25,8 +25,8 @@ class PipeOperator(PipeAbstract, Generic[PipeOperatorOutputType]):job_metadata: JobMetadata,working_memory: WorkingMemory,pipe_run_params: PipeRunParams,- output_name: Optional[str] = None,- print_intermediate_outputs: Optional[bool] = False,+ output_name: str | None = None,+ print_intermediate_outputs: bool | None = False,) -> PipeOutput:pipe_run_params.push_pipe_to_stack(pipe_code=self.code)self.monitor_pipe_stack(pipe_run_params=pipe_run_params)@@ -70,7 +70,7 @@ class PipeOperator(PipeAbstract, Generic[PipeOperatorOutputType]):activity_report=ActivityReport(job_metadata=job_metadata,content=pipe_output.main_stuff,- )+ ),)pipe_run_params.pop_pipe_from_stack(pipe_code=self.code)@@ -83,7 +83,7 @@ class PipeOperator(PipeAbstract, Generic[PipeOperatorOutputType]):job_metadata: JobMetadata,working_memory: WorkingMemory,pipe_run_params: PipeRunParams,- output_name: Optional[str] = None,+ output_name: str | None = None,) -> PipeOperatorOutputType:pass@@ -93,6 +93,6 @@ class PipeOperator(PipeAbstract, Generic[PipeOperatorOutputType]):job_metadata: JobMetadata,working_memory: WorkingMemory,pipe_run_params: PipeRunParams,- output_name: Optional[str] = None,+ output_name: str | None = None,) -> PipeOperatorOutputType:passdiff --git a/pipelex/pipe_works/__init__.py b/pipelex/pipe_works/__init__.pyindex 8b1378917..e69de29bb 100644--- a/pipelex/pipe_works/__init__.py+++ b/pipelex/pipe_works/__init__.py@@ -1 +0,0 @@-diff --git a/pipelex/pipe_works/pipe_dry.py b/pipelex/pipe_works/pipe_dry.pyindex eb7695b9f..f5285278a 100644--- a/pipelex/pipe_works/pipe_dry.py+++ b/pipelex/pipe_works/pipe_dry.py@@ -2,7 +2,6 @@ import asyncioimport functoolsimport timefrom concurrent.futures import ThreadPoolExecutor-from typing import Dict, List, Optionalfrom pydantic import BaseModel@@ -22,26 +21,32 @@ from pipelex.types import StrEnumclass DryRunError(Exception):"""Raised when a dry run fails due to missing inputs or other validation issues."""- pass-class DryRunStatus(StrEnum):SUCCESS = "SUCCESS"FAILURE = "FAILURE"WARNING = "WARNING"+ @property+ def is_failure(self) -> bool:+ match self:+ case DryRunStatus.FAILURE:+ return True+ case DryRunStatus.SUCCESS:+ return False+ case DryRunStatus.WARNING:+ return False+class DryRunOutput(BaseModel):pipe_code: strstatus: DryRunStatus- error_message: Optional[str] = None- warning_message: Optional[str] = None+ error_message: str | None = None+ warning_message: str | None = Noneasync def dry_run_pipe(pipe: PipeAbstract, raise_on_failure: bool = False) -> DryRunOutput:- """- Dry run a single pipe directly without parallelization.- """+ """Dry run a single pipe directly without parallelization."""allowed_to_fail_pipes = get_config().pipelex.dry_run_config.allowed_to_fail_pipesneeded_inputs_for_factory = _convert_to_working_memory_format(needed_inputs_spec=pipe.needed_inputs())@@ -58,23 +63,24 @@ async def dry_run_pipe(pipe: PipeAbstract, raise_on_failure: bool = False) -> Drwarning_message = f"Allowed to fail dry run for pipe '{pipe.code}': {exc}"log.warning(warning_message)return DryRunOutput(pipe_code=pipe.code, status=DryRunStatus.WARNING, warning_message=warning_message)- elif raise_on_failure:- raise exc- else:- error_message = f"Dry run failed for pipe '{pipe.code}': {exc}"- log.error(error_message)- return DryRunOutput(pipe_code=pipe.code, status=DryRunStatus.FAILURE, error_message=error_message)++ if raise_on_failure:+ raise++ error_message = f"Dry run failed for pipe '{pipe.code}': {exc}"+ log.error(error_message)+ return DryRunOutput(pipe_code=pipe.code, status=DryRunStatus.FAILURE, error_message=error_message)log.info(f"Pipe '{pipe.code}' dry run completed successfully")return DryRunOutput(pipe_code=pipe.code, status=DryRunStatus.SUCCESS)-async def dry_run_pipes(pipes: List[PipeAbstract], run_in_parallel: bool = True, raise_on_failure: bool = True) -> Dict[str, DryRunOutput]:- """- Dry run pipes with optional parallelization.+async def dry_run_pipes(pipes: list[PipeAbstract], run_in_parallel: bool = True, raise_on_failure: bool = True) -> dict[str, DryRunOutput]:+ """Dry run pipes with optional parallelization.Args:pipes: List of pipes to dry runrun_in_parallel: If True, run pipes in parallel using ThreadPoolExecutor. If False, run sequentially.+ raise_on_failure: If True, raise an exception if any pipe fails.For each pipe, this method:1. Gets the pipe's needed inputs@@ -83,10 +89,10 @@ async def dry_run_pipes(pipes: List[PipeAbstract], run_in_parallel: bool = True,Returns:Dict mapping pipe codes to their dry run status ("SUCCESS" or error message)- """+ """start_time = time.time()- results: Dict[str, DryRunOutput] = {}+ results: dict[str, DryRunOutput] = {}allowed_to_fail_pipes = get_config().pipelex.dry_run_config.allowed_to_fail_pipesif run_in_parallel:@@ -113,30 +119,30 @@ async def dry_run_pipes(pipes: List[PipeAbstract], run_in_parallel: bool = True,log.info(f"Dry run completed: '{len(successful_pipes)}' successful, '{len(failed_pipes)}' failed, "- f"'{len(allowed_to_fail_pipes)}' allowed to fail, in '{time.time() - start_time:.2f}' seconds"+ f"'{len(allowed_to_fail_pipes)}' allowed to fail, in '{time.time() - start_time:.2f}' seconds",)if unexpected_failures:unexpected_failures_details = "\n".join([f"'{pipe_code}': {results[pipe_code]}" for pipe_code in unexpected_failures])if raise_on_failure:- raise DryRunError(f"Dry run failed with '{len(unexpected_failures)}' unexpected pipe failures:\n{unexpected_failures_details}")- else:- log.error(f"Dry run failed with '{len(unexpected_failures)}' unexpected pipe failures:\n{unexpected_failures_details}")- return results+ msg = f"Dry run failed with '{len(unexpected_failures)}' unexpected pipe failures:\n{unexpected_failures_details}"+ raise DryRunError(msg)+ log.error(f"Dry run failed with '{len(unexpected_failures)}' unexpected pipe failures:\n{unexpected_failures_details}")+ return resultsreturn results-def _convert_to_working_memory_format(needed_inputs_spec: PipeInputSpec) -> List[TypedNamedInputRequirement]:- """- Convert PipeInputSpec to the format needed by WorkingMemoryFactory.make_for_dry_run.+def _convert_to_working_memory_format(needed_inputs_spec: PipeInputSpec) -> list[TypedNamedInputRequirement]:+ """Convert PipeInputSpec to the format needed by WorkingMemoryFactory.make_for_dry_run.Args:needed_inputs_spec: PipeInputSpec with detailed_requirementsReturns:List of tuples (variable_name, concept_code, structure_class)+"""- needed_inputs_for_factory: List[TypedNamedInputRequirement] = []+ needed_inputs_for_factory: list[TypedNamedInputRequirement] = []class_registry = get_class_registry()for named_input_requirement in needed_inputs_spec.named_input_requirements:@@ -158,7 +164,7 @@ def _convert_to_working_memory_format(needed_inputs_spec: PipeInputSpec) -> List# Fallback to TextContent if we can't get the proper classlog.warning(f"Could not get structure class '{structure_class_name}' for "- f"concept '{named_input_requirement.concept.code}', falling back to TextContent"+ f"concept '{named_input_requirement.concept.code}', falling back to TextContent",)text_typed_named_input_requirement = TypedNamedInputRequirement.make_from_named(named=named_input_requirement,diff --git a/pipelex/pipe_works/pipe_job.py b/pipelex/pipe_works/pipe_job.pyindex d25b0f61a..172b44ce9 100644--- a/pipelex/pipe_works/pipe_job.py+++ b/pipelex/pipe_works/pipe_job.py@@ -1,5 +1,3 @@-from typing import Optional-from pydantic import BaseModel, Fieldfrom pipelex.core.memory.working_memory import WorkingMemory@@ -13,4 +11,4 @@ class PipeJob(BaseModel):working_memory: WorkingMemory = Field(default_factory=WorkingMemory)pipe_run_params: PipeRunParamsjob_metadata: JobMetadata- output_name: Optional[str] = None+ output_name: str | None = Nonediff --git a/pipelex/pipe_works/pipe_job_factory.py b/pipelex/pipe_works/pipe_job_factory.pyindex 73340757e..8f6f2f185 100644--- a/pipelex/pipe_works/pipe_job_factory.py+++ b/pipelex/pipe_works/pipe_job_factory.py@@ -1,5 +1,3 @@-from typing import Optional-from pipelex.core.memory.working_memory import WorkingMemoryfrom pipelex.core.memory.working_memory_factory import WorkingMemoryFactoryfrom pipelex.core.pipes.pipe_abstract import PipeAbstract@@ -14,10 +12,10 @@ class PipeJobFactory:def make_pipe_job(cls,pipe: PipeAbstract,- pipe_run_params: Optional[PipeRunParams] = None,- working_memory: Optional[WorkingMemory] = None,- job_metadata: Optional[JobMetadata] = None,- output_name: Optional[str] = None,+ pipe_run_params: PipeRunParams | None = None,+ working_memory: WorkingMemory | None = None,+ job_metadata: JobMetadata | None = None,+ output_name: str | None = None,) -> PipeJob:job_metadata = job_metadata or JobMetadata()working_memory = working_memory or WorkingMemoryFactory.make_empty()diff --git a/pipelex/pipe_works/pipe_router_protocol.py b/pipelex/pipe_works/pipe_router_protocol.pyindex ee1e3793f..2977bd4f6 100644--- a/pipelex/pipe_works/pipe_router_protocol.py+++ b/pipelex/pipe_works/pipe_router_protocol.py@@ -35,6 +35,7 @@ class PipeRouterProtocol(Protocol):) -> None:payload: PayloadType = {"pipe_job": pipe_job,+ "error": error,}await self.observer_provider.observe_after_failing_run(payload)@@ -48,7 +49,7 @@ class PipeRouterProtocol(Protocol):pipe_output = await self._run_pipe_job(pipe_job)except Exception as exc:await self._after_failing_run(pipe_job, exc)- raise exc+ raiseawait self._after_successful_run(pipe_job, pipe_output)diff --git a/pipelex/pipelex.py b/pipelex/pipelex.pyindex e9926031e..4a66d11f1 100644--- a/pipelex/pipelex.py+++ b/pipelex/pipelex.py@@ -1,14 +1,13 @@import inspectimport osfrom importlib.metadata import metadata-from typing import Optional, Type, cast+from typing import castfrom kajson.class_registry import ClassRegistryfrom kajson.class_registry_abstract import ClassRegistryAbstractfrom kajson.kajson_manager import KajsonManagerfrom kajson.singleton import MetaSingletonfrom pydantic import ValidationError-from typing_extensions import Selffrom pipelex import logfrom pipelex.cogt.content_generation.content_generator import ContentGenerator@@ -53,6 +52,7 @@ from pipelex.tools.secrets.secrets_provider_abstract import SecretsProviderAbstrfrom pipelex.tools.storage.storage_provider_abstract import StorageProviderAbstractfrom pipelex.tools.templating.template_library import TemplateLibraryfrom pipelex.tools.typing.pydantic_utils import format_pydantic_validation_error+from pipelex.types import SelfPACKAGE_NAME = __name__.split(".", maxsplit=1)[0]PACKAGE_VERSION = metadata(PACKAGE_NAME)["Version"]@@ -63,16 +63,16 @@ class Pipelex(metaclass=MetaSingleton):self,config_dir_path: str,# Dependency injection- pipelex_hub: Optional[PipelexHub] = None,- config_cls: Optional[Type[ConfigRoot]] = None,- class_registry: Optional[ClassRegistryAbstract] = None,- template_provider: Optional[TemplateLibrary] = None,- models_manager: Optional[ModelManagerAbstract] = None,- inference_manager: Optional[InferenceManager] = None,- pipeline_manager: Optional[PipelineManager] = None,- pipeline_tracker: Optional[PipelineTracker] = None,- activity_manager: Optional[ActivityManagerProtocol] = None,- reporting_delegate: Optional[ReportingProtocol] = None,+ pipelex_hub: PipelexHub | None = None,+ config_cls: type[ConfigRoot] | None = None,+ class_registry: ClassRegistryAbstract | None = None,+ template_provider: TemplateLibrary | None = None,+ models_manager: ModelManagerAbstract | None = None,+ inference_manager: InferenceManager | None = None,+ pipeline_manager: PipelineManager | None = None,+ pipeline_tracker: PipelineTracker | None = None,+ activity_manager: ActivityManagerProtocol | None = None,+ reporting_delegate: ReportingProtocol | None = None,) -> None:self.config_dir_path = config_dir_pathself.pipelex_hub = pipelex_hub or PipelexHub()@@ -82,8 +82,9 @@ class Pipelex(metaclass=MetaSingleton):try:self.pipelex_hub.setup_config(config_cls=config_cls or PipelexConfig)except ValidationError as exc:- error_msg = format_pydantic_validation_error(exc)- raise PipelexConfigError(f"Could not setup config because of: {error_msg}") from exc+ formatted_error_msg = format_pydantic_validation_error(exc)+ msg = f"Could not setup config because of: {formatted_error_msg}"+ raise PipelexConfigError(msg) from exclog.configure(project_name=get_config().project_name or "unknown_project",@@ -157,10 +158,10 @@ class Pipelex(metaclass=MetaSingleton):def setup(self,- secrets_provider: Optional[SecretsProviderAbstract] = None,- content_generator: Optional[ContentGeneratorProtocol] = None,- pipe_router: Optional[PipeRouterProtocol] = None,- storage_provider: Optional[StorageProviderAbstract] = None,+ secrets_provider: SecretsProviderAbstract | None = None,+ content_generator: ContentGeneratorProtocol | None = None,+ pipe_router: PipeRouterProtocol | None = None,+ storage_provider: StorageProviderAbstract | None = None,):# toolsself.pipelex_hub.set_secrets_provider(secrets_provider or EnvSecretsProvider())@@ -170,9 +171,8 @@ class Pipelex(metaclass=MetaSingleton):try:self.models_manager.setup()except RoutingProfileLibraryNotFoundError as routing_profile_library_exc:- raise PipelexSetupError(- "The routing library could not be found, please call `pipelex init config` to create it"- ) from routing_profile_library_exc+ msg = "The routing library could not be found, please call `pipelex init config` to create it"+ raise PipelexSetupError(msg) from routing_profile_library_excexcept InferenceBackendCredentialsError as credentials_exc:backend_name = credentials_exc.backend_namevar_name = credentials_exc.key_name@@ -219,16 +219,18 @@ class Pipelex(metaclass=MetaSingleton):self.library_manager.setup()self.library_manager.load_libraries()except ValidationError as exc:- error_msg = format_pydantic_validation_error(exc)- raise PipelexSetupError(f"Could not setup libraries because of: {error_msg}") from exc+ formatted_error_msg = format_pydantic_validation_error(exc)+ msg = f"Could not setup libraries because of: {formatted_error_msg}"+ raise PipelexSetupError(msg) from exclog.debug(f"{PACKAGE_NAME} version {PACKAGE_VERSION} setup libraries done for {get_config().project_name}")def validate_libraries(self):try:self.library_manager.validate_libraries()except ValidationError as exc:- error_msg = format_pydantic_validation_error(exc)- raise PipelexSetupError(f"Could not validate libraries because of: {error_msg}") from exc+ formatted_error_msg = format_pydantic_validation_error(exc)+ msg = f"Could not validate libraries because of: {formatted_error_msg}"+ raise PipelexSetupError(msg) from exclog.debug(f"{PACKAGE_NAME} version {PACKAGE_VERSION} validate libraries done for {get_config().project_name}")def teardown(self):@@ -259,9 +261,9 @@ class Pipelex(metaclass=MetaSingleton):@classmethoddef make(cls,- relative_config_folder_path: Optional[str] = None,- absolute_config_folder_path: Optional[str] = None,- from_file: Optional[bool] = True,+ relative_config_folder_path: str | None = None,+ absolute_config_folder_path: str | None = None,+ from_file: bool | None = True,) -> Self:"""Create and initialize a Pipelex instance.@@ -283,17 +285,21 @@ class Pipelex(metaclass=MetaSingleton):Note:If neither path is provided, defaults to "./pipelex_libraries".+"""if relative_config_folder_path is not None and absolute_config_folder_path is not None:- raise PipelexSetupError("Cannot specify both relative_config_folder_path and absolute_config_folder_path")+ msg = "Cannot specify both relative_config_folder_path and absolute_config_folder_path"+ raise PipelexSetupError(msg)if relative_config_folder_path is not None:if from_file:current_frame = inspect.currentframe()if current_frame is None:- raise PipelexSetupError("Could not find relative config folder path because of: Failed to get current frame")+ msg = "Could not find relative config folder path because of: Failed to get current frame"+ raise PipelexSetupError(msg)if current_frame.f_back is None:- raise PipelexSetupError("Could not find relative config folder path because of: Failed to get caller frame")+ msg = "Could not find relative config folder path because of: Failed to get caller frame"+ raise PipelexSetupError(msg)caller_file = current_frame.f_back.f_code.co_filenamecaller_dir = os.path.dirname(os.path.abspath(caller_file))config_dir_path = os.path.abspath(os.path.join(caller_dir, relative_config_folder_path))@@ -310,13 +316,14 @@ class Pipelex(metaclass=MetaSingleton):return pipelex_instance@classmethod- def get_optional_instance(cls) -> Optional[Self]:+ def get_optional_instance(cls) -> Self | None:instance = MetaSingleton.instances.get(cls)- return cast(Optional[Self], instance)+ return cast("Self | None", instance)@classmethoddef get_instance(cls) -> Self:instance = MetaSingleton.instances.get(cls)if instance is None:- raise RuntimeError("Pipelex is not initialized")- return cast(Self, instance)+ msg = "Pipelex is not initialized"+ raise RuntimeError(msg)+ return cast("Self", instance)diff --git a/pipelex/pipeline/__init__.py b/pipelex/pipeline/__init__.pynew file mode 100644index 000000000..e69de29bbdiff --git a/pipelex/pipeline/activity/activity_handler.py b/pipelex/pipeline/activity/activity_handler.pyindex 9b896d5ae..68b74da7e 100644--- a/pipelex/pipeline/activity/activity_handler.py+++ b/pipelex/pipeline/activity/activity_handler.py@@ -1,5 +1,5 @@import os-from typing import Set, cast+from typing import castfrom pipelex import logfrom pipelex.config import get_config@@ -28,7 +28,7 @@ class ActivityHandlerForResultFiles:img_gen_config = get_config().cogt.img_gen_configimg_gen_param_defaults = img_gen_config.img_gen_param_defaultsself.image_output_format = img_gen_param_defaults.output_format- self.already_handled_stuff: Set[str] = set()+ self.already_handled_stuff: set[str] = set()def _generate_stuff_id(self, stuff: Stuff) -> str:# Use name if available, otherwise just use code@@ -66,7 +66,7 @@ class ActivityHandlerForResultFiles:self._handle_structured_content(content=stuff.content, stuff_id=stuff_id)elif isinstance(stuff.content, ListContent):# TODO: check that all items are StuffContent- self._handle_list_content(content=cast(ListContent[StuffContent], stuff.content), stuff_id=stuff_id) # type: ignore+ self._handle_list_content(content=cast("ListContent[StuffContent]", stuff.content), stuff_id=stuff_id) # pyright: ignore[reportUnknownMemberType]else:log.error(f"Unhandled stuff content type: {type(stuff.content)}")@@ -88,7 +88,7 @@ class ActivityHandlerForResultFiles:# Save the imageimage_path = os.path.join(self.images_dir_path, f"{stuff_id}.{self.image_output_format}")if content.url.startswith("http"):- image_bytes: bytes = fetch_file_from_url_httpx(url=content.url, timeout=10)+ image_bytes: bytes = fetch_file_from_url_httpx(url=content.url, request_timeout=10)with open(image_path, "wb") as image_file:image_file.write(image_bytes)else:diff --git a/pipelex/pipeline/activity/activity_manager.py b/pipelex/pipeline/activity/activity_manager.pyindex b2ba80029..662b74deb 100644--- a/pipelex/pipeline/activity/activity_manager.py+++ b/pipelex/pipeline/activity/activity_manager.py@@ -1,5 +1,3 @@-from typing import Dict-from pydantic import Fieldfrom typing_extensions import override@@ -15,7 +13,7 @@ class ActivityManagerError(RootException):class ActivityManager(ActivityManagerProtocol):def __init__(self) -> None:- self.activity_callbacks: Dict[str, ActivityCallback] = Field(default_factory=dict)+ self.activity_callbacks: dict[str, ActivityCallback] = Field(default_factory=dict)@overridedef setup(self) -> None:diff --git a/pipelex/pipeline/activity/activity_models.py b/pipelex/pipeline/activity/activity_models.pyindex afc8a37b9..fdb878e74 100644--- a/pipelex/pipeline/activity/activity_models.py+++ b/pipelex/pipeline/activity/activity_models.py@@ -1,4 +1,5 @@-from typing import Any, Callable+from collections.abc import Callable+from typing import Anyfrom pydantic import BaseModeldiff --git a/pipelex/pipeline/execute.py b/pipelex/pipeline/execute.pyindex a83e83c4c..8cb52f1ba 100644--- a/pipelex/pipeline/execute.py+++ b/pipelex/pipeline/execute.py@@ -1,5 +1,3 @@-from typing import List, Optional-from pipelex.client.protocol import ImplicitMemoryfrom pipelex.core.memory.working_memory import WorkingMemoryfrom pipelex.core.memory.working_memory_factory import WorkingMemoryFactory@@ -24,13 +22,13 @@ from pipelex.tools.environment import get_optional_envasync def execute_pipeline(pipe_code: str,- working_memory: Optional[WorkingMemory] = None,- input_memory: Optional[ImplicitMemory] = None,- search_domains: Optional[List[str]] = None,- output_name: Optional[str] = None,- output_multiplicity: Optional[PipeOutputMultiplicity] = None,- dynamic_output_concept_code: Optional[str] = None,- pipe_run_mode: Optional[PipeRunMode] = None,+ working_memory: WorkingMemory | None = None,+ input_memory: ImplicitMemory | None = None,+ search_domains: list[str] | None = None,+ output_name: str | None = None,+ output_multiplicity: PipeOutputMultiplicity | None = None,+ dynamic_output_concept_code: str | None = None,+ pipe_run_mode: PipeRunMode | None = None,) -> PipeOutput:"""Execute a pipeline and wait for its completion.@@ -57,11 +55,14 @@ async def execute_pipeline(If not specified, the pipe run mode is inferred from the environment variable``PIPELEX_FORCE_DRY_RUN_MODE``. If the environment variable is not set,the pipe run mode is ``PipeRunMode.LIVE``.+ search_domains:+ List of domains to search for pipes.- Returns+ Returns:-------Tuple[PipeOutput, str]A tuple containing the pipe output and the pipeline run ID.+"""search_domains = search_domains or []pipe = get_required_pipe(pipe_code=pipe_code)@@ -70,8 +71,9 @@ async def execute_pipeline(# Can be either working_memory or compact_memory or neither, but not bothif working_memory and input_memory:- raise PipelineInputError(f"Cannot pass both working_memory and input_memory to `execute_pipeline` {pipe_code=}")- elif input_memory:+ msg = f"Cannot pass both working_memory and input_memory to `execute_pipeline` {pipe_code=}"+ raise PipelineInputError(msg)+ if input_memory:working_memory = WorkingMemoryFactory.make_from_implicit_memory(implicit_memory=input_memory,search_domains=search_domains,diff --git a/pipelex/pipeline/job_metadata.py b/pipelex/pipeline/job_metadata.pyindex f3f94b4dc..60b214c3e 100644--- a/pipelex/pipeline/job_metadata.py+++ b/pipelex/pipeline/job_metadata.py@@ -1,5 +1,4 @@from datetime import datetime-from typing import List, Optionalfrom pydantic import BaseModel, Field@@ -24,19 +23,19 @@ class UnitJobId(StrEnum):class JobMetadata(BaseModel):- job_name: Optional[str] = None+ job_name: str | None = Nonepipeline_run_id: str = Field(default=SpecialPipelineId.UNTITLED)- pipe_job_ids: Optional[List[str]] = None+ pipe_job_ids: list[str] | None = None- content_generation_job_id: Optional[str] = None- unit_job_id: Optional[str] = None- job_category: Optional[JobCategory] = None+ content_generation_job_id: str | None = None+ unit_job_id: str | None = None+ job_category: JobCategory | None = None- started_at: Optional[datetime] = Field(default_factory=lambda: datetime.now())- completed_at: Optional[datetime] = None+ started_at: datetime | None = Field(default_factory=lambda: datetime.now())+ completed_at: datetime | None = None@property- def duration(self) -> Optional[float]:+ def duration(self) -> float | None:if self.started_at is not None and self.completed_at is not None:return (self.completed_at - self.started_at).total_seconds()return Nonediff --git a/pipelex/pipeline/pipeline_manager.py b/pipelex/pipeline/pipeline_manager.pyindex 99600a1c0..919ff4fce 100644--- a/pipelex/pipeline/pipeline_manager.py+++ b/pipelex/pipeline/pipeline_manager.py@@ -1,5 +1,3 @@-from typing import Dict, Optional-from pydantic import Field, RootModelfrom typing_extensions import override@@ -8,7 +6,7 @@ from pipelex.pipeline.pipeline import Pipelinefrom pipelex.pipeline.pipeline_factory import PipelineFactoryfrom pipelex.pipeline.pipeline_manager_abstract import PipelineManagerAbstract-PipelineManagerRoot = Dict[str, Pipeline]+PipelineManagerRoot = dict[str, Pipeline]class PipelineManager(PipelineManagerAbstract, RootModel[PipelineManagerRoot]):@@ -23,14 +21,15 @@ class PipelineManager(PipelineManagerAbstract, RootModel[PipelineManagerRoot]):self.root.clear()@override- def get_optional_pipeline(self, pipeline_run_id: str) -> Optional[Pipeline]:+ def get_optional_pipeline(self, pipeline_run_id: str) -> Pipeline | None:return self.root.get(pipeline_run_id)@overridedef get_pipeline(self, pipeline_run_id: str) -> Pipeline:pipeline = self.get_optional_pipeline(pipeline_run_id=pipeline_run_id)if pipeline is None:- raise PipelineManagerNotFoundError(f"Pipeline {pipeline_run_id} not found")+ msg = f"Pipeline {pipeline_run_id} not found"+ raise PipelineManagerNotFoundError(msg)return pipelinedef _set_pipeline(self, pipeline_run_id: str, pipeline: Pipeline) -> Pipeline:diff --git a/pipelex/pipeline/pipeline_manager_abstract.py b/pipelex/pipeline/pipeline_manager_abstract.pyindex 1c7e13878..9e3e7a972 100644--- a/pipelex/pipeline/pipeline_manager_abstract.py+++ b/pipelex/pipeline/pipeline_manager_abstract.py@@ -1,5 +1,4 @@from abc import ABC, abstractmethod-from typing import Optionalfrom pipelex.pipeline.pipeline import Pipeline@@ -14,7 +13,7 @@ class PipelineManagerAbstract(ABC):pass@abstractmethod- def get_optional_pipeline(self, pipeline_run_id: str) -> Optional[Pipeline]:+ def get_optional_pipeline(self, pipeline_run_id: str) -> Pipeline | None:pass@abstractmethoddiff --git a/pipelex/pipeline/start.py b/pipelex/pipeline/start.pyindex 1de22d885..67b92b1dc 100644--- a/pipelex/pipeline/start.py+++ b/pipelex/pipeline/start.py@@ -1,5 +1,4 @@import asyncio-from typing import Optionalfrom pipelex.client.protocol import CompactMemoryfrom pipelex.core.memory.working_memory import WorkingMemory@@ -7,7 +6,7 @@ from pipelex.core.memory.working_memory_factory import WorkingMemoryFactoryfrom pipelex.core.pipes.pipe_output import PipeOutputfrom pipelex.core.pipes.pipe_run_params import PipeOutputMultiplicity, PipeRunModefrom pipelex.core.pipes.pipe_run_params_factory import PipeRunParamsFactory-from pipelex.exceptions import StartPipelineException+from pipelex.exceptions import StartPipelineErrorfrom pipelex.hub import get_pipe_router, get_pipeline_manager, get_report_delegate, get_required_pipefrom pipelex.pipe_works.pipe_job_factory import PipeJobFactoryfrom pipelex.pipeline.job_metadata import JobMetadata@@ -15,11 +14,11 @@ from pipelex.pipeline.job_metadata import JobMetadataasync def start_pipeline(pipe_code: str,- working_memory: Optional[WorkingMemory] = None,- input_memory: Optional[CompactMemory] = None,- output_name: Optional[str] = None,- output_multiplicity: Optional[PipeOutputMultiplicity] = None,- dynamic_output_concept_code: Optional[str] = None,+ working_memory: WorkingMemory | None = None,+ input_memory: CompactMemory | None = None,+ output_name: str | None = None,+ output_multiplicity: PipeOutputMultiplicity | None = None,+ dynamic_output_concept_code: str | None = None,pipe_run_mode: PipeRunMode = PipeRunMode.LIVE,) -> asyncio.Task[PipeOutput]:"""Start a pipeline in the background.@@ -45,15 +44,17 @@ async def start_pipeline(Override the dynamic output concept code.pipe_run_mode:Pipe run mode: ``PipeRunMode.LIVE`` or ``PipeRunMode.DRY``.- Returns++ Returns:-------Tuple[str, asyncio.Task[PipeOutput]]The ``pipeline_run_id`` of the newly started pipeline and a task thatcan be awaited to get the pipe output.- """+ """if working_memory and input_memory:- raise StartPipelineException(f"Cannot pass both working_memory and input_memory to `start_pipeline` {pipe_code=}")+ msg = f"Cannot pass both working_memory and input_memory to `start_pipeline` {pipe_code=}"+ raise StartPipelineError(msg)if input_memory:working_memory = WorkingMemoryFactory.make_from_compact_memory(input_memory)diff --git a/pipelex/pipeline/track/flow_chart.py b/pipelex/pipeline/track/flow_chart.pyindex 6736fb75e..77e145114 100644--- a/pipelex/pipeline/track/flow_chart.py+++ b/pipelex/pipeline/track/flow_chart.py@@ -3,7 +3,7 @@# pyright: reportUnknownMemberType=false# pyright: reportUnknownParameterType=false# pyright: reportMissingTypeArgument=false-from typing import Any, Dict, List, Optional, Tuple+from typing import Anyimport networkx as nximport yaml@@ -37,7 +37,7 @@ class PipelineFlowChart:self._tracker_config = tracker_configself.is_debug_mode = tracker_config.is_debug_modeself.start_node = start_node- self.sub_graph_class_defs: List[SubGraphClassDef] = []+ self.sub_graph_class_defs: list[SubGraphClassDef] = []for sub_graph_index, sub_graph_color in enumerate(self._tracker_config.sub_graph_colors):class_def_letter = chr(ord("a") + sub_graph_index)class_def_name = f"sub_{class_def_letter}"@@ -45,14 +45,15 @@ class PipelineFlowChart:def generate_mermaid_flowchart(self,- title: Optional[str] = None,- subtitle: Optional[str] = None,- ) -> Tuple[str, str]:+ title: str | None = None,+ subtitle: str | None = None,+ ) -> tuple[str, str]:nb_nodes = len(self.nx_graph.nodes)if nb_nodes == 0:- raise JobHistoryError("Graph has no nodes")+ msg = "Graph has no nodes"+ raise JobHistoryError(msg)log.debug(f"Generating mermaid flowchart for the whole graph which holds {nb_nodes} nodes")- mermaid_settings: Dict[str, Any] = {}+ mermaid_settings: dict[str, Any] = {}if title:mermaid_settings["title"] = titlemermaid_settings["config"] = {}@@ -73,12 +74,14 @@ class PipelineFlowChart:for node in self.nx_graph.nodes:node_attributes = self.nx_graph.nodes[node]if not node_attributes:- raise JobHistoryError(f"Node attributes are empty for node '{node}'")+ msg = f"Node attributes are empty for node '{node}'"+ raise JobHistoryError(msg)node_pipe_layer = node_attributes.get(NodeAttributeKey.SUBGRAPH)if not node_pipe_layer:node_pipe_layer = "Unknown"elif not isinstance(node_pipe_layer, str):- raise JobHistoryError(f"Node '{node}' has no pipe stack: {node_attributes}")+ msg = f"Node '{node}' has no pipe stack: {node_attributes}"+ raise JobHistoryError(msg)sub_graph = node_pipe_layer or "root"# Split sub_graph by "-" and keep the last token@@ -98,7 +101,8 @@ class PipelineFlowChart:if subtitle:# this is a hack to add something that looks like a subtitle, but it's actually a node with no stroke and no visible link# if self.start_node is None:- # raise JobHistoryError("Start node is not set")+ # msg = "Start node is not set"+ # raise JobHistoryError(msg)mermaid_code += f"""classDef subtitleNodeClass fill:transparent,stroke:#333,stroke-width:0px;__subtitle__["{subtitle}"]@@ -123,7 +127,8 @@ class PipelineFlowChart:edge_tag = nice_edge_tag(pipe_code)mermaid_code += f" {source} -- {edge_tag} {self._tracker_config.pipe_edge_style} {target}\n"else:- raise JobHistoryError(f"Pipe edge missing pipe code: {edge_data}")+ msg = f"Pipe edge missing pipe code: {edge_data}"+ raise JobHistoryError(msg)case EdgeCategory.BATCH:mermaid_code += f" {source} {self._tracker_config.branch_edge_style} {target}\n"case EdgeCategory.AGGREGATE:@@ -131,35 +136,38 @@ class PipelineFlowChart:case EdgeCategory.CONDITION:condition_expression = edge_data.get(EdgeAttributeKey.CONDITION_EXPRESSION)if not condition_expression:- raise JobHistoryError(f"Condition edge missing condition expression: {edge_data}")+ msg = f"Condition edge missing condition expression: {edge_data}"+ raise JobHistoryError(msg)edge_tag = nice_edge_tag(condition_expression)mermaid_code += f" {source} {self._tracker_config.condition_edge_style} {target}\n"case EdgeCategory.CHOICE:chosen_pipe_code = edge_data.get(EdgeAttributeKey.CHOSEN_PIPE)if not chosen_pipe_code:- raise JobHistoryError(f"No chosen pipe code set for edge {source} --- {target}")+ msg = f"No chosen pipe code set for edge {source} --- {target}"+ raise JobHistoryError(msg)edge_tag = nice_edge_tag(chosen_pipe_code)mermaid_code += f" {source} -- {edge_tag} {self._tracker_config.choice_edge_style} {target}\n"url = make_mermaid_url(mermaid_code)return mermaid_code, url- def generate_subgraph_lines(self, graph_tree: GraphTree) -> List[str]:- subgraph_lines: List[str] = []- subgraph_class_lines: List[str] = []+ def generate_subgraph_lines(self, graph_tree: GraphTree) -> list[str]:+ subgraph_lines: list[str] = []+ subgraph_class_lines: list[str] = []- cycle = 0- for subgraph_name, nodes in graph_tree.nodes_by_subgraph.items():- node_lines: List[str] = []+ for cycle, (subgraph_name, nodes) in enumerate(graph_tree.nodes_by_subgraph.items()):+ node_lines: list[str] = []for node in nodes:# log.debug(f"generate_subgraph_lines for node '{node}'")node_attributes = self.nx_graph.nodes[node]if not node_attributes:- raise JobHistoryError(f"Node attributes are empty for node '{node}'")+ msg = f"Node attributes are empty for node '{node}'"+ raise JobHistoryError(msg)node_category = NodeCategory(node_attributes[NodeAttributeKey.CATEGORY])node_tag = node_attributes[NodeAttributeKey.TAG]if not node_tag:- raise JobHistoryError(f"Node tag is empty for node '{node}'")+ msg = f"Node tag is empty for node '{node}'"+ raise JobHistoryError(msg)node_text = node_tagif self.is_debug_mode:if node_comment := node_attributes.get(NodeAttributeKey.COMMENT):@@ -175,29 +183,27 @@ class PipelineFlowChart:node_lines.append(f'{node}["{node_text}"]')case NodeCategory.CONDITION:node_lines.append(f'{node}{{"{node_text}"}}')- if self._tracker_config.is_include_interactivity:- if node_description := node_attributes.get(NodeAttributeKey.DESCRIPTION):- if not isinstance(node_description, str):- raise JobHistoryError(f"Node description is not a string: {node_description}")- node_description = clean_str_for_mermaid_node_title(node_description)- node_lines.append(f'click {node} stuff_node_callback "{node_description}"')+ if node_description := node_attributes.get(NodeAttributeKey.DESCRIPTION) and self._tracker_config.is_include_interactivity:+ if not isinstance(node_description, str):+ msg = f"Node description is not a string: {node_description}"+ raise JobHistoryError(msg)+ node_description = clean_str_for_mermaid_node_title(node_description)+ node_lines.append(f'click {node} stuff_node_callback "{node_description}"')if not node_lines:- raise JobHistoryError(f"No node lines found for subgraph '{subgraph_name}'")+ msg = f"No node lines found for subgraph '{subgraph_name}'"+ raise JobHistoryError(msg)if subgraph_name == "root":- for mermaid_line in node_lines:- subgraph_lines.append(_indent_line(mermaid_line, 2))+ subgraph_lines.extend(_indent_line(mermaid_line, 2) for mermaid_line in node_lines)else:subgraph_lines.append(_indent_line(f'subgraph "{subgraph_name}"', 1))subgraph_lines.append(_indent_line("direction LR", 1))- for mermaid_line in node_lines:- subgraph_lines.append(_indent_line(mermaid_line, 2))+ subgraph_lines.extend(_indent_line(mermaid_line, 2) for mermaid_line in node_lines)subgraph_lines.append(_indent_line("end", 1))class_def = self.sub_graph_class_defs[cycle % len(self.sub_graph_class_defs)]subgraph_class_lines.append(f"class {subgraph_name} {class_def.name};")- cycle += 1subgraph_lines.extend(subgraph_class_lines)diff --git a/pipelex/pipeline/track/pipeline_tracker.py b/pipelex/pipeline/track/pipeline_tracker.pyindex af4cb3a81..9ae145f36 100644--- a/pipelex/pipeline/track/pipeline_tracker.py+++ b/pipelex/pipeline/track/pipeline_tracker.py@@ -2,7 +2,7 @@# pyright: reportUnknownArgumentType=false# pyright: reportUnknownMemberType=false# pyright: reportMissingTypeArgument=false-from typing import Any, Dict, List, Optional+from typing import Anyimport networkx as nxfrom typing_extensions import override@@ -33,7 +33,7 @@ class PipelineTracker(PipelineTrackerProtocol):self._is_debug_mode = tracker_config.is_debug_modeself.is_active: bool = Falseself.nx_graph: nx.DiGraph = nx.DiGraph()- self.start_node: Optional[str] = None+ self.start_node: str | None = None@overridedef setup(self):@@ -50,20 +50,20 @@ class PipelineTracker(PipelineTrackerProtocol):self.teardown()self.setup()- def _get_node_name(self, node: str) -> Optional[str]:+ def _get_node_name(self, node: str) -> str | None:node_attributes = self.nx_graph.nodes[node]node_name = node_attributes[NodeAttributeKey.NAME]if isinstance(node_name, str):return node_name- else:- raise JobHistoryError(f"Node name is not a string: {node_name}")+ msg = f"Node name is not a string: {node_name}"+ raise JobHistoryError(msg)- def _pipe_layer_to_subgraph_name(self, pipe_layer: List[str]) -> str:+ def _pipe_layer_to_subgraph_name(self, pipe_layer: list[str]) -> str:return "-".join(pipe_layer)def _add_start_node(self) -> str:node = SpecialNodeName.START- node_attributes: Dict[str, Any] = {+ node_attributes: dict[str, Any] = {NodeAttributeKey.CATEGORY: NodeCategory.SPECIAL,NodeAttributeKey.TAG: "Start",NodeAttributeKey.NAME: "Start",@@ -74,7 +74,7 @@ class PipelineTracker(PipelineTrackerProtocol):def _make_stuff_node_tag(self,stuff: Stuff,- as_item_index: Optional[int] = None,+ as_item_index: int | None = None,) -> str:concept_display = Concept.sentence_from_concept(concept=stuff.concept)log.debug(f"Concept display: {stuff.concept.code} -> {concept_display}")@@ -82,18 +82,18 @@ class PipelineTracker(PipelineTrackerProtocol):concept_display = f"List of [{concept_display}]"if as_item_index is not None:return f"**{concept_display}** #{as_item_index + 1}"- else:- name = stuff.stuff_name- if not name:- raise JobHistoryError(f"Stuff name is empty for stuff {stuff}")- return f"{name}:<br>**{concept_display}**"+ name = stuff.stuff_name+ if not name:+ msg = f"Stuff name is empty for stuff {stuff}"+ raise JobHistoryError(msg)+ return f"{name}:<br>**{concept_display}**"def _add_stuff_node(self,stuff: Stuff,- pipe_layer: List[str],+ pipe_layer: list[str],comment: str,- as_item_index: Optional[int] = None,+ as_item_index: int | None = None,) -> str:node = stuff.stuff_codeis_existing = self.nx_graph.has_node(node)@@ -116,7 +116,7 @@ class PipelineTracker(PipelineTrackerProtocol):if stuff.is_text and self._tracker_config.is_include_text_preview:node_tag += f"<br/>{stuff_content_rendered[:100]}"pipe_layer_str = self._pipe_layer_to_subgraph_name(pipe_layer)- node_attributes: Dict[str, Any] = {+ node_attributes: dict[str, Any] = {NodeAttributeKey.CATEGORY: NodeCategory.STUFF,NodeAttributeKey.TAG: node_tag,NodeAttributeKey.NAME: stuff.stuff_name,@@ -133,19 +133,23 @@ class PipelineTracker(PipelineTrackerProtocol):from_node: str,to_node: str,edge_category: EdgeCategory,- attributes: Optional[Dict[str, Any]] = None,+ attributes: dict[str, Any] | None = None,):# Ensure both nodes exist with attributesif not self.nx_graph.has_node(from_node):- raise JobHistoryError(f"Source node '{from_node}' does not exist")+ msg = f"Source node '{from_node}' does not exist"+ raise JobHistoryError(msg)if not self.nx_graph.has_node(to_node):- raise JobHistoryError(f"Target node '{to_node}' does not exist")+ msg = f"Target node '{to_node}' does not exist"+ raise JobHistoryError(msg)if not self.nx_graph.nodes[from_node]:- raise JobHistoryError(f"Source node '{from_node}' exists but has no attributes")+ msg = f"Source node '{from_node}' exists but has no attributes"+ raise JobHistoryError(msg)if not self.nx_graph.nodes[to_node]:- raise JobHistoryError(f"Target node '{to_node}' exists but has no attributes")+ msg = f"Target node '{to_node}' exists but has no attributes"+ raise JobHistoryError(msg)- edge_attributes: Dict[str, Any] = {+ edge_attributes: dict[str, Any] = {EdgeAttributeKey.EDGE_CATEGORY: edge_category,}if attributes:@@ -155,12 +159,12 @@ class PipelineTracker(PipelineTrackerProtocol):@overridedef add_pipe_step(self,- from_stuff: Optional[Stuff],+ from_stuff: Stuff | None,to_stuff: Stuff,pipe_code: str,comment: str,- pipe_layer: List[str],- as_item_index: Optional[int] = None,+ pipe_layer: list[str],+ as_item_index: int | None = None,is_with_edge: bool = True,):if not self.is_active:@@ -186,7 +190,7 @@ class PipelineTracker(PipelineTrackerProtocol):edge_caption = pipe_codeif self._is_debug_mode:edge_caption += f" ({comment})"- edge_attributes: Dict[str, Any] = {+ edge_attributes: dict[str, Any] = {EdgeAttributeKey.PIPE_CODE: edge_caption,}if is_with_edge:@@ -200,10 +204,10 @@ class PipelineTracker(PipelineTrackerProtocol):@overridedef add_batch_step(self,- from_stuff: Optional[Stuff],+ from_stuff: Stuff | None,to_stuff: Stuff,to_branch_index: int,- pipe_layer: List[str],+ pipe_layer: list[str],comment: str,):if not self.is_active:@@ -236,7 +240,7 @@ class PipelineTracker(PipelineTrackerProtocol):self,from_stuff: Stuff,to_stuff: Stuff,- pipe_layer: List[str],+ pipe_layer: list[str],comment: str,):if not self.is_active:@@ -257,11 +261,11 @@ class PipelineTracker(PipelineTrackerProtocol):edge_category=EdgeCategory.AGGREGATE,)- def _add_condition_node(self, condition: PipeConditionDetails, pipe_layer: List[str]) -> str:+ def _add_condition_node(self, condition: PipeConditionDetails, pipe_layer: list[str]) -> str:node = condition.codecondition_node_tag = f"Condition:<br>**{condition.test_expression}<br>= {condition.evaluated_expression}**"pipe_layer_str = self._pipe_layer_to_subgraph_name(pipe_layer)- node_attributes: Dict[str, Any] = {+ node_attributes: dict[str, Any] = {NodeAttributeKey.CATEGORY: NodeCategory.CONDITION,NodeAttributeKey.TAG: condition_node_tag,NodeAttributeKey.NAME: condition.code,@@ -276,7 +280,7 @@ class PipelineTracker(PipelineTrackerProtocol):from_stuff: Stuff,to_condition: PipeConditionDetails,condition_expression: str,- pipe_layer: List[str],+ pipe_layer: list[str],comment: str,):if not self.is_active:@@ -287,7 +291,7 @@ class PipelineTracker(PipelineTrackerProtocol):comment=comment,)to_node = self._add_condition_node(condition=to_condition, pipe_layer=pipe_layer)- edge_attributes: Dict[str, Any] = {+ edge_attributes: dict[str, Any] = {EdgeAttributeKey.CONDITION_EXPRESSION: condition_expression,}self._add_edge(@@ -302,7 +306,7 @@ class PipelineTracker(PipelineTrackerProtocol):self,from_condition: PipeConditionDetails,to_stuff: Stuff,- pipe_layer: List[str],+ pipe_layer: list[str],comment: str,):if not self.is_active:@@ -312,7 +316,7 @@ class PipelineTracker(PipelineTrackerProtocol):pipe_layer=pipe_layer,comment=comment,)- edge_attributes: Dict[str, Any] = {+ edge_attributes: dict[str, Any] = {EdgeAttributeKey.CHOSEN_PIPE: from_condition.chosen_pipe_code,}self._add_edge(@@ -322,12 +326,13 @@ class PipelineTracker(PipelineTrackerProtocol):attributes=edge_attributes,)- def _print_mermaid_flowchart_code_and_url(self, title: Optional[str] = None, subtitle: Optional[str] = None):+ def _print_mermaid_flowchart_code_and_url(self, title: str | None = None, subtitle: str | None = None):if not self.nx_graph.nodes:log.debug("No nodes in the pipeline tracker")returnif self.start_node is None:- raise JobHistoryError("Start node is not set")+ msg = "Start node is not set"+ raise JobHistoryError(msg)flowchart = PipelineFlowChart(nx_graph=self.nx_graph, start_node=self.start_node, tracker_config=self._tracker_config)mermaid_code, url = flowchart.generate_mermaid_flowchart(title=title, subtitle=subtitle)print(mermaid_code)@@ -336,12 +341,13 @@ class PipelineTracker(PipelineTrackerProtocol):title_to_print += f" for {title}"print_mermaid_url(url=url, title=title_to_print)- def _print_mermaid_flowchart_url(self, title: Optional[str] = None, subtitle: Optional[str] = None) -> Optional[str]:+ def _print_mermaid_flowchart_url(self, title: str | None = None, subtitle: str | None = None) -> str | None:if not self.nx_graph.nodes:log.debug("No nodes in the pipeline tracker")return Noneif self.start_node is None:- raise JobHistoryError("Start node is not set")+ msg = "Start node is not set"+ raise JobHistoryError(msg)flowchart = PipelineFlowChart(nx_graph=self.nx_graph, start_node=self.start_node, tracker_config=self._tracker_config)_, url = flowchart.generate_mermaid_flowchart(title=title, subtitle=subtitle)title_to_print = "Mermaid flowchart URL"@@ -353,10 +359,10 @@ class PipelineTracker(PipelineTrackerProtocol):@overridedef output_flowchart(self,- title: Optional[str] = None,- subtitle: Optional[str] = None,+ title: str | None = None,+ subtitle: str | None = None,is_detailed: bool = False,- ) -> Optional[str]:+ ) -> str | None:if is_detailed:self._print_mermaid_flowchart_code_and_url(title=title, subtitle=subtitle)else:diff --git a/pipelex/pipeline/track/pipeline_tracker_protocol.py b/pipelex/pipeline/track/pipeline_tracker_protocol.pyindex 374540725..55bfbfc98 100644--- a/pipelex/pipeline/track/pipeline_tracker_protocol.py+++ b/pipelex/pipeline/track/pipeline_tracker_protocol.py@@ -2,7 +2,7 @@# pyright: reportUnknownArgumentType=false# pyright: reportUnknownMemberType=false# pyright: reportMissingTypeArgument=false-from typing import List, Optional, Protocol+from typing import Protocolfrom typing_extensions import override@@ -19,21 +19,21 @@ class PipelineTrackerProtocol(Protocol):def add_pipe_step(self,- from_stuff: Optional[Stuff],+ from_stuff: Stuff | None,to_stuff: Stuff,pipe_code: str,comment: str,- pipe_layer: List[str],- as_item_index: Optional[int] = None,+ pipe_layer: list[str],+ as_item_index: int | None = None,is_with_edge: bool = True,): ...def add_batch_step(self,- from_stuff: Optional[Stuff],+ from_stuff: Stuff | None,to_stuff: Stuff,to_branch_index: int,- pipe_layer: List[str],+ pipe_layer: list[str],comment: str,): ...@@ -41,7 +41,7 @@ class PipelineTrackerProtocol(Protocol):self,from_stuff: Stuff,to_stuff: Stuff,- pipe_layer: List[str],+ pipe_layer: list[str],comment: str,): ...@@ -50,7 +50,7 @@ class PipelineTrackerProtocol(Protocol):from_stuff: Stuff,to_condition: PipeConditionDetails,condition_expression: str,- pipe_layer: List[str],+ pipe_layer: list[str],comment: str,): ...@@ -58,16 +58,16 @@ class PipelineTrackerProtocol(Protocol):self,from_condition: PipeConditionDetails,to_stuff: Stuff,- pipe_layer: List[str],+ pipe_layer: list[str],comment: str,): ...def output_flowchart(self,- title: Optional[str] = None,- subtitle: Optional[str] = None,+ title: str | None = None,+ subtitle: str | None = None,is_detailed: bool = False,- ) -> Optional[str]: ...+ ) -> str | None: ...class PipelineTrackerNoOp(PipelineTrackerProtocol):@@ -90,12 +90,12 @@ class PipelineTrackerNoOp(PipelineTrackerProtocol):@overridedef add_pipe_step(self,- from_stuff: Optional[Stuff],+ from_stuff: Stuff | None,to_stuff: Stuff,pipe_code: str,comment: str,- pipe_layer: List[str],- as_item_index: Optional[int] = None,+ pipe_layer: list[str],+ as_item_index: int | None = None,is_with_edge: bool = True,) -> None:pass@@ -103,10 +103,10 @@ class PipelineTrackerNoOp(PipelineTrackerProtocol):@overridedef add_batch_step(self,- from_stuff: Optional[Stuff],+ from_stuff: Stuff | None,to_stuff: Stuff,to_branch_index: int,- pipe_layer: List[str],+ pipe_layer: list[str],comment: str,) -> None:pass@@ -116,7 +116,7 @@ class PipelineTrackerNoOp(PipelineTrackerProtocol):self,from_stuff: Stuff,to_stuff: Stuff,- pipe_layer: List[str],+ pipe_layer: list[str],comment: str,) -> None:pass@@ -127,7 +127,7 @@ class PipelineTrackerNoOp(PipelineTrackerProtocol):from_stuff: Stuff,to_condition: PipeConditionDetails,condition_expression: str,- pipe_layer: List[str],+ pipe_layer: list[str],comment: str,) -> None:pass@@ -137,7 +137,7 @@ class PipelineTrackerNoOp(PipelineTrackerProtocol):self,from_condition: PipeConditionDetails,to_stuff: Stuff,- pipe_layer: List[str],+ pipe_layer: list[str],comment: str,) -> None:pass@@ -145,8 +145,8 @@ class PipelineTrackerNoOp(PipelineTrackerProtocol):@overridedef output_flowchart(self,- title: Optional[str] = None,- subtitle: Optional[str] = None,+ title: str | None = None,+ subtitle: str | None = None,is_detailed: bool = False,) -> None:passdiff --git a/pipelex/pipeline/track/tracker_config.py b/pipelex/pipeline/track/tracker_config.pyindex e74a14129..39abd0f2a 100644--- a/pipelex/pipeline/track/tracker_config.py+++ b/pipelex/pipeline/track/tracker_config.py@@ -1,4 +1,4 @@-from typing import List, Literal, Optional, Union+from typing import Literalfrom pipelex.tools.config.config_model import ConfigModel@@ -7,11 +7,11 @@ class TrackerConfig(ConfigModel):is_debug_mode: boolis_include_text_preview: boolis_include_interactivity: bool- theme: Union[str, Literal["auto"]]- layout: Union[str, Literal["auto"]]- wrapping_width: Union[int, Literal["auto"]]- nb_items_limit: Union[int, Literal["unlimited"]]- sub_graph_colors: List[str]+ theme: str | Literal["auto"]+ layout: str | Literal["auto"]+ wrapping_width: int | Literal["auto"]+ nb_items_limit: int | Literal["unlimited"]+ sub_graph_colors: list[str]pipe_edge_style: strbranch_edge_style: straggregate_edge_style: str@@ -19,29 +19,25 @@ class TrackerConfig(ConfigModel):choice_edge_style: str@property- def applied_theme(self) -> Optional[str]:+ def applied_theme(self) -> str | None:if self.theme == "auto":return None- else:- return self.theme+ return self.theme@property- def applied_layout(self) -> Optional[str]:+ def applied_layout(self) -> str | None:if self.layout == "auto":return None- else:- return self.layout+ return self.layout@property- def applied_wrapping_width(self) -> Optional[int]:+ def applied_wrapping_width(self) -> int | None:if self.wrapping_width == "auto":return None- else:- return self.wrapping_width+ return self.wrapping_width@property- def applied_nb_items_limit(self) -> Optional[int]:+ def applied_nb_items_limit(self) -> int | None:if self.nb_items_limit == "unlimited":return None- else:- return self.nb_items_limit+ return self.nb_items_limitdiff --git a/pipelex/pipeline/track/tracker_models.py b/pipelex/pipeline/track/tracker_models.pyindex cc1d8561a..49a2712f1 100644--- a/pipelex/pipeline/track/tracker_models.py+++ b/pipelex/pipeline/track/tracker_models.py@@ -1,5 +1,3 @@-from typing import Dict, List-from pydantic import BaseModelfrom pipelex.types import StrEnum@@ -41,7 +39,7 @@ class EdgeAttributeKey(StrEnum):class GraphTree(BaseModel):- nodes_by_subgraph: Dict[str, List[str]]+ nodes_by_subgraph: dict[str, list[str]]class SubGraphClassDef(BaseModel):diff --git a/pipelex/plugins/anthropic/anthropic_factory.py b/pipelex/plugins/anthropic/anthropic_factory.pyindex f7df8abc9..a59d76f56 100644--- a/pipelex/plugins/anthropic/anthropic_factory.py+++ b/pipelex/plugins/anthropic/anthropic_factory.py@@ -1,11 +1,9 @@import asyncio-from typing import List, Optional, Union+from typing import TYPE_CHECKINGfrom anthropic import AsyncAnthropic, AsyncAnthropicBedrockfrom anthropic.types import Usage-from anthropic.types.image_block_param import ImageBlockParamfrom anthropic.types.message_param import MessageParam-from anthropic.types.text_block_param import TextBlockParamfrom openai.types.chat import (ChatCompletionMessageParam,ChatCompletionSystemMessageParam,@@ -31,6 +29,10 @@ from pipelex.tools.misc.base_64_utils import load_binary_as_base64_asyncfrom pipelex.tools.misc.filetype_utils import detect_file_type_from_base64from pipelex.types import StrEnum+if TYPE_CHECKING:+ from anthropic.types.image_block_param import ImageBlockParam+ from anthropic.types.text_block_param import TextBlockParam+class AnthropicFactoryError(CogtError):pass@@ -46,11 +48,12 @@ class AnthropicFactory:def make_anthropic_client(plugin: Plugin,backend: InferenceBackend,- ) -> Union[AsyncAnthropic, AsyncAnthropicBedrock]:+ ) -> AsyncAnthropic | AsyncAnthropicBedrock:try:sdk_variant = AnthropicSdkVariant(plugin.sdk)- except ValueError:- raise AnthropicFactoryError(f"Plugin '{plugin}' is not supported by AnthropicFactory")+ except ValueError as exc:+ msg = f"Plugin '{plugin}' is not supported by AnthropicFactory"+ raise AnthropicFactoryError(msg) from excmatch sdk_variant:case AnthropicSdkVariant.ANTHROPIC:@@ -73,7 +76,7 @@ class AnthropicFactory:llm_job: LLMJob,) -> MessageParam:message: MessageParam- content: List[Union[TextBlockParam, ImageBlockParam]] = []+ content: list[TextBlockParam | ImageBlockParam] = []if llm_job.llm_prompt.user_text:text_block_param: TextBlockParam = {@@ -93,9 +96,9 @@ class AnthropicFactory:"type": "image","source": {"type": "base64",- "media_type": mime, # type: ignore+ "media_type": mime, # type: ignore[typeddict-item]"data": prepped_image.base_64.decode("utf-8"),- }, # type: ignore+ }, # pyright: ignore[reportAssignmentType]}elif isinstance(prepped_image, str): # pyright: ignore[reportUnnecessaryIsInstance]url = prepped_image@@ -107,7 +110,8 @@ class AnthropicFactory:},}else:- raise AnthropicFactoryError(f"Unsupported PromptImageTypedBytesOrUrl type: '{type(prepped_image).__name__}'")+ msg = f"Unsupported PromptImageTypedBytesOrUrl type: '{type(prepped_image).__name__}'"+ raise AnthropicFactoryError(msg)content.append(image_block_param)message = {@@ -121,13 +125,13 @@ class AnthropicFactory:@staticmethoddef openai_typed_user_message(user_content_txt: str,- prepped_user_images: Optional[List[PromptImageTypedUrlOrBase64]] = None,+ prepped_user_images: list[PromptImageTypedUrlOrBase64] | None = None,) -> ChatCompletionMessageParam:text_block_param: TextBlockParam = {"type": "text", "text": user_content_txt}message: MessageParamif prepped_user_images is not None:log.debug(prepped_user_images)- images_block_params: List[ImageBlockParam] = []+ images_block_params: list[ImageBlockParam] = []for prepped_image in prepped_user_images:image_block_param_in_loop: ImageBlockParamif isinstance(prepped_image, PromptImageTypedBase64):@@ -136,9 +140,9 @@ class AnthropicFactory:"type": "image","source": {"type": "base64",- "media_type": mime, # type: ignore+ "media_type": mime, # type: ignore[typeddict-item]"data": prepped_image.base_64.decode("utf-8"),- }, # type: ignore+ }, # pyright: ignore[reportAssignmentType]}elif isinstance(prepped_image, str): # pyright: ignore[reportUnnecessaryIsInstance]url = prepped_image@@ -150,10 +154,11 @@ class AnthropicFactory:},}else:- raise AnthropicFactoryError(f"Unsupported PromptImageTypedBytesOrUrl type: '{type(prepped_image).__name__}'")+ msg = f"Unsupported PromptImageTypedBytesOrUrl type: '{type(prepped_image).__name__}'"+ raise AnthropicFactoryError(msg)images_block_params.append(image_block_param_in_loop)- content: List[Union[TextBlockParam, ImageBlockParam]] = images_block_params + [text_block_param]+ content: list[TextBlockParam | ImageBlockParam] = [*images_block_params, text_block_param]message = {"role": "user","content": content,@@ -165,7 +170,7 @@ class AnthropicFactory:"content": [text_block_param],}- return message # type: ignore+ return message # type: ignore[return-value, valid-type] # pyright: ignore[reportReturnType]@classmethodasync def _prep_image_for_anthropic(@@ -183,24 +188,23 @@ class AnthropicFactory:b64 = await load_binary_as_base64_async(prompt_image.file_path)typed_bytes_or_url = PromptImageTypedBase64(base_64=b64, file_type=prompt_image.get_file_type())else:- raise AnthropicFactoryError(f"Unsupported PromptImage type: '{type(prompt_image).__name__}'")+ msg = f"Unsupported PromptImage type: '{type(prompt_image).__name__}'"+ raise AnthropicFactoryError(msg)return typed_bytes_or_url@classmethodasync def make_simple_messages(cls,llm_job: LLMJob,- ) -> List[ChatCompletionMessageParam]:- """- Makes a list of messages with a system message (if provided) and followed by a user message.- """+ ) -> list[ChatCompletionMessageParam]:+ """Makes a list of messages with a system message (if provided) and followed by a user message."""llm_prompt = llm_job.llm_prompt- messages: List[ChatCompletionMessageParam] = []+ messages: list[ChatCompletionMessageParam] = []#### System message ####if system_content := llm_prompt.system_text:messages.append(ChatCompletionSystemMessageParam(role="system", content=system_content))- prepped_user_images: Optional[List[PromptImageTypedUrlOrBase64]]+ prepped_user_images: list[PromptImageTypedUrlOrBase64] | Noneif llm_prompt.user_images:tasks_to_prep_images = [cls._prep_image_for_anthropic(prompt_image) for prompt_image in llm_prompt.user_images]prepped_user_images = await asyncio.gather(*tasks_to_prep_images)@@ -212,7 +216,7 @@ class AnthropicFactory:AnthropicFactory.openai_typed_user_message(user_content_txt=llm_prompt.user_text if llm_prompt.user_text else "",prepped_user_images=prepped_user_images,- )+ ),)return messagesdiff --git a/pipelex/plugins/anthropic/anthropic_llm_worker.py b/pipelex/plugins/anthropic/anthropic_llm_worker.pyindex 1f7c1037a..77b67798b 100644--- a/pipelex/plugins/anthropic/anthropic_llm_worker.py+++ b/pipelex/plugins/anthropic/anthropic_llm_worker.py@@ -1,7 +1,7 @@-from typing import Any, Dict, Optional, Type+from typing import Anyimport instructor-from anthropic import NOT_GIVEN, AsyncAnthropic, AsyncAnthropicBedrock+from anthropic import AsyncAnthropic, AsyncAnthropicBedrock, omitfrom typing_extensions import overridefrom pipelex import log@@ -24,29 +24,23 @@ class AnthropicExtraField(StrEnum):class AnthropicLLMWorkerError(Exception):"""Base exception for Anthropic LLM Worker errors."""- pass-class AnthropicBadRequestError(AnthropicLLMWorkerError):"""Raised when Anthropic API returns a BadRequestError."""- pass-class AnthropicInstructorError(AnthropicLLMWorkerError):"""Raised when Instructor encounters an error with Anthropic."""- pass-class AnthropicLLMWorker(LLMWorkerInternalAbstract):def __init__(self,sdk_instance: Any,- extra_config: Dict[str, Any],+ extra_config: dict[str, Any],inference_model: InferenceModelSpec,- structure_method: Optional[StructureMethod] = None,- reporting_delegate: Optional[ReportingProtocol] = None,+ structure_method: StructureMethod | None = None,+ reporting_delegate: ReportingProtocol | None = None,):LLMWorkerInternalAbstract.__init__(self,@@ -54,23 +48,24 @@ class AnthropicLLMWorker(LLMWorkerInternalAbstract):structure_method=structure_method,reporting_delegate=reporting_delegate,)- self.extra_config: Dict[str, Any] = extra_config+ self.extra_config: dict[str, Any] = extra_configself.default_max_tokens: int = 0if inference_model.max_tokens:self.default_max_tokens = inference_model.max_tokenselse:- raise AnthropicWorkerConfigurationError(- f"No max_tokens provided for llm model '{self.inference_model.desc}', but it is required for Anthropic"- )+ msg = f"No max_tokens provided for llm model '{self.inference_model.desc}', but it is required for Anthropic"+ raise AnthropicWorkerConfigurationError(msg)# Verify if the sdk_instance is compatible with the current LLM platformif isinstance(sdk_instance, (AsyncAnthropic, AsyncAnthropicBedrock)):- if inference_model.sdk == AnthropicSdkVariant.ANTHROPIC and not (isinstance(sdk_instance, AsyncAnthropic)):- raise SdkTypeError(f"Provided sdk_instance does not match LLMEngine platform:{sdk_instance}")- elif inference_model.sdk == AnthropicSdkVariant.BEDROCK_ANTHROPIC and not (isinstance(sdk_instance, AsyncAnthropicBedrock)):- raise SdkTypeError(f"Provided sdk_instance does not match LLMEngine platform:{sdk_instance}")+ if (inference_model.sdk == AnthropicSdkVariant.ANTHROPIC and not (isinstance(sdk_instance, AsyncAnthropic))) or (+ inference_model.sdk == AnthropicSdkVariant.BEDROCK_ANTHROPIC and not (isinstance(sdk_instance, AsyncAnthropicBedrock))+ ):+ msg = f"Provided sdk_instance does not match LLMEngine platform:{sdk_instance}"+ raise SdkTypeError(msg)else:- raise SdkTypeError(f"Provided sdk_instance does not match LLMEngine platform:{sdk_instance}")+ msg = f"Provided sdk_instance does not match LLMEngine platform:{sdk_instance}"+ raise SdkTypeError(msg)self.anthropic_async_client = sdk_instanceif structure_method:@@ -85,14 +80,15 @@ class AnthropicLLMWorker(LLMWorkerInternalAbstract):########################################################## TODO: implement streaming behind the scenes to avoid timeout/streaming errors with Claude 4 and high tokens- def _adapt_max_tokens(self, max_tokens: Optional[int]) -> int:+ def _adapt_max_tokens(self, max_tokens: int | None) -> int:max_tokens = max_tokens or self.default_max_tokensif (claude_4_tokens_limit := self.extra_config.get(AnthropicExtraField.CLAUDE_4_TOKENS_LIMIT)) and max_tokens > claude_4_tokens_limit:max_tokens = claude_4_tokens_limitlog.warning(f"Max tokens is greater than the claude 4 reduced tokens limit, reducing to {max_tokens}")if not max_tokens:- raise AnthropicWorkerConfigurationError(f"Max tokens is None for model {self.inference_model.desc}")+ msg = f"Max tokens is None for model {self.inference_model.desc}"+ raise AnthropicWorkerConfigurationError(msg)return max_tokens@override@@ -104,7 +100,7 @@ class AnthropicLLMWorker(LLMWorkerInternalAbstract):max_tokens = self._adapt_max_tokens(max_tokens=llm_job.job_params.max_tokens)response = await self.anthropic_async_client.messages.create(messages=[message],- system=llm_job.llm_prompt.system_text or NOT_GIVEN,+ system=llm_job.llm_prompt.system_text or omit,model=self.inference_model.model_id,temperature=llm_job.job_params.temperature,max_tokens=max_tokens,@@ -112,12 +108,14 @@ class AnthropicLLMWorker(LLMWorkerInternalAbstract):single_content_block = response.content[0]if single_content_block.type != "text":- raise LLMCompletionError(f"Unexpected content block type: {single_content_block.type}\nmodel: {self.inference_model.desc}")+ msg = f"Unexpected content block type: {single_content_block.type}\nmodel: {self.inference_model.desc}"+ raise LLMCompletionError(msg)full_reply_content = single_content_block.textsingle_content_block = response.content[0]if single_content_block.type != "text":- raise LLMCompletionError(f"Unexpected content block type: {single_content_block.type}\nmodel: {self.inference_model.desc}")+ msg = f"Unexpected content block type: {single_content_block.type}\nmodel: {self.inference_model.desc}"+ raise LLMCompletionError(msg)full_reply_content = single_content_block.textif (llm_tokens_usage := llm_job.job_report.llm_tokens_usage) and (usage := response.usage):@@ -129,7 +127,7 @@ class AnthropicLLMWorker(LLMWorkerInternalAbstract):async def _gen_object(self,llm_job: LLMJob,- schema: Type[BaseModelTypeVar],+ schema: type[BaseModelTypeVar],) -> BaseModelTypeVar:messages = await AnthropicFactory.make_simple_messages(llm_job=llm_job)max_tokens = self._adapt_max_tokens(max_tokens=llm_job.job_params.max_tokens)diff --git a/pipelex/plugins/anthropic/anthropic_llms.py b/pipelex/plugins/anthropic/anthropic_llms.pyindex 7c9436e5f..95e20a5e7 100644--- a/pipelex/plugins/anthropic/anthropic_llms.py+++ b/pipelex/plugins/anthropic/anthropic_llms.py@@ -1,5 +1,3 @@-from typing import List-from anthropic import AsyncAnthropicfrom anthropic.types import ModelInfo@@ -9,22 +7,26 @@ from pipelex.plugins.anthropic.anthropic_factory import AnthropicFactoryfrom pipelex.plugins.plugin_sdk_registry import Plugin-async def anthropic_list_anthropic_models(plugin: Plugin, backend: InferenceBackend) -> List[ModelInfo]:- """- List available Anthropic models.+async def anthropic_list_anthropic_models(plugin: Plugin, backend: InferenceBackend) -> list[ModelInfo]:+ """List available Anthropic models.Returns:List[ModelInfo]: A list of Anthropic model information objects+"""anthropic_client = AnthropicFactory.make_anthropic_client(plugin=plugin, backend=backend)if not hasattr(anthropic_client, "models"):- raise AnthropicSDKUnsupportedError(f"{type(anthropic_client).__name__} does not support listing models")+ msg = f"{type(anthropic_client).__name__} does not support listing models"+ raise AnthropicSDKUnsupportedError(msg)if not isinstance(anthropic_client, AsyncAnthropic):- raise AnthropicSDKUnsupportedError("We only support the standard Anthropic client for listing models")+ msg = "We only support the standard Anthropic client for listing models"+ raise AnthropicSDKUnsupportedError(msg)models_response = await anthropic_client.models.list()if not models_response:- raise AnthropicModelListingError("No models found")+ msg = "No models found"+ raise AnthropicModelListingError(msg)models_list = models_response.dataif not models_list:- raise AnthropicModelListingError("No models found")+ msg = "No models found"+ raise AnthropicModelListingError(msg)return sorted(models_list, key=lambda model: model.created_at)diff --git a/pipelex/plugins/bedrock/bedrock_client_aioboto3.py b/pipelex/plugins/bedrock/bedrock_client_aioboto3.pyindex 7d0dbe647..435d20882 100644--- a/pipelex/plugins/bedrock/bedrock_client_aioboto3.py+++ b/pipelex/plugins/bedrock/bedrock_client_aioboto3.py@@ -1,4 +1,4 @@-from typing import Any, Dict, Optional, Tuple, cast+from typing import TYPE_CHECKING, Any, castimport aioboto3from types_aiobotocore_bedrock_runtime.type_defs import ConverseResponseTypeDef@@ -9,6 +9,9 @@ from pipelex.cogt.usage.token_category import NbTokensByCategoryDict, TokenCategfrom pipelex.plugins.bedrock.bedrock_client_protocol import BedrockClientProtocolfrom pipelex.plugins.bedrock.bedrock_message import BedrockMessageDictList+if TYPE_CHECKING:+ from types_aiobotocore_bedrock_runtime.type_defs import ConverseResponseTypeDef+class BedrockClientAioboto3(BedrockClientProtocol):def __init__(self, aws_region: str):@@ -20,12 +23,12 @@ class BedrockClientAioboto3(BedrockClientProtocol):async def chat(self,messages: BedrockMessageDictList,- system_text: Optional[str],+ system_text: str | None,model: str,temperature: float,- max_tokens: Optional[int] = None,- ) -> Tuple[str, NbTokensByCategoryDict]:- params: Dict[str, Any] = {+ max_tokens: int | None = None,+ ) -> tuple[str, NbTokensByCategoryDict]:+ params: dict[str, Any] = {"modelId": model,"messages": messages,"inferenceConfig": {@@ -36,10 +39,10 @@ class BedrockClientAioboto3(BedrockClientProtocol):if system_text:params["system"] = [{"text": system_text}]- async with self.session.client("bedrock-runtime", region_name=self.aws_region) as aioboto3_session: # type: ignore+ async with self.session.client("bedrock-runtime", region_name=self.aws_region) as aioboto3_session: # pyright: ignore[reportUnknownMemberType]conversation_response: ConverseResponseTypeDef = await aioboto3_session.converse(**params)- resp_dict: Dict[str, Any] = cast(Dict[str, Any], conversation_response)- usage_dict: Dict[str, Any] = resp_dict["usage"]+ resp_dict: dict[str, Any] = cast("dict[str, Any]", conversation_response)+ usage_dict: dict[str, Any] = resp_dict["usage"]nb_tokens_by_category: NbTokensByCategoryDict = {TokenCategory.INPUT: usage_dict["inputTokens"],TokenCategory.OUTPUT: usage_dict["outputTokens"],diff --git a/pipelex/plugins/bedrock/bedrock_client_boto3.py b/pipelex/plugins/bedrock/bedrock_client_boto3.pyindex 1a7183238..b47c62a9a 100644--- a/pipelex/plugins/bedrock/bedrock_client_boto3.py+++ b/pipelex/plugins/bedrock/bedrock_client_boto3.py@@ -1,5 +1,5 @@import asyncio-from typing import Any, Dict, Optional, Tuple+from typing import Anyimport boto3from typing_extensions import override@@ -13,18 +13,18 @@ from pipelex.plugins.bedrock.bedrock_message import BedrockMessageDictListclass BedrockClientBoto3(BedrockClientProtocol):def __init__(self, aws_region: str):log.debug(f"Initializing BedrockClientBoto3 with region '{aws_region}'")- self.boto3_client = boto3.client(service_name="bedrock-runtime", region_name=aws_region) # pyright: ignore+ self.boto3_client = boto3.client(service_name="bedrock-runtime", region_name=aws_region) # pyright: ignore[reportUnknownMemberType]@overrideasync def chat(self,messages: BedrockMessageDictList,- system_text: Optional[str],+ system_text: str | None,model: str,temperature: float,- max_tokens: Optional[int] = None,- ) -> Tuple[str, NbTokensByCategoryDict]:- params: Dict[str, Any] = {+ max_tokens: int | None = None,+ ) -> tuple[str, NbTokensByCategoryDict]:+ params: dict[str, Any] = {"modelId": model,"messages": messages,"inferenceConfig": {@@ -36,9 +36,9 @@ class BedrockClientBoto3(BedrockClientProtocol):params["system"] = [{"text": system_text}]loop: asyncio.AbstractEventLoop = asyncio.get_event_loop()- resp_dict: Dict[str, Any] = await loop.run_in_executor(None, lambda: self.boto3_client.converse(**params)) # pyright: ignore+ resp_dict: dict[str, Any] = await loop.run_in_executor(None, lambda: self.boto3_client.converse(**params)) # pyright: ignore[reportUnknownLambdaType, reportUnknownMemberType]- usage_dict: Dict[str, Any] = resp_dict["usage"]+ usage_dict: dict[str, Any] = resp_dict["usage"]nb_tokens_by_category: NbTokensByCategoryDict = {TokenCategory.INPUT: usage_dict["inputTokens"],TokenCategory.OUTPUT: usage_dict["outputTokens"],diff --git a/pipelex/plugins/bedrock/bedrock_client_protocol.py b/pipelex/plugins/bedrock/bedrock_client_protocol.pyindex c3d7bd447..6ce5290b3 100644--- a/pipelex/plugins/bedrock/bedrock_client_protocol.py+++ b/pipelex/plugins/bedrock/bedrock_client_protocol.py@@ -1,4 +1,4 @@-from typing import Optional, Protocol, Tuple, runtime_checkable+from typing import Protocol, runtime_checkablefrom pipelex.cogt.usage.token_category import NbTokensByCategoryDictfrom pipelex.plugins.bedrock.bedrock_message import BedrockMessageDictList@@ -9,8 +9,8 @@ class BedrockClientProtocol(Protocol):async def chat(self,messages: BedrockMessageDictList,- system_text: Optional[str],+ system_text: str | None,model: str,temperature: float,- max_tokens: Optional[int] = None,- ) -> Tuple[str, NbTokensByCategoryDict]: ...+ max_tokens: int | None = None,+ ) -> tuple[str, NbTokensByCategoryDict]: ...diff --git a/pipelex/plugins/bedrock/bedrock_factory.py b/pipelex/plugins/bedrock/bedrock_factory.pyindex f12bb584c..dd5a8691e 100644--- a/pipelex/plugins/bedrock/bedrock_factory.py+++ b/pipelex/plugins/bedrock/bedrock_factory.py@@ -34,20 +34,21 @@ class BedrockFactory:) -> BedrockClientProtocol:try:sdk_variant = BedrockSdkVariant(plugin.sdk)- except ValueError:- raise BedrockFactoryError(f"Plugin '{plugin}' is not supported by BedrockFactory")+ except ValueError as exc:+ msg = f"Plugin '{plugin}' is not supported by BedrockFactory"+ raise BedrockFactoryError(msg) from excbedrock_async_client: BedrockClientProtocollog.verbose(f"Using '{sdk_variant}' for BedrockClient")match sdk_variant:case BedrockSdkVariant.AIBOTO3:- from pipelex.plugins.bedrock.bedrock_client_aioboto3 import BedrockClientAioboto3+ from pipelex.plugins.bedrock.bedrock_client_aioboto3 import BedrockClientAioboto3 # noqa: PLC0415bedrock_async_client = BedrockClientAioboto3(aws_region=backend.extra_config[BedrockExtraField.AWS_REGION],)case BedrockSdkVariant.BOTO3:- from pipelex.plugins.bedrock.bedrock_client_boto3 import BedrockClientBoto3+ from pipelex.plugins.bedrock.bedrock_client_boto3 import BedrockClientBoto3 # noqa: PLC0415bedrock_async_client = BedrockClientBoto3(aws_region=backend.extra_config[BedrockExtraField.AWS_REGION],@@ -61,13 +62,12 @@ class BedrockFactory:@classmethoddef make_simple_message(cls, llm_job: LLMJob) -> BedrockMessage:- """- Makes a list of messages with a system message (if provided) and followed by a user message.- """+ """Makes a list of messages with a system message (if provided) and followed by a user message."""message = BedrockMessage(role="user", content=[])if user_text := llm_job.llm_prompt.user_text:message.content.append(BedrockContentItem(text=user_text))if llm_job.llm_prompt.user_images:- raise LLMCapabilityError("BedrockFactory does not support images. Skipping images.")+ msg = "BedrockFactory does not support images. Skipping images."+ raise LLMCapabilityError(msg)return messagediff --git a/pipelex/plugins/bedrock/bedrock_llm_worker.py b/pipelex/plugins/bedrock/bedrock_llm_worker.pyindex da02b300b..5098dbee7 100644--- a/pipelex/plugins/bedrock/bedrock_llm_worker.py+++ b/pipelex/plugins/bedrock/bedrock_llm_worker.py@@ -1,4 +1,4 @@-from typing import Any, Optional, Type+from typing import Anyfrom typing_extensions import override@@ -23,8 +23,8 @@ class BedrockLLMWorker(LLMWorkerInternalAbstract):self,sdk_instance: Any,inference_model: InferenceModelSpec,- structure_method: Optional[StructureMethod] = None,- reporting_delegate: Optional[ReportingProtocol] = None,+ structure_method: StructureMethod | None = None,+ reporting_delegate: ReportingProtocol | None = None,):LLMWorkerInternalAbstract.__init__(self,@@ -34,16 +34,14 @@ class BedrockLLMWorker(LLMWorkerInternalAbstract):)if not isinstance(sdk_instance, BedrockClientProtocol):- raise SdkTypeError(- f"Provided sdk_instance for {self.__class__.__name__} is not of type BedrockClientProtocol: it's a '{type(sdk_instance)}'"- )+ msg = f"Provided sdk_instance for {self.__class__.__name__} is not of type BedrockClientProtocol: it's a '{type(sdk_instance)}'"+ raise SdkTypeError(msg)if default_max_tokens := inference_model.max_tokens:self.default_max_tokens = default_max_tokenselse:- raise BedrockWorkerConfigurationError(- f"No max_tokens provided for llm model '{self.inference_model.desc}', but it is required for Bedrock"- )+ msg = f"No max_tokens provided for llm model '{self.inference_model.desc}', but it is required for Bedrock"+ raise BedrockWorkerConfigurationError(msg)self.bedrock_client_for_text = sdk_instance@override@@ -70,7 +68,8 @@ class BedrockLLMWorker(LLMWorkerInternalAbstract):async def _gen_object(self,llm_job: LLMJob,- schema: Type[BaseModelTypeVar],+ schema: type[BaseModelTypeVar],) -> BaseModelTypeVar:# TODO: try with the newest instructor release- raise LLMCapabilityError(f"It is not possible to generate objects with a {self.__class__.__name__}.")+ msg = f"It is not possible to generate objects with a {self.__class__.__name__}."+ raise LLMCapabilityError(msg)diff --git a/pipelex/plugins/bedrock/bedrock_message.py b/pipelex/plugins/bedrock/bedrock_message.pyindex 425e59518..eb21b0163 100644--- a/pipelex/plugins/bedrock/bedrock_message.py+++ b/pipelex/plugins/bedrock/bedrock_message.py@@ -1,4 +1,4 @@-from typing import Any, Dict, List, Literal, Optional+from typing import Any, Literalfrom pydantic import BaseModel@@ -98,8 +98,8 @@ class BedrockImage(BaseModel):class BedrockContentItem(BaseModel):- text: Optional[str] = None- image: Optional[BedrockImage] = None+ text: str | None = None+ image: BedrockImage | None = None# document: Optional[BedrockDocument] = None# video: Optional[BedrockVideo] = None# toolUse: Optional[BedrockToolUse] = None@@ -107,13 +107,13 @@ class BedrockContentItem(BaseModel):# guardContent: Optional[GuardContent] = None-BedrockMessageDict = Dict[str, Any]-BedrockMessageDictList = List[BedrockMessageDict]+BedrockMessageDict = dict[str, Any]+BedrockMessageDictList = list[BedrockMessageDict]class BedrockMessage(BaseModel):role: Literal["user", "assistant"]- content: List[BedrockContentItem]+ content: list[BedrockContentItem]def to_dict(self) -> BedrockMessageDict:return self.model_dump(exclude_none=True)diff --git a/pipelex/plugins/fal/fal_config.py b/pipelex/plugins/fal/fal_config.pyindex ea3d9e252..da52401e6 100644--- a/pipelex/plugins/fal/fal_config.py+++ b/pipelex/plugins/fal/fal_config.py@@ -1,5 +1,3 @@-from typing import Dict-from pydantic import field_validatorfrom pipelex.cogt.img_gen.img_gen_job_components import Quality@@ -8,20 +6,23 @@ from pipelex.tools.exceptions import ConfigValidationErrorclass FalConfig(ConfigModel):- flux_map_quality_to_steps: Dict[str, int]- sdxl_lightning_map_quality_to_steps: Dict[str, int]+ flux_map_quality_to_steps: dict[str, int]+ sdxl_lightning_map_quality_to_steps: dict[str, int]@field_validator("flux_map_quality_to_steps", "sdxl_lightning_map_quality_to_steps")@classmethod- def validate_quality_mapping(cls, value: Dict[str, int]) -> Dict[str, int]:+ def validate_quality_mapping(cls, value: dict[str, int]) -> dict[str, int]:valid_qualities = {quality.value for quality in Quality}missing_qualities = valid_qualities - set(value.keys())invalid_qualities = set(value.keys()) - valid_qualitiesif missing_qualities and invalid_qualities:- raise ConfigValidationError(f"Missing ({missing_qualities}) and invalid ({invalid_qualities}) quality levels in mapping")- elif missing_qualities:- raise ConfigValidationError(f"Missing quality levels in mapping: {missing_qualities}")- elif invalid_qualities:- raise ConfigValidationError(f"Invalid quality levels in mapping: {invalid_qualities}")+ msg = f"Missing ({missing_qualities}) and invalid ({invalid_qualities}) quality levels in mapping"+ raise ConfigValidationError(msg)+ if missing_qualities:+ msg = f"Missing quality levels in mapping: {missing_qualities}"+ raise ConfigValidationError(msg)+ if invalid_qualities:+ msg = f"Invalid quality levels in mapping: {invalid_qualities}"+ raise ConfigValidationError(msg)return valuediff --git a/pipelex/plugins/fal/fal_factory.py b/pipelex/plugins/fal/fal_factory.pyindex f94705e92..577a44c83 100644--- a/pipelex/plugins/fal/fal_factory.py+++ b/pipelex/plugins/fal/fal_factory.py@@ -1,4 +1,4 @@-from typing import Any, Dict, List, Optional+from typing import Anyfrom pipelex import logfrom pipelex.cogt.exceptions import ImgGenGeneratedTypeError, ImgGenParameterError@@ -37,7 +37,8 @@ class FalFactory:case AspectRatio.PORTRAIT_9_21:return "portrait_21_9"case AspectRatio.LANDSCAPE_3_2 | AspectRatio.PORTRAIT_2_3:- raise ImgGenParameterError(f"Aspect ratio '{aspect_ratio}' is not supported by Flux-1 image generation model")+ msg = f"Aspect ratio '{aspect_ratio}' is not supported by Flux-1 image generation model"+ raise ImgGenParameterError(msg)@classmethoddef make_aspect_ratio_for_flux_1_1_ultra(cls, aspect_ratio: AspectRatio) -> str:@@ -57,7 +58,8 @@ class FalFactory:case AspectRatio.PORTRAIT_9_21:return "9:21"case AspectRatio.LANDSCAPE_3_2 | AspectRatio.PORTRAIT_2_3:- raise ImgGenParameterError(f"Aspect ratio '{aspect_ratio}' is not supported by Flux-1.1 Ultra image generation model")+ msg = f"Aspect ratio '{aspect_ratio}' is not supported by Flux-1.1 Ultra image generation model"+ raise ImgGenParameterError(msg)@classmethoddef make_output_format_for_flux(cls, output_format: OutputFormat) -> str:@@ -67,7 +69,8 @@ class FalFactory:case OutputFormat.JPG:return "jpeg"case OutputFormat.WEBP:- raise ImgGenParameterError("Output format WebP is not supported for Flux")+ msg = "Output format WebP is not supported for Flux"+ raise ImgGenParameterError(msg)@classmethoddef make_fal_arguments(@@ -75,16 +78,17 @@ class FalFactory:fal_application: str,img_gen_job: ImgGenJob,nb_images: int,- ) -> Dict[str, Any]:+ ) -> dict[str, Any]:params = img_gen_job.job_params- args_dict: Dict[str, Any]- num_inference_steps: Optional[int]+ args_dict: dict[str, Any]+ num_inference_steps: int | Nonematch fal_application:case "fal-ai/flux-pro" | "fal-ai/flux-pro/v1.1":num_inference_steps = params.nb_stepsif not num_inference_steps:if not params.quality:- raise ImgGenParameterError(f"Either nb_steps or quality must be set for image generation with '{fal_application}'")+ msg = f"Either nb_steps or quality must be set for image generation with '{fal_application}'"+ raise ImgGenParameterError(msg)num_inference_steps = cls.make_nb_steps_from_quality_for_flux_pro(quality=params.quality)args_dict = {@@ -129,49 +133,55 @@ class FalFactory:"sync_mode": img_gen_job.job_config.is_sync_mode,}case _:- raise ImgGenParameterError(f"Invalid fal application: '{fal_application}'")+ msg = f"Invalid fal application: '{fal_application}'"+ raise ImgGenParameterError(msg)return args_dict@staticmethod- def make_generated_image(fal_result: Dict[str, Any]) -> GeneratedImage:+ def make_generated_image(fal_result: dict[str, Any]) -> GeneratedImage:images = fal_result["images"]fal_image_dict = images[0]image_url = fal_image_dict["url"]if not isinstance(image_url, str):- raise ImgGenGeneratedTypeError("Image url is not a string")+ msg = "Image url is not a string"+ raise ImgGenGeneratedTypeError(msg)# TODO: if the url is actual image data, send it to cloud storage?width = fal_image_dict["width"]if not isinstance(width, int):- raise ImgGenGeneratedTypeError("Image width is not an integer")+ msg = "Image width is not an integer"+ raise ImgGenGeneratedTypeError(msg)height = fal_image_dict["height"]if not isinstance(height, int):- raise ImgGenGeneratedTypeError("Image height is not an integer")+ msg = "Image height is not an integer"+ raise ImgGenGeneratedTypeError(msg)- generated_image = GeneratedImage(+ return GeneratedImage(url=image_url,width=width,height=height,)- return generated_image@staticmethod- def make_generated_image_list(fal_result: Dict[str, Any]) -> List[GeneratedImage]:+ def make_generated_image_list(fal_result: dict[str, Any]) -> list[GeneratedImage]:fal_image_dicts = fal_result["images"]- generated_image_list: List[GeneratedImage] = []+ generated_image_list: list[GeneratedImage] = []for fal_image_dict in fal_image_dicts:image_url = fal_image_dict["url"]if not isinstance(image_url, str):- raise ImgGenGeneratedTypeError("Image url is not a string")+ msg = "Image url is not a string"+ raise ImgGenGeneratedTypeError(msg)width = fal_image_dict["width"]if not isinstance(width, int):- raise ImgGenGeneratedTypeError("Image width is not an integer")+ msg = "Image width is not an integer"+ raise ImgGenGeneratedTypeError(msg)height = fal_image_dict["height"]if not isinstance(height, int):- raise ImgGenGeneratedTypeError("Image height is not an integer")+ msg = "Image height is not an integer"+ raise ImgGenGeneratedTypeError(msg)generated_image = GeneratedImage(url=image_url,diff --git a/pipelex/plugins/fal/fal_img_gen_worker.py b/pipelex/plugins/fal/fal_img_gen_worker.pyindex 3e9df2102..cf023d174 100644--- a/pipelex/plugins/fal/fal_img_gen_worker.py+++ b/pipelex/plugins/fal/fal_img_gen_worker.py@@ -1,4 +1,4 @@-from typing import Any, List, Optional+from typing import Anyfrom fal_client import AsyncClient, InProgressfrom typing_extensions import override@@ -18,12 +18,13 @@ class FalImgGenWorker(ImgGenWorkerAbstract):self,sdk_instance: Any,inference_model: InferenceModelSpec,- reporting_delegate: Optional[ReportingProtocol] = None,+ reporting_delegate: ReportingProtocol | None = None,):super().__init__(inference_model=inference_model, reporting_delegate=reporting_delegate)if not isinstance(sdk_instance, AsyncClient):- raise SdkTypeError(f"Provided Imgg sdk_instance is not of type fal_client.AsyncClient: it's a '{type(sdk_instance)}'")+ msg = f"Provided ImgGen sdk_instance is not of type fal_client.AsyncClient: it's a '{type(sdk_instance)}'"+ raise SdkTypeError(msg)self.fal_async_client = sdk_instance@@ -64,7 +65,7 @@ class FalImgGenWorker(ImgGenWorkerAbstract):self,img_gen_job: ImgGenJob,nb_images: int,- ) -> List[GeneratedImage]:+ ) -> list[GeneratedImage]:application = self.inference_model.model_idarguments = FalFactory.make_fal_arguments(fal_application=application,diff --git a/pipelex/plugins/google/google_factory.py b/pipelex/plugins/google/google_factory.pyindex 06a77b7fe..35e17430a 100644--- a/pipelex/plugins/google/google_factory.py+++ b/pipelex/plugins/google/google_factory.py@@ -1,5 +1,4 @@import asyncio-from typing import List, Optionalfrom google import genaifrom google.genai import types as genai_types@@ -38,24 +37,23 @@ class GoogleFactory:image_bytes = prompt_image.get_decoded_bytes()mime_type = prompt_image.get_mime_type()return genai_types.Part.from_bytes(data=image_bytes, mime_type=mime_type)- elif isinstance(prompt_image, PromptImagePath):+ if isinstance(prompt_image, PromptImagePath):image_bytes = await load_binary_async(prompt_image.file_path)mime_type = prompt_image.get_mime_type()return genai_types.Part.from_bytes(data=image_bytes, mime_type=mime_type)- elif isinstance(prompt_image, PromptImageUrl):+ if isinstance(prompt_image, PromptImageUrl):prompt_image_binary = await PromptImageFactory.make_promptimagebinary_from_url_async(prompt_image)image_bytes = prompt_image_binary.binarymime_type = prompt_image_binary.get_mime_type()return genai_types.Part.from_bytes(data=image_bytes, mime_type=mime_type)- else:- raise GoogleFactoryError(f"Unsupported PromptImage type: '{type(prompt_image).__name__}'")+ msg = f"Unsupported PromptImage type: '{type(prompt_image).__name__}'"+ raise GoogleFactoryError(msg)@classmethodasync def prepare_user_contents(cls, llm_prompt: LLMPrompt) -> genai_types.ContentListUnion:"""Prepare contents for Google genai API."""-# Build list of parts for multimodal content- parts: List[genai_types.Part] = []+ parts: list[genai_types.Part] = []# Add text content if presentif llm_prompt.user_text:@@ -71,7 +69,7 @@ class GoogleFactory:return genai_types.Content(parts=parts, role="user")@classmethod- def extract_token_usage(cls, usage_metadata: Optional[genai_types.GenerateContentResponseUsageMetadata]) -> NbTokensByCategoryDict:+ def extract_token_usage(cls, usage_metadata: genai_types.GenerateContentResponseUsageMetadata | None) -> NbTokensByCategoryDict:"""Extract token usage from Google's usage metadata."""if not usage_metadata:return {}diff --git a/pipelex/plugins/google/google_llm_worker.py b/pipelex/plugins/google/google_llm_worker.pyindex 9b03e9d6e..f49d8ddd0 100644--- a/pipelex/plugins/google/google_llm_worker.py+++ b/pipelex/plugins/google/google_llm_worker.py@@ -1,9 +1,11 @@-from typing import Optional, Type, cast+from typing import TYPE_CHECKING, castimport instructorfrom google import genaifrom google.genai import types-from openai.types.chat import ChatCompletionMessageParam++if TYPE_CHECKING:+ from openai.types.chat import ChatCompletionMessageParamfrom typing_extensions import overridefrom pipelex import log@@ -21,16 +23,14 @@ from pipelex.tools.typing.pydantic_utils import BaseModelTypeVarclass GoogleLLMWorkerError(Exception):"""Base exception for Google LLM Worker errors."""- pass-class GoogleLLMWorker(LLMWorkerInternalAbstract):def __init__(self,sdk_instance: genai.Client,inference_model: InferenceModelSpec,- structure_method: Optional[StructureMethod] = None,- reporting_delegate: Optional[ReportingProtocol] = None,+ structure_method: StructureMethod | None = None,+ reporting_delegate: ReportingProtocol | None = None,):super().__init__(inference_model=inference_model,@@ -75,16 +75,19 @@ class GoogleLLMWorker(LLMWorkerInternalAbstract):# Extract text from responseif not response.candidates:- raise LLMCompletionError(f"No candidates returned from model: {self.inference_model.desc}")+ msg = f"No candidates returned from model: {self.inference_model.desc}"+ raise LLMCompletionError(msg)candidate = response.candidates[0]if not candidate.content or not candidate.content.parts:- raise LLMCompletionError(f"No content parts in response from model: {self.inference_model.desc}")+ msg = f"No content parts in response from model: {self.inference_model.desc}"+ raise LLMCompletionError(msg)# Extract text from the first parttext_content = candidate.content.parts[0].textif not text_content:- raise LLMCompletionError(f"No text content in response from model: {self.inference_model.desc}")+ msg = f"No text content in response from model: {self.inference_model.desc}"+ raise LLMCompletionError(msg)# Track token usage if availableif llm_job.job_report.llm_tokens_usage and response.usage_metadata:@@ -96,7 +99,7 @@ class GoogleLLMWorker(LLMWorkerInternalAbstract):async def _gen_object(self,llm_job: LLMJob,- schema: Type[BaseModelTypeVar],+ schema: type[BaseModelTypeVar],) -> BaseModelTypeVar:"""Generate structured output using Google Gemini API with instructor."""# Prepare contents (text and images)@@ -111,14 +114,15 @@ class GoogleLLMWorker(LLMWorkerInternalAbstract):)result_object, completion = await self.instructor_for_objects.chat.completions.create_with_completion(- messages=[cast(ChatCompletionMessageParam, contents)],+ messages=[cast("ChatCompletionMessageParam", contents)],response_model=schema,max_retries=llm_job.job_config.max_retries,model=self.inference_model.model_id,generation_config=generation_config,)if not isinstance(result_object, schema):- raise GoogleLLMWorkerError(f"Google Gemini API returned an object that is not of type {schema}: {result_object}")+ msg = f"Google Gemini API returned an object that is not of type {schema}: {result_object}"+ raise GoogleLLMWorkerError(msg)# Track token usage if available from completionif llm_job.job_report.llm_tokens_usage:diff --git a/pipelex/plugins/mistral/mistral_factory.py b/pipelex/plugins/mistral/mistral_factory.pyindex 100934d72..e08322db4 100644--- a/pipelex/plugins/mistral/mistral_factory.py+++ b/pipelex/plugins/mistral/mistral_factory.py@@ -1,5 +1,3 @@-from typing import Dict, List-from mistralai import Mistral, OCRImageObject, OCRResponsefrom mistralai.models import (ContentChunk,@@ -47,17 +45,14 @@ class MistralFactory:#########################################################@classmethod- def make_simple_messages(cls, llm_job: LLMJob) -> List[Messages]:- """- Makes a list of messages with a system message (if provided) and followed by a user message.- """- messages: List[Messages] = []- user_content: List[ContentChunk] = []+ def make_simple_messages(cls, llm_job: LLMJob) -> list[Messages]:+ """Makes a list of messages with a system message (if provided) and followed by a user message."""+ messages: list[Messages] = []+ user_content: list[ContentChunk] = []if user_text := llm_job.llm_prompt.user_text:user_content.append(TextChunk(text=user_text))if user_images := llm_job.llm_prompt.user_images:- for user_image in user_images:- user_content.append(cls.make_mistral_image_url(user_image))+ user_content.extend(cls.make_mistral_image_url(user_image) for user_image in user_images)if user_content:messages.append(UserMessage(content=user_content))@@ -70,28 +65,26 @@ class MistralFactory:def make_mistral_image_url(cls, prompt_image: PromptImage) -> ImageURLChunk:if isinstance(prompt_image, PromptImageUrl):return ImageURLChunk(image_url=prompt_image.url)- elif isinstance(prompt_image, PromptImagePath):+ if isinstance(prompt_image, PromptImagePath):image_bytes = load_binary_as_base64(prompt_image.file_path).decode("utf-8")file_type = detect_file_type_from_path(prompt_image.file_path)return ImageURLChunk(image_url=f"data:{file_type.mime};base64,{image_bytes}")- elif isinstance(prompt_image, PromptImageBase64):+ if isinstance(prompt_image, PromptImageBase64):image_bytes = prompt_image.base_64.decode("utf-8")file_type = detect_file_type_from_base64(prompt_image.base_64)return ImageURLChunk(image_url=f"data:{file_type.mime};base64,{image_bytes}")- else:- raise PromptImageFormatError(f"prompt_image of type {type(prompt_image)} is not supported")+ msg = f"prompt_image of type {type(prompt_image)} is not supported"+ raise PromptImageFormatError(msg)@classmethoddef make_simple_messages_openai_typed(cls,llm_job: LLMJob,- ) -> List[ChatCompletionMessageParam]:- """- Makes a list of messages with a system message (if provided) and followed by a user message.- """+ ) -> list[ChatCompletionMessageParam]:+ """Makes a list of messages with a system message (if provided) and followed by a user message."""llm_prompt = llm_job.llm_prompt- messages: List[ChatCompletionMessageParam] = []- user_contents: List[ChatCompletionContentPartParam] = []+ messages: list[ChatCompletionMessageParam] = []+ user_contents: list[ChatCompletionContentPartParam] = []if system_content := llm_prompt.system_text:messages.append(ChatCompletionSystemMessageParam(role="system", content=system_content))# TODO: confirm that we can prompt without user_contents, for instance if we have only images,@@ -124,7 +117,7 @@ class MistralFactory:should_include_images: bool = False,# export_dir: Optional[str] = None,) -> OcrOutput:- pages: Dict[int, Page] = {}+ pages: dict[int, Page] = {}for ocr_response_page in mistral_ocr_response.pages:page = Page(text=ocr_response_page.markdown,@@ -145,7 +138,7 @@ class MistralFactory:cls,mistral_ocr_image_obj: OCRImageObject,) -> ExtractedImageFromPage:- extracted_image = ExtractedImageFromPage(+ return ExtractedImageFromPage(image_id=mistral_ocr_image_obj.id,top_left_x=mistral_ocr_image_obj.top_left_x,top_left_y=mistral_ocr_image_obj.top_left_y,@@ -153,4 +146,3 @@ class MistralFactory:bottom_right_y=mistral_ocr_image_obj.bottom_right_y,base_64=mistral_ocr_image_obj.image_base64 if mistral_ocr_image_obj.image_base64 else None,)- return extracted_imagediff --git a/pipelex/plugins/mistral/mistral_llm_worker.py b/pipelex/plugins/mistral/mistral_llm_worker.pyindex f0de7ad25..ef4160c3f 100644--- a/pipelex/plugins/mistral/mistral_llm_worker.py+++ b/pipelex/plugins/mistral/mistral_llm_worker.py@@ -1,8 +1,10 @@-from typing import Any, Optional, Type+from typing import TYPE_CHECKING, Anyimport instructorfrom mistralai import Mistral-from mistralai.models import ChatCompletionResponse++if TYPE_CHECKING:+ from mistralai.models import ChatCompletionResponsefrom typing_extensions import overridefrom pipelex import log@@ -22,8 +24,8 @@ class MistralLLMWorker(LLMWorkerInternalAbstract):self,sdk_instance: Any,inference_model: InferenceModelSpec,- structure_method: Optional[StructureMethod] = None,- reporting_delegate: Optional[ReportingProtocol] = None,+ structure_method: StructureMethod | None = None,+ reporting_delegate: ReportingProtocol | None = None,):LLMWorkerInternalAbstract.__init__(self,@@ -33,14 +35,14 @@ class MistralLLMWorker(LLMWorkerInternalAbstract):)if not isinstance(sdk_instance, Mistral):- raise SdkTypeError(f"Provided LLM sdk_instance for {self.__class__.__name__} is not of type Mistral: it's a '{type(sdk_instance)}'")+ msg = f"Provided LLM sdk_instance for {self.__class__.__name__} is not of type Mistral: it's a '{type(sdk_instance)}'"+ raise SdkTypeError(msg)if default_max_tokens := inference_model.max_tokens:self.default_max_tokens = default_max_tokenselse:- raise MistralWorkerConfigurationError(- f"No max_tokens provided for llm model '{self.inference_model.desc}', but it is required for Mistral"- )+ msg = f"No max_tokens provided for llm model '{self.inference_model.desc}', but it is required for Mistral"+ raise MistralWorkerConfigurationError(msg)self.mistral_client_for_text: Mistral = sdk_instanceif structure_method:@@ -56,19 +58,22 @@ class MistralLLMWorker(LLMWorkerInternalAbstract):llm_job: LLMJob,) -> str:messages = MistralFactory.make_simple_messages(llm_job=llm_job)- response: Optional[ChatCompletionResponse] = await self.mistral_client_for_text.chat.complete_async(+ response: ChatCompletionResponse | None = await self.mistral_client_for_text.chat.complete_async(messages=messages,model=self.inference_model.model_id,temperature=llm_job.job_params.temperature,max_tokens=llm_job.job_params.max_tokens or self.default_max_tokens,)if not response:- raise LLMCompletionError("Mistral response is None")+ msg = "Mistral response is None"+ raise LLMCompletionError(msg)if not response.choices:- raise LLMCompletionError("Mistral response.choices is None")+ msg = "Mistral response.choices is None"+ raise LLMCompletionError(msg)mistral_response_content = response.choices[0].message.contentif not isinstance(mistral_response_content, str):- raise LLMCompletionError("Mistral response.choices[0].message.content is not a string")+ msg = "Mistral response.choices[0].message.content is not a string"+ raise LLMCompletionError(msg)if (llm_tokens_usage := llm_job.job_report.llm_tokens_usage) and (usage := response.usage):llm_tokens_usage.nb_tokens_by_category = MistralFactory.make_nb_tokens_by_category(usage=usage)@@ -79,7 +84,7 @@ class MistralLLMWorker(LLMWorkerInternalAbstract):async def _gen_object(self,llm_job: LLMJob,- schema: Type[BaseModelTypeVar],+ schema: type[BaseModelTypeVar],) -> BaseModelTypeVar:result_object, completion = await self.instructor_for_objects.chat.completions.create_with_completion(response_model=schema,diff --git a/pipelex/plugins/mistral/mistral_llms.py b/pipelex/plugins/mistral/mistral_llms.pyindex c9a22d23f..6179927f3 100644--- a/pipelex/plugins/mistral/mistral_llms.py+++ b/pipelex/plugins/mistral/mistral_llms.py@@ -1,5 +1,3 @@-from typing import List-from mistralai.models import Datafrom pipelex.hub import get_models_manager@@ -7,13 +5,15 @@ from pipelex.plugins.mistral.mistral_exceptions import MistralModelListingErrorfrom pipelex.plugins.mistral.mistral_factory import MistralFactory-def list_mistral_models() -> List[Data]:+def list_mistral_models() -> list[Data]:backend = get_models_manager().get_required_inference_backend("mistral")mistral_client = MistralFactory.make_mistral_client(backend=backend)models_list_response = mistral_client.models.list()if not models_list_response:- raise MistralModelListingError("No models found")+ msg = "No models found"+ raise MistralModelListingError(msg)models_list = models_list_response.dataif not models_list:- raise MistralModelListingError("No models found")+ msg = "No models found"+ raise MistralModelListingError(msg)return sorted(models_list, key=lambda model: model.id)diff --git a/pipelex/plugins/mistral/mistral_ocr_worker.py b/pipelex/plugins/mistral/mistral_ocr_worker.pyindex c63004b01..6a1c43b0c 100644--- a/pipelex/plugins/mistral/mistral_ocr_worker.py+++ b/pipelex/plugins/mistral/mistral_ocr_worker.py@@ -1,4 +1,4 @@-from typing import Any, Dict, Optional+from typing import Anyfrom mistralai import Mistralfrom typing_extensions import override@@ -22,9 +22,9 @@ class MistralOcrWorker(OcrWorkerAbstract):def __init__(self,sdk_instance: Any,- extra_config: Dict[str, Any],+ extra_config: dict[str, Any],inference_model: InferenceModelSpec,- reporting_delegate: Optional[ReportingProtocol] = None,+ reporting_delegate: ReportingProtocol | None = None,):super().__init__(extra_config=extra_config,@@ -33,7 +33,8 @@ class MistralOcrWorker(OcrWorkerAbstract):)if not isinstance(sdk_instance, Mistral):- raise SdkTypeError(f"Provided OCR sdk_instance for {self.__class__.__name__} is not of type Mistral: it's a '{type(sdk_instance)}'")+ msg = f"Provided OCR sdk_instance for {self.__class__.__name__} is not of type Mistral: it's a '{type(sdk_instance)}'"+ raise SdkTypeError(msg)self.mistral_client: Mistral = sdk_instance@@ -57,7 +58,8 @@ class MistralOcrWorker(OcrWorkerAbstract):should_include_page_views=ocr_job.job_params.should_include_page_views,)else:- raise OcrInputError("No image nor PDF URI provided in OcrJob")+ msg = "No image nor PDF URI provided in OcrJob"+ raise OcrInputError(msg)return ocr_outputasync def _make_ocr_output_from_image(@@ -66,17 +68,17 @@ class MistralOcrWorker(OcrWorkerAbstract):should_caption_image: bool = False,) -> OcrOutput:if should_caption_image:- raise NotImplementedError("Captioning is not implemented for Mistral OCR.")+ msg = "Captioning is not implemented for Mistral OCR."+ raise NotImplementedError(msg)image_path, image_url = clarify_path_or_url(path_or_uri=image_uri)if image_url:return await self.extract_from_image_url(image_url=image_url,)- else:- assert image_path is not None # Type narrowing for mypy- return await self.extract_from_image_file(- image_path=image_path,- )+ assert image_path is not None+ return await self.extract_from_image_file(+ image_path=image_path,+ )async def _make_ocr_output_from_pdf(self,@@ -86,14 +88,15 @@ class MistralOcrWorker(OcrWorkerAbstract):should_include_page_views: bool,) -> OcrOutput:if should_caption_images:- raise OcrCapabilityError("Captioning is not implemented for Mistral OCR.")+ msg = "Captioning is not implemented for Mistral OCR."+ raise OcrCapabilityError(msg)if should_include_page_views:log.debug("Page views are not implemented for Mistral OCR.")# TODO: use a model capability flag to check possibility before asking for it# it it's asked and not available, raise# the caller will be responsible to get the page views using other solution if needed# raise OcrCapabilityError("Page views are not implemented for Mistral OCR.")- pdf_path, pdf_url = clarify_path_or_url(path_or_uri=pdf_uri) # pyright: ignore+ pdf_path, pdf_url = clarify_path_or_url(path_or_uri=pdf_uri)ocr_output: OcrOutputif pdf_url:ocr_output = await self.extract_from_pdf_url(@@ -101,7 +104,7 @@ class MistralOcrWorker(OcrWorkerAbstract):should_include_images=should_include_images,)else: # pdf_path must be provided based on validation- assert pdf_path is not None # Type narrowing for mypy+ assert pdf_path is not Noneocr_output = await self.extract_from_pdf_file(pdf_path=pdf_path,should_include_images=should_include_images,@@ -119,10 +122,9 @@ class MistralOcrWorker(OcrWorkerAbstract):"image_url": image_url,},)- ocr_output = await MistralFactory.make_ocr_output_from_mistral_response(+ return await MistralFactory.make_ocr_output_from_mistral_response(mistral_ocr_response=ocr_response,)- return ocr_outputasync def extract_from_image_file(self,@@ -137,10 +139,9 @@ class MistralOcrWorker(OcrWorkerAbstract):model=self.inference_model.model_id,document={"type": "image_url", "image_url": f"data:{mime_type};base64,{b64.decode('utf-8')}"},)- ocr_output = await MistralFactory.make_ocr_output_from_mistral_response(+ return await MistralFactory.make_ocr_output_from_mistral_response(mistral_ocr_response=ocr_response,)- return ocr_outputasync def extract_from_pdf_url(self,@@ -156,11 +157,10 @@ class MistralOcrWorker(OcrWorkerAbstract):include_image_base64=should_include_images,)- ocr_output = await MistralFactory.make_ocr_output_from_mistral_response(+ return await MistralFactory.make_ocr_output_from_mistral_response(mistral_ocr_response=ocr_response,should_include_images=should_include_images,)- return ocr_outputasync def extract_from_pdf_file(self,diff --git a/pipelex/plugins/mistral/mistral_utils.py b/pipelex/plugins/mistral/mistral_utils.pyindex fbbb43ffe..d09e37c01 100644--- a/pipelex/plugins/mistral/mistral_utils.py+++ b/pipelex/plugins/mistral/mistral_utils.py@@ -8,16 +8,17 @@ async def upload_file_for_ocr(mistral_client: Mistral,file_path: str,) -> str:- """- Upload a local file to Mistral.+ """Upload a local file to Mistral.Args:- pdf_path: Path to the local file to upload+ file_path: Path to the local file to upload+ mistral_client: Mistral clientReturns:ID of the uploaded file+"""- async with aiofiles.open(file_path, "rb") as file: # type: ignore[reportUnknownMemberType]+ async with aiofiles.open(file_path, "rb") as file: # pyright: ignore[reportUnknownMemberType]file_content = await file.read()uploaded_file = await mistral_client.files.upload_async(diff --git a/pipelex/plugins/openai/openai_factory.py b/pipelex/plugins/openai/openai_factory.pyindex 9bf8427a2..fe28215e8 100644--- a/pipelex/plugins/openai/openai_factory.py+++ b/pipelex/plugins/openai/openai_factory.py@@ -1,5 +1,3 @@-from typing import Dict, List-import openaifrom openai.types.chat import (ChatCompletionContentPartImageParam,@@ -45,15 +43,17 @@ class OpenAIFactory:) -> openai.AsyncClient:try:sdk_variant = OpenAISdkVariant(plugin.sdk)- except ValueError:- raise OpenAIFactoryError(f"Plugin '{plugin}' is not supported by OpenAIFactory")+ except ValueError as exc:+ msg = f"Plugin '{plugin}' is not supported by OpenAIFactory"+ raise OpenAIFactoryError(msg) from excthe_client: openai.AsyncOpenAImatch sdk_variant:case OpenAISdkVariant.AZURE_OPENAI:log.debug(f"Making AsyncOpenAI client with endpoint: {backend.endpoint}")if backend.endpoint is None:- raise OpenAIFactoryError("Azure OpenAI endpoint is not set")+ msg = "Azure OpenAI endpoint is not set"+ raise OpenAIFactoryError(msg)the_client = openai.AsyncAzureOpenAI(azure_endpoint=backend.endpoint,api_key=backend.api_key,@@ -73,13 +73,11 @@ class OpenAIFactory:def make_simple_messages(cls,llm_job: LLMJob,- ) -> List[ChatCompletionMessageParam]:- """- Makes a list of messages with a system message (if provided) and followed by a user message.- """+ ) -> list[ChatCompletionMessageParam]:+ """Makes a list of messages with a system message (if provided) and followed by a user message."""llm_prompt = llm_job.llm_prompt- messages: List[ChatCompletionMessageParam] = []- user_contents: List[ChatCompletionContentPartParam] = []+ messages: list[ChatCompletionMessageParam] = []+ user_contents: list[ChatCompletionContentPartParam] = []if system_content := llm_prompt.system_text:messages.append(ChatCompletionSystemMessageParam(role="system", content=system_content))# TODO: confirm that we can prompt without user_contents, for instance if we have only images,@@ -109,12 +107,13 @@ class OpenAIFactory:image_bytes = load_binary_as_base64(path=prompt_image.file_path)return cls.make_openai_image_url(PromptImageBase64(base_64=image_bytes))else:- raise LLMPromptParameterError(f"prompt_image of type {type(prompt_image)} is not supported")+ msg = f"prompt_image of type {type(prompt_image)} is not supported"+ raise LLMPromptParameterError(msg)return openai_image_url@staticmethoddef make_openai_error_info(exception: Exception) -> str:- error_mapping: Dict[type, str] = {+ error_mapping: dict[type, str] = {openai.BadRequestError: "OpenAI API request was invalid.",openai.InternalServerError: "OpenAI is having trouble. Please try again later.",openai.RateLimitError: "OpenAI API request exceeded rate limit.",@@ -125,8 +124,7 @@ class OpenAIFactory:openai.APIConnectionError: "OpenAI API request failed to connect.",openai.APIError: "OpenAI API returned an API Error.",}- error_info = error_mapping.get(type(exception), "An unexpected error occurred with the OpenAI API.")- return error_info+ return error_mapping.get(type(exception), "An unexpected error occurred with the OpenAI API.")# reference:# https://help.openai.com/en/articles/5247780-using-logit-bias-to-define-token-probabilitydiff --git a/pipelex/plugins/openai/openai_func.py b/pipelex/plugins/openai/openai_func.pyindex 598dc0212..71aeb42cc 100644--- a/pipelex/plugins/openai/openai_func.py+++ b/pipelex/plugins/openai/openai_func.py@@ -1,42 +1,51 @@import inspect-from typing import Any, Callable, Dict, List, Tuple, Type+from collections.abc import Callable+from typing import Anyfrom instructor import OpenAISchema, openai_schemafrom openai.types.chat import ChatCompletionMessagefrom pydantic import BaseModel, create_model-def create_pydantic_model_from_function(func: Callable[..., Any]) -> Type[BaseModel]:- """- Creates a Pydantic BaseModel from a function's signature.+def create_pydantic_model_from_function(func: Callable[..., Any]) -> type[BaseModel]:+ """Create a Pydantic BaseModel from a function's signature.- Parameters:- func (Callable): The function from which to derive the BaseModel.+ Args:+ func: The function to inspect. Its parameters become model fields.Returns:- Type[BaseModel]: A dynamically created Pydantic BaseModel class.+ type[BaseModel]: A dynamically created Pydantic model class.++ Raises:+ ValueError: If the function has unsupported parameter kinds.+"""sig = inspect.signature(func)- fields: Dict[str, Tuple[Type[Any], Any]] = {+ fields: dict[str, tuple[type[Any], Any]] = {name: (param.annotation, param.default if param.default is not inspect.Parameter.empty else ...) for name, param in sig.parameters.items()}model_name = func.__name__- return create_model(__model_name=model_name, **fields) # type: ignore+ return create_model(__model_name=model_name, **fields) # type: ignore[name-defined,call-overload,no-any-return] # pyright: ignore[reportCallIssue, reportUnknownVariableType]-def create_openai_schema_from_function(func: Callable[..., Any]) -> Dict[str, Any]:- """- Creates an OpenAI schema from a function.+def create_openai_schema_from_function(func: Callable[..., Any]) -> dict[str, Any]:+ """Creates an OpenAI schema from a function.- Parameters:- func (Callable): The function from which to derive the OpenAI schema.+ Parameters+ ----------+ func : Callable[..., Any]+ The Python function to introspect and convert into an OpenAI tool schema.Returns:- Dict[str, Any]: The OpenAI schema as a dictionary.+ -------+ dict[str, Any]+ The OpenAI-compatible schema describing the function's name, description,+ and parameters.+"""- model: Type[BaseModel] = create_pydantic_model_from_function(func)+ model: type[BaseModel] = create_pydantic_model_from_function(func)the_openai_schema: OpenAISchema = openai_schema(model)- as_openai_schema: Dict[str, Any] = the_openai_schema.openai_schema+ as_openai_schema: dict[str, Any] = the_openai_schema.openai_schema# replace description using first non-empty line of func.__doc__ descriptionif func.__doc__:@@ -47,10 +56,10 @@ def create_openai_schema_from_function(func: Callable[..., Any]) -> Dict[str, Anreturn as_openai_schema-def list_openai_tools(openai_message: ChatCompletionMessage) -> List[str]:+def list_openai_tools(openai_message: ChatCompletionMessage) -> list[str]:if not openai_message.tool_calls:return []- tools: List[str] = []+ tools: list[str] = []for tool_call in openai_message.tool_calls:if tool_call.type == "function":tools.append(tool_call.function.name)diff --git a/pipelex/plugins/openai/openai_img_gen_factory.py b/pipelex/plugins/openai/openai_img_gen_factory.pyindex 4d99ca48d..00279f979 100644--- a/pipelex/plugins/openai/openai_img_gen_factory.py+++ b/pipelex/plugins/openai/openai_img_gen_factory.py@@ -28,7 +28,8 @@ class OpenAIImgGenFactory:| AspectRatio.PORTRAIT_9_16| AspectRatio.PORTRAIT_9_21):- raise ImgGenParameterError(f"Aspect ratio '{aspect_ratio}' is not supported by GPT Image 1 model")+ msg = f"Aspect ratio '{aspect_ratio}' is not supported by GPT Image 1 model"+ raise ImgGenParameterError(msg)@classmethoddef output_format_for_gpt_image_1(cls, output_format: OutputFormat) -> GptImage1OutputFormatType:diff --git a/pipelex/plugins/openai/openai_img_gen_worker.py b/pipelex/plugins/openai/openai_img_gen_worker.pyindex 4b8720153..c87d949e6 100644--- a/pipelex/plugins/openai/openai_img_gen_worker.py+++ b/pipelex/plugins/openai/openai_img_gen_worker.py@@ -1,4 +1,4 @@-from typing import Any, List, Optional+from typing import Anyimport openaiimport shortuuid@@ -24,12 +24,13 @@ class OpenAIImgGenWorker(ImgGenWorkerAbstract):self,sdk_instance: Any,inference_model: InferenceModelSpec,- reporting_delegate: Optional[ReportingProtocol] = None,+ reporting_delegate: ReportingProtocol | None = None,):super().__init__(inference_model=inference_model, reporting_delegate=reporting_delegate)if not isinstance(sdk_instance, openai.AsyncOpenAI):- raise SdkTypeError(f"Provided Imgg sdk_instance is not of type openai.AsyncOpenAI: it's a '{type(sdk_instance)}'")+ msg = f"Provided ImgGen sdk_instance is not of type openai.AsyncOpenAI: it's a '{type(sdk_instance)}'"+ raise SdkTypeError(msg)self.openai_client = sdk_instance@@ -39,15 +40,14 @@ class OpenAIImgGenWorker(ImgGenWorkerAbstract):img_gen_job: ImgGenJob,) -> GeneratedImage:one_image_list = await self.gen_image_list(img_gen_job=img_gen_job, nb_images=1)- generated_image = one_image_list[0]- return generated_image+ return one_image_list[0]@overrideasync def _gen_image_list(self,img_gen_job: ImgGenJob,nb_images: int,- ) -> List[GeneratedImage]:+ ) -> list[GeneratedImage]:image_size = OpenAIImgGenFactory.image_size_for_gpt_image_1(aspect_ratio=img_gen_job.job_params.aspect_ratio)output_format = OpenAIImgGenFactory.output_format_for_gpt_image_1(output_format=img_gen_job.job_params.output_format)moderation = OpenAIImgGenFactory.moderation_for_gpt_image_1(is_moderated=img_gen_job.job_params.is_moderated)@@ -66,14 +66,16 @@ class OpenAIImgGenWorker(ImgGenWorkerAbstract):n=nb_images,)if not result.data:- raise ImgGenGenerationError("No result from OpenAI")+ msg = "No result from OpenAI"+ raise ImgGenGenerationError(msg)- generated_image_list: List[GeneratedImage] = []+ generated_image_list: list[GeneratedImage] = []image_id = shortuuid.uuid()[:4]for image_index, image_data in enumerate(result.data):image_base64 = image_data.b64_jsonif not image_base64:- raise ImgGenGenerationError("No base64 image data received from OpenAI")+ msg = "No base64 image data received from OpenAI"+ raise ImgGenGenerationError(msg)folder_path = TEMP_OUTPUTS_DIRensure_path(folder_path)@@ -85,6 +87,6 @@ class OpenAIImgGenWorker(ImgGenWorkerAbstract):url=img_path,width=1024,height=1024,- )+ ),)return generated_image_listdiff --git a/pipelex/plugins/openai/openai_llm_worker.py b/pipelex/plugins/openai/openai_llm_worker.pyindex cd4380647..f3e6cd9bb 100644--- a/pipelex/plugins/openai/openai_llm_worker.py+++ b/pipelex/plugins/openai/openai_llm_worker.py@@ -1,10 +1,12 @@-from typing import Any, Optional, Type+from typing import TYPE_CHECKING, Anyimport instructorimport openaifrom instructor.exceptions import InstructorRetryExceptionfrom openai import NOT_GIVEN, APIConnectionError, BadRequestError, NotFoundError-from openai.types.chat import ChatCompletionMessage++if TYPE_CHECKING:+ from openai.types.chat import ChatCompletionMessagefrom typing_extensions import overridefrom pipelex import log@@ -24,8 +26,8 @@ class OpenAILLMWorker(LLMWorkerInternalAbstract):self,sdk_instance: Any,inference_model: InferenceModelSpec,- structure_method: Optional[StructureMethod],- reporting_delegate: Optional[ReportingProtocol] = None,+ structure_method: StructureMethod | None,+ reporting_delegate: ReportingProtocol | None = None,):LLMWorkerInternalAbstract.__init__(self,@@ -35,9 +37,8 @@ class OpenAILLMWorker(LLMWorkerInternalAbstract):)if not isinstance(sdk_instance, openai.AsyncOpenAI):- raise SdkTypeError(- f"Provided LLM sdk_instance for {self.__class__.__name__} is not of type openai.AsyncOpenAI: it's a '{type(sdk_instance)}'"- )+ msg = f"Provided LLM sdk_instance for {self.__class__.__name__} is not of type openai.AsyncOpenAI: it's a '{type(sdk_instance)}'"+ raise SdkTypeError(msg)self.openai_client_for_text: openai.AsyncOpenAI = sdk_instanceif structure_method:@@ -48,6 +49,13 @@ class OpenAILLMWorker(LLMWorkerInternalAbstract):self.instructor_for_objects = instructor.from_openai(client=sdk_instance)#########################################################+ @override+ def setup(self):+ pass++ @override+ def teardown(self):+ pass@overrideasync def _gen_text(@@ -72,18 +80,20 @@ class OpenAILLMWorker(LLMWorkerInternalAbstract):)except NotFoundError as not_found_error:# TODO: record llm config so it can be displayed here- raise LLMModelNotFoundError(- f"OpenAI model or deployment not found:\n{self.inference_model.desc}\nmodel: {self.inference_model.desc}\n{not_found_error}"- ) from not_found_error+ msg = f"OpenAI model or deployment not found:\n{self.inference_model.desc}\nmodel: {self.inference_model.desc}\n{not_found_error}"+ raise LLMModelNotFoundError(msg) from not_found_errorexcept APIConnectionError as api_connection_error:- raise LLMCompletionError(f"OpenAI API connection error: {api_connection_error}") from api_connection_error+ msg = f"OpenAI API connection error: {api_connection_error}"+ raise LLMCompletionError(msg) from api_connection_errorexcept BadRequestError as bad_request_error:- raise LLMCompletionError(f"OpenAI bad request error with model: {self.inference_model.desc}:\n{bad_request_error}") from bad_request_error+ msg = f"OpenAI bad request error with model: {self.inference_model.desc}:\n{bad_request_error}"+ raise LLMCompletionError(msg) from bad_request_erroropenai_message: ChatCompletionMessage = response.choices[0].messageresponse_text = openai_message.contentif response_text is None:- raise LLMCompletionError(f"OpenAI response message content is None: {response}\nmodel: {self.inference_model.desc}")+ msg = f"OpenAI response message content is None: {response}\nmodel: {self.inference_model.desc}"+ raise LLMCompletionError(msg)if (llm_tokens_usage := llm_job.job_report.llm_tokens_usage) and (usage := response.usage):llm_tokens_usage.nb_tokens_by_category = OpenAIFactory.make_nb_tokens_by_category(usage=usage)@@ -93,7 +103,7 @@ class OpenAILLMWorker(LLMWorkerInternalAbstract):async def _gen_object(self,llm_job: LLMJob,- schema: Type[BaseModelTypeVar],+ schema: type[BaseModelTypeVar],) -> BaseModelTypeVar:messages = OpenAIFactory.make_simple_messages(llm_job=llm_job)try:@@ -114,13 +124,14 @@ class OpenAILLMWorker(LLMWorkerInternalAbstract):max_retries=llm_job.job_config.max_retries,)except InstructorRetryException as exc:- raise LLMCompletionError(- f"OpenAI instructor failed with model: {self.inference_model.desc} trying to generate schema: {schema} with error: {exc}"- ) from exc+ msg = f"OpenAI instructor failed with model: {self.inference_model.desc} trying to generate schema: {schema} with error: {exc}"+ raise LLMCompletionError(msg) from excexcept NotFoundError as exc:- raise LLMCompletionError(f"OpenAI model or deployment '{self.inference_model.model_id}' not found: {exc}") from exc+ msg = f"OpenAI model or deployment '{self.inference_model.model_id}' not found: {exc}"+ raise LLMCompletionError(msg) from excexcept BadRequestError as bad_request_error:- raise LLMCompletionError(f"OpenAI bad request error with model: {self.inference_model.desc}:\n{bad_request_error}") from bad_request_error+ msg = f"OpenAI bad request error with model: {self.inference_model.desc}:\n{bad_request_error}"+ raise LLMCompletionError(msg) from bad_request_errorif (llm_tokens_usage := llm_job.job_report.llm_tokens_usage) and (usage := completion.usage):llm_tokens_usage.nb_tokens_by_category = OpenAIFactory.make_nb_tokens_by_category(usage=usage)diff --git a/pipelex/plugins/openai/openai_llms.py b/pipelex/plugins/openai/openai_llms.pyindex 61f5a13ca..d790e3fe3 100644--- a/pipelex/plugins/openai/openai_llms.py+++ b/pipelex/plugins/openai/openai_llms.py@@ -1,5 +1,3 @@-from typing import List-from openai.types import Modelfrom pipelex.cogt.model_backends.backend import InferenceBackend@@ -10,7 +8,7 @@ from pipelex.plugins.plugin_sdk_registry import Pluginasync def openai_list_available_models(plugin: Plugin,backend: InferenceBackend,-) -> List[Model]:+) -> list[Model]:openai_client_async = OpenAIFactory.make_openai_client(plugin=plugin,backend=backend,@@ -18,5 +16,4 @@ async def openai_list_available_models(models = await openai_client_async.models.list()data = models.data- sorted_data = sorted(data, key=lambda model: model.id)- return sorted_data+ return sorted(data, key=lambda model: model.id)diff --git a/pipelex/plugins/openai/vertexai_factory.py b/pipelex/plugins/openai/vertexai_factory.pyindex e8b5a71e0..3adb49e5a 100644--- a/pipelex/plugins/openai/vertexai_factory.py+++ b/pipelex/plugins/openai/vertexai_factory.py@@ -1,4 +1,4 @@-from typing import Any, Dict, Tuple+from typing import Anyfrom pipelex.cogt.exceptions import CogtError, MissingDependencyErrorfrom pipelex.tools.config.config_model import ConfigModel@@ -26,22 +26,25 @@ class VertexAIFactory(ConfigModel):@classmethoddef make_endpoint_and_api_key(cls,- extra_config: Dict[str, Any],- ) -> Tuple[str, str]:+ extra_config: dict[str, Any],+ ) -> tuple[str, str]:"""Configure and return endpoint, and API key."""gcp_project_id = extra_config.get(VertexAIExtraField.GCP_PROJECT_ID)if not gcp_project_id:- raise VertexAIConfigError("GCP project ID is not set in VertexAI config")+ msg = "GCP project ID is not set in VertexAI config"+ raise VertexAIConfigError(msg)gcp_location = extra_config.get(VertexAIExtraField.GCP_LOCATION)if not gcp_location:- raise VertexAIConfigError("GCP location is not set in VertexAI config")+ msg = "GCP location is not set in VertexAI config"+ raise VertexAIConfigError(msg)endpoint = cls._make_endpoint(gcp_project_id=gcp_project_id, gcp_location=gcp_location)gcp_credentials_file_path = extra_config.get(VertexAIExtraField.GCP_CREDENTIALS_FILE_PATH)if not gcp_credentials_file_path or value_is_placeholder(gcp_credentials_file_path):- raise VertexAIConfigError("GCP credentials file path is not properly set for VertexAI.")+ msg = "GCP credentials file path is not properly set for VertexAI."+ raise VertexAIConfigError(msg)api_key = cls._make_api_key(gcp_credentials_file_path=gcp_credentials_file_path)@@ -55,31 +58,31 @@ class VertexAIFactory(ConfigModel):def _make_api_key(cls, gcp_credentials_file_path: str) -> str:"""Get GCP credentials and return access token."""try:- from google.auth.transport.requests import Request- from google.oauth2.service_account import Credentials+ from google.auth.transport.requests import Request # noqa: PLC0415+ from google.oauth2.service_account import Credentials # noqa: PLC0415except ImportError as exc:- raise MissingDependencyError(- "google-auth-oauthlib",- "google",- (- "The google-auth-oauthlib SDK is required to use Google connection. "- "You can install it with 'pip install pipelex[google]', or use this model via another provider "- "(such as Azure OpenAI, OpenAI, anthropic or bedrock)."- ),- ) from exc+ lib_name = "google-auth-oauthlib"+ lib_extra_name = "google"+ msg = (+ "The google-auth-oauthlib SDK is required to use Google connection. "+ "You can install it with 'pip install pipelex[google]', or use this model via another provider "+ "(such as Azure OpenAI, OpenAI, anthropic or bedrock)."+ )+ raise MissingDependencyError(lib_name, lib_extra_name, msg) from exctry:- credentials_dict: Dict[str, Any] = load_json_dict_from_path(path=gcp_credentials_file_path)+ credentials_dict: dict[str, Any] = load_json_dict_from_path(path=gcp_credentials_file_path)except FileNotFoundError as exc:- raise VertexAICredentialsError(- f"Could not get VertexAI credentials from GCP credentials file: File not found: {gcp_credentials_file_path}"- ) from exc+ msg = f"Could not get VertexAI credentials from GCP credentials file: File not found: {gcp_credentials_file_path}"+ raise VertexAICredentialsError(msg) from exccredentials = Credentials.from_service_account_info( # pyright: ignore[reportUnknownMemberType]- credentials_dict, scopes=["https://www.googleapis.com/auth/cloud-platform"]+ credentials_dict,+ scopes=["https://www.googleapis.com/auth/cloud-platform"],)auth_req = Request()credentials.refresh(auth_req) # pyright: ignore[reportUnknownMemberType]if not isinstance(credentials.token, str): # pyright: ignore[reportUnknownMemberType]- raise VertexAICredentialsError("Token is not a string")+ msg = "Token is not a string"+ raise VertexAICredentialsError(msg)return credentials.tokendiff --git a/pipelex/plugins/plugin_sdk_registry.py b/pipelex/plugins/plugin_sdk_registry.pyindex 3fd7f8168..0cb5f84de 100644--- a/pipelex/plugins/plugin_sdk_registry.py+++ b/pipelex/plugins/plugin_sdk_registry.py@@ -1,4 +1,4 @@-from typing import Any, Dict, Optional+from typing import Anyfrom pydantic import BaseModel, Field, RootModel@@ -21,7 +21,7 @@ class Plugin(BaseModel):)-PluginSdkRegistryRoot = Dict[str, Any]+PluginSdkRegistryRoot = dict[str, Any]class PluginSdkRegistry(RootModel[PluginSdkRegistryRoot]):@@ -33,7 +33,7 @@ class PluginSdkRegistry(RootModel[PluginSdkRegistryRoot]):sdk_instance.teardown()self.root = {}- def get_sdk_instance(self, plugin: Plugin) -> Optional[Any]:+ def get_sdk_instance(self, plugin: Plugin) -> Any | None:return self.root.get(plugin.sdk_handle)def set_sdk_instance(self, plugin: Plugin, sdk_instance: Any) -> Any:diff --git a/pipelex/plugins/pypdfium2/__init__.py b/pipelex/plugins/pypdfium2/__init__.pynew file mode 100644index 000000000..e69de29bbdiff --git a/pipelex/plugins/pypdfium2/pypdfium2_worker.py b/pipelex/plugins/pypdfium2/pypdfium2_worker.pyindex 399cd55c8..388178efa 100644--- a/pipelex/plugins/pypdfium2/pypdfium2_worker.py+++ b/pipelex/plugins/pypdfium2/pypdfium2_worker.py@@ -1,4 +1,4 @@-from typing import Any, Dict, Optional+from typing import Anyfrom typing_extensions import override@@ -15,10 +15,9 @@ from pipelex.tools.pdf.pypdfium2_renderer import pypdfium2_rendererclass Pypdfium2Worker(OcrWorkerAbstract):def __init__(self,- sdk_instance: Any,- extra_config: Dict[str, Any],+ extra_config: dict[str, Any],inference_model: InferenceModelSpec,- reporting_delegate: Optional[ReportingProtocol] = None,+ reporting_delegate: ReportingProtocol | None = None,):super().__init__(extra_config=extra_config, inference_model=inference_model, reporting_delegate=reporting_delegate)@@ -28,10 +27,11 @@ class Pypdfium2Worker(OcrWorkerAbstract):ocr_job: OcrJob,) -> OcrOutput:if ocr_job.ocr_input.image_uri:- raise NotImplementedError("Pypdfium2 only extracts text from PDFs, not images")+ msg = "Pypdfium2 only extracts text from PDFs, not images"+ raise NotImplementedError(msg)- elif pdf_uri := ocr_job.ocr_input.pdf_uri:- pdf_path, pdf_url = clarify_path_or_url(path_or_uri=pdf_uri) # pyright: ignore+ if pdf_uri := ocr_job.ocr_input.pdf_uri:+ pdf_path, pdf_url = clarify_path_or_url(path_or_uri=pdf_uri)ocr_output: OcrOutputif pdf_url:ocr_output = await self.extract_from_pdf_url(@@ -43,7 +43,8 @@ class Pypdfium2Worker(OcrWorkerAbstract):pdf_path=pdf_path,)else:- raise OcrInputError("No PDF URI provided in OcrJob")+ msg = "No PDF URI provided in OcrJob"+ raise OcrInputError(msg)return ocr_outputasync def extract_from_pdf_url(@@ -51,27 +52,25 @@ class Pypdfium2Worker(OcrWorkerAbstract):pdf_url: str,) -> OcrOutput:page_texts = await pypdfium2_renderer.get_text_from_pdf_pages_from_uri(pdf_uri=pdf_url)- pages: Dict[int, Page] = {}+ pages: dict[int, Page] = {}for page_index, page_text in enumerate(page_texts):pages[page_index + 1] = Page(text=page_text,)- ocr_output = OcrOutput(+ return OcrOutput(pages=pages,)- return ocr_outputasync def extract_from_pdf_file(self,pdf_path: str,) -> OcrOutput:page_texts = await pypdfium2_renderer.get_text_from_pdf_pages(pdf_input=pdf_path)- pages: Dict[int, Page] = {}+ pages: dict[int, Page] = {}for page_index, page_text in enumerate(page_texts):pages[page_index + 1] = Page(text=page_text,)- ocr_output = OcrOutput(+ return OcrOutput(pages=pages,)- return ocr_outputdiff --git a/pipelex/reporting/__init__.py b/pipelex/reporting/__init__.pynew file mode 100644index 000000000..e69de29bbdiff --git a/pipelex/reporting/reporting_manager.py b/pipelex/reporting/reporting_manager.pyindex 85c063824..8169a7865 100644--- a/pipelex/reporting/reporting_manager.py+++ b/pipelex/reporting/reporting_manager.py@@ -1,5 +1,3 @@-from typing import Dict, List, Optional-from pydantic import Field, RootModelfrom typing_extensions import override@@ -13,12 +11,13 @@ from pipelex.config import ReportingConfigfrom pipelex.pipeline.pipeline_models import SpecialPipelineIdfrom pipelex.reporting.reporting_protocol import ReportingProtocolfrom pipelex.tools.misc.file_utils import ensure_path, get_incremental_file_path+from pipelex.tools.typing.pydantic_utils import empty_list_factory_of-LLMUsageRegistryRoot = List[LLMTokensUsage]+LLMUsageRegistryRoot = list[LLMTokensUsage]class UsageRegistry(RootModel[LLMUsageRegistryRoot]):- root: LLMUsageRegistryRoot = Field(default_factory=list)+ root: LLMUsageRegistryRoot = Field(default_factory=empty_list_factory_of(LLMTokensUsage))def get_current_tokens_usage(self) -> LLMUsageRegistryRoot:return self.root@@ -29,7 +28,7 @@ class UsageRegistry(RootModel[LLMUsageRegistryRoot]):class ReportingManager(ReportingProtocol):def __init__(self, reporting_config: ReportingConfig):- self._usage_registries: Dict[str, UsageRegistry] = {}+ self._usage_registries: dict[str, UsageRegistry] = {}self._reporting_config = reporting_config############################################################@@ -51,7 +50,8 @@ class ReportingManager(ReportingProtocol):def _get_registry(self, pipeline_run_id: str) -> UsageRegistry:if pipeline_run_id not in self._usage_registries:- raise ReportingManagerError(f"Registry for pipeline '{pipeline_run_id}' does not exist")+ msg = f"Registry for pipeline '{pipeline_run_id}' does not exist"+ raise ReportingManagerError(msg)return self._usage_registries[pipeline_run_id]def _report_llm_job(self, llm_job: LLMJob):@@ -61,7 +61,7 @@ class ReportingManager(ReportingProtocol):log.warning("LLM job has no llm_tokens_usage")return- llm_token_cost_report: Optional[LLMTokenCostReport] = None+ llm_token_cost_report: LLMTokenCostReport | None = Noneif self._reporting_config.is_log_costs_to_console:llm_token_cost_report = CostRegistry.complete_cost_report(llm_tokens_usage=llm_tokens_usage)@@ -79,7 +79,8 @@ class ReportingManager(ReportingProtocol):@overridedef open_registry(self, pipeline_run_id: str):if pipeline_run_id in self._usage_registries:- raise ReportingManagerError(f"Registry for pipeline '{pipeline_run_id}' already exists")+ msg = f"Registry for pipeline '{pipeline_run_id}' already exists"+ raise ReportingManagerError(msg)self._usage_registries[pipeline_run_id] = UsageRegistry()@override@@ -93,8 +94,8 @@ class ReportingManager(ReportingProtocol):self._report_llm_job(llm_job=llm_job)@override- def generate_report(self, pipeline_run_id: Optional[str] = None):- cost_report_file_path: Optional[str] = None+ def generate_report(self, pipeline_run_id: str | None = None):+ cost_report_file_path: str | None = Noneif self._reporting_config.is_generate_cost_report_file_enabled:ensure_path(self._reporting_config.cost_report_dir_path)cost_report_file_path = get_incremental_file_path(@@ -103,7 +104,7 @@ class ReportingManager(ReportingProtocol):extension=self._reporting_config.cost_report_extension,)- registries_to_process: Dict[str, UsageRegistry] = {}+ registries_to_process: dict[str, UsageRegistry] = {}if pipeline_run_id:registries_to_process = {pipeline_run_id: self._get_registry(pipeline_run_id)}else:diff --git a/pipelex/reporting/reporting_protocol.py b/pipelex/reporting/reporting_protocol.pyindex 2c63e4d9a..9d8664a5e 100644--- a/pipelex/reporting/reporting_protocol.py+++ b/pipelex/reporting/reporting_protocol.py@@ -1,4 +1,4 @@-from typing import Optional, Protocol+from typing import Protocolfrom typing_extensions import override@@ -10,7 +10,7 @@ class ReportingProtocol(Protocol):def report_inference_job(self, inference_job: InferenceJobAbstract): ...- def generate_report(self, pipeline_run_id: Optional[str] = None): ...+ def generate_report(self, pipeline_run_id: str | None = None): ...def close_registry(self, pipeline_run_id: str): ...@@ -29,7 +29,7 @@ class ReportingNoOp(ReportingProtocol):pass@override- def generate_report(self, pipeline_run_id: Optional[str] = None):+ def generate_report(self, pipeline_run_id: str | None = None):pass@overridediff --git a/pipelex/test_extras/registry_test_models.py b/pipelex/test_extras/registry_test_models.pyindex d453f729c..33a7bf088 100644--- a/pipelex/test_extras/registry_test_models.py+++ b/pipelex/test_extras/registry_test_models.py@@ -1,4 +1,4 @@-from typing import ClassVar, List+from typing import ClassVarfrom pipelex.core.stuffs.stuff_content import StructuredContentfrom pipelex.tools.registry_models import ModelType, RegistryModels@@ -12,4 +12,4 @@ class FictionCharacter(StructuredContent):class PipelexTestModels(RegistryModels):- TEST_MODELS: ClassVar[List[ModelType]] = [FictionCharacter]+ TEST_MODELS: ClassVar[list[ModelType]] = [FictionCharacter]diff --git a/pipelex/test_extras/shared_pytest_plugins.py b/pipelex/test_extras/shared_pytest_plugins.pyindex 6e5dd3880..6c9ed48f8 100644--- a/pipelex/test_extras/shared_pytest_plugins.py+++ b/pipelex/test_extras/shared_pytest_plugins.py@@ -1,9 +1,8 @@import os-from typing import Listimport pytestfrom pytest import FixtureRequest, Parser-from rich import print+from rich import print # noqa: A004from pipelex.core.pipes.pipe_run_params import PipeRunModefrom pipelex.tools.environment import is_env_var_set, set_env@@ -61,7 +60,7 @@ def pipe_run_mode(request: FixtureRequest) -> PipeRunMode:return PipeRunMode(mode_str)-def _setup_env_var_placeholders(env_var_keys: List[str]) -> None:+def _setup_env_var_placeholders(env_var_keys: list[str]) -> None:"""Set placeholder environment variables when running in CI to prevent import failures.These placeholders allow the code to import successfully, while actual inference tests@@ -69,6 +68,7 @@ def _setup_env_var_placeholders(env_var_keys: List[str]) -> None:Args:env_var_keys: List of environment variable keys that need placeholders+"""# Set placeholders for env vars who's presence is required for the code to run properly# even if their value is not used in the test@@ -83,7 +83,7 @@ def _setup_env_var_placeholders(env_var_keys: List[str]) -> None:print(f"[yellow]Set {substitutions_counter} placeholder environment variables[/yellow]")-def _cleanup_placeholder_env_vars(env_var_keys: List[str]) -> None:+def _cleanup_placeholder_env_vars(env_var_keys: list[str]) -> None:"""Remove placeholder environment variables that were set during CI testing.This function identifies and removes any environment variables that contain@@ -91,6 +91,7 @@ def _cleanup_placeholder_env_vars(env_var_keys: List[str]) -> None:Args:env_var_keys: List of environment variable keys to check for placeholders+"""removed_counter = 0diff --git a/pipelex/tools/aws/aws_config.py b/pipelex/tools/aws/aws_config.pyindex b3aa13c63..cace6d869 100644--- a/pipelex/tools/aws/aws_config.py+++ b/pipelex/tools/aws/aws_config.py@@ -1,5 +1,3 @@-from typing import Tuple-from pydantic import Fieldfrom pipelex import log@@ -28,10 +26,10 @@ AWS_REGION_VAR_NAME = "AWS_REGION"class AwsConfig(ConfigModel):api_key_method: AwsKeyMethod = Field(strict=False)- def get_aws_access_keys(self) -> Tuple[str, str, str]:+ def get_aws_access_keys(self) -> tuple[str, str, str]:return self.get_aws_access_keys_with_method(api_key_method=self.api_key_method)- def get_aws_access_keys_with_method(self, api_key_method: AwsKeyMethod) -> Tuple[str, str, str]:+ def get_aws_access_keys_with_method(self, api_key_method: AwsKeyMethod) -> tuple[str, str, str]:match api_key_method:case AwsKeyMethod.ENV:log.debug("Getting AWS access keys from environment (key id and secret access key).")@@ -40,7 +38,8 @@ class AwsConfig(ConfigModel):aws_secret_access_key = get_required_env(AWS_SECRET_ACCESS_KEY_VAR_NAME)aws_region = get_required_env(AWS_REGION_VAR_NAME)except EnvVarNotFoundError as exc:- raise AwsCredentialsError(f"Error getting AWS access keys from environment: {exc}") from exc+ msg = f"Error getting AWS access keys from environment: {exc}"+ raise AwsCredentialsError(msg) from exclog.debug("Getting AWS region from environment (priority override) or from aws_config.")case AwsKeyMethod.SECRET_PROVIDER:@@ -50,7 +49,8 @@ class AwsConfig(ConfigModel):aws_secret_access_key = get_secret(AWS_SECRET_ACCESS_KEY_VAR_NAME)aws_region = get_secret(AWS_REGION_VAR_NAME)except SecretNotFoundError as exc:- raise AwsCredentialsError("Error getting AWS access keys from secrets provider.") from exc+ msg = "Error getting AWS access keys from secrets provider."+ raise AwsCredentialsError(msg) from exclog.debug("Getting AWS region from environment (priority override) or from aws_config.")return aws_access_key_id, aws_secret_access_key, aws_regiondiff --git a/pipelex/tools/class_registry_utils.py b/pipelex/tools/class_registry_utils.pyindex f44b33592..18e7384b8 100644--- a/pipelex/tools/class_registry_utils.py+++ b/pipelex/tools/class_registry_utils.py@@ -1,18 +1,24 @@+import typesfrom pathlib import Path-from typing import Any, Dict, List, Optional, Type, Union, get_args, get_origin+from typing import TYPE_CHECKING, Annotated, Any, Union, get_args, get_originfrom kajson.kajson_manager import KajsonManager-from pydantic.fields import FieldInfo++if TYPE_CHECKING:+ from pydantic.fields import FieldInfofrom pipelex.tools.typing.module_inspector import find_classes_in_module, import_module_from_file+_NoneType = type(None)+_UnionType = getattr(types, "UnionType", None) # Py3.10+: types.UnionType+class ClassRegistryUtils:@classmethoddef register_classes_in_file(cls,file_path: str,- base_class: Optional[Type[Any]],+ base_class: type[Any] | None,is_include_imported: bool,) -> None:"""Processes a Python file to find and register classes."""@@ -31,23 +37,21 @@ class ClassRegistryUtils:def register_classes_in_folder(cls,folder_path: str,- base_class: Optional[Type[Any]] = None,+ base_class: type[Any] | None = None,is_recursive: bool = True,is_include_imported: bool = False,) -> None:- """- Registers all classes in Python files within folders that are subclasses of base_class.+ """Registers all classes in Python files within folders that are subclasses of base_class.If base_class is None, registers all classes.Args:- folder_paths: List of paths to folders containing Python files+ folder_path: Path to folder containing Python filesbase_class: Optional base class to filter registerable classes- recursive: Whether to search recursively in subdirectories- exclude_files: List of filenames to exclude- exclude_dirs: List of directory names to exclude+ is_recursive: Whether to search recursively in subdirectoriesinclude_imported: Whether to include classes imported from other modules- """+ is_include_imported: Whether to include classes imported from other modules+ """python_files = cls.find_files_in_dir(dir_path=folder_path,pattern="*.py",@@ -62,46 +66,45 @@ class ClassRegistryUtils:)@classmethod- def find_files_in_dir(cls, dir_path: str, pattern: str, is_recursive: bool) -> List[Path]:- """- Find files matching a pattern in a directory.+ def find_files_in_dir(cls, dir_path: str, pattern: str, is_recursive: bool) -> list[Path]:+ """Find files matching a pattern in a directory.Args:dir_path: Directory path to search inpattern: File pattern to match (e.g. "*.py")- recursive: Whether to search recursively in subdirectories+ is_recursive: Whether to search recursively in subdirectoriesReturns:List of matching Path objects+"""path = Path(dir_path)if is_recursive:return list(path.rglob(pattern))- else:- return list(path.glob(pattern))+ return list(path.glob(pattern))@staticmethod- def are_classes_equivalent(class_1: Type[Any], class_2: Type[Any]) -> bool:+ def are_classes_equivalent(class_1: type[Any], class_2: type[Any]) -> bool:"""Check if two Pydantic classes are equivalent (same fields, types, descriptions)."""if not (hasattr(class_1, "model_fields") and hasattr(class_2, "model_fields")):return class_1 == class_2# Compare model schemas using Pydantic's built-in capabilitiestry:- schema_1: Dict[str, Any] = class_1.model_json_schema() # type: ignore[attr-defined]- schema_2: Dict[str, Any] = class_2.model_json_schema() # type: ignore[attr-defined]+ schema_1: dict[str, Any] = class_1.model_json_schema()+ schema_2: dict[str, Any] = class_2.model_json_schema()return schema_1 == schema_2except Exception:# Fallback to manual field comparison if schema comparison fails- fields_1: Dict[str, FieldInfo] = class_1.model_fields # type: ignore[attr-defined]- fields_2: Dict[str, FieldInfo] = class_2.model_fields # type: ignore[attr-defined]+ fields_1: dict[str, FieldInfo] = class_1.model_fields+ fields_2: dict[str, FieldInfo] = class_2.model_fieldsif set(fields_1.keys()) != set(fields_2.keys()):return False- for field_name in fields_1.keys():- field_1: FieldInfo = fields_1[field_name]- field_2: FieldInfo = fields_2[field_name]+ for field_1_name, field_1_info in fields_1.items():+ field_1: FieldInfo = field_1_info+ field_2: FieldInfo = fields_2[field_1_name]# Compare field typesif field_1.annotation != field_2.annotation:@@ -118,32 +121,34 @@ class ClassRegistryUtils:return True@staticmethod- def has_compatible_field(class_1: Type[Any], class_2: Type[Any]) -> bool:- """Check if class_1 has a field that is compatible with class_2."""+ def has_compatible_field(class_1: type[Any], class_2: type[Any]) -> bool:+ """Check if class_1 has a field whose (possibly wrapped) type matches/subclasses class_2."""if not hasattr(class_1, "model_fields"):return False- fields: Dict[str, FieldInfo] = class_1.model_fields # type: ignore[attr-defined]- for _field_name, field_info in fields.items():- field_type = field_info.annotation+ fields: dict[str, FieldInfo] = class_1.model_fields # type: ignore[attr-defined]- # Handle Optional types by extracting the inner type- origin = get_origin(field_type)- if origin is Union:- args = get_args(field_type)- # Check if this is Optional[T] (Union[T, None])- if len(args) == 2 and type(None) in args:- field_type = args[0] if args[1] is type(None) else args[1]+ def _is_compatible(t: Any) -> bool:+ # Unwrap Annotated[T, ...]+ if get_origin(t) is Annotated:+ t = get_args(t)[0]- # Check if the field type matches class_2- if field_type == class_2:- return True+ origin = get_origin(t)- # Check if field_type is a subclass of class_2+ # Handle unions, including PEP 604 (T | None)+ if origin in (Union, _UnionType):+ for arg in get_args(t):+ if arg is _NoneType:+ continue+ if _is_compatible(arg):+ return True+ return False++ # Base case: direct match / subclasstry:- if isinstance(field_type, type) and issubclass(field_type, class_2):- return True+ return t is class_2 or (isinstance(t, type) and issubclass(t, class_2))except TypeError:- pass+ # Not a class type (e.g., typing constructs you don't care about)+ return False- return False+ return any(_is_compatible(field.annotation) for field in fields.values())diff --git a/pipelex/tools/config/config_model.py b/pipelex/tools/config/config_model.pyindex 64a3f7e0d..f92ae20b9 100644--- a/pipelex/tools/config/config_model.py+++ b/pipelex/tools/config/config_model.py@@ -1,4 +1,4 @@-from typing import Dict, List, Optional, Type, TypeVar, Union+from typing import TypeVarfrom pydantic import BaseModel, ConfigDict@@ -13,12 +13,11 @@ class ConfigModel(BaseModel):@staticmethoddef transform_dict_str_to_enum(- input_dict: Dict[str, str],- key_enum_cls: Optional[Type[StrEnumType]] = None,- value_enum_cls: Optional[Type[StrEnumType]] = None,- ) -> Union[Dict[str, StrEnumType], Dict[StrEnumType, str], Dict[StrEnumType, StrEnumType]]:- """- Transforms a dictionary with str values into a dictionary with enum values.+ input_dict: dict[str, str],+ key_enum_cls: type[StrEnumType] | None = None,+ value_enum_cls: type[StrEnumType] | None = None,+ ) -> dict[str, StrEnumType] | dict[StrEnumType, str] | dict[StrEnumType, StrEnumType]:+ """Transforms a dictionary with str values into a dictionary with enum values.Args:input_dict: Dictionary with string values to be transformed.@@ -27,24 +26,24 @@ class ConfigModel(BaseModel):Returns:A dictionary where the values are converted to the given StrEnum type.+"""# return {key: value_enum_cls(value) for key, value in input_dict.items()}if key_enum_cls and value_enum_cls:return {key_enum_cls(key): value_enum_cls(value) for key, value in input_dict.items()}- elif key_enum_cls:+ if key_enum_cls:return {key_enum_cls(key): value for key, value in input_dict.items()}- elif value_enum_cls:+ if value_enum_cls:return {key: value_enum_cls(value) for key, value in input_dict.items()}- else:- raise ConfigModelError("Either key_enum_cls or value_enum_cls must be provided.")+ msg = "Either key_enum_cls or value_enum_cls must be provided."+ raise ConfigModelError(msg)@staticmethoddef transform_dict_of_floats_str_to_enum(- input_dict: Dict[str, float],- key_enum_cls: Type[StrEnumType],- ) -> Dict[StrEnumType, float]:- """- Transforms a dictionary with str keys and float values into a dictionary with enum keys and float values.+ input_dict: dict[str, float],+ key_enum_cls: type[StrEnumType],+ ) -> dict[StrEnumType, float]:+ """Transforms a dictionary with str keys and float values into a dictionary with enum keys and float values.Args:input_dict: Dictionary with string values to be transformed.@@ -52,12 +51,13 @@ class ConfigModel(BaseModel):Returns:A dictionary where the keys are converted to the given StrEnum type.+"""return {key_enum_cls(key): value for key, value in input_dict.items()}@staticmethoddef transform_list_of_str_to_enum(- input_list: List[str],- enum_cls: Type[StrEnumType],- ) -> List[StrEnumType]:+ input_list: list[str],+ enum_cls: type[StrEnumType],+ ) -> list[StrEnumType]:return [enum_cls(item) for item in input_list]diff --git a/pipelex/tools/config/config_root.py b/pipelex/tools/config/config_root.pyindex fcf42c180..65096c4cf 100644--- a/pipelex/tools/config/config_root.py+++ b/pipelex/tools/config/config_root.py@@ -1,4 +1,4 @@-from typing import Any, Optional+from typing import Anyfrom pydantic import ValidationError@@ -18,26 +18,25 @@ class SecretMethod(StrEnum):class ConfigRoot(ConfigModel):- """- Main configuration class for the project.+ """Main configuration class for the project.Attributes:project_name (str): Name of the current project.+"""- project_name: Optional[str] = None+ project_name: str | None = Nonedef __init__(self, **kwargs: Any):- """- Initialize the Config instance.+ """Initialize the Config instance.Args:**kwargs: Keyword arguments for configuration.Raises:ConfigValidationError: If the provided data is invalid.- """+ """try:super().__init__(**kwargs)except ValidationError as exc:diff --git a/pipelex/tools/config/manager.py b/pipelex/tools/config/manager.pyindex 53f1079a1..15376ec22 100644--- a/pipelex/tools/config/manager.py+++ b/pipelex/tools/config/manager.py@@ -1,7 +1,7 @@import importlibimport osfrom configparser import ConfigParser-from typing import Any, Dict, Optional+from typing import Anyimport toml@@ -19,7 +19,7 @@ CONFIG_TEMPLATE_SUBPATH = "config_template"INFERENCE_CONFIG_SUBPATH = "inference"-class ConfigException(Exception):+class ConfigError(Exception):pass@@ -45,7 +45,8 @@ class ConfigManager:@propertydef local_root_dir(self) -> str:"""Get the root directory of the project using pipelex.- This is the directory from where the command is being run."""+ This is the directory from where the command is being run.+ """return os.path.abspath(os.getcwd())@property@@ -56,29 +57,28 @@ class ConfigManager:def pipelex_specific_config_file_path(self) -> str:return os.path.join(self.pipelex_config_dir, CONFIG_NAME)- def get_pipelex_config(self) -> Dict[str, Any]:+ def get_pipelex_config(self) -> dict[str, Any]:"""Get the pipelex configuration from pipelex.toml.Returns:Dict[str, Any]: The configuration dictionary from pipelex.toml+"""config_path = self.pipelex_root_config_path- config = load_toml_from_path(config_path)- return config+ return load_toml_from_path(config_path)- def get_local_config(self) -> Dict[str, Any]:+ def get_local_config(self) -> dict[str, Any]:"""Get the local pipelex configuration from pipelex.toml in the project root.Returns:Dict[str, Any]: The configuration dictionary from the local pipelex.toml+"""config_path = os.path.join(self.local_root_dir, CONFIG_NAME)- config = failable_load_toml_from_path(config_path)- return config or {}+ return failable_load_toml_from_path(config_path) or {}- def load_inheritance_config(self, the_pipelex_config: Dict[str, Any]):- """- Load the config by inheritance in a pyproject.toml file.+ def load_inheritance_config(self, the_pipelex_config: dict[str, Any]):+ """Load the config by inheritance in a pyproject.toml file.This will be removed in the future.Requires to have a pyproject.toml file in the project root."""@@ -87,7 +87,7 @@ class ConfigManager:print(f"pyproject.toml not found in {self.local_root_dir}")return- def _find_package_path(package_name: str) -> Optional[str]:+ def _find_package_path(package_name: str) -> str | None:"""Find package path by importing it"""try:module = importlib.import_module(package_name)@@ -109,7 +109,7 @@ class ConfigManager:if config:deep_update(the_pipelex_config, config)- def load_config(self, specific_config_path: Optional[str] = None) -> Dict[str, Any]:+ def load_config(self, specific_config_path: str | None = None) -> dict[str, Any]:"""Load and merge configurations from pipelex and local config files.The configuration is loaded and merged in the following order:@@ -122,8 +122,8 @@ class ConfigManager:Returns:Dict[str, Any]: The merged configuration dictionary- """+ """#################### 1. Load pipelex config ####################pipelex_config = self.get_pipelex_config()@@ -138,9 +138,12 @@ class ConfigManager:deep_update(pipelex_config, local_config)#################### 4. Load overrides for the current project ####################- list_of_overrides = (- CONFIG_BASE_OVERRIDES_BEFORE_ENV + [runtime_manager.environment] + [runtime_manager.run_mode] + CONFIG_BASE_OVERRIDES_AFTER_ENV- )+ list_of_overrides = [+ *CONFIG_BASE_OVERRIDES_BEFORE_ENV,+ runtime_manager.environment,+ runtime_manager.run_mode,+ *CONFIG_BASE_OVERRIDES_AFTER_ENV,+ ]for override in list_of_overrides:if override:if override == runtime_manager.run_mode.UNIT_TEST:@@ -156,11 +159,12 @@ class ConfigManager:if config:deep_update(pipelex_config, config)else:- raise ConfigException(f"Failed to load specific config from {specific_config_path}")+ msg = f"Failed to load specific config from {specific_config_path}"+ raise ConfigError(msg)return pipelex_config- def get_project_name(self) -> Optional[str]:+ def get_project_name(self) -> str | None:"""Get the project name from configuration files.Checks the following files in order:@@ -170,15 +174,15 @@ class ConfigManager:4. setup.pyReturns:- Optional[str]: The project name or None if not found+ str | None: The project name or None if not found+"""# First check pipelex's pyproject.tomlpipelex_pyproject_path = os.path.join(os.path.dirname(self.local_root_dir), "pyproject.toml")try:pyproject = toml.load(pipelex_pyproject_path)- if project_name := pyproject.get("project", {}).get("name"):- if isinstance(project_name, str):- return project_name+ if (project_name := pyproject.get("project", {}).get("name")) and isinstance(project_name, str):+ return str(project_name)except FileNotFoundError:passexcept toml.TomlDecodeError as exc:@@ -190,9 +194,9 @@ class ConfigManager:pyproject_path = os.path.join(self.local_root_dir, "pyproject.toml")try:pyproject = toml.load(pyproject_path)- if project_name := pyproject.get("project", {}).get("name") or pyproject.get("tool", {}).get("poetry", {}).get("name"):- if isinstance(project_name, str):- return project_name+ name_obj: object = pyproject.get("project", {}).get("name") or pyproject.get("tool", {}).get("poetry", {}).get("name")+ if isinstance(name_obj, str):+ return name_objexcept FileNotFoundError as exc:print(f"Local pyproject.toml not found at {pyproject_path}: {exc}")except toml.TomlDecodeError as exc:@@ -205,9 +209,8 @@ class ConfigManager:try:config = ConfigParser()config.read(setup_cfg_path)- if config.has_section("metadata"):- if cfg_name := config.get("metadata", "name", fallback=None):- return cfg_name+ if (cfg_name := config.get("metadata", "name", fallback=None)) and config.has_section("metadata"):+ return cfg_nameexcept FileNotFoundError as exc:print(f"setup.cfg not found at {setup_cfg_path}: {exc}")except (ValueError, OSError) as exc:@@ -230,7 +233,7 @@ class ConfigManager:return line[start + 1 : end]except FileNotFoundError as exc:print(f"setup.py not found at {setup_py_path}: {exc}")- except (IOError, UnicodeDecodeError) as exc:+ except (OSError, UnicodeDecodeError) as exc:print(f"Failed to read setup.py at {setup_py_path}: {exc}")print("Could not find project name in any of the configuration files")diff --git a/pipelex/tools/environment.py b/pipelex/tools/environment.pyindex 97f14fc59..ec2cef78f 100644--- a/pipelex/tools/environment.py+++ b/pipelex/tools/environment.py@@ -1,5 +1,4 @@import os-from typing import List, Optionalfrom dotenv import load_dotenv@@ -16,27 +15,24 @@ class EnvVarNotFoundError(ToolException):def get_required_env(key: str) -> str:value = os.getenv(key)if not value:- raise EnvVarNotFoundError(f"Environment variable '{key}' is required but not set")+ msg = f"Environment variable '{key}' is required but not set"+ raise EnvVarNotFoundError(msg)return value-def get_optional_env(key: str) -> Optional[str]:- value = os.getenv(key)- return value+def get_optional_env(key: str) -> str | None:+ return os.getenv(key)def is_env_var_set(key: str) -> bool:return os.getenv(key) is not None-def all_env_vars_are_set(keys: List[str]) -> bool:- for each_key in keys:- if not is_env_var_set(each_key):- return False- return True+def all_env_vars_are_set(keys: list[str]) -> bool:+ return all(is_env_var_set(each_key) for each_key in keys)-def any_env_var_is_placeholder(keys: List[str]) -> bool:+def any_env_var_is_placeholder(keys: list[str]) -> bool:for each_key in keys:env_value = os.getenv(each_key)if value_is_placeholder(env_value):@@ -46,4 +42,3 @@ def any_env_var_is_placeholder(keys: List[str]) -> bool:def set_env(key: str, value: str) -> None:os.environ[key] = value- return Nonediff --git a/pipelex/tools/func_registry.py b/pipelex/tools/func_registry.pyindex 223e19cb9..34b58e97e 100644--- a/pipelex/tools/func_registry.py+++ b/pipelex/tools/func_registry.py@@ -1,6 +1,7 @@import inspectimport logging-from typing import Any, Callable, Dict, List, Optional, TypeVar, get_type_hints+from collections.abc import Callable+from typing import Any, TypeVar, get_type_hintsfrom pydantic import Field, PrivateAttr, RootModel@@ -10,7 +11,7 @@ FUNC_REGISTRY_LOGGER_CHANNEL_NAME = "func_registry"# Type variable for generic function typesT = TypeVar("T")-FuncRegistryDict = Dict[str, Callable[..., Any]]+FuncRegistryDict = dict[str, Callable[..., Any]]class FuncRegistryError(ToolException):@@ -34,7 +35,7 @@ class FuncRegistry(RootModel[FuncRegistryDict]):def register_function(self,func: Callable[..., Any],- name: Optional[str] = None,+ name: str | None = None,should_warn_if_already_registered: bool = True,) -> None:"""Registers a function in the registry with a name if it meets eligibility criteria."""@@ -46,7 +47,8 @@ class FuncRegistry(RootModel[FuncRegistryDict]):if should_warn_if_already_registered:self.log(f"Function '{key}' already exists in registry")else:- raise FuncRegistryError(f"Function '{key}' already exists in registry")+ msg = f"Function '{key}' already exists in registry"+ raise FuncRegistryError(msg)else:self.log(f"Registered new single function '{key}' in registry")self.root[key] = func@@ -55,52 +57,56 @@ class FuncRegistry(RootModel[FuncRegistryDict]):"""Unregisters a function from the registry."""key = func.__name__if key not in self.root:- raise FuncRegistryError(f"Function '{key}' not found in registry")+ msg = f"Function '{key}' not found in registry"+ raise FuncRegistryError(msg)del self.root[key]self.log(f"Unregistered single function '{key}' from registry")def unregister_function_by_name(self, name: str) -> None:"""Unregisters a function from the registry by its name."""if name not in self.root:- raise FuncRegistryError(f"Function '{name}' not found in registry")+ msg = f"Function '{name}' not found in registry"+ raise FuncRegistryError(msg)del self.root[name]- def register_functions_dict(self, functions: Dict[str, Callable[..., Any]]) -> None:+ def register_functions_dict(self, functions: dict[str, Callable[..., Any]]) -> None:"""Registers multiple functions in the registry with names if they meet eligibility criteria."""for name, func in functions.items():self.register_function(func=func, name=name, should_warn_if_already_registered=False)- def register_functions(self, functions: List[Callable[..., Any]]) -> None:+ def register_functions(self, functions: list[Callable[..., Any]]) -> None:"""Registers multiple functions in the registry with names if they meet eligibility criteria."""for func in functions:self.register_function(func=func, should_warn_if_already_registered=False)- def get_function(self, name: str) -> Optional[Callable[..., Any]]:+ def get_function(self, name: str) -> Callable[..., Any] | None:"""Retrieves a function from the registry by its name. Returns None if not found."""return self.root.get(name)def get_required_function(self, name: str) -> Callable[..., Any]:"""Retrieves a function from the registry by its name. Raises an error if not found."""if name not in self.root:- raise FuncRegistryError(- f"Function '{name}' not found in registry: \- See how to register a function here: https://docs.pipelex.com/pages/build-reliable-ai-workflows-with-pipelex/pipe-operators/PipeFunc"+ msg = (+ f"Function '{name}' not found in registry:"+ "See how to register a function here: https://docs.pipelex.com/pages/build-reliable-ai-workflows-with-pipelex/pipe-operators/PipeFunc")+ raise FuncRegistryError(msg)return self.root[name]- def get_required_function_with_signature(self, name: str, expected_signature: Callable[..., T]) -> Callable[..., T]:- """- Retrieves a function from the registry by its name and verifies it matches the expected signature.+ def get_required_function_with_signature(self, name: str) -> Callable[..., object]:+ """Retrieves a function from the registry by its name and verifies it matches the expected signature.Raises an error if not found or if signature doesn't match."""if name not in self.root:- raise FuncRegistryError(f"Function '{name}' not found in registry")+ msg = f"Function '{name}' not found in registry"+ raise FuncRegistryError(msg)func = self.root[name]# Note: This is a basic signature check. For more thorough type checking,# you might want to use typing.get_type_hints() or a more sophisticated type checkerif not callable(func):- raise FuncRegistryError(f"'{name}' is not a callable function")+ msg = f"'{name}' is not a callable function"+ raise FuncRegistryError(msg)return funcdef has_function(self, name: str) -> bool:@@ -108,15 +114,14 @@ class FuncRegistry(RootModel[FuncRegistryDict]):return name in self.rootdef is_eligible_function(self, func: Callable[..., Any]) -> bool:- """- Checks if a function matches the criteria for PipeFunc registration:+ """Checks if a function matches the criteria for PipeFunc registration:- Exactly 1 parameter named "working_memory" with type WorkingMemory- Return type that is a subclass of StuffContent"""try:# Import here to avoid circular imports- from pipelex.core.memory.working_memory import WorkingMemory- from pipelex.core.stuffs.stuff_content import StuffContent+ from pipelex.core.memory.working_memory import WorkingMemory # noqa: PLC0415+ from pipelex.core.stuffs.stuff_content import StuffContent # noqa: PLC0415# Get function signaturesig = inspect.signature(func)@@ -153,7 +158,7 @@ class FuncRegistry(RootModel[FuncRegistryDict]):return True# Handle generic types like ListContent[SomeType]if hasattr(return_type, "__origin__"):- origin = getattr(return_type, "__origin__")+ origin = return_type.__origin__if inspect.isclass(origin) and issubclass(origin, StuffContent):return Trueexcept TypeError:diff --git a/pipelex/tools/func_registry_utils.py b/pipelex/tools/func_registry_utils.pyindex c16bd06f9..469f8646d 100644--- a/pipelex/tools/func_registry_utils.py+++ b/pipelex/tools/func_registry_utils.py@@ -1,6 +1,7 @@import inspect+from collections.abc import Callablefrom pathlib import Path-from typing import Any, Callable, List+from typing import Anyfrom pipelex.tools.func_registry import func_registryfrom pipelex.tools.typing.module_inspector import import_module_from_file@@ -13,8 +14,7 @@ class FuncRegistryUtils:folder_path: str,is_recursive: bool = True,) -> None:- """- Discovers and attempts to register all functions in Python files within a folder.+ """Discovers and attempts to register all functions in Python files within a folder.Only functions that meet the eligibility criteria will be registered:- Must be an async function- Exactly 1 parameter named "working_memory" with type WorkingMemory@@ -25,6 +25,7 @@ class FuncRegistryUtils:Args:folder_path: Path to folder containing Python filesis_recursive: Whether to search recursively in subdirectories+"""python_files = cls._find_files_in_dir(dir_path=folder_path,@@ -55,11 +56,9 @@ class FuncRegistryUtils:print(f"Error processing file {file_path}: {e}")@classmethod- def _find_functions_in_module(cls, module: Any) -> List[Callable[..., Any]]:- """- Finds all functions in a module (eligibility will be checked during registration).- """- functions: List[Callable[..., Any]] = []+ def _find_functions_in_module(cls, module: Any) -> list[Callable[..., Any]]:+ """Finds all functions in a module (eligibility will be checked during registration)."""+ functions: list[Callable[..., Any]] = []module_name = module.__name__# Find all functions in the module (not imported ones)@@ -74,9 +73,8 @@ class FuncRegistryUtils:return functions@classmethod- def _find_files_in_dir(cls, dir_path: str, pattern: str, is_recursive: bool) -> List[Path]:- """- Find files matching a pattern in a directory.+ def _find_files_in_dir(cls, dir_path: str, pattern: str, is_recursive: bool) -> list[Path]:+ """Find files matching a pattern in a directory.Args:dir_path: Directory path to search in@@ -85,9 +83,9 @@ class FuncRegistryUtils:Returns:List of matching Path objects+"""path = Path(dir_path)if is_recursive:return list(path.rglob(pattern))- else:- return list(path.glob(pattern))+ return list(path.glob(pattern))diff --git a/pipelex/tools/log/log.py b/pipelex/tools/log/log.pyindex 8e418d9be..32da57727 100644--- a/pipelex/tools/log/log.py+++ b/pipelex/tools/log/log.py@@ -1,5 +1,5 @@import logging-from typing import Any, Dict, Optional, Union+from typing import Anyfrom pipelex.tools.log.log_config import LogConfig, LogModefrom pipelex.tools.log.log_dispatch import LogDispatch@@ -8,22 +8,18 @@ from pipelex.tools.log.log_levels import LOGGING_LEVEL_DEV, LOGGING_LEVEL_OFF, Lclass Log:- """- A class for managing logging configurations and operations.- """+ """A class for managing logging configurations and operations."""######################################################### Init and Configure########################################################def __init__(self):- """- Initialize the Log class with default attributes.- """- self.project_name: Optional[str] = None- self._log_config_instance: Optional[LogConfig] = None- self.rich_handler: Optional[logging.Handler] = None- self.poor_handler: Optional[logging.Handler] = None+ """Initialize the Log class with default attributes."""+ self.project_name: str | None = None+ self._log_config_instance: LogConfig | None = None+ self.rich_handler: logging.Handler | None = None+ self.poor_handler: logging.Handler | None = Noneself.log_dispatch: LogDispatch = LogDispatch()def set_log_mode(self, mode: LogMode):@@ -31,23 +27,22 @@ class Log:@propertydef _log_config(self) -> LogConfig:- """- Get the log configuration, raising an error if it's not set.+ """Get the log configuration, raising an error if it's not set.Returns:LogConfig: The current log configuration.Raises:RuntimeError: If the log configuration is not set.+"""if self._log_config_instance is None:- raise RuntimeError("LogConfig is not set. You must initialize Pipelex first.")+ msg = "LogConfig is not set. You must initialize Pipelex first."+ raise RuntimeError(msg)return self._log_config_instancedef reset(self):- """- Reset the logging system.- """+ """Reset the logging system."""# Remove all handlers from the root loggerroot_logger = logging.getLogger()for handler in root_logger.handlers[:]:@@ -56,7 +51,10 @@ class Log:# Remove handlers from poor loggersif self._log_config_instance:- poor_loggers = set(self._log_config_instance.poor_loggers + [self._log_config_instance.generic_poor_logger])+ poor_loggers = {+ *self._log_config_instance.poor_loggers,+ self._log_config_instance.generic_poor_logger,+ }for logger_name in poor_loggers:logger = logging.getLogger(logger_name)for handler in logger.handlers[:]:@@ -75,8 +73,7 @@ class Log:project_name: str,log_config: LogConfig,):- """- Configure the logging system with the given project name and log configuration.+ """Configure the logging system with the given project name and log configuration.Args:project_name (str): The name of the project.@@ -84,9 +81,11 @@ class Log:Raises:RuntimeError: If the log configuration is already set.+"""if self._log_config_instance is not None:- raise RuntimeError("LogConfig is already set. You can only call log.configure() once.")+ msg = "LogConfig is already set. You can only call log.configure() once."+ raise RuntimeError(msg)self.log_dispatch.configure(project_name=project_name,@@ -106,7 +105,7 @@ class Log:self.poor_handler = logging.StreamHandler()self.poor_handler.setFormatter(LevelAndEmojiLogFormatter())- poor_loggers = set(log_config.poor_loggers + [log_config.generic_poor_logger])+ poor_loggers = {*log_config.poor_loggers, log_config.generic_poor_logger}for logger_name in poor_loggers:logger = logging.getLogger(logger_name)logger.setLevel(log_config.default_log_level.int_logging_level)@@ -119,50 +118,48 @@ class Log:self.verbose(f"Config set for {project_name}")def set_poor_log_formatter(self, formatter: logging.Formatter):- """- Set the formatter for the poor log handler.+ """Set the formatter for the poor log handler.Args:formatter (logging.Formatter): The formatter to use for poor logging.+"""if self.poor_handler is None:- raise RuntimeError("Poor log handler is not set.")+ msg = "Poor log handler is not set."+ raise RuntimeError(msg)self.poor_handler.setFormatter(formatter)- def _should_ignore(self, problem_id: Optional[str] = None) -> bool:- """- Check if a log message should be ignored based on the problem ID.+ def _should_ignore(self, problem_id: str | None = None) -> bool:+ """Check if a log message should be ignored based on the problem ID.Args:- problem_id (Optional[str]): The problem ID to check.+ problem_id (str | None): The problem ID to check.Returns:bool: True if the message should be ignored, False otherwise.+"""- if problem_id and problem_id in self._log_config.silenced_problem_ids:- return True- else:- return False+ return bool(problem_id) and problem_id in self._log_config.silenced_problem_ids######################################################### Public methods########################################################def set_level_by_int(self, level_int: int):- """- Set the log level using an integer value.+ """Set the log level using an integer value.Args:level_int (int): The integer representation of the log level.+"""logging.getLogger().setLevel(level_int)def set_level_by_name(self, level_name: str):- """- Set the log level using a string name.+ """Set the log level using a string name.Args:level_name (str): The name of the log level.+"""if level_name.upper() == LogLevel.DEV:level = LOGGING_LEVEL_DEV@@ -173,122 +170,123 @@ class Log:self.set_level_by_int(level)def set_level(self, level: LogLevel):- """- Set the default log level for all loggers.+ """Set the default log level for all loggers.+Args:level (LogLevel): The log level to set.+"""self.set_level_by_int(level_int=level.int_logging_level)def set_level_for_package(self, package_name: str, level: LogLevel):- """- Set the log level for a specific package.+ """Set the log level for a specific package.Args:package_name (str): The name of the package.level (LogLevel): The log level to set for the package.+"""logger_name = package_name.replace("-", ".")logging.getLogger(logger_name).setLevel(level.int_logging_level)- def set_levels_for_packages(self, package_log_levels: Dict[str, LogLevel]):- """- Set log levels for multiple packages.+ def set_levels_for_packages(self, package_log_levels: dict[str, LogLevel]):+ """Set log levels for multiple packages.Args:package_log_levels (Dict[str, LogLevel]): A dictionary mapping package names to log levels.+"""for package_name, level in package_log_levels.items():self.set_level_for_package(package_name=package_name, level=level)def verbose(self,- content: Union[str, Any],- title: Optional[str] = None,- inline: Optional[str] = None,+ content: str | Any,+ title: str | None = None,+ inline: str | None = None,):- """- Log a verbose message.+ """Log a verbose message.Args:content (Union[str, Any]): The content to log.- title (Optional[str], optional): The title of the log message. Defaults to None.- inline (Optional[str], optional): Inline title for the log message. Defaults to None.+ title (str | None, optional): The title of the log message. Defaults to None.+ inline (str | None, optional): Inline title for the log message. Defaults to None.Used to display the title inline, only if the title arg is None.+"""severity = LOGGING_LEVEL_VERBOSEself.log_dispatch.dispatch(content=content, severity=severity, title=title, inline=inline)def debug(self,- content: Union[str, Any],- title: Optional[str] = None,- inline: Optional[str] = None,+ content: str | Any,+ title: str | None = None,+ inline: str | None = None,):- """- Log a debug message.+ """Log a debug message.Args:content (Union[str, Any]): The content to log.- title (Optional[str], optional): The title of the log message. Defaults to None.- inline (Optional[str], optional): Inline title for the log message. Defaults to None.+ title (str | None, optional): The title of the log message. Defaults to None.+ inline (str | None, optional): Inline title for the log message. Defaults to None.Used to display the title inline, only if the title arg is None.+"""severity = logging.DEBUGself.log_dispatch.dispatch(content=content, severity=severity, title=title, inline=inline)def dev(self,- content: Union[str, Any],- title: Optional[str] = None,- inline: Optional[str] = None,+ content: str | Any,+ title: str | None = None,+ inline: str | None = None,):- """- Log a development message.+ """Log a development message.Args:content (Union[str, Any]): The content to log.- title (Optional[str], optional): The title of the log message. Defaults to None.- inline (Optional[str], optional): Inline title for the log message. Defaults to None.+ title (str | None, optional): The title of the log message. Defaults to None.+ inline (str | None, optional): Inline title for the log message. Defaults to None.Used to display the title inline, only if the title arg is None.+"""severity = LOGGING_LEVEL_DEVself.log_dispatch.dispatch(content=content, severity=severity, title=title, inline=inline)def info(self,- content: Union[str, Any],- title: Optional[str] = None,- inline: Optional[str] = None,+ content: str | Any,+ title: str | None = None,+ inline: str | None = None,):- """- Log an info message.+ """Log an info message.Args:content (Union[str, Any]): The content to log.- title (Optional[str], optional): The title of the log message. Defaults to None.- inline (Optional[str], optional): Inline title for the log message. Defaults to None.+ title (str | None, optional): The title of the log message. Defaults to None.+ inline (str | None, optional): Inline title for the log message. Defaults to None.Used to display the title inline, only if the title arg is None.+"""severity = logging.INFOself.log_dispatch.dispatch(content=content, severity=severity, title=title, inline=inline)def warning(self,- content: Union[str, Any],- title: Optional[str] = None,- inline: Optional[str] = None,- problem_id: Optional[str] = None,+ content: str | Any,+ title: str | None = None,+ inline: str | None = None,+ problem_id: str | None = None,):- """- Log a warning message.+ """Log a warning message.Args:content (Union[str, Any]): The content to log.- title (Optional[str], optional): The title of the log message. Defaults to None.- inline (Optional[str], optional): Inline title for the log message. Defaults to None.+ title (str | None, optional): The title of the log message. Defaults to None.+ inline (str | None, optional): Inline title for the log message. Defaults to None.Used to display the title inline, only if the title arg is None.- problem_id (Optional[str], optional): A problem ID to associate with the warning. Defaults to None.+ problem_id (str | None, optional): A problem ID to associate with the warning. Defaults to None.+"""if self._should_ignore(problem_id=problem_id):return@@ -297,22 +295,22 @@ class Log:def error(self,- content: Union[str, Any],- title: Optional[str] = None,- inline: Optional[str] = None,+ content: str | Any,+ title: str | None = None,+ inline: str | None = None,include_exception: bool = False,- problem_id: Optional[str] = None,+ problem_id: str | None = None,):- """- Log an error message.+ """Log an error message.Args:content (Union[str, Any]): The content to log.- title (Optional[str], optional): The title of the log message. Defaults to None.- inline (Optional[str], optional): Inline title for the log message. Defaults to None.+ title (str | None, optional): The title of the log message. Defaults to None.+ inline (str | None, optional): Inline title for the log message. Defaults to None.Used to display the title inline, only if the title arg is None.include_exception (bool, optional): Whether to include exception information. Defaults to False.- problem_id (Optional[str], optional): A problem ID to associate with the error. Defaults to None.+ problem_id (str | None, optional): A problem ID to associate with the error. Defaults to None.+"""if self._should_ignore(problem_id=problem_id):return@@ -327,22 +325,22 @@ class Log:def critical(self,- content: Union[str, Any],- title: Optional[str] = None,- inline: Optional[str] = None,+ content: str | Any,+ title: str | None = None,+ inline: str | None = None,include_exception: bool = False,- problem_id: Optional[str] = None,+ problem_id: str | None = None,):- """- Log a critical message.+ """Log a critical message.Args:content (Union[str, Any]): The content to log.- title (Optional[str], optional): The title of the log message. Defaults to None.- inline (Optional[str], optional): Inline title for the log message. Defaults to None.+ title (str | None, optional): The title of the log message. Defaults to None.+ inline (str | None, optional): Inline title for the log message. Defaults to None.Used to display the title inline, only if the title arg is None.include_exception (bool, optional): Whether to include exception information. Defaults to False.- problem_id (Optional[str], optional): A problem ID to associate with the critical message. Defaults to None.+ problem_id (str | None, optional): A problem ID to associate with the critical message. Defaults to None.+"""if self._should_ignore(problem_id=problem_id):returndiff --git a/pipelex/tools/log/log_config.py b/pipelex/tools/log/log_config.pyindex 1a49e6295..c2ae2e5ac 100644--- a/pipelex/tools/log/log_config.py+++ b/pipelex/tools/log/log_config.py@@ -1,5 +1,5 @@from enum import Enum-from typing import Dict, List, cast+from typing import castfrom pydantic import Field, field_validatorfrom rich.highlighter import Highlighter, JSONHighlighter, ReprHighlighter@@ -20,7 +20,7 @@ class HighlighterName(StrEnum):REPR = "repr"-class PROBLEM_IDS(StrEnum):+class ProblemIds(StrEnum):AZURE_OPENAI_NO_STREAM_OPTIONS = "Azure OpenAI no stream_options"@@ -35,7 +35,7 @@ class CallerInfoTemplate(Enum):@classmethoddef for_template_key(cls, key: "CallerInfoTemplate") -> str:- templates: Dict[CallerInfoTemplate, str] = {+ templates: dict[CallerInfoTemplate, str] = {cls.FILE_LINE: "{file}:{line}",cls.FILE_LINE_FUNC: "{file}:{line} {func}",cls.FUNC: "{func}",@@ -56,8 +56,8 @@ class RichLogConfig(ConfigModel):is_rich_tracebacks: boolis_tracebacks_word_wrap: boolis_tracebacks_show_locals: bool- tracebacks_suppress: List[str]- keywords_to_hilight: List[str]+ tracebacks_suppress: list[str]+ keywords_to_hilight: list[str]def make_rich_handler(self) -> RichHandler:highlighter: Highlighter@@ -83,7 +83,7 @@ class RichLogConfig(ConfigModel):class LogConfig(ConfigModel):default_log_level: LogLevel = Field(strict=False)- package_log_levels: Dict[str, LogLevel]+ package_log_levels: dict[str, LogLevel]log_mode: LogMode = Field(strict=False)is_console_logging_enabled: bool@@ -93,21 +93,18 @@ class LogConfig(ConfigModel):is_caller_info_enabled: boolcaller_info_template: CallerInfoTemplate = Field(strict=False)- silenced_problem_ids: List[str]+ silenced_problem_ids: list[str]rich_log_config: RichLogConfig# logger name to use for safe logging without fancy features like code filepath and stuffgeneric_poor_logger: str- poor_loggers: List[str]+ poor_loggers: list[str]@field_validator("package_log_levels", mode="before")- def validate_package_log_levels(cls, value: Dict[str, str]) -> Dict[str, LogLevel]:- """- Validate the package log levels dictionary.- """- the_dict = cast(- Dict[str, LogLevel],+ @staticmethod+ def validate_package_log_levels(value: dict[str, str]) -> dict[str, LogLevel]:+ return cast(+ "dict[str, LogLevel]",ConfigModel.transform_dict_str_to_enum(input_dict=value, value_enum_cls=LogLevel),)- return the_dictdiff --git a/pipelex/tools/log/log_dispatch.py b/pipelex/tools/log/log_dispatch.pyindex c9a7f2e5b..a902afbf6 100644--- a/pipelex/tools/log/log_dispatch.py+++ b/pipelex/tools/log/log_dispatch.py@@ -2,49 +2,47 @@ import inspectimport loggingimport osimport traceback-from typing import Any, List, Optional, Union+from pathlib import Path+from typing import Any, castfrom pipelex.tools.log.log_config import CallerInfoTemplate, LogConfig, LogModefrom pipelex.tools.misc.json_utils import purify_json, purify_json_dict, purify_json_listclass LogDispatch:- """- A class for handling log dispatching to both console and Google Cloud.- """+ """A class for handling log dispatching to both console and Google Cloud."""######################################################### Init and Configure######################################################### TODO: more elegant init for log_dispatch / logdef __init__(self):- self.project_name: Optional[str] = None- self._log_config_instance: Optional[LogConfig] = None+ self.project_name: str | None = None+ self._log_config_instance: LogConfig | None = Noneself.log_mode: LogMode = LogMode.RICHdef set_log_mode(self, mode: LogMode):self.log_mode = modedef reset(self):- """- Reset the log dispatch.- """+ """Reset the log dispatch."""self.project_name = Noneself._log_config_instance = None@propertydef _log_config(self) -> LogConfig:- """- Retrieves the log configuration.+ """Retrieves the log configuration.Raises:RuntimeError: If LogConfig is not set.Returns:LogConfig: The current log configuration.+"""if self._log_config_instance is None:- raise RuntimeError("LogConfig is not set. You must call pipelex_hub.set_config().")+ msg = "LogConfig is not set. You must call pipelex_hub.set_config()."+ raise RuntimeError(msg)return self._log_config_instancedef configure(@@ -52,8 +50,7 @@ class LogDispatch:project_name: str,log_config: LogConfig,):- """- Configures the LogDispatch with project name and log configuration.+ """Configures the LogDispatch with project name and log configuration.Args:project_name (str): The name of the project.@@ -61,9 +58,11 @@ class LogDispatch:Raises:RuntimeError: If LogConfig is already set.+"""if self._log_config_instance is not None:- raise RuntimeError("LogConfig is already set. You can only call log.configure() once.")+ msg = "LogConfig is already set. You can only call log.configure() once."+ raise RuntimeError(msg)self._log_config_instance = log_configself.project_name = project_nameself.log_mode = log_config.log_mode@@ -74,24 +73,24 @@ class LogDispatch:def dispatch(self,- content: Union[str, Any],+ content: str | Any,severity: int,- title: Optional[str] = None,- inline: Optional[str] = None,+ title: str | None = None,+ inline: str | None = None,include_exception: bool = False,):- """- Dispatches a log message to appropriate logging methods based on content type.+ """Dispatches a log message to appropriate logging methods based on content type.Args:content (Union[str, Any]): The content to be logged.severity (int): The severity level of the log message.- title (Optional[str], optional): The title of the log message. Defaults to None.- inline (Optional[str], optional): Inline title for the log message. Defaults to None.+ title (str | None, optional): The title of the log message. Defaults to None.+ inline (str | None, optional): Inline title for the log message. Defaults to None.Used to display the title inline, only if the title arg is None.include_exception (bool, optional): Whether to include exception traceback. Defaults to False.+"""- caller_info_str: Optional[str] = None+ caller_info_str: str | None = Noneif ((self._log_config.is_caller_info_enabled)and (frame0 := inspect.currentframe())@@ -100,10 +99,9 @@ class LogDispatch:and (caller_frame := frame1.f_back)):caller_info = inspect.getframeinfo(caller_frame)- caller_file = caller_info.filename- cwd = os.getcwd()+ caller_path = Path(caller_info.filename)try:- caller_file = os.path.relpath(caller_file, cwd)+ caller_path = caller_path.relative_to(Path.cwd())except ValueError:# This can happen if the file is on a different drive (on Windows)# In this case, we'll keep the absolute path@@ -111,7 +109,7 @@ class LogDispatch:caller_line = caller_info.linenocaller_func = caller_info.functiontemplate_str = CallerInfoTemplate.for_template_key(key=self._log_config.caller_info_template)- caller_info_str = template_str.format(file=caller_file, line=caller_line, func=caller_func)+ caller_info_str = template_str.format(file=str(caller_path), line=caller_line, func=caller_func)if isinstance(content, str):self._log_message(@@ -135,22 +133,22 @@ class LogDispatch:self,message: str,severity: int,- caller_info_str: Optional[str],- title: Optional[str] = None,- inline: Optional[str] = None,+ caller_info_str: str | None,+ title: str | None = None,+ inline: str | None = None,include_exception: bool = False,):- """- Logs a message to both console and Google Cloud.+ """Logs a message to both console and Google Cloud.Args:message (str): The message to be logged.severity (int): The severity level of the log message.- caller_info_str (Optional[str]): Information about the caller.- title (Optional[str], optional): The title of the log message. Defaults to None.- inline (Optional[str], optional): Inline title for the log message. Defaults to None.+ caller_info_str (str | None): Information about the caller.+ title (str | None, optional): The title of the log message. Defaults to None.+ inline (str | None, optional): Inline title for the log message. Defaults to None.Used to display the title inline, only if the title arg is None.include_exception (bool, optional): Whether to include exception traceback. Defaults to False.+"""if title is not None:message = f"{title}:\n{message}"@@ -169,19 +167,19 @@ class LogDispatch:self,data: Any,severity: int,- caller_info_str: Optional[str],- title: Optional[str] = None,+ caller_info_str: str | None,+ title: str | None = None,include_exception: bool = False,):- """- Logs potentially structured data (maybe it's a dict or a list) to both console and Google Cloud.+ """Logs potentially structured data (maybe it's a dict or a list) to both console and Google Cloud.Args:data (Any): The data to be logged.severity (int): The severity level of the log message.- caller_info_str (Optional[str]): Information about the caller.- title (Optional[str], optional): The title of the log message. Defaults to None.+ caller_info_str (str | None): Information about the caller.+ title (str | None, optional): The title of the log message. Defaults to None.include_exception (bool, optional): Whether to include exception traceback. Defaults to False.+"""if data is None:message = "None"@@ -208,7 +206,7 @@ class LogDispatch:message += f"\n{traceback.format_exc()}"self._log_to_console(message=message, severity=severity)elif isinstance(data, list):- list_data: List[Any] = data+ list_data = cast("list[Any]", data)_, list_string = purify_json_list(data=list_data,indent=self._log_config.json_logs_indent,@@ -239,12 +237,12 @@ class LogDispatch:self._log_to_console(message=message, severity=severity)def _log_to_console(self, message: str, severity: int):- """- Logs a message to the console.+ """Logs a message to the console.Args:message (str): The message to be logged.severity (int): The severity level of the log message.+"""if not self._log_config.is_console_logging_enabled:return@@ -258,7 +256,7 @@ class LogDispatch:stack = inspect.stack()try:- logging_module_path = os.path.abspath(__file__)+ logging_module_path = Path(__file__).absolute()log_origin_name = "unknown"for frame_info in stack[1:]:@@ -276,14 +274,14 @@ class LogDispatch:if module_file == logging_module_path or module_file.endswith("/log.py"):continue+ if module.__name__ == "__main__":+ if self.project_name is None:+ msg = "Project name is not set. You must call initialize Pipelex first."+ raise RuntimeError(msg)+ log_origin_name = self.project_nameelse:- if module.__name__ == "__main__":- if self.project_name is None:- raise RuntimeError("Project name is not set. You must call initialize Pipelex first.")- log_origin_name = self.project_name- else:- log_origin_name = module.__name__.split(sep=".", maxsplit=1)[0]- break+ log_origin_name = module.__name__.split(sep=".", maxsplit=1)[0]+ breakfinally:del frame_infologger = logging.getLogger(log_origin_name)diff --git a/pipelex/tools/log/log_formatter.py b/pipelex/tools/log/log_formatter.pyindex ac4de0704..fea7a3191 100644--- a/pipelex/tools/log/log_formatter.py+++ b/pipelex/tools/log/log_formatter.py@@ -1,5 +1,4 @@import logging-from typing import Dict, Optionalfrom typing_extensions import override@@ -8,14 +7,14 @@ from pipelex.tools.misc.terminal_utils import BOLD_FONT, RESET_FONT, TerminalCol# TODO: move these to the config-def emoji_for_channel(channel_name: str) -> Optional[str]:- channel_emojis: Dict[str, str] = {+def emoji_for_channel(channel_name: str) -> str | None:+ channel_emojis: dict[str, str] = {"root": "","werkzeug": "📡","urllib3.connectionpool": "⚡️",}- emoji = channel_emojis.get(channel_name, None)+ emoji = channel_emojis.get(channel_name)if emoji == "":# blank emoji is OKreturn emoji@@ -53,7 +52,7 @@ class EmojiLogFormatter(logging.Formatter):return formatter.format(record)-log_level_color: Dict[int, TerminalColor] = {+log_level_color: dict[int, TerminalColor] = {LOGGING_LEVEL_VERBOSE: TerminalColor.WHITE,logging.DEBUG: TerminalColor.GREEN,LOGGING_LEVEL_DEV: TerminalColor.CYAN,@@ -64,7 +63,7 @@ log_level_color: Dict[int, TerminalColor] = {}# added spaces and truncated to 5 characters to make it look better-log_level_tag: Dict[int, str] = {+log_level_tag: dict[int, str] = {LOGGING_LEVEL_VERBOSE: "VERBO",logging.DEBUG: "DEBUG",LOGGING_LEVEL_DEV: "DEV ",diff --git a/pipelex/tools/misc/attribute_utils.py b/pipelex/tools/misc/attribute_utils.pyindex 8cfec1008..11a64369f 100644--- a/pipelex/tools/misc/attribute_utils.py+++ b/pipelex/tools/misc/attribute_utils.py@@ -1,4 +1,4 @@-from typing import Any, ClassVar, Union+from typing import Any, ClassVarclass AttributePolisher:@@ -25,19 +25,17 @@ class AttributePolisher:if not isinstance(value, (str, bytes)):return False- if name == "base_64" and len(value) > cls.base_64_truncate_length:- return True- elif name == "url" and isinstance(value, str) and value.startswith("data:image/") and len(value) > cls.url_truncate_length:- return True- return False+ return (name == "base_64" and len(value) > cls.base_64_truncate_length) or (+ name == "url" and isinstance(value, str) and value.startswith("data:image/") and len(value) > cls.url_truncate_length+ )@classmethod- def get_truncated_value(cls, name: str, value: Union[str, bytes]) -> Union[str, bytes]:+ def get_truncated_value(cls, name: str, value: str | bytes) -> str | bytes:"""Get the truncated value based on the field name and value type."""if isinstance(value, bytes):return cls._truncate_bytes(value, cls.base_64_truncate_length)- elif name == "base_64":+ if name == "base_64":return cls._truncate_string(value, cls.base_64_truncate_length)- elif name == "url" and value.startswith("data:image/"):+ if name == "url" and value.startswith("data:image/"):return cls._truncate_string(value, cls.url_truncate_length)return valuediff --git a/pipelex/tools/misc/base_64_utils.py b/pipelex/tools/misc/base_64_utils.pyindex a95d5ebaa..14cda8867 100644--- a/pipelex/tools/misc/base_64_utils.py+++ b/pipelex/tools/misc/base_64_utils.py@@ -12,26 +12,23 @@ def load_binary_as_base64(path: str) -> bytes:async def load_binary_as_base64_async(path: str) -> bytes:- async with aiofiles.open(path, "rb") as fp: # type: ignore[reportUnknownMemberType]+ async with aiofiles.open(path, "rb") as fp: # pyright: ignore[reportUnknownMemberType]data_bytes = await fp.read()return base64.b64encode(data_bytes)async def load_binary_async(path: str) -> bytes:- async with aiofiles.open(path, "rb") as fp: # type: ignore[reportUnknownMemberType]- data_bytes = await fp.read()- return data_bytes+ async with aiofiles.open(path, "rb") as fp: # pyright: ignore[reportUnknownMemberType]+ return await fp.read()def encode_to_base64(data_bytes: bytes) -> bytes:- b64 = base64.b64encode(data_bytes)- return b64+ return base64.b64encode(data_bytes)async def encode_to_base64_async(data_bytes: bytes) -> bytes:# Use asyncio.to_thread to run the CPU-bound task in a separate thread- b64 = await asyncio.to_thread(base64.b64encode, data_bytes)- return b64+ return await asyncio.to_thread(base64.b64encode, data_bytes)def save_base64_to_binary_file(diff --git a/pipelex/tools/misc/context_provider_abstract.py b/pipelex/tools/misc/context_provider_abstract.pyindex 8bfd24567..1cff3ac83 100644--- a/pipelex/tools/misc/context_provider_abstract.py+++ b/pipelex/tools/misc/context_provider_abstract.py@@ -1,5 +1,5 @@from abc import ABC, abstractmethod-from typing import Any, Dict, Optional, Type+from typing import Anyfrom pipelex.tools.exceptions import ToolException@@ -11,15 +11,14 @@ class ContextProviderException(ToolException):class ContextProviderAbstract(ABC):- """- A ContextProvider provides context to templating engine. This interface is implemented by WorkingMemory.+ """A ContextProvider provides context to templating engine. This interface is implemented by WorkingMemory.It exists to make these features available to lower level classes."""@abstractmethod- def get_typed_object_or_attribute(self, name: str, wanted_type: Optional[Type[Any]] = None) -> Any:+ def get_typed_object_or_attribute(self, name: str, wanted_type: type[Any] | None = None) -> Any:pass@abstractmethod- def generate_context(self) -> Dict[str, Any]:+ def generate_context(self) -> dict[str, Any]:passdiff --git a/pipelex/tools/misc/dict_utils.py b/pipelex/tools/misc/dict_utils.pyindex 1d54df1e7..fe8c88a83 100644--- a/pipelex/tools/misc/dict_utils.py+++ b/pipelex/tools/misc/dict_utils.py@@ -1,14 +1,14 @@"""Dictionary utility functions for manipulating dictionary order and structure."""-from typing import Any, Callable, Dict, List, TypeVar, cast+from collections.abc import Callable+from typing import Any, TypeVar, castK = TypeVar("K")V = TypeVar("V")-def insert_before(dictionary: Dict[K, V], target_key: K, new_key: K, new_value: V) -> Dict[K, V]:- """- Insert a new key-value pair before a target key in a dictionary.+def insert_before(dictionary: dict[K, V], target_key: K, new_key: K, new_value: V) -> dict[K, V]:+ """Insert a new key-value pair before a target key in a dictionary.Creates a new dictionary with the new item positioned before the target key.If the target key doesn't exist, the new item is added at the end.@@ -26,8 +26,9 @@ def insert_before(dictionary: Dict[K, V], target_key: K, new_key: K, new_value:>>> d = {'a': 1, 'c': 3}>>> insert_before(d, 'c', 'b', 2){'a': 1, 'b': 2, 'c': 3}+"""- result: Dict[K, V] = {}+ result: dict[K, V] = {}inserted = Falsefor key, value in dictionary.items():@@ -43,9 +44,8 @@ def insert_before(dictionary: Dict[K, V], target_key: K, new_key: K, new_value:return result-def apply_to_strings_recursive(data: Any, transform_func: Callable[[str], str]) -> Dict[str, Any]:- """- Recursively traverse a data structure and apply a transformation function to all string values.+def apply_to_strings_recursive(data: Any, transform_func: Callable[[str], str]) -> dict[str, Any]:+ """Recursively traverse a data structure and apply a transformation function to all string values.This function walks through dictionaries, lists, and other nested structures,applying the provided transformation function only to string values while@@ -62,13 +62,14 @@ def apply_to_strings_recursive(data: Any, transform_func: Callable[[str], str])>>> data = {'a': 'hello ${USER}', 'b': [1, 'world ${HOME}'], 'c': {'d': 'test ${PATH}'}}>>> result = apply_to_strings_recursive(data, lambda s: s.replace('${USER}', 'john'))>>> # Returns: {'a': 'hello john', 'b': [1, 'world ${HOME}'], 'c': {'d': 'test ${PATH}'}}+"""- result: Dict[str, Any] = {}+ result: dict[str, Any] = {}for key, value in data.items():if isinstance(value, dict):result[key] = apply_to_strings_recursive(value, transform_func)elif isinstance(value, list):- list_value: List[Any] = cast(List[Any], value)+ list_value: list[Any] = cast("list[Any]", value)result[key] = apply_to_strings_in_list(list_value, transform_func)elif isinstance(value, str):result[key] = transform_func(value)@@ -78,14 +79,14 @@ def apply_to_strings_recursive(data: Any, transform_func: Callable[[str], str])return result-def apply_to_strings_in_list(data: List[Any], transform_func: Callable[[str], str]) -> List[Any]:+def apply_to_strings_in_list(data: list[Any], transform_func: Callable[[str], str]) -> list[Any]:"""Helper function to apply string transformation to items in a list."""- result: List[Any] = []+ result: list[Any] = []for item in data:if isinstance(item, dict):result.append(apply_to_strings_recursive(item, transform_func))elif isinstance(item, list):- list_item: List[Any] = cast(List[Any], item)+ list_item: list[Any] = cast("list[Any]", item)result.append(apply_to_strings_in_list(list_item, transform_func))elif isinstance(item, str):result.append(transform_func(item))diff --git a/pipelex/tools/misc/file_fetch_utils.py b/pipelex/tools/misc/file_fetch_utils.pyindex 2d9b73683..cb295dbe2 100644--- a/pipelex/tools/misc/file_fetch_utils.py+++ b/pipelex/tools/misc/file_fetch_utils.py@@ -1,33 +1,30 @@-from typing import Optional-import httpxfrom httpx import Responseasync def fetch_file_from_url_httpx_async(url: str,- timeout: Optional[int] = None,+ request_timeout: int | None = None,) -> bytes:async with httpx.AsyncClient() as client:response: Response = await client.get(url,- timeout=timeout,+ timeout=request_timeout,follow_redirects=True,)response.raise_for_status() # Raise exception for 4XX/5XX status codes- bytes_content: bytes = response.content- return bytes_content+ return response.contentdef fetch_file_from_url_httpx(url: str,- timeout: Optional[int] = None,+ request_timeout: int | None = None,) -> bytes:with httpx.Client() as client:response: Response = client.get(url,- timeout=timeout,+ timeout=request_timeout,follow_redirects=True,)response.raise_for_status() # Raise exception for 4XX/5XX status codesdiff --git a/pipelex/tools/misc/file_utils.py b/pipelex/tools/misc/file_utils.pyindex e87e5404d..2aa7e5a09 100644--- a/pipelex/tools/misc/file_utils.py+++ b/pipelex/tools/misc/file_utils.py@@ -2,7 +2,6 @@ import importlib.resourcesimport osimport shutilfrom pathlib import Path-from typing import List, Optional######################################################### Save & Load@@ -10,15 +9,17 @@ from typing import List, Optionaldef save_bytes_to_binary_file(file_path: str, byte_data: bytes, create_directory: bool = False) -> str:- """- Write binary data to a file.+ """Write binary data to a file.Args:file_path (str): Path where the binary data will be savedbyte_data (bytes): Binary data to be written+ create_directory (bool, optional): Whether to create the directory if it doesn't exist.+ Defaults to False.Returns:str: Path to the saved file+"""# Ensure the directory existsif create_directory:@@ -30,8 +31,7 @@ def save_bytes_to_binary_file(file_path: str, byte_data: bytes, create_directorydef save_text_to_path(text: str, path: str, create_directory: bool = False):- """- Writes text content to a file at the specified path.+ """Writes text content to a file at the specified path.This function opens a file in write mode and writes the provided text to it.If the file already exists, it will be overwritten.@@ -44,6 +44,7 @@ def save_text_to_path(text: str, path: str, create_directory: bool = False):Raises:IOError: If there are issues writing to the file (e.g., permission denied).+"""if create_directory:directory = os.path.dirname(path)@@ -55,8 +56,7 @@ def save_text_to_path(text: str, path: str, create_directory: bool = False):def load_text_from_path(path: str) -> str:- """- Reads and returns the entire contents of a text file.+ """Reads and returns the entire contents of a text file.This function opens a file in text mode using UTF-8 encoding and readsits entire contents into a string.@@ -69,14 +69,14 @@ def load_text_from_path(path: str) -> str:Raises:FileNotFoundError: If the file does not exist.+"""with open(path, encoding="utf-8") as file:return file.read()-def failable_load_text_from_path(path: str) -> Optional[str]:- """- Attempts to read a text file, returning None if the file doesn't exist.+def failable_load_text_from_path(path: str) -> str | None:+ """Attempts to read a text file, returning None if the file doesn't exist.This function is a safer version of load_text_from_path that handles missing filesgracefully by returning None instead of raising an error.@@ -86,6 +86,7 @@ def failable_load_text_from_path(path: str) -> Optional[str]:Returns:Optional[str]: The complete contents of the file as a string, or None if the file doesn't exist.+"""if not path_exists(path):return None@@ -98,8 +99,7 @@ def failable_load_text_from_path(path: str) -> Optional[str]:def copy_file(source_path: str, target_path: str, overwrite: bool = True) -> None:- """- Copies a file from the source path to the target path.+ """Copies a file from the source path to the target path.Creates any necessary parent directories for the target path if they don't exist.@@ -107,6 +107,7 @@ def copy_file(source_path: str, target_path: str, overwrite: bool = True) -> Nonsource_path (str): The path to the source file.target_path (str): The path to the target file.overwrite (bool, optional): Whether to overwrite existing files. Defaults to True.+"""# Ensure the target directory existstarget_dir = os.path.dirname(target_path)@@ -123,9 +124,7 @@ def copy_file_from_package(target_path: str,overwrite: bool = True,) -> None:- """- Copies a file from a package to a target directory.- """+ """Copies a file from a package to a target directory."""file_path = str(importlib.resources.files(package_name).joinpath(file_path_in_package))copy_file(source_path=file_path,@@ -139,10 +138,9 @@ def copy_folder_from_package(folder_path_in_package: str,target_dir: str,overwrite: bool = True,- non_overwrite_files: Optional[List[str]] = None,+ non_overwrite_files: list[str] | None = None,) -> None:- """- Copies a folder from a package to a target directory.+ """Copies a folder from a package to a target directory.This function walks through the specified folder in the package and copiesall files and directories to the target directory, preserving the directory@@ -153,6 +151,8 @@ def copy_folder_from_package(folder_path_in_package (str): The path to the folder in the package to copy.target_dir (str): The target directory to copy the folder to.overwrite (bool, optional): Whether to overwrite existing files. Defaults to True.+ non_overwrite_files (Optional[List[str]], optional): List of files to not overwrite. Defaults to None.+"""os.makedirs(target_dir, exist_ok=True)@@ -177,7 +177,7 @@ def copy_folder_from_package(dest_file = os.path.join(target_subdir, file)# Check if the file exists and respect the overwrite parameter- if not os.path.exists(dest_file) or (overwrite and file not in non_overwrite_files):+ if not Path(dest_file).exists() or (overwrite and file not in non_overwrite_files):copy_file(source_path=src_file,target_path=dest_file,@@ -187,8 +187,7 @@ def copy_folder_from_package(def remove_file(file_path: str):- """- Removes a file if it exists at the specified path.+ """Removes a file if it exists at the specified path.This function checks if a file exists before attempting to remove it,preventing errors from trying to remove non-existent files.@@ -198,22 +197,23 @@ def remove_file(file_path: str):Note:This function silently succeeds if the file doesn't exist.+"""if path_exists(file_path):- os.remove(file_path)+ Path(file_path).unlink()def remove_folder(folder_path: str) -> None:- """- Removes a folder if it exists at the specified path.+ """Removes a folder if it exists at the specified path.This function checks if a folder exists before attempting to remove it,preventing errors from trying to remove non-existent folders.Args:folder_path (str): The path to the folder to be removed.+"""- if os.path.exists(folder_path):+ if Path(folder_path).exists():shutil.rmtree(folder_path)@@ -223,18 +223,17 @@ def remove_folder(folder_path: str) -> None:def ensure_directory_exists(directory_path: str) -> None:- """- Creates a directory and any necessary parent directories if they don't exist.+ """Creates a directory and any necessary parent directories if they don't exist.Args:directory_path (str): The path to the directory to create.+"""- os.makedirs(directory_path, exist_ok=True)+ Path(directory_path).mkdir(parents=True, exist_ok=True)def ensure_path(path: str) -> bool:- """- Ensures a directory exists at the specified path, creating it if necessary.+ """Ensures a directory exists at the specified path, creating it if necessary.This function checks if a directory exists at the given path. If it doesn't exist,it creates the directory and any necessary parent directories.@@ -244,17 +243,16 @@ def ensure_path(path: str) -> bool:Returns:bool: True if the directory was created, False if it already existed.+"""- typed_path = Path(path)- if typed_path.exists():+ if Path(path).exists():return False- typed_path.mkdir(parents=True, exist_ok=True)+ Path(path).mkdir(parents=True, exist_ok=True)return Truedef path_exists(path_str: str) -> bool:- """- Checks if a file or directory exists at the specified path.+ """Checks if a file or directory exists at the specified path.This function converts the input string path to a Path object and checksif anything exists at that location in the filesystem.@@ -264,14 +262,13 @@ def path_exists(path_str: str) -> bool:Returns:bool: True if a file or directory exists at the path, False otherwise.+"""- path = Path(path_str)- return path.exists()+ return Path(path_str).exists()def get_incremental_directory_path(base_path: str, base_name: str, start_at: int = 1) -> str:- """- Generates a unique directory path by incrementing a counter until an unused path is found.+ """Generates a unique directory path by incrementing a counter until an unused path is found.This function creates a directory path in the format 'base_path/base_name_XX' where XXis a two-digit number that starts at start_at and increments until an unused path is found.@@ -284,6 +281,7 @@ def get_incremental_directory_path(base_path: str, base_name: str, start_at: intReturns:str: The path to the newly created directory.+"""counter = start_atwhile True:@@ -302,8 +300,7 @@ def get_incremental_file_path(start_at: int = 1,avoid_suffix_if_possible: bool = False,) -> str:- """- Generates a unique file path by incrementing a counter until an unused path is found.+ """Generates a unique file path by incrementing a counter until an unused path is found.This function creates a file path in the format 'base_path/base_name_XX.extension' where XXis a two-digit number that starts at start_at and increments until an unused path is found.@@ -314,9 +311,11 @@ def get_incremental_file_path(base_name (str): The base name for the file (will be appended with _XX).extension (str): The file extension (without the dot).start_at (int, optional): The number to start counting from. Defaults to 1.+ avoid_suffix_if_possible (bool, optional): If True, avoids adding a suffix if possible. Defaults to False.Returns:str: A unique file path that does not exist in the filesystem.+"""if avoid_suffix_if_possible:# try without adding the suffix@@ -339,20 +338,19 @@ def get_incremental_file_path(########################################################-def find_files_in_dir(dir_path: str, pattern: str, is_recursive: bool) -> List[Path]:- """- Find files matching a pattern in a directory.+def find_files_in_dir(dir_path: str, pattern: str, is_recursive: bool) -> list[Path]:+ """Find files matching a pattern in a directory.Args:dir_path: Directory path to search inpattern: File pattern to match (e.g. "*.py")- recursive: Whether to search recursively in subdirectories+ is_recursive: Whether to search recursively in subdirectoriesReturns:List of matching Path objects+"""path = Path(dir_path)if is_recursive:return list(path.rglob(pattern))- else:- return list(path.glob(pattern))+ return list(path.glob(pattern))diff --git a/pipelex/tools/misc/filetype_utils.py b/pipelex/tools/misc/filetype_utils.pyindex 1ee664d8f..af274d770 100644--- a/pipelex/tools/misc/filetype_utils.py+++ b/pipelex/tools/misc/filetype_utils.py@@ -1,7 +1,6 @@import base64import binasciifrom pathlib import Path-from typing import Unionimport filetypefrom pydantic import BaseModel@@ -18,9 +17,8 @@ class FileType(BaseModel):mime: str-def detect_file_type_from_path(path: Union[str, Path]) -> FileType:- """- Detect the file type of a file at a given path.+def detect_file_type_from_path(path: str | Path) -> FileType:+ """Detect the file type of a file at a given path.Args:path: The path to the file to detect the type of.@@ -30,18 +28,19 @@ def detect_file_type_from_path(path: Union[str, Path]) -> FileType:Raises:FileTypeException: If the file type cannot be identified.+"""kind = filetype.guess(path) # pyright: ignore[reportUnknownMemberType]if kind is None:- raise FileTypeException(f"Could not identify file type of '{path!s}'")+ msg = f"Could not identify file type of '{path!s}'"+ raise FileTypeException(msg)extension = f"{kind.extension}"mime = f"{kind.mime}"return FileType(extension=extension, mime=mime)def detect_file_type_from_bytes(buf: bytes) -> FileType:- """- Detect the file type of a given bytes object.+ """Detect the file type of a given bytes object.Args:buf: The bytes object to detect the type of.@@ -51,18 +50,19 @@ def detect_file_type_from_bytes(buf: bytes) -> FileType:Raises:FileTypeException: If the file type cannot be identified.+"""kind = filetype.guess(buf) # pyright: ignore[reportUnknownMemberType]if kind is None:- raise FileTypeException(f"Could not identify file type of given bytes: {buf[:300]!r}")+ msg = f"Could not identify file type of given bytes: {buf[:300]!r}"+ raise FileTypeException(msg)extension = f"{kind.extension}"mime = f"{kind.mime}"return FileType(extension=extension, mime=mime)-def detect_file_type_from_base64(b64: Union[str, bytes]) -> FileType:- """- Detect the file type of a given Base-64-encoded string.+def detect_file_type_from_base64(b64: str | bytes) -> FileType:+ """Detect the file type of a given Base-64-encoded string.Args:b64: The Base-64-encoded bytes or string to detect the type of.@@ -72,6 +72,7 @@ def detect_file_type_from_base64(b64: Union[str, bytes]) -> FileType:Raises:FileTypeException: If the file type cannot be identified.+"""# Normalise to bytes holding only the Base-64 alphabetif isinstance(b64, bytes):@@ -84,6 +85,7 @@ def detect_file_type_from_base64(b64: Union[str, bytes]) -> FileType:try:raw = base64.b64decode(b64_bytes, validate=True)except binascii.Error as exc: # malformed Base-64- raise FileTypeException(f"Could not identify file type of given bytes because input is not valid Base-64: {exc}") from exc+ msg = f"Could not identify file type of given bytes because input is not valid Base-64: {exc}"+ raise FileTypeException(msg) from excreturn detect_file_type_from_bytes(buf=raw)diff --git a/pipelex/tools/misc/json_utils.py b/pipelex/tools/misc/json_utils.pyindex a1c528128..8a230f122 100644--- a/pipelex/tools/misc/json_utils.py+++ b/pipelex/tools/misc/json_utils.py@@ -1,5 +1,6 @@import json-from typing import Any, Dict, List, Mapping, Optional, Tuple, Union, cast+from collections.abc import Mapping+from typing import Any, Union, castfrom kajson import kajsonfrom pydantic import BaseModel@@ -8,7 +9,7 @@ from pipelex.tools.exceptions import ToolExceptionfrom pipelex.tools.misc.file_utils import save_text_to_pathfrom pipelex.tools.typing.pydantic_utils import CustomBaseModel-JsonContent = Union[Dict[str, Any], List[Any]]+JsonContent = Union[dict[str, Any], list[Any]]class ArgumentTypeError(ToolException):@@ -19,9 +20,8 @@ class JsonTypeError(ToolException):pass-def json_str(some_object: Any, title: Optional[str] = None, is_spaced: bool = False) -> str:- """- Creates a formatted JSON string representation of any Python object with optional title and spacing.+def json_str(some_object: Any, title: str | None = None, is_spaced: bool = False) -> str:+ """Creates a formatted JSON string representation of any Python object with optional title and spacing.This function is a higher-level wrapper around purify_json that provides additional formattingoptions. It always uses 4-space indentation and disables warning wrapping for non-serializable@@ -30,7 +30,7 @@ def json_str(some_object: Any, title: Optional[str] = None, is_spaced: bool = FaArgs:some_object (Any): The object to convert to a JSON string. Can be any type supportedby purify_json.- title (Optional[str], optional): A title to prepend to the JSON string. If provided,+ title (str | None, optional): A title to prepend to the JSON string. If provided,the output will be in the format "title: {json_string}". Defaults to None.is_spaced (bool, optional): If True, adds newlines before and after the JSON stringfor better readability. Defaults to False.@@ -59,11 +59,10 @@ def json_str(some_object: Any, title: Optional[str] = None, is_spaced: bool = Fadef save_as_json_to_path(object_to_save: Any,path: str,- indent: Optional[int] = 4,+ indent: int | None = 4,is_warning_enabled: bool = True,):- """- Saves a Python object as a JSON file at the specified path.+ """Saves a Python object as a JSON file at the specified path.This function converts a Python object to a JSON string and saves it to a file. The objectis first purified to ensure JSON compatibility before saving.@@ -71,19 +70,19 @@ def save_as_json_to_path(Args:object_to_save (Any): The Python object to be saved as JSON. Can be any JSON-serializable object.path (str): The file path where the JSON file will be saved.- indent (Optional[int], optional): Number of spaces for JSON formatting indentation. Defaults to 4.+ indent (int | None, optional): Number of spaces for JSON formatting indentation. Defaults to 4.is_warning_enabled (bool, optional): Whether to show warnings during JSON purification. Defaults to True.Returns:None+"""_, json_string = purify_json(object_to_save, indent=indent, is_warning_enabled=is_warning_enabled)save_text_to_path(json_string, path)def load_json_from_path(path: str) -> JsonContent:- """- Loads and parses a JSON file from the specified path.+ """Loads and parses a JSON file from the specified path.This function reads a JSON file and returns its contents as a Python object.The file is read using UTF-8 encoding.@@ -97,15 +96,15 @@ def load_json_from_path(path: str) -> JsonContent:Raises:FileNotFoundError: If the file does not exist.json.JSONDecodeError: If the file contains invalid JSON.+"""with open(path, encoding="utf-8") as file:json_content: JsonContent = json.load(file)return json_content-def load_json_dict_from_path(path: str) -> Dict[Any, Any]:- """- Loads a JSON file and ensures it contains a dictionary.+def load_json_dict_from_path(path: str) -> dict[Any, Any]:+ """Loads a JSON file and ensures it contains a dictionary.This function reads a JSON file and verifies that its content is a dictionary.It uses load_json_from_path internally and adds type checking.@@ -120,17 +119,17 @@ def load_json_dict_from_path(path: str) -> Dict[Any, Any]:JsonTypeError: If the JSON content is not a dictionary.FileNotFoundError: If the file does not exist.json.JSONDecodeError: If the file contains invalid JSON.+"""json_content: JsonContent = load_json_from_path(path)if isinstance(json_content, dict):return json_content- else:- raise JsonTypeError(f"{path} is not a dict")+ msg = f"{path} is not a dict"+ raise JsonTypeError(msg)-def load_json_list_from_path(path: str) -> List[Any]:- """- Loads a JSON file and ensures it contains a list.+def load_json_list_from_path(path: str) -> list[Any]:+ """Loads a JSON file and ensures it contains a list.This function reads a JSON file and verifies that its content is a list.It uses load_json_from_path internally and adds type checking.@@ -145,17 +144,17 @@ def load_json_list_from_path(path: str) -> List[Any]:JsonTypeError: If the JSON content is not a list.FileNotFoundError: If the file does not exist.json.JSONDecodeError: If the file contains invalid JSON.+"""json_content: JsonContent = load_json_from_path(path)if isinstance(json_content, list):return json_content- else:- raise JsonTypeError(f"{path} is not a list")+ msg = f"{path} is not a list"+ raise JsonTypeError(msg)-def deep_update(target_dict: Dict[str, Any], updates: Dict[str, Any]):- """- Recursively updates a dictionary with values from another dictionary.+def deep_update(target_dict: dict[str, Any], updates: dict[str, Any]):+ """Recursively updates a dictionary with values from another dictionary.This function performs a deep merge of two dictionaries, handling nesteddictionaries and lists. For dictionaries, it recursively updates values,@@ -172,19 +171,19 @@ def deep_update(target_dict: Dict[str, Any], updates: Dict[str, Any]):>>> deep_update(base, updates)>>> print(base){'a': 1, 'b': {'x': 2, 'y': 4, 'z': 5}, 'c': [1, 2, 3, 4]}+"""for key, value in updates.items():if isinstance(value, dict) and key in target_dict and isinstance(target_dict[key], dict):- deep_update(target_dict[key], value) # type: ignore+ deep_update(target_dict[key], value) # pyright: ignore[reportUnknownArgumentType]elif isinstance(value, list) and key in target_dict and isinstance(target_dict[key], list):- target_dict[key] = list(target_dict[key] + value) # type: ignore+ target_dict[key] = list(target_dict[key] + value)else:target_dict[key] = valuedef remove_none_values(json_content: JsonContent | Any) -> JsonContent | Any:- """- Recursively removes all None values from a JSON-compatible data structure.+ """Recursively removes all None values from a JSON-compatible data structure.This function traverses dictionaries and lists, removing any keys with None valuesfrom dictionaries and processing nested structures. It preserves the structure of@@ -212,37 +211,37 @@ def remove_none_values(json_content: JsonContent | Any) -> JsonContent | Any:"nested": {"key": "exists"},"list": [1, {}]}+"""if isinstance(json_content, dict):- json_content = cast(Dict[str, Any], json_content) # pyright: ignore[reportUnnecessaryCast]- cleaned_dict: Dict[str, Any] = {}+ json_content = cast("dict[str, Any]", json_content) # pyright: ignore[reportUnnecessaryCast]+ cleaned_dict: dict[str, Any] = {}for key, value in json_content.items():if value is not None:cleaned_dict[key] = remove_none_values(json_content=value)return cleaned_dictelif isinstance(json_content, list):- json_content = cast(List[Any], json_content) # pyright: ignore[reportUnnecessaryCast]+ json_content = cast("list[Any]", json_content) # pyright: ignore[reportUnnecessaryCast]return [remove_none_values(item) for item in json_content]else:return json_content-def remove_none_values_from_dict(data: Mapping[str, Any]) -> Dict[str, Any]:+def remove_none_values_from_dict(data: Mapping[str, Any]) -> dict[str, Any]:processed = remove_none_values(json_content=data)if not isinstance(processed, dict):- raise JsonTypeError("Removing None values from a dict, we expected a dict in return")- processed = cast(Dict[str, Any], processed) # pyright: ignore[reportUnnecessaryCast]- return processed+ msg = "Removing None values from a dict, we expected a dict in return"+ raise JsonTypeError(msg)+ return cast("dict[str, Any]", processed) # pyright: ignore[reportUnnecessaryCast]def purify_json(data: Any,- indent: Optional[int] = None,+ indent: int | None = None,is_truncate_bytes_enabled: bool = False,is_warning_enabled: bool = True,-) -> Tuple[Union[Dict[Any, Any], List[Any]], str]:- """- Converts any Python object into a JSON-serializable format and its string representation.+) -> tuple[dict[Any, Any] | list[Any], str]:+ """Converts any Python object into a JSON-serializable format and its string representation.This function handles various types of input data:- Pydantic BaseModel instances are converted using their model_dump method@@ -252,7 +251,8 @@ def purify_json(Args:data (Any): The data to convert. Can be a Pydantic model, list, dict, or any other type.- indent (Optional[int], optional): Number of spaces for JSON formatting indentation. Defaults to None.+ indent (int | None, optional): Number of spaces for JSON formatting indentation. Defaults to None.+ is_truncate_bytes_enabled (bool, optional): If True, truncates bytes values to a string representation. Defaults to False.is_warning_enabled (bool, optional): If True, wraps non-serializable data in a warning object.Defaults to True.@@ -266,6 +266,7 @@ def purify_json(>>> data, json_str = purify_json(model)>>> print(json_str)'{"name": "test"}'+"""dict_string: strif isinstance(data, CustomBaseModel) and is_truncate_bytes_enabled:@@ -275,7 +276,7 @@ def purify_json(is_truncate_bytes_enabled=is_truncate_bytes_enabled,is_warning_enabled=is_warning_enabled,)- elif isinstance(data, BaseModel):+ if isinstance(data, BaseModel):return purify_json(data.model_dump(serialize_as_any=True),indent=indent,@@ -284,28 +285,29 @@ def purify_json()if isinstance(data, list):- the_list = data # type: ignore+ the_list = data # pyright: ignore[reportUnknownVariableType]if not the_list:return [], "[]"if isinstance(the_list[0], CustomBaseModel) and is_truncate_bytes_enabled:- the_list_of_custom_base_models: List[CustomBaseModel] = the_list+ the_list_of_custom_base_models = cast("list[CustomBaseModel]", the_list)pure_list = [item.model_dump_truncated(serialize_as_any=True) for item in the_list_of_custom_base_models]dict_string = json.dumps(pure_list, indent=indent, default=str)return pure_list, dict_string- elif isinstance(the_list[0], BaseModel):- the_list_of_base_models: List[BaseModel] = the_list+ if isinstance(the_list[0], BaseModel):+ the_list_of_base_models = cast("list[BaseModel]", the_list)pure_list = [item.model_dump(serialize_as_any=True) for item in the_list_of_base_models]dict_string = json.dumps(pure_list, indent=indent, default=str)return pure_list, dict_stringtry:dict_string = json.dumps(data, indent=indent)- pure_dict: Union[Dict[Any, Any], List[Any]] = data+ pure_dict = cast("dict[Any, Any] | list[Any]", data)except TypeError:try:dict_string = kajson.dumps(data, indent=indent) # pyright: ignore[reportUnknownMemberType]except Exception:if is_warning_enabled:+ data = cast("dict[Any, Any] | list[Any]", data)data = {"!": data}dict_string = json.dumps(data, indent=indent, default=str)pure_dict = json.loads(dict_string)@@ -313,12 +315,11 @@ def purify_json(def purify_json_list(- data: List[Any],- indent: Optional[int] = None,+ data: list[Any],+ indent: int | None = None,is_truncate_bytes_enabled: bool = False,-) -> Tuple[List[Any], str]:- """- Converts a list of Python objects into a JSON-serializable list and its string representation.+) -> tuple[list[Any], str]:+ """Converts a list of Python objects into a JSON-serializable list and its string representation.This function specifically handles lists and provides specialized processing for lists ofPydantic BaseModel instances. It attempts multiple serialization methods to ensure successful@@ -326,8 +327,9 @@ def purify_json_list(Args:data (List[Any]): The list to convert. Can contain Pydantic models or other types.- indent (Optional[int], optional): Number of spaces for JSON formatting indentation.+ indent (int | None, optional): Number of spaces for JSON formatting indentation.Defaults to None.+ is_truncate_bytes_enabled (bool, optional): If True, truncates bytes values to a string representation. Defaults to False.Returns:Tuple[List[Any], str]: A tuple containing:@@ -339,19 +341,20 @@ def purify_json_list(>>> data, json_str = purify_json_list(models)>>> print(json_str)'[{"name": "test1"}, {"name": "test2"}]'+"""list_string: str- pure_list: List[Any]+ pure_list: list[Any]if not data:return [], "[]"if isinstance(data[0], CustomBaseModel) and is_truncate_bytes_enabled:- the_list_of_custom_base_models: List[CustomBaseModel] = data+ the_list_of_custom_base_models = cast("list[CustomBaseModel]", data)pure_list = [item.model_dump_truncated(serialize_as_any=True) for item in the_list_of_custom_base_models]list_string = json.dumps(pure_list, indent=indent, default=str)return pure_list, list_string- elif isinstance(data[0], BaseModel):- the_list_of_base_models: List[BaseModel] = data+ if isinstance(data[0], BaseModel):+ the_list_of_base_models: list[BaseModel] = datapure_list = [item.model_dump(serialize_as_any=True) for item in the_list_of_base_models]list_string = json.dumps(pure_list, indent=indent, default=str)return pure_list, list_string@@ -368,9 +371,8 @@ def purify_json_list(return pure_list, list_string-def purify_json_dict(data: Any, indent: Optional[int] = None, is_warning_enabled: bool = True) -> Tuple[Dict[str, Any], str]:- """- Converts any Python object into a JSON-serializable dictionary and its string representation.+def purify_json_dict(data: Any, indent: int | None = None, is_warning_enabled: bool = True) -> tuple[dict[str, Any], str]:+ """Converts any Python object into a JSON-serializable dictionary and its string representation.This function specifically handles dictionary-like objects and Pydantic BaseModel instances,converting them to pure dictionaries that can be JSON serialized. It includes multiple@@ -378,7 +380,7 @@ def purify_json_dict(data: Any, indent: Optional[int] = None, is_warning_enabledArgs:data (Any): The data to convert. Can be a Pydantic model or dictionary-like object.- indent (Optional[int], optional): Number of spaces for JSON formatting indentation.+ indent (int | None, optional): Number of spaces for JSON formatting indentation.Defaults to None.is_warning_enabled (bool, optional): If True, wraps non-serializable data in a warningobject with a "!" key. Defaults to True.@@ -396,6 +398,7 @@ def purify_json_dict(data: Any, indent: Optional[int] = None, is_warning_enabled>>> data, json_str = purify_json_dict(model)>>> print(json_str)'{"name": "test", "value": 123}'+"""dict_string: strif isinstance(data, BaseModel):@@ -406,11 +409,12 @@ def purify_json_dict(data: Any, indent: Optional[int] = None, is_warning_enabled)if isinstance(data, list):- raise ArgumentTypeError("The data is a list, not a dict")+ msg = "The data is a list, not a dict"+ raise ArgumentTypeError(msg)try:dict_string = json.dumps(data, indent=indent)- pure_dict: Dict[str, Any] = data+ pure_dict: dict[str, Any] = dataexcept TypeError:try:dict_string = kajson.dumps(data, indent=indent) # pyright: ignore[reportUnknownMemberType]@@ -422,9 +426,8 @@ def purify_json_dict(data: Any, indent: Optional[int] = None, is_warning_enabledreturn pure_dict, dict_string-def pure_json_str(data: Any, indent: Optional[int] = None, is_warning_enabled: bool = True) -> str:- """- Converts any Python object directly to its JSON string representation.+def pure_json_str(data: Any, indent: int | None = None, is_warning_enabled: bool = True) -> str:+ """Converts any Python object directly to its JSON string representation.This is a convenience wrapper around purify_json that returns only the stringrepresentation, discarding the intermediate data structure. It inherits all the@@ -433,7 +436,7 @@ def pure_json_str(data: Any, indent: Optional[int] = None, is_warning_enabled: bArgs:data (Any): The data to convert to a JSON string. Can be any type supportedby purify_json.- indent (Optional[int], optional): Number of spaces for JSON formatting indentation.+ indent (int | None, optional): Number of spaces for JSON formatting indentation.Defaults to None.is_warning_enabled (bool, optional): If True, wraps non-serializable data in awarning object. Defaults to True.@@ -448,6 +451,7 @@ def pure_json_str(data: Any, indent: Optional[int] = None, is_warning_enabled: b'{"name": "test"}'+"""_, json_string = purify_json(data, indent=indent, is_warning_enabled=is_warning_enabled)return json_stringdiff --git a/pipelex/tools/misc/markdown_utils.py b/pipelex/tools/misc/markdown_utils.pyindex c0695ed8e..d0f69ee9d 100644--- a/pipelex/tools/misc/markdown_utils.py+++ b/pipelex/tools/misc/markdown_utils.py@@ -1,29 +1,25 @@-from typing import Any, Dict, List, Optional+from typing import Any, castfrom pipelex.tools.misc.attribute_utils import AttributePolisherfrom pipelex.tools.misc.json_utils import purify_json_dictfrom pipelex.tools.misc.string_utils import snake_to_capitalize_first_letter-def convert_to_markdown(data: Any, level: int = 1, is_pretty: bool = False, key: Optional[str] = None) -> str:- """- Convert arbitrary JSON-compatible Python data to a Markdown string+def convert_to_markdown(data: Any, level: int = 1, is_pretty: bool = False, key: str | None = None) -> str:+ """Convert arbitrary JSON-compatible Python data to a Markdown stringwithout needing to specify the markdown type explicitly."""if isinstance(data, dict):- the_dict: Dict[str, Any] = data+ the_dict = cast("dict[str, Any]", data)# Treat keys as headings and values as their content- dict_result_lines: List[str] = []- for key, value in the_dict.items():+ dict_result_lines: list[str] = []+ for _key, _value in the_dict.items():heading_prefix = "#" * min(level, 6) # Limit heading levels to 6# Use the key as a heading- if is_pretty:- converted_line = f"{heading_prefix} {snake_to_capitalize_first_letter(key)}"- else:- converted_line = f"{heading_prefix} {key}"+ converted_line = f"{heading_prefix} {snake_to_capitalize_first_letter(_key)}" if is_pretty else f"{heading_prefix} {_key}"# Convert the value recursively, increasing the heading level# dict_result_lines.append(convert_to_markdown(data=value, level=level + 1))- converted_value = convert_to_markdown(data=value, level=level + 1, key=key)+ converted_value = convert_to_markdown(data=_value, level=level + 1, key=_key)converted_value_nb_lines = len(converted_value.split("\n"))if converted_value_nb_lines > 1:dict_result_lines.append(converted_line)@@ -38,8 +34,8 @@ def convert_to_markdown(data: Any, level: int = 1, is_pretty: bool = False, key:# they become list items.if not data:return ""- the_list: List[Any] = data- list_result_lines: List[str] = []+ the_list = cast("list[Any]", data)+ list_result_lines: list[str] = []for item in the_list:# Convert the item firstitem_md = convert_to_markdown(item, level=level)diff --git a/pipelex/tools/misc/mermaid_utils.py b/pipelex/tools/misc/mermaid_utils.pyindex b88f9e43a..994a6a158 100644--- a/pipelex/tools/misc/mermaid_utils.py+++ b/pipelex/tools/misc/mermaid_utils.py@@ -24,13 +24,11 @@ def make_mermaid_url(mermaid_code: str) -> str:},}encoded = encode_pako_from_string(json.dumps(as_dict))- url = f"https://mermaid.ink/img/{encoded}"- return url+ return f"https://mermaid.ink/img/{encoded}"def clean_str_for_mermaid_node_title(text: str) -> str:- """- Cleans a string to be safely used as a Mermaid node title by replacing quotes+ """Cleans a string to be safely used as a Mermaid node title by replacing quoteswith similar Unicode characters that won't interfere with Mermaid syntax.Args:@@ -38,11 +36,11 @@ def clean_str_for_mermaid_node_title(text: str) -> str:Returns:The cleaned string with quotes replaced+"""# Replace single and double quotes with similar Unicode characterstext = text.replace('"', "″") # Replace with prime symbol- text = text.replace("'", "′") # Replace with curly quote- return text+ return text.replace("'", "′") # Replace with curly quotedef print_mermaid_url(url: str, title: str):diff --git a/pipelex/tools/misc/path_utils.py b/pipelex/tools/misc/path_utils.pyindex 052cd3002..fbf894d29 100644--- a/pipelex/tools/misc/path_utils.py+++ b/pipelex/tools/misc/path_utils.py@@ -1,6 +1,5 @@import osimport urllib.parse-from typing import Optional, Tuplefrom pipelex.types import StrEnum@@ -28,8 +27,7 @@ class InterpretedPathOrUrl(StrEnum):def interpret_path_or_url(path_or_uri: str) -> InterpretedPathOrUrl:- """- Determines whether a string represents a file URI, URL, or file path.+ """Determines whether a string represents a file URI, URL, or file path.This function analyzes the input string to categorize it as one of three types:- File URI (starts with "file://")@@ -55,6 +53,7 @@ def interpret_path_or_url(path_or_uri: str) -> InterpretedPathOrUrl:InterpretedPathOrUrl.URL>>> interpret_path_or_url("/home/user/file.txt")InterpretedPathOrUrl.FILE_PATH+"""if path_or_uri.startswith("file://"):return InterpretedPathOrUrl.FILE_URI@@ -66,9 +65,8 @@ def interpret_path_or_url(path_or_uri: str) -> InterpretedPathOrUrl:return InterpretedPathOrUrl.FILE_NAME-def clarify_path_or_url(path_or_uri: str) -> Tuple[Optional[str], Optional[str]]:- """- Separates a path_or_uri string into either a file path or online URL component.+def clarify_path_or_url(path_or_uri: str) -> tuple[str | None, str | None]:+ """Separates a path_or_uri string into either a file path or online URL component.This function processes the input string to determine its type and returnsthe appropriate components. For file URIs, it converts them to regular file paths.@@ -90,9 +88,10 @@ def clarify_path_or_url(path_or_uri: str) -> Tuple[Optional[str], Optional[str]](None, 'https://example.com')>>> clarify_path_or_url("/home/user/file.txt")('/home/user/file.txt', None)+"""- file_path: Optional[str]- url: Optional[str]+ file_path: str | None+ url: str | Nonematch interpret_path_or_url(path_or_uri):case InterpretedPathOrUrl.FILE_URI:parsed_uri = urllib.parse.urlparse(path_or_uri)@@ -109,5 +108,6 @@ def clarify_path_or_url(path_or_uri: str) -> Tuple[Optional[str], Optional[str]]file_path = path_or_uriurl = Nonecase InterpretedPathOrUrl.BASE_64:- raise NotImplementedError("Base 64 is not supported yet by clarify_path_or_url")+ msg = "Base 64 is not supported yet by clarify_path_or_url"+ raise NotImplementedError(msg)return file_path, urldiff --git a/pipelex/tools/misc/placeholder.py b/pipelex/tools/misc/placeholder.pyindex a8275ebfe..db84ab0af 100644--- a/pipelex/tools/misc/placeholder.py+++ b/pipelex/tools/misc/placeholder.py@@ -1,5 +1,3 @@-from typing import Optional-PLACEHOLDER_PREFIX = "placeholder"@@ -11,11 +9,12 @@ def make_placeholder_value(key: str) -> str:Returns:A placeholder value in the format "placeholder-for-{key}"+"""return f"{PLACEHOLDER_PREFIX}-for-{key}"-def value_is_placeholder(value: Optional[str]) -> bool:+def value_is_placeholder(value: str | None) -> bool:"""Check if a value is a placeholder based on the prefix.Args:@@ -23,5 +22,6 @@ def value_is_placeholder(value: Optional[str]) -> bool:Returns:True if the value starts with the placeholder prefix, False otherwise+"""return value is not None and value.startswith(PLACEHOLDER_PREFIX)diff --git a/pipelex/tools/misc/pretty.py b/pipelex/tools/misc/pretty.pyindex 49d030275..42a9e4d19 100644--- a/pipelex/tools/misc/pretty.py+++ b/pipelex/tools/misc/pretty.py@@ -1,5 +1,5 @@import shutil-from typing import Any, ClassVar, List, Optional, Union+from typing import Any, ClassVarfrom rich import print as rich_printfrom rich.markdown import Markdown@@ -18,19 +18,19 @@ BORDER_COLOR = TerminalColor.YELLOWdef pretty_print(- content: Union[str, Any],- title: Optional[TextType] = None,- subtitle: Optional[TextType] = None,- border_style: Optional[StyleType] = None,+ content: str | Any,+ title: TextType | None = None,+ subtitle: TextType | None = None,+ border_style: StyleType | None = None,):PrettyPrinter.pretty_print(content=content, title=title, subtitle=subtitle, border_style=border_style)def pretty_print_md(content: str,- title: Optional[TextType] = None,- subtitle: Optional[TextType] = None,- border_style: Optional[StyleType] = None,+ title: TextType | None = None,+ subtitle: TextType | None = None,+ border_style: StyleType | None = None,):md_content = Markdown(content)PrettyPrinter.pretty_print(content=md_content, title=title, subtitle=subtitle, border_style=border_style)@@ -47,10 +47,10 @@ class PrettyPrinter:@classmethoddef pretty_print(cls,- content: Union[str, Any],- title: Optional[TextType] = None,- subtitle: Optional[TextType] = None,- border_style: Optional[StyleType] = None,+ content: str | Any,+ title: TextType | None = None,+ subtitle: TextType | None = None,+ border_style: StyleType | None = None,):match cls.mode:case PrettyPrintMode.RICH:@@ -61,10 +61,10 @@ class PrettyPrinter:@classmethoddef pretty_print_using_rich(cls,- content: Union[str, Any],- title: Optional[TextType] = None,- subtitle: Optional[TextType] = None,- border_style: Optional[StyleType] = None,+ content: str | Any,+ title: TextType | None = None,+ subtitle: TextType | None = None,+ border_style: StyleType | None = None,):if isinstance(content, str):if content.startswith(("http://", "https://")):@@ -99,9 +99,9 @@ class PrettyPrinter:@classmethoddef pretty_print_without_rich(cls,- content: Union[str, Any],- title: Optional[TextType] = None,- subtitle: Optional[TextType] = None,+ content: str | Any,+ title: TextType | None = None,+ subtitle: TextType | None = None,):if isinstance(content, str) and content.startswith(("http://", "https://")):cls.pretty_print_url_without_rich(content=content, title=title, subtitle=subtitle)@@ -112,7 +112,7 @@ class PrettyPrinter:terminal_width = shutil.get_terminal_size().columnscontent_str = f"{content}"max_content_width = terminal_width - len(title) - 8 # Accounting for frame and padding- wrapped_lines: List[str] = []+ wrapped_lines: list[str] = []for line in content_str.splitlines():while len(line) > max_content_width:wrapped_lines.append(line[:max_content_width])@@ -137,9 +137,9 @@ class PrettyPrinter:@classmethoddef pretty_print_url_without_rich(cls,- content: Union[str, Any],- title: Optional[TextType] = None,- subtitle: Optional[TextType] = None,+ content: str | Any,+ title: TextType | None = None,+ subtitle: TextType | None = None,):title = title or ""if subtitle:diff --git a/pipelex/tools/misc/string_utils.py b/pipelex/tools/misc/string_utils.pyindex 6237f1dfe..48cddf388 100644--- a/pipelex/tools/misc/string_utils.py+++ b/pipelex/tools/misc/string_utils.py@@ -1,10 +1,9 @@import re-from typing import Any, List, Optional+from typing import Anydef has_text(text: str) -> bool:- """- Checks if a string contains any alphanumeric characters.+ """Checks if a string contains any alphanumeric characters.This function uses a regex pattern to check for the presence of any alphanumericcharacter, including non-ASCII characters (e.g., é, ñ, etc.).@@ -14,41 +13,42 @@ def has_text(text: str) -> bool:Returns:bool: True if the string contains at least one alphanumeric character, False otherwise.+"""return bool(re.search(r"\w", text))-def is_none_or_has_text(text: Optional[str]) -> bool:- """- Checks if a string is None or contains alphanumeric characters.+def is_none_or_has_text(text: str | None) -> bool:+ """Checks if a string is None or contains alphanumeric characters.This function combines a None check with the has_text function to validate thata string is either None or contains meaningful content.Args:- text (Optional[str]): The string to check, which can be None.+ text (str | None): The string to check, which can be None.Returns:bool: True if the string is None or contains at least one alphanumeric character,False if it's an empty string or contains only non-alphanumeric characters.+"""return text is None or has_text(text)-def can_inject_text(value: Optional[Any]) -> bool:- """- Checks if a value can be safely converted to a string containing alphanumeric characters.+def can_inject_text(value: Any | None) -> bool:+ """Checks if a value can be safely converted to a string containing alphanumeric characters.This function attempts to convert any value to a string and checks if it containsalphanumeric characters. It handles None values and exceptions gracefully, making itsafe for type-uncertain inputs.Args:- value (Optional[Any]): Any value that might be convertible to a string, including None.+ value (Any | None): Any value that might be convertible to a string, including None.Returns:bool: True if the value can be converted to a string containing alphanumeric characters,False if the value is None, empty, or cannot be safely converted to a string.+"""if not value:return False@@ -58,26 +58,25 @@ def can_inject_text(value: Optional[Any]) -> bool:return False-def is_not_none_and_has_text(text: Optional[str]) -> bool:- """- Checks if a string is not None and contains alphanumeric characters.+def is_not_none_and_has_text(text: str | None) -> bool:+ """Checks if a string is not None and contains alphanumeric characters.This function combines a None check with the has_text function to validate thata string exists and contains meaningful content. It's the opposite of is_none_or_has_text.Args:- text (Optional[str]): The string to check, which can be None.+ text (str | None): The string to check, which can be None.Returns:bool: True if the string is not None and contains at least one alphanumeric character,False if it's None, empty, or contains only non-alphanumeric characters.+"""return text is not None and has_text(text)def camel_to_snake_case(name: str) -> str:- """- Converts a camelCase string to snake_case format.+ """Converts a camelCase string to snake_case format.This function identifies word boundaries in camelCase by looking for transitionsbetween lowercase and uppercase letters, then joins the words with underscores.@@ -91,14 +90,14 @@ def camel_to_snake_case(name: str) -> str:Example:>>> camel_to_snake_case("thisIsATest")'this_is_a_test'+"""s1 = re.sub("(.)([A-Z][a-z]+)", r"\1_\2", name)return re.sub("([a-z0-9])([A-Z])", r"\1_\2", s1).lower()def pascal_case_to_snake_case(name: str) -> str:- """- Converts a PascalCase string to snake_case format.+ """Converts a PascalCase string to snake_case format.This function is a wrapper around camel_to_snake_case, as the same conversionrules apply to both PascalCase and camelCase strings.@@ -112,13 +111,13 @@ def pascal_case_to_snake_case(name: str) -> str:Example:>>> pascal_case_to_snake_case("ThisIsATest")'this_is_a_test'+"""return camel_to_snake_case(name=name)def pascal_case_to_sentence(name: str) -> str:- """- Converts a PascalCase string to a capitalized sentence.+ """Converts a PascalCase string to a capitalized sentence.This function splits a PascalCase string at word boundaries and spaces.- Preserves fully uppercase words (e.g., "BOB", "JSON")@@ -137,10 +136,11 @@ def pascal_case_to_sentence(name: str) -> str:'BOB low key'>>> pascal_case_to_sentence("ParseJSONData")'Parse JSON data'+"""# First split by spaces- space_parts: List[str] = name.split()- result_parts: List[str] = []+ space_parts: list[str] = name.split()+ result_parts: list[str] = []for part in space_parts:if part.isupper():@@ -148,16 +148,15 @@ def pascal_case_to_sentence(name: str) -> str:result_parts.append(part)else:# Handle PascalCase parts- words: List[str] = re.findall(r"([A-Z][a-z]+|[A-Z]{2,}(?=[A-Z][a-z]|\d|\W|$)|[A-Z]{2,}|[0-9]+)", part)- processed_words: List[str] = [word if word.isupper() else word.lower() for word in words]+ words: list[str] = re.findall(r"([A-Z][a-z]+|[A-Z]{2,}(?=[A-Z][a-z]|\d|\W|$)|[A-Z]{2,}|[0-9]+)", part)+ processed_words: list[str] = [word if word.isupper() else word.lower() for word in words]result_parts.append(" ".join(processed_words))return " ".join(result_parts).capitalize()def snake_to_pascal_case(snake_str: str) -> str:- """- Converts a snake_case string to PascalCase format.+ """Converts a snake_case string to PascalCase format.This function splits the string at underscores, capitalizes the first letterof each component, and joins them together without separators.@@ -171,14 +170,14 @@ def snake_to_pascal_case(snake_str: str) -> str:Example:>>> snake_to_pascal_case("hello_world")'HelloWorld'+"""components = snake_str.split("_")return "".join(component.title() for component in components)def snake_to_capitalize_first_letter(snake_str: str) -> str:- """- Converts a snake_case string to a space-separated string with the first letter capitalized.+ """Converts a snake_case string to a space-separated string with the first letter capitalized.This function splits the string at underscores, joins the components with spaces,and capitalizes only the first letter of the entire string.@@ -192,6 +191,7 @@ def snake_to_capitalize_first_letter(snake_str: str) -> str:Example:>>> snake_to_capitalize_first_letter("this_is_a_test")'This is a test'+"""components = snake_str.split("_")phrase = " ".join(components)@@ -230,11 +230,11 @@ def matches_wildcard_pattern(text: str, pattern: str) -> bool:True>>> matches_wildcard_pattern("exact-match", "exact-match")True+"""if pattern == "*":return True-- if pattern.startswith("*") and pattern.endswith("*"):+ elif pattern.startswith("*") and pattern.endswith("*"):# Pattern like "*sonnet*"middle = pattern[1:-1]return middle in textdiff --git a/pipelex/tools/misc/toml_utils.py b/pipelex/tools/misc/toml_utils.pyindex c14e5c286..d20b5bec8 100644--- a/pipelex/tools/misc/toml_utils.py+++ b/pipelex/tools/misc/toml_utils.py@@ -1,7 +1,8 @@from __future__ import annotationsimport re-from typing import Any, Dict, List, Mapping, Optional, cast+from collections.abc import Mapping+from typing import Any, castimport tomlimport tomlkit@@ -16,13 +17,11 @@ from pipelex.tools.misc.json_utils import remove_none_values_from_dictclass TOMLValidationError(Exception):"""Raised when TOML file has formatting issues that could cause problems."""- pass--def validate_toml_content(content: str, file_path: Optional[str] = None) -> None:+def validate_toml_content(content: str, file_path: str | None = None) -> None:"""Validate TOML content for common formatting issues."""lines = content.splitlines()- issues: List[str] = []+ issues: list[str] = []for line_num, line in enumerate(lines, 1):# Check for trailing whitespace@@ -55,8 +54,9 @@ def validate_toml_file(path: str) -> None:Raises:TOMLValidationError: If formatting issues are detected+"""- with open(path, "r", encoding="utf-8") as file:+ with open(path, encoding="utf-8") as file:content = file.read()validate_toml_content(content, path)@@ -74,6 +74,7 @@ def clean_trailing_whitespace(content: str) -> str:Returns:The cleaned TOML content with trailing whitespace removed and an empty line at EOF+"""# Split into lines and clean each linelines = [line.rstrip() for line in content.splitlines()]@@ -90,7 +91,7 @@ def clean_trailing_whitespace(content: str) -> str:return "\n".join(lines) + "\n\n"-def load_toml_from_path(path: str) -> Dict[str, Any]:+def load_toml_from_path(path: str) -> dict[str, Any]:"""Load TOML from path.Args:@@ -101,9 +102,10 @@ def load_toml_from_path(path: str) -> Dict[str, Any]:Raises:toml.TomlDecodeError: If TOML parsing fails, with file path included+"""try:- with open(path, "r", encoding="utf-8") as file:+ with open(path, encoding="utf-8") as file:content = file.read()cleaned_content = clean_trailing_whitespace(content)@@ -114,14 +116,13 @@ def load_toml_from_path(path: str) -> Dict[str, Any]:file.write(cleaned_content)# Parse TOML first- dict_from_toml = toml.loads(cleaned_content)-- return dict_from_toml+ return toml.loads(cleaned_content)except toml.TomlDecodeError as exc:- raise toml.TomlDecodeError(f"TOML parsing error in file '{path}': {exc}", exc.doc, exc.pos) from exc+ msg = f"TOML parsing error in file '{path}': {exc}"+ raise toml.TomlDecodeError(msg, exc.doc, exc.pos) from exc-def failable_load_toml_from_path(path: str) -> Optional[Dict[str, Any]]:+def failable_load_toml_from_path(path: str) -> dict[str, Any] | None:"""Load TOML from path with failure handling."""if not path_exists(path):return None@@ -139,8 +140,7 @@ def make_toml_string(ensure_trailing_newline: bool = True,ensure_leading_blank_line_in_value: bool = False,):- """- Build a tomlkit string node.+ r"""Build a tomlkit string node.- If `force_multiline` or the text contains '\\n', we emit a triple-quoted multiline string.- When multiline, `ensure_trailing_newline` puts the closing quotes on their own line.- When multiline, `ensure_leading_blank_line_in_value` inserts a real blank line at the start of the value.@@ -163,15 +163,10 @@ def _convert_to_inline(value: Any) -> Any:# Handle Pydantic models by converting them to dict firstif isinstance(value, BaseModel):# For RootModel, use the root attribute; for regular models, use model_dump()- if hasattr(value, "root"):- # This is a RootModel, use its root value- value = getattr(value, "root")- else:- # This is a regular BaseModel, convert to dict- value = value.model_dump()+ value = value.root if hasattr(value, "root") else value.model_dump() # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType, reportUnknownVariableType]if isinstance(value, Mapping):- value = cast(Mapping[str, Any], value)+ value = cast("Mapping[str, Any]", value)inline_table_obj = inline_table()for key, value_item in value.items():inline_table_obj[key] = _convert_to_inline(value_item)@@ -179,12 +174,12 @@ def _convert_to_inline(value: Any) -> Any:return inline_table_objif isinstance(value, list):- value = cast(List[Any], value)+ value = cast("list[Any]", value)array_obj = array()array_obj.multiline(True) # set to False for single-line arraysfor element in value:if isinstance(element, Mapping):- element = cast(Mapping[str, Any], element)+ element = cast("Mapping[str, Any]", element)inline_element = inline_table()for inner_key, inner_value in element.items():inline_element[inner_key] = _convert_to_inline(inner_value)@@ -199,32 +194,30 @@ def _convert_to_inline(value: Any) -> Any:# This looks like a prompt template - escape newlines for single-line format# Return the string directly and let tomlkit handle the escaping properlyreturn value- else:- # Triple quotes if needed (or forced); closing quotes on their own line.- return make_toml_string(- value,- prefer_literal=False, # flip to True for '''...'''- force_multiline=False, # flip to True to force """...""" even without \n- ensure_trailing_newline=True, # keep closing """ on its own line- ensure_leading_blank_line_in_value=False, # flip to True to keep a blank first line- )- return value+ # Triple quotes if needed (or forced); closing quotes on their own line.+ return make_toml_string(+ value,+ prefer_literal=False, # flip to True for '''...'''+ force_multiline=False, # flip to True to force """...""" even without \n+ ensure_trailing_newline=True, # keep closing """ on its own line+ ensure_leading_blank_line_in_value=False, # flip to True to keep a blank first line+ )+ return value # pyright: ignore[reportUnknownVariableType]def _filter_empty_values(value: Any) -> Any:"""Filter out empty lists and None values from data structures."""if isinstance(value, dict):- filtered: Dict[str, Any] = {}- for k, v in cast(Dict[str, Any], value).items():+ filtered: dict[str, Any] = {}+ for k, v in cast("dict[str, Any]", value).items():filtered_v = _filter_empty_values(v)# Keep empty dicts but skip empty lists and None valuesif filtered_v is not None and (not isinstance(filtered_v, list) or filtered_v):filtered[k] = filtered_vreturn filtered- elif isinstance(value, list):- return [_filter_empty_values(item) for item in cast(List[Any], value) if _filter_empty_values(item) is not None]- else:- return value+ if isinstance(value, list):+ return [_filter_empty_values(item) for item in cast("list[Any]", value) if _filter_empty_values(item) is not None]+ return valuedef _create_ordered_inline_table(data: Mapping[str, Any]) -> Any:@@ -268,7 +261,7 @@ def dict_to_toml(data: Mapping[str, Any]) -> str:# Handle sections (concepts, pipes)for section_key, section_value in data.items():if isinstance(section_value, Mapping):- section_value = cast(Mapping[str, Any], section_value)+ section_value = cast("Mapping[str, Any]", section_value)# Skip empty sectionsif not section_value:@@ -284,7 +277,7 @@ def dict_to_toml(data: Mapping[str, Any]) -> str:section_table.add(item_key, _convert_to_inline(item_value))elif isinstance(item_value, Mapping):# Complex object that needs its own table- item_value = cast(Mapping[str, Any], item_value)+ item_value = cast("Mapping[str, Any]", item_value)item_table = table()# Handle the structure field specially@@ -293,9 +286,9 @@ def dict_to_toml(data: Mapping[str, Any]) -> str:if field_key == "structure" and isinstance(field_value, Mapping):# Structure should be its own table [section.item.structure]structure_table = table()- for struct_key, struct_value in cast(Mapping[str, Any], field_value).items():+ for struct_key, struct_value in cast("Mapping[str, Any]", field_value).items():if isinstance(struct_value, Mapping):- structure_table.add(struct_key, _create_ordered_inline_table(cast(Mapping[str, Any], struct_value)))+ structure_table.add(struct_key, _create_ordered_inline_table(cast("Mapping[str, Any]", struct_value)))else:structure_table.add(struct_key, _convert_to_inline(struct_value))item_table.add(field_key, structure_table)@@ -377,17 +370,16 @@ def dict_to_toml(data: Mapping[str, Any]) -> str:return "\n".join(processed_lines)- dumped_content = fix_inline_table_spacing(dumped_content)+ return fix_inline_table_spacing(dumped_content)- return dumped_content--def save_toml_to_path(data: Dict[str, Any], path: str) -> None:+def save_toml_to_path(data: dict[str, Any], path: str) -> None:"""Save dictionary as TOML to file path.Args:data: Dictionary to save as TOMLpath: Path where to save the TOML file+"""data_cleaned = datawith open(path, "w", encoding="utf-8") as file:diff --git a/pipelex/tools/pdf/pypdfium2_renderer.py b/pipelex/tools/pdf/pypdfium2_renderer.pyindex 4df3eb70e..35a996a97 100644--- a/pipelex/tools/pdf/pypdfium2_renderer.py+++ b/pipelex/tools/pdf/pypdfium2_renderer.py@@ -2,16 +2,17 @@ from __future__ import annotationsimport asyncioimport pathlib-from typing import List+from typing import TYPE_CHECKINGimport pypdfium2 as pdfium-from PIL import Imagefrom pypdfium2.raw import FPDFBitmap_BGRAfrom pipelex.tools.exceptions import ToolExceptionfrom pipelex.tools.misc.file_fetch_utils import fetch_file_from_url_httpx_asyncfrom pipelex.tools.misc.path_utils import clarify_path_or_url+if TYPE_CHECKING:+ from PIL import ImagePDFIUM2_REFERENCE_DPI = 72@@ -23,8 +24,7 @@ PdfInput = str | pathlib.Path | bytesclass PyPdfium2Renderer:- """- Thread-safe PDF page renderer built on pypdfium2.+ """Thread-safe PDF page renderer built on pypdfium2.• All entry into the native PDFium library is protected by a singleasyncio.Lock, so the enclosing *process* is safe even if other@@ -39,9 +39,9 @@ class PyPdfium2Renderer:# ---- internal blocking helper ------------------------------------# TODO: Needs UT@staticmethod- def _render_pdf_pages_sync(pdf_input: PdfInput, scale: float) -> List[Image.Image]:+ def _render_pdf_pages_sync(pdf_input: PdfInput, scale: float) -> list[Image.Image]:pdf_doc = pdfium.PdfDocument(pdf_input)- images: List[Image.Image] = []+ images: list[Image.Image] = []for index in range(len(pdf_doc)):# page = doc.get_page(index)page = pdf_doc[index]@@ -59,9 +59,9 @@ class PyPdfium2Renderer:# TODO: Needs UT@staticmethod- def _get_text_from_pdf_pages_sync(pdf_input: PdfInput) -> List[str]:+ def _get_text_from_pdf_pages_sync(pdf_input: PdfInput) -> list[str]:pdf_doc = pdfium.PdfDocument(pdf_input)- texts: List[str] = []+ texts: list[str] = []for index in range(len(pdf_doc)):page = pdf_doc[index]text = page.get_textpage().get_text_bounded() # pyright: ignore[reportUnknownMemberType]@@ -71,37 +71,37 @@ class PyPdfium2Renderer:return texts# ---- public async façade ------------------------------------------ async def render_pdf_pages(self, pdf_input: PdfInput, dpi: int) -> List[Image.Image]:+ async def render_pdf_pages(self, pdf_input: PdfInput, dpi: int) -> list[Image.Image]:scale = dpi / PDFIUM2_REFERENCE_DPI"""Render *one* page and return PNG bytes."""async with self._pdfium_lock:return await asyncio.to_thread(self._render_pdf_pages_sync, pdf_input, scale)- async def get_text_from_pdf_pages(self, pdf_input: PdfInput) -> List[str]:+ async def get_text_from_pdf_pages(self, pdf_input: PdfInput) -> list[str]:"""Extract text from all pages of a PDF."""async with self._pdfium_lock:return await asyncio.to_thread(self._get_text_from_pdf_pages_sync, pdf_input)- async def render_pdf_pages_from_uri(self, pdf_uri: str, dpi: int) -> List[Image.Image]:- pdf_path, pdf_url = clarify_path_or_url(path_or_uri=pdf_uri) # pyright: ignore+ async def render_pdf_pages_from_uri(self, pdf_uri: str, dpi: int) -> list[Image.Image]:+ pdf_path, pdf_url = clarify_path_or_url(path_or_uri=pdf_uri)if pdf_url:pdf_bytes = await fetch_file_from_url_httpx_async(url=pdf_url)return await self.render_pdf_pages(pdf_input=pdf_bytes, dpi=dpi)- elif pdf_path:+ if pdf_path:return await self.render_pdf_pages(pdf_input=pdf_path, dpi=dpi)- else:- raise PyPdfium2RendererError(f"Invalid PDF URI: {pdf_uri}")+ msg = f"Invalid PDF URI: {pdf_uri}"+ raise PyPdfium2RendererError(msg)- async def get_text_from_pdf_pages_from_uri(self, pdf_uri: str) -> List[str]:+ async def get_text_from_pdf_pages_from_uri(self, pdf_uri: str) -> list[str]:"""Extract text from all pages of a PDF from URI."""- pdf_path, pdf_url = clarify_path_or_url(path_or_uri=pdf_uri) # pyright: ignore+ pdf_path, pdf_url = clarify_path_or_url(path_or_uri=pdf_uri)if pdf_url:pdf_bytes = await fetch_file_from_url_httpx_async(url=pdf_url)return await self.get_text_from_pdf_pages(pdf_input=pdf_bytes)- elif pdf_path:+ if pdf_path:return await self.get_text_from_pdf_pages(pdf_input=pdf_path)- else:- raise PyPdfium2RendererError(f"Invalid PDF URI: {pdf_uri}")+ msg = f"Invalid PDF URI: {pdf_uri}"+ raise PyPdfium2RendererError(msg)pypdfium2_renderer = PyPdfium2Renderer()diff --git a/pipelex/tools/registry_models.py b/pipelex/tools/registry_models.pyindex d4f175d5f..76d087a0a 100644--- a/pipelex/tools/registry_models.py+++ b/pipelex/tools/registry_models.py@@ -1,15 +1,16 @@-from typing import Any, Callable, List, Set, Type+from collections.abc import Callable+from typing import Anyfrom pydantic import BaseModel-ModelType = Type[BaseModel]+ModelType = type[BaseModel]class RegistryModels:@classmethod- def get_all_models(cls) -> List[ModelType]:- model_lists: List[List[ModelType]] = [getattr(cls, attr) for attr in dir(cls) if isinstance(getattr(cls, attr), list)]- all_models: Set[ModelType] = set()+ def get_all_models(cls) -> list[ModelType]:+ model_lists: list[list[ModelType]] = [getattr(cls, attr) for attr in dir(cls) if isinstance(getattr(cls, attr), list)]+ all_models: set[ModelType] = set()for model_list in model_lists:all_models.update(model_list)@@ -18,8 +19,8 @@ class RegistryModels:class RegistryFuncs:@classmethod- def get_all_functions(cls) -> List[Callable[..., Any]]:- functions: List[Callable] = [] # pyright: ignore[reportMissingTypeArgument, reportUnknownVariableType]+ def get_all_functions(cls) -> list[Callable[..., Any]]:+ functions: list[Callable] = [] # pyright: ignore[reportMissingTypeArgument, reportUnknownVariableType]for attr in dir(cls):attr_value = getattr(cls, attr)if callable(attr_value):diff --git a/pipelex/tools/runtime_manager.py b/pipelex/tools/runtime_manager.pyindex 2d0874e29..6a3fb9f02 100644--- a/pipelex/tools/runtime_manager.py+++ b/pipelex/tools/runtime_manager.py@@ -1,5 +1,3 @@-from typing import Optional-from pydantic import BaseModelfrom pipelex.tools.environment import get_optional_env@@ -17,14 +15,11 @@ class RunMode(StrEnum):def get_from_env_var(cls) -> "RunMode":if mode_str := get_optional_env(RUN_MODE_ENV_VAR_KEY):return RunMode(mode_str)- else:- return RunMode.NORMAL+ return RunMode.NORMALclass WorkerMode(StrEnum):- """- Used for external worker, note that it can be run "for unit tests" even if it's not a unit test.- """+ """Used for external worker, note that it can be run "for unit tests" even if it's not a unit test."""NORMAL = "normal"UNIT_TEST = "unit_test"@@ -56,7 +51,7 @@ class ProblemReactions(BaseModel):class RuntimeManager(BaseModel):_environment: RunEnvironment = RunEnvironment.get_from_env_var()_run_mode: RunMode = RunMode.get_from_env_var()- _worker_mode: Optional[WorkerMode] = None+ _worker_mode: WorkerMode | None = Noneproblem_reactions: ProblemReactions = ProblemReactions(template_inputs=ProblemReaction.LOG,@@ -79,7 +74,7 @@ class RuntimeManager(BaseModel):return self._run_mode@property- def worker_mode(self) -> Optional[WorkerMode]:+ def worker_mode(self) -> WorkerMode | None:return self._worker_mode@propertydiff --git a/pipelex/tools/secrets/env_secrets_provider.py b/pipelex/tools/secrets/env_secrets_provider.pyindex dc12a9a8a..14ccf218e 100644--- a/pipelex/tools/secrets/env_secrets_provider.py+++ b/pipelex/tools/secrets/env_secrets_provider.py@@ -1,5 +1,3 @@-from typing import Optional-from typing_extensions import overridefrom pipelex.tools.environment import EnvVarNotFoundError, get_optional_env, get_required_env@@ -13,19 +11,22 @@ class EnvSecretsProvider(SecretsProviderAbstract):try:return get_required_env(key=secret_id)except EnvVarNotFoundError as exc:- raise SecretNotFoundError(f"Secret '{secret_id}' not found") from exc+ msg = f"Secret '{secret_id}' not found"+ raise SecretNotFoundError(msg) from exc@override- def get_optional_secret(self, secret_id: str) -> Optional[str]:+ def get_optional_secret(self, secret_id: str) -> str | None:return get_optional_env(key=secret_id)@overridedef get_required_secret_specific_version(self, secret_id: str, version_id: str) -> str:- raise NotImplementedError("EnvSecretsProvider does not support specific versions")+ msg = "EnvSecretsProvider does not support specific versions"+ raise NotImplementedError(msg)@override- def get_optional_secret_specific_version(self, secret_id: str, version_id: str) -> Optional[str]:- raise NotImplementedError("EnvSecretsProvider does not support specific versions")+ def get_optional_secret_specific_version(self, secret_id: str, version_id: str) -> str | None:+ msg = "EnvSecretsProvider does not support specific versions"+ raise NotImplementedError(msg)@overridedef set_secret_as_env_var(self, secret_id: str, version_id: str = "latest"):diff --git a/pipelex/tools/secrets/secrets_provider_abstract.py b/pipelex/tools/secrets/secrets_provider_abstract.pyindex a5ce8ca94..dd83a6733 100644--- a/pipelex/tools/secrets/secrets_provider_abstract.py+++ b/pipelex/tools/secrets/secrets_provider_abstract.py@@ -1,5 +1,4 @@from abc import ABC, abstractmethod-from typing import OptionalLATEST_SECRET_VERSION_NAME = "latest"@@ -9,13 +8,13 @@ class SecretsProviderAbstract(ABC):def get_required_secret(self, secret_id: str) -> str: ...@abstractmethod- def get_optional_secret(self, secret_id: str) -> Optional[str]: ...+ def get_optional_secret(self, secret_id: str) -> str | None: ...@abstractmethoddef get_required_secret_specific_version(self, secret_id: str, version_id: str) -> str: ...@abstractmethod- def get_optional_secret_specific_version(self, secret_id: str, version_id: str) -> Optional[str]: ...+ def get_optional_secret_specific_version(self, secret_id: str, version_id: str) -> str | None: ...@abstractmethoddef set_secret_as_env_var(self, secret_id: str, version_id: str = LATEST_SECRET_VERSION_NAME): ...diff --git a/pipelex/tools/secrets/secrets_utils.py b/pipelex/tools/secrets/secrets_utils.pyindex 50d2c400c..30d952d41 100644--- a/pipelex/tools/secrets/secrets_utils.py+++ b/pipelex/tools/secrets/secrets_utils.py@@ -49,6 +49,7 @@ def substitute_vars(content: str) -> str:Raises:VarNotFoundError: If required variable is missing from all specified sources+"""def replace_var(match: re.Match[str]) -> str:@@ -65,11 +66,12 @@ def substitute_vars(content: str) -> str:try:prefix = VarPrefix(prefix_str)- except ValueError:+ except ValueError as exc:+ msg = f"Unknown variable prefix: '{prefix_str}'"raise UnknownVarPrefixError(var_name=var_name,- message=f"Unknown variable prefix: '{prefix_str}'",- )+ message=msg,+ ) from excmatch prefix:case VarPrefix.ENV:@@ -97,11 +99,12 @@ def _handle_fallback_pattern(var_spec: str) -> str:try:prefix = VarPrefix(prefix_str)- except ValueError:+ except ValueError as exc:+ msg = f"Unknown variable prefix: '{prefix_str}'"raise UnknownVarPrefixError(var_name=var_name,- message=f"Unknown variable prefix: '{prefix_str}'",- )+ message=msg,+ ) from excmatch prefix:case VarPrefix.ENV:@@ -119,10 +122,8 @@ def _handle_fallback_pattern(var_spec: str) -> str:return get_secrets_provider().get_secret(secret_id=part)except SecretNotFoundError:continue # Try next option-- raise VarFallbackPatternError(- message=f"Could not get variable from fallback pattern: {var_spec}",- )+ msg = f"Could not get variable from fallback pattern: {var_spec}"+ raise VarFallbackPatternError(message=msg)def _get_env_var(var_name: str) -> str:@@ -130,7 +131,8 @@ def _get_env_var(var_name: str) -> str:try:return get_required_env(var_name)except EnvVarNotFoundError as exc:- raise VarNotFoundError(message=f"Could not get variable '{var_name}': {str(exc)}", var_name=var_name) from exc+ msg = f"Could not get variable '{var_name}': {exc!s}"+ raise VarNotFoundError(message=msg, var_name=var_name) from excdef _get_secret(secret_name: str) -> str:@@ -138,4 +140,5 @@ def _get_secret(secret_name: str) -> str:try:return get_secrets_provider().get_secret(secret_id=secret_name)except SecretNotFoundError as exc:- raise VarNotFoundError(message=f"Could not get variable '{secret_name}': {str(exc)}", var_name=secret_name) from exc+ msg = f"Could not get variable '{secret_name}': {exc!s}"+ raise VarNotFoundError(message=msg, var_name=secret_name) from excdiff --git a/pipelex/tools/templating/jinja2_blueprint.py b/pipelex/tools/templating/jinja2_blueprint.pyindex ac136d9d9..8210ba811 100644--- a/pipelex/tools/templating/jinja2_blueprint.py+++ b/pipelex/tools/templating/jinja2_blueprint.py@@ -1,4 +1,4 @@-from typing import Any, Dict, Optional+from typing import Anyfrom pydantic import BaseModel, Field@@ -7,11 +7,11 @@ from pipelex.tools.templating.templating_models import PromptingStyleclass Jinja2Blueprint(BaseModel):- jinja2_name: Optional[str] = Field(default=None, description="Name of the Jinja2 template to use")- jinja2: Optional[str] = Field(default=None, description="Raw Jinja2 template string")- prompting_style: Optional[PromptingStyle] = Field(default=None, description="Style of prompting to use (typically for different LLMs)")+ jinja2_name: str | None = Field(default=None, description="Name of the Jinja2 template to use")+ jinja2: str | None = Field(default=None, description="Raw Jinja2 template string")+ prompting_style: PromptingStyle | None = Field(default=None, description="Style of prompting to use (typically for different LLMs)")template_category: Jinja2TemplateCategory = Field(default=Jinja2TemplateCategory.LLM_PROMPT,description="Category of the template (could also be HTML, MARKDOWN, MERMAID, etc.), influences Jinja2 rendering environment config",)- extra_context: Optional[Dict[str, Any]] = Field(default=None, description="Additional context variables for template rendering")+ extra_context: dict[str, Any] | None = Field(default=None, description="Additional context variables for template rendering")diff --git a/pipelex/tools/templating/jinja2_environment.py b/pipelex/tools/templating/jinja2_environment.pyindex 6d513780a..ba8dd6262 100644--- a/pipelex/tools/templating/jinja2_environment.py+++ b/pipelex/tools/templating/jinja2_environment.py@@ -1,5 +1,3 @@-from typing import Tuple-from jinja2 import BaseLoader, Environment, PackageLoaderfrom pipelex.tools.templating.jinja2_template_category import Jinja2TemplateCategory@@ -51,7 +49,7 @@ def make_jinja2_env_from_package(template_category: Jinja2TemplateCategory,package_name: str,package_path: str,-) -> Tuple[Environment, BaseLoader]:+) -> tuple[Environment, BaseLoader]:full_package_path = f"{package_path}/jinja2_{template_category}"loader = PackageLoader(package_name=package_name,@@ -65,18 +63,17 @@ def make_jinja2_env_without_loader(template_category: Jinja2TemplateCategory,) -> Environment:loader = BaseLoader()- jinja2_env = make_jinja2_env_from_loader(template_category=template_category, loader=loader)- return jinja2_env+ return make_jinja2_env_from_loader(template_category=template_category, loader=loader)def make_jinja2_env_from_template_provider(template_category: Jinja2TemplateCategory,template_provider: TemplateProviderAbstract,-) -> Tuple[Environment, BaseLoader]:+) -> tuple[Environment, BaseLoader]:loader = Jinja2TemplateLoader(template_provider=template_provider)jinja2_env = make_jinja2_env_from_loader(template_category=template_category, loader=loader)filters = template_category.filtersfor filter_name, filter_function in filters.items():- jinja2_env.filters[filter_name] = filter_function # type: ignore+ jinja2_env.filters[filter_name] = filter_function # pyright: ignore[reportArgumentType]return jinja2_env, loaderdiff --git a/pipelex/tools/templating/jinja2_errors.py b/pipelex/tools/templating/jinja2_errors.pyindex 79bf0195f..97ac0c3cb 100644--- a/pipelex/tools/templating/jinja2_errors.py+++ b/pipelex/tools/templating/jinja2_errors.py@@ -1,5 +1,3 @@-from typing import Optional-from pipelex.tools.exceptions import ToolException@@ -23,7 +21,7 @@ class Jinja2DetectVariablesError(ToolException):pass-def make_jinja2_error_explanation(jinja2_name: Optional[str], template_text: Optional[str]) -> str:+def make_jinja2_error_explanation(jinja2_name: str | None, template_text: str | None) -> str:explanation = ""if jinja2_name:explanation += f"\nJinja2 name: '{jinja2_name}'\n"diff --git a/pipelex/tools/templating/jinja2_filters.py b/pipelex/tools/templating/jinja2_filters.pyindex 63f3a391e..c798c5293 100644--- a/pipelex/tools/templating/jinja2_filters.py+++ b/pipelex/tools/templating/jinja2_filters.py@@ -1,4 +1,4 @@-from typing import Any, Optional+from typing import Anyfrom jinja2 import pass_contextfrom jinja2.runtime import Context, Undefined@@ -17,40 +17,39 @@ ALLOWED_FILTERS = ["tag", "format"]# Filter to format some Stuff or any object with the appropriate text formatting methods@pass_context-def text_format(context: Context, value: Any, text_format: Optional[TextFormat] = None) -> Any:+def text_format(context: Context, value: Any, text_format: TextFormat | None = None) -> Any:if text_format:if isinstance(text_format, str): # pyright: ignore[reportUnnecessaryIsInstance]applied_text_format = TextFormat(text_format)elif isinstance(text_format, TextFormat): # pyright: ignore[reportUnnecessaryIsInstance]applied_text_format = text_formatelse:- raise Jinja2ContextError(f"Invalid text format: '{text_format}'")+ msg = f"Invalid text format: '{text_format}'"+ raise Jinja2ContextError(msg)else:applied_text_format = TextFormat(context.get(Jinja2ContextKey.TEXT_FORMAT, default=TextFormat.PLAIN))if hasattr(value, "rendered_str"):- rendered_str = value.rendered_str(text_format=applied_text_format)- return rendered_str- elif hasattr(value, applied_text_format.render_method_name):+ return value.rendered_str(text_format=applied_text_format)+ if hasattr(value, applied_text_format.render_method_name):render_method = getattr(value, applied_text_format.render_method_name)- rendered_str = render_method()- return rendered_str- elif isinstance(value, StrEnum):+ return render_method()+ if isinstance(value, StrEnum):return value.value- else:- return value+ return value# TODO: better separate tag and render# Filter to tag the variable with a tag style and a provided name, appropriate for tagging in a prompt@pass_context-def tag(context: Context, value: Any, tag_name: Optional[str] = None) -> Any:+def tag(context: Context, value: Any, tag_name: str | None = None) -> Any:if isinstance(value, Undefined):# maybe we don't need this checkif tag_name:- raise Jinja2ContextError(f"Jinja2 undefined value with tag_name '{tag_name}'")- else:- raise Jinja2ContextError("Jinja2 undefined value.")+ msg = f"Jinja2 undefined value with tag_name '{tag_name}'"+ raise Jinja2ContextError(msg)+ msg = "Jinja2 undefined value."+ raise Jinja2ContextError(msg)if isinstance(value, Jinja2TaggableAbstract):value, tag_name = value.render_tagged_for_jinja2(context=context, tag_name=tag_name)@@ -58,7 +57,7 @@ def tag(context: Context, value: Any, tag_name: Optional[str] = None) -> Any:return render_any_tagged_for_jinja2(context=context, value=value, tag_name=tag_name)-def render_any_tagged_for_jinja2(context: Context, value: Any, tag_name: Optional[str] = None) -> Any:+def render_any_tagged_for_jinja2(context: Context, value: Any, tag_name: str | None = None) -> Any:tag_style_str = context.get(Jinja2ContextKey.TAG_STYLE)tag_style: TagStyleif tag_style_str:diff --git a/pipelex/tools/templating/jinja2_models.py b/pipelex/tools/templating/jinja2_models.pyindex 09a129413..29aa180f4 100644--- a/pipelex/tools/templating/jinja2_models.py+++ b/pipelex/tools/templating/jinja2_models.py@@ -1,5 +1,5 @@from abc import ABC, abstractmethod-from typing import Any, Optional, Tuple+from typing import Anyfrom jinja2.runtime import Context@@ -18,5 +18,5 @@ class Jinja2ContextKey(StrEnum):class Jinja2TaggableAbstract(ABC):@abstractmethod- def render_tagged_for_jinja2(self, context: Context, tag_name: Optional[str] = None) -> Tuple[Any, Optional[str]]:+ def render_tagged_for_jinja2(self, context: Context, tag_name: str | None = None) -> tuple[Any, str | None]:passdiff --git a/pipelex/tools/templating/jinja2_parsing.py b/pipelex/tools/templating/jinja2_parsing.pyindex 901b426fc..eb6a773b7 100644--- a/pipelex/tools/templating/jinja2_parsing.py+++ b/pipelex/tools/templating/jinja2_parsing.py@@ -16,4 +16,5 @@ def check_jinja2_parsing(try:jinja2_env.parse(jinja2_template_source)except TemplateSyntaxError as exc:- raise Jinja2ParsingError(f"Could not parse Jinja2 template because of: {exc}. The template is:\n{jinja2_template_source}") from exc+ msg = f"Could not parse Jinja2 template because of: {exc}. The template is:\n{jinja2_template_source}"+ raise Jinja2ParsingError(msg) from excdiff --git a/pipelex/tools/templating/jinja2_rendering.py b/pipelex/tools/templating/jinja2_rendering.pyindex 2b1f42e54..fd2cc23cf 100644--- a/pipelex/tools/templating/jinja2_rendering.py+++ b/pipelex/tools/templating/jinja2_rendering.py@@ -1,4 +1,4 @@-from typing import Any, Dict, Optional+from typing import Anyfrom jinja2 import Template, metafrom jinja2.exceptions import (@@ -27,19 +27,20 @@ from pipelex.tools.templating.templating_models import PromptingStyle########################################################################################-def _add_to_templating_context(temlating_context: Dict[str, Any], jinja2_context_key: Jinja2ContextKey, value: Any) -> None:+def _add_to_templating_context(temlating_context: dict[str, Any], jinja2_context_key: Jinja2ContextKey, value: Any) -> None:if jinja2_context_key in temlating_context:- raise Jinja2StuffError(f"Jinja2 context key '{jinja2_context_key}' already in temlating_context")+ msg = f"Jinja2 context key '{jinja2_context_key}' already in temlating_context"+ raise Jinja2StuffError(msg)temlating_context[jinja2_context_key] = valueasync def render_jinja2(template_category: Jinja2TemplateCategory,template_provider: TemplateProviderAbstract,- temlating_context: Dict[str, Any],- jinja2_name: Optional[str] = None,- jinja2: Optional[str] = None,- prompting_style: Optional[PromptingStyle] = None,+ temlating_context: dict[str, Any],+ jinja2_name: str | None = None,+ jinja2: str | None = None,+ prompting_style: PromptingStyle | None = None,) -> str:jinja2_env, loader = make_jinja2_env_from_template_provider(template_category=template_category,@@ -56,10 +57,12 @@ async def render_jinja2(template = jinja2_env.get_template(jinja2_name)template_source = loader.get_source(jinja2_env, jinja2_name)[0]else:- raise Jinja2StuffError("No jinja2 or jinja2_name in Jinja2Assignment")+ msg = "No jinja2 or jinja2_name in Jinja2Assignment"+ raise Jinja2StuffError(msg)except TemplateAssertionError as exc:explanation = make_jinja2_error_explanation(jinja2_name=jinja2_name, template_text=jinja2)- raise Jinja2RenderError(f"Jinja2 render error: '{exc}' {explanation}") from exc+ msg = f"Jinja2 render error: '{exc}' {explanation}"+ raise Jinja2RenderError(msg) from exctemplate_source = preprocess_template(template_source)check_jinja2_parsing(@@ -97,14 +100,18 @@ async def render_jinja2(generated_text: str = await template.render_async(**temlating_context)except Jinja2StuffError as stuff_error:explanation = make_jinja2_error_explanation(jinja2_name=jinja2_name, template_text=template_source)- raise Jinja2RenderError(f"Jinja2 render — stuff error: '{stuff_error}' {explanation}") from stuff_error+ msg = f"Jinja2 render — stuff error: '{stuff_error}' {explanation}"+ raise Jinja2RenderError(msg) from stuff_errorexcept TemplateSyntaxError as syntax_error:explanation = make_jinja2_error_explanation(jinja2_name=jinja2_name, template_text=template_source)- raise Jinja2RenderError(f"Jinja2 render — syntax error: '{syntax_error}' {explanation}") from syntax_error+ msg = f"Jinja2 render — syntax error: '{syntax_error}' {explanation}"+ raise Jinja2RenderError(msg) from syntax_errorexcept UndefinedError as undef_error:explanation = make_jinja2_error_explanation(jinja2_name=jinja2_name, template_text=template_source)- raise Jinja2RenderError(f"Jinja2 render — undefined error: '{undef_error}' {explanation}") from undef_error+ msg = f"Jinja2 render — undefined error: '{undef_error}' {explanation}"+ raise Jinja2RenderError(msg) from undef_errorexcept Jinja2ContextError as context_error:explanation = make_jinja2_error_explanation(jinja2_name=jinja2_name, template_text=template_source)- raise Jinja2RenderError(f"Jinja2 render — context error: '{context_error}' {explanation}") from context_error+ msg = f"Jinja2 render — context error: '{context_error}' {explanation}"+ raise Jinja2RenderError(msg) from context_errorreturn generated_textdiff --git a/pipelex/tools/templating/jinja2_required_variables.py b/pipelex/tools/templating/jinja2_required_variables.pyindex 346d57bd9..e901a79d6 100644--- a/pipelex/tools/templating/jinja2_required_variables.py+++ b/pipelex/tools/templating/jinja2_required_variables.py@@ -1,5 +1,3 @@-from typing import Optional, Set-from jinja2 import metafrom jinja2.exceptions import (TemplateSyntaxError,@@ -15,15 +13,14 @@ from pipelex.tools.templating.template_provider_abstract import TemplateProviderdef detect_jinja2_required_variables(template_category: Jinja2TemplateCategory,template_provider: TemplateProviderAbstract,- jinja2_name: Optional[str] = None,- jinja2: Optional[str] = None,-) -> Set[str]:- """- Returns a list of variables required by the Jinja2 template.+ jinja2_name: str | None = None,+ jinja2: str | None = None,+) -> set[str]:+ """Returns a list of variables required by the Jinja2 template.Args:template_category: Category of the template (HTML, MARKDOWN, etc.), used to set the appropriate jinja2 environment settings- template_library: Library containing templates+ template_provider: Library containing templatesjinja2_name: Name of template in library (optional)jinja2: Direct Jinja2 template string (optional)@@ -32,6 +29,7 @@ def detect_jinja2_required_variables(Raises:Jinja2StuffError: If neither jinja2 nor jinja2_name is provided+"""jinja2_env, loader = make_jinja2_env_from_template_provider(template_category=template_category,@@ -44,19 +42,23 @@ def detect_jinja2_required_variables(elif jinja2_name:template_source = loader.get_source(jinja2_env, jinja2_name)[0]else:- raise Jinja2StuffError("No jinja2 or jinja2_name provided")+ msg = "No jinja2 or jinja2_name provided"+ raise Jinja2StuffError(msg)try:parsed_ast = jinja2_env.parse(template_source)undeclared_variables = meta.find_undeclared_variables(parsed_ast)except Jinja2StuffError as stuff_error:explanation = make_jinja2_error_explanation(jinja2_name=jinja2_name, template_text=template_source)- raise Jinja2DetectVariablesError(f"Jinja2 detect variables — stuff error: '{stuff_error}' {explanation}") from stuff_error+ msg = f"Jinja2 detect variables — stuff error: '{stuff_error}' {explanation}"+ raise Jinja2DetectVariablesError(msg) from stuff_errorexcept TemplateSyntaxError as syntax_error:explanation = make_jinja2_error_explanation(jinja2_name=jinja2_name, template_text=template_source)- raise Jinja2DetectVariablesError(f"Jinja2 detect variables — syntax error: '{syntax_error}' {explanation}") from syntax_error+ msg = f"Jinja2 detect variables — syntax error: '{syntax_error}' {explanation}"+ raise Jinja2DetectVariablesError(msg) from syntax_errorexcept UndefinedError as undef_error:explanation = make_jinja2_error_explanation(jinja2_name=jinja2_name, template_text=template_source)- raise Jinja2DetectVariablesError(f"Jinja2 detect variables — undefined error: '{undef_error}' {explanation}") from undef_error+ msg = f"Jinja2 detect variables — undefined error: '{undef_error}' {explanation}"+ raise Jinja2DetectVariablesError(msg) from undef_errorreturn undeclared_variablesdiff --git a/pipelex/tools/templating/jinja2_template_category.py b/pipelex/tools/templating/jinja2_template_category.pyindex dd609d6fd..cae802315 100644--- a/pipelex/tools/templating/jinja2_template_category.py+++ b/pipelex/tools/templating/jinja2_template_category.py@@ -1,4 +1,5 @@-from typing import Any, Callable, Dict, Optional+from collections.abc import Callable+from typing import Anyfrom jinja2.runtime import Context@@ -15,7 +16,7 @@ class Jinja2TemplateCategory(StrEnum):LLM_PROMPT = "llm_prompt"@property- def filters(self) -> Dict[Jinja2FilterName, Callable[[Context, Any, Optional[TextFormat]], Any]]:+ def filters(self) -> dict[Jinja2FilterName, Callable[[Context, Any, TextFormat | None], Any]]:match self:case Jinja2TemplateCategory.MERMAID:return {}diff --git a/pipelex/tools/templating/jinja2_template_loader.py b/pipelex/tools/templating/jinja2_template_loader.pyindex fcdbf7361..417d36254 100644--- a/pipelex/tools/templating/jinja2_template_loader.py+++ b/pipelex/tools/templating/jinja2_template_loader.py@@ -1,4 +1,4 @@-from typing import Callable, Optional, Tuple+from collections.abc import Callablefrom jinja2 import BaseLoader, Environmentfrom typing_extensions import override@@ -13,7 +13,7 @@ class Jinja2TemplateLoader(BaseLoader):self.template_provider = template_provider@override- def get_source(self, environment: Environment, template: str) -> Tuple[str, Optional[str], Optional[Callable[[], bool]]]:+ def get_source(self, environment: Environment, template: str) -> tuple[str, str | None, Callable[[], bool] | None]:the_template = self.template_provider.get_template(template_name=template)log.debug(f"TemplateLoader.get_source: template='{template}'")return the_template, None, Nonediff --git a/pipelex/tools/templating/template_library.py b/pipelex/tools/templating/template_library.pyindex 70e7a8624..82132d484 100644--- a/pipelex/tools/templating/template_library.py+++ b/pipelex/tools/templating/template_library.py@@ -1,4 +1,4 @@-from typing import Any, ClassVar, Dict+from typing import Any, ClassVar, castfrom jinja2 import TemplateSyntaxErrorfrom pydantic import Field, RootModel, ValidationError@@ -14,7 +14,7 @@ from pipelex.tools.templating.template_preprocessor import preprocess_templatefrom pipelex.tools.templating.template_provider_abstract import TemplateNotFoundError, TemplateProviderAbstractfrom pipelex.tools.typing.pydantic_utils import format_pydantic_validation_error-TemplateLibraryRoot = Dict[str, str]+TemplateLibraryRoot = dict[str, str]class TemplateLibraryError(ToolException):@@ -46,7 +46,8 @@ class TemplateLibrary(TemplateProviderAbstract, RootModel[TemplateLibraryRoot]):try:return self.root[template_name]except KeyError as exc:- raise TemplateNotFoundError(f"Template '{template_name}' not found in template library") from exc+ msg = f"Template '{template_name}' not found in template library"+ raise TemplateNotFoundError(msg) from excdef _set_template(self, template: str, name: str):preprocessed_template = preprocess_template(template)@@ -54,7 +55,8 @@ class TemplateLibrary(TemplateProviderAbstract, RootModel[TemplateLibraryRoot]):def _add_new_template(self, template: str, name: str):if name in self.root:- raise TemplateLibraryError(f"Template '{name}' already exists in the library")+ msg = f"Template '{name}' already exists in the library"+ raise TemplateLibraryError(msg)self._set_template(template=template, name=name)def _load_from_toml(self, toml_path: str):@@ -65,7 +67,7 @@ class TemplateLibrary(TemplateProviderAbstract, RootModel[TemplateLibraryRoot]):toml_name = toml_path.split("/")[-1]log.debug(f"Loaded {len(self.root) - nb_concepts_before} templates from '{toml_name}'")- def _load_from_recursive_dict(self, domain: str, recursive_dict: Dict[str, Any]):+ def _load_from_recursive_dict(self, domain: str, recursive_dict: dict[str, Any]):for name, obj in recursive_dict.items():try:if isinstance(obj, str):@@ -74,14 +76,16 @@ class TemplateLibrary(TemplateProviderAbstract, RootModel[TemplateLibraryRoot]):self._add_new_template(template=template, name=name)elif isinstance(obj, dict):# this is not a templae but a subdomain- sub_recursive_dict: Dict[str, str] = obj+ sub_recursive_dict = cast("dict[str, str]", obj)domain = f"{domain}/{name}"self._load_from_recursive_dict(domain=domain, recursive_dict=sub_recursive_dict)else:- raise TemplateLibraryError(f"Unexpected type for key '{name}' in recursive_dict: {type(obj)}")+ msg = f"Unexpected type for key '{name}' in recursive_dict: {type(obj)}"+ raise TemplateLibraryError(msg)except ValidationError as exc:error_msg = format_pydantic_validation_error(exc)- raise TemplateLibraryError(f"Error loading concept '{name}' of domain '{domain}' because of: {error_msg}") from exc+ msg = f"Error loading concept '{name}' of domain '{domain}' because of: {error_msg}"+ raise TemplateLibraryError(msg) from excdef validate_templates(self, template_category: Jinja2TemplateCategory):for template_name, template in self.root.items():@@ -96,4 +100,5 @@ class TemplateLibrary(TemplateProviderAbstract, RootModel[TemplateLibraryRoot]):error_msg += f"\nThe template is:\n{template}"else:error_msg += "The template is empty."- raise TemplateLibraryError(error_msg) from exc+ msg = error_msg+ raise TemplateLibraryError(msg) from excdiff --git a/pipelex/tools/templating/template_preprocessor.py b/pipelex/tools/templating/template_preprocessor.pyindex 115e97859..a7b82174b 100644--- a/pipelex/tools/templating/template_preprocessor.py+++ b/pipelex/tools/templating/template_preprocessor.py@@ -1,6 +1,5 @@import refrom re import Match-from typing import Listfrom pipelex.tools.templating.jinja2_errors import Jinja2TemplateErrorfrom pipelex.tools.templating.jinja2_filters import ALLOWED_FILTERS@@ -14,6 +13,7 @@ def _detect_non_existent_filters(template_str: str) -> None:Raises:Jinja2TemplateError: If any non-allowed filters are found+"""# Pattern to match Jinja2 filter syntax: {{ variable|filter() }} or {{ variable|filter(param) }}# This handles:@@ -24,14 +24,15 @@ def _detect_non_existent_filters(template_str: str) -> None:filter_pattern = r"\{\{\s*[^|}]+\|\s*([a-zA-Z0-9_]+)(?:\([^)]*\))?\s*\}\}"matches = re.finditer(filter_pattern, template_str)- invalid_filters: List[str] = []+ invalid_filters: list[str] = []for match in matches:filter_name = match.group(1)if filter_name not in ALLOWED_FILTERS:invalid_filters.append(filter_name)if invalid_filters:- raise Jinja2TemplateError(f"Invalid Jinja2 filters found: {invalid_filters}. Only the following filters are allowed: {ALLOWED_FILTERS}")+ msg = f"Invalid Jinja2 filters found: {invalid_filters}. Only the following filters are allowed: {ALLOWED_FILTERS}"+ raise Jinja2TemplateError(msg)# Handle @variable patterns@@ -57,8 +58,7 @@ def replace_dollar_variable(match: Match[str]) -> str:def preprocess_template(template: str) -> str:- """- Preprocess a template string to interpret our syntax patterns and convert them to Jinja2 syntax.+ """Preprocess a template string to interpret our syntax patterns and convert them to Jinja2 syntax.Also, detect the use of non-existent filters."""_detect_non_existent_filters(template_str=template)diff --git a/pipelex/tools/templating/templating_models.py b/pipelex/tools/templating/templating_models.pyindex cd95b0439..098128a61 100644--- a/pipelex/tools/templating/templating_models.py+++ b/pipelex/tools/templating/templating_models.py@@ -1,7 +1,7 @@from pydantic import BaseModel, Field-from typing_extensions import Self, override+from typing_extensions import override-from pipelex.types import StrEnum+from pipelex.types import Self, StrEnumclass TextFormat(StrEnum):diff --git a/pipelex/tools/typing/__init__.py b/pipelex/tools/typing/__init__.pynew file mode 100644index 000000000..e69de29bbdiff --git a/pipelex/tools/typing/module_inspector.py b/pipelex/tools/typing/module_inspector.pyindex bbe6dac70..e86a143bc 100644--- a/pipelex/tools/typing/module_inspector.py+++ b/pipelex/tools/typing/module_inspector.py@@ -2,14 +2,12 @@ import importlib.utilimport inspectimport osimport sys-from typing import Any, List, Optional, Type+from typing import Anyclass ModuleFileError(Exception):"""Exception raised for errors related to module file operations."""- pass-def import_module_from_file(file_path: str) -> Any:"""Imports a module from a file path.@@ -22,10 +20,12 @@ def import_module_from_file(file_path: str) -> Any:Raises:ModuleFileError: If the file is not a Python file or cannot be loaded+"""# Validate that the file is a Python fileif not file_path.endswith(".py"):- raise ModuleFileError(f"File {file_path} is not a Python file (must end with .py)")+ msg = f"File {file_path} is not a Python file (must end with .py)"+ raise ModuleFileError(msg)# Convert file path to module-style path to use as the actual module namemodule_name = _convert_file_path_to_module_path(file_path)@@ -37,7 +37,8 @@ def import_module_from_file(file_path: str) -> Any:# Use importlib.util to load the module from file pathspec = importlib.util.spec_from_file_location(module_name, file_path)if spec is None or spec.loader is None:- raise ModuleFileError(f"Could not load module from {file_path}")+ msg = f"Could not load module from {file_path}"+ raise ModuleFileError(msg)module = importlib.util.module_from_spec(spec)@@ -53,25 +54,21 @@ def import_module_from_file(file_path: str) -> Any:def _convert_file_path_to_module_path(file_path: str) -> str:"""Convert a file path to a module-style path."""# Remove .py extension- module_path = file_path[:-3] if file_path.endswith(".py") else file_path+ module_path = file_path.removesuffix(".py")# Replace path separators with dotsmodule_path = module_path.replace(os.sep, ".")# Handle __init__.py files by removing the __init__ part- if module_path.endswith(".__init__"):- module_path = module_path[:-9]-- return module_path+ return module_path.removesuffix(".__init__")def find_classes_in_module(module: Any,- base_class: Optional[Type[Any]],+ base_class: type[Any] | None,include_imported: bool,-) -> List[Type[Any]]:- """- Finds all classes in a module that match the criteria.+) -> list[type[Any]]:+ """Finds all classes in a module that match the criteria.Args:module: The module to search for classes@@ -80,8 +77,9 @@ def find_classes_in_module(Returns:List of class types that match the criteria+"""- classes: List[Type[Any]] = []+ classes: list[type[Any]] = []module_name = module.__name__# Find all classes in the modulediff --git a/pipelex/tools/typing/pydantic_utils.py b/pipelex/tools/typing/pydantic_utils.pyindex d4115b02d..8190dc636 100644--- a/pipelex/tools/typing/pydantic_utils.py+++ b/pipelex/tools/typing/pydantic_utils.py@@ -1,4 +1,5 @@-from typing import Any, Dict, List, Optional, Sequence, Set, TypeVar, Union+from collections.abc import Callable, Sequence+from typing import Any, TypeVar, castfrom pydantic import BaseModel, ValidationErrorfrom rich.repr import Result as RichReprResult@@ -9,6 +10,15 @@ from pipelex.types import StrEnumBaseModelTypeVar = TypeVar("BaseModelTypeVar", bound=BaseModel)+T = TypeVar("T")+++def empty_list_factory_of(_: type[T]) -> Callable[[], list[T]]:+ def _factory() -> list[T]:+ return []++ return _factory+def format_pydantic_validation_error(exc: ValidationError) -> str:"""Format a Pydantic ValidationError into a readable string with detailed error information.@@ -18,6 +28,7 @@ def format_pydantic_validation_error(exc: ValidationError) -> str:Returns:A formatted string containing categorized validation errors+"""error_msg = "Validation error(s):"@@ -29,7 +40,7 @@ def format_pydantic_validation_error(exc: ValidationError) -> str:enum_errors = [f"{'.'.join(map(str, err['loc']))}: invalid enum value '{err.get('input', 'unknown')}'" for err in exc.errors() if err["type"] == "enum"]- model_type_errors: List[str] = []+ model_type_errors: list[str] = []for err in exc.errors():if err["type"] == "model_type":field_path = ".".join(map(str, err["loc"]))@@ -63,18 +74,19 @@ def format_pydantic_validation_error(exc: ValidationError) -> str:def convert_strenum_to_str(- obj: Dict[str, Any] | List[Any] | StrEnum | Any,-) -> Dict[str, Any] | List[Any] | str | Any:+ obj: dict[str, Any] | list[Any] | StrEnum | Any,+) -> dict[str, Any] | list[Any] | str | Any:if isinstance(obj, dict):- obj_dict: Dict[str, Any] = obj+ obj_dict = cast("dict[str, Any]", obj)return {str(key): convert_strenum_to_str(value) for key, value in obj_dict.items()}elif isinstance(obj, list):- obj_list: List[Any] = obj+ obj_list = cast("list[Any]", obj)return [convert_strenum_to_str(item) for item in obj_list]elif isinstance(obj, StrEnum):if hasattr(obj, "display_name"):- return obj.display_name() # type: ignore- return str(obj)+ return obj.display_name() # pyright: ignore[reportUnknownVariableType, reportUnknownMemberType, reportAttributeAccessIssue]+ else:+ return str(obj)else:return obj@@ -89,25 +101,24 @@ class FieldVisibility(StrEnum):ONLY_HIDDEN_FIELDS = "only_hidden_fields"-def clean_model_to_dict(obj: BaseModel) -> Dict[str, Any]:+def clean_model_to_dict(obj: BaseModel) -> dict[str, Any]:dict_dump = serialize_model(obj=obj,field_visibility=FieldVisibility.NO_HIDDEN_FIELDS,is_stringify_enums=True,)if not isinstance(dict_dump, dict):- raise TypeError(f"Expected dict, got {type(dict_dump)}")- result_dict: Dict[str, Any] = dict_dump- return result_dict+ msg = f"Expected dict, got {type(dict_dump)}"+ raise TypeError(msg)+ return cast("dict[str, Any]", dict_dump)def serialize_model(obj: Any,field_visibility: FieldVisibility = FieldVisibility.NO_HIDDEN_FIELDS,is_stringify_enums: bool = True,-) -> Union[Dict[str, Any], List[Any], Any]:- """- Recursively serialize a Pydantic BaseModel (and its nested BaseModels)+) -> dict[str, Any] | list[Any] | Any:+ """Recursively serialize a Pydantic BaseModel (and its nested BaseModels)into a dictionary, omitting any fields marked with'json_schema_extra={ExtraFieldAttribute.IS_HIDDEN: True}'.@@ -120,11 +131,16 @@ def serialize_model(return obj# Identify which fields should be excluded- fields_to_exclude: Set[str] = set()+ fields_to_exclude: set[str] = set()for field_name, field_info in obj.__class__.model_fields.items():json_schema_extra = field_info.json_schema_extra- is_hidden = json_schema_extra and isinstance(json_schema_extra, dict) and json_schema_extra.get(ExtraFieldAttribute.IS_HIDDEN) is True+ is_hidden: bool+ if json_schema_extra and isinstance(json_schema_extra, dict):+ typed_json_schema_extra = cast("dict[str, Any]", json_schema_extra)+ is_hidden = typed_json_schema_extra.get(ExtraFieldAttribute.IS_HIDDEN) is True+ else:+ is_hidden = Falsematch field_visibility:case FieldVisibility.ALL_FIELDS:pass@@ -136,8 +152,8 @@ def serialize_model(fields_to_exclude.add(field_name)# Build a dict, omitting hidden fields. Recursively handle nested models.- data: Dict[str, Any] = {}- for field_name, _ in obj.__class__.model_fields.items():+ data: dict[str, Any] = {}+ for field_name in obj.__class__.model_fields:if field_name in fields_to_exclude:continue # Skip hidden fields@@ -153,7 +169,7 @@ def serialize_model(# If it's a list, we recurse for each itemelif isinstance(value, list):- value_list: List[Any] = value+ value_list = cast("list[Any]", value)data[field_name] = [serialize_model(obj=item,@@ -165,7 +181,7 @@ def serialize_model(# If it's a dict, we can similarly recurse for any nested BaseModels inside the dictelif isinstance(value, dict):- value_dict: Dict[str, Any] = value+ value_dict = cast("dict[str, Any]", value)data[field_name] = {key: serialize_model(obj=value,@@ -177,7 +193,7 @@ def serialize_model(elif is_stringify_enums and isinstance(value, StrEnum):if hasattr(value, "display_name"):- data[field_name] = value.display_name() # type: ignore+ data[field_name] = value.display_name() # pyright: ignore[reportUnknownMemberType, reportAttributeAccessIssue]else:data[field_name] = str(value)@@ -190,10 +206,10 @@ def serialize_model(class CustomBaseModel(BaseModel):@override- def __rich_repr__(self) -> RichReprResult: # type: ignore- for item in super().__rich_repr__(): # type: ignore+ def __rich_repr__(self) -> RichReprResult:+ for item in super().__rich_repr__(): # type: ignore[misc]if isinstance(item, tuple):- tuple_item: tuple[Any, ...] = item+ tuple_item = cast("tuple[Any, ...]", item)if len(tuple_item) >= 2:name = tuple_item[0]value = tuple_item[1]@@ -209,8 +225,8 @@ class CustomBaseModel(BaseModel):yield item@override- def __repr_args__(self) -> Sequence[tuple[Optional[str], Any]]:- processed_args: list[tuple[Optional[str], Any]] = []+ def __repr_args__(self) -> Sequence[tuple[str | None, Any]]:+ processed_args: list[tuple[str | None, Any]] = []for name, value in super().__repr_args__():if name and AttributePolisher.should_truncate(name=name, value=value):truncated_value = AttributePolisher.get_truncated_value(name, value)@@ -220,8 +236,8 @@ class CustomBaseModel(BaseModel):return processed_argsdef model_dump_truncated(self, **kwargs: Any) -> Any:- """- Dump the model to a dictionary with serialize_as_any=True and apply+ """Dump the model to a dictionary with serialize_as_any=True and apply+AttributePolisher truncation to fields that should be truncated.Handles nested attributes recursively.@@ -230,6 +246,7 @@ class CustomBaseModel(BaseModel):Returns:Dictionary with truncated values where appropriate+"""# Get the model dump with serialize_as_any=Truedumped_data = self.model_dump(**kwargs)@@ -237,9 +254,8 @@ class CustomBaseModel(BaseModel):# Apply truncation logic recursivelyreturn self._apply_truncation_recursive(dumped_data)- def _apply_truncation_recursive(self, obj: Any, name: Optional[str] = None) -> Any:- """- Recursively apply AttributePolisher truncation logic to a data structure.+ def _apply_truncation_recursive(self, obj: Any, name: str | None = None) -> Any:+ """Recursively apply AttributePolisher truncation logic to a data structure.Args:obj: The object to process@@ -247,6 +263,7 @@ class CustomBaseModel(BaseModel):Returns:The processed object with truncation applied where appropriate+"""# First check if this specific object should be truncatedif name and AttributePolisher.should_truncate(name=name, value=obj):@@ -254,21 +271,20 @@ class CustomBaseModel(BaseModel):# If it's a dictionary, recurse into its valuesif isinstance(obj, dict):- obj_dict: Dict[str, Any] = obj- truncated_dict: Dict[str, Any] = {}+ obj_dict = cast("dict[str, Any]", obj)+ truncated_dict: dict[str, Any] = {}for key, value in obj_dict.items():truncated_dict[key] = self._apply_truncation_recursive(value, name=key)return truncated_dict# If it's a list, recurse into its items- elif isinstance(obj, list):- obj_list: List[Any] = obj+ if isinstance(obj, list):+ obj_list = cast("list[Any]", obj)return [self._apply_truncation_recursive(item, name=name) for item in obj_list]# If it's a tuple, recurse into its items and return as tuple- elif isinstance(obj, tuple):- return tuple(self._apply_truncation_recursive(item, name=name) for item in obj) # type: ignore+ if isinstance(obj, tuple):+ return tuple(self._apply_truncation_recursive(item, name=name) for item in obj) # pyright: ignore[reportUnknownVariableType]# For all other types, return as-is- else:- return obj+ return objdiff --git a/pipelex/tools/typing/structure_printer.py b/pipelex/tools/typing/structure_printer.pyindex 982064336..7cc8c4a0e 100644--- a/pipelex/tools/typing/structure_printer.py+++ b/pipelex/tools/typing/structure_printer.py@@ -1,5 +1,6 @@+import typesfrom enum import Enum-from typing import Any, Dict, List, Optional, Set, Type, Union, get_args, get_origin, get_type_hints+from typing import Any, Union, get_args, get_origin, get_type_hintsfrom pydantic import BaseModel@@ -9,24 +10,27 @@ from pipelex.types import StrEnumclass StructurePrinter:def pretty_type(self, tp: object) -> str:"""Pretty print a type, with special handling for containers, literals and enums."""- origin = getattr(tp, "__origin__", None)args = getattr(tp, "__args__", None)+ origin = get_origin(tp)if origin is None:if isinstance(tp, type):return tp.__name__return str(tp)- if origin is Union and args:+ args = get_args(tp)++ # was: if origin is Union and args:+ if origin in (Union, types.UnionType) and args:non_none = [a for a in args if a is not type(None)]- if len(non_none) == 1 and len(args) == 2:- return f"Optional[{self.pretty_type(non_none[0])}]"- return f"Union[{', '.join(self.pretty_type(a) for a in args)}]"+ if len(args) == 2 and len(non_none) == 1:+ return f"{self.pretty_type(non_none[0])} | None"+ return " | ".join(self.pretty_type(a) for a in args)if str(origin).endswith("Literal") and args: # Handle both typing.Literal and typing_extensions.Literal# For enum values, just get their values- values: List[str] = []+ values: list[str] = []for arg in args:- if isinstance(arg, Enum) or isinstance(arg, StrEnum):+ if isinstance(arg, (Enum, StrEnum)):values.append(f'"{arg.value}"')else:values.append(repr(arg))@@ -35,22 +39,21 @@ class StructurePrinter:return "Literal[\n " + ",\n ".join(values) + ",\n ]"return f"Literal[{', '.join(values)}]"- if (origin is list or origin is List) and args:+ if (origin is list or origin is list) and args:return f"List[{self.pretty_type(args[0])}]"- if (origin is dict or origin is Dict) and args:+ if (origin is dict or origin is dict) and args:return f"Dict[{self.pretty_type(args[0])}, {self.pretty_type(args[1])}]"return str(tp)def get_type_structure(self,- tp: Type[Any],- seen_types: Optional[Set[str]] = None,- collected_types: Optional[Dict[str, Type[Any]]] = None,- collected_enums: Optional[Dict[str, Type[Enum]]] = None,- base_class: Type[Any] = BaseModel,- ) -> List[str]:- """- Get the structure of a type, listing referenced subclasses of base_class and enums.+ tp: type[Any],+ seen_types: set[str] | None = None,+ collected_types: dict[str, type[Any]] | None = None,+ collected_enums: dict[str, type[Enum]] | None = None,+ base_class: type[Any] = BaseModel,+ ) -> list[str]:+ """Get the structure of a type, listing referenced subclasses of base_class and enums.Args:tp: The type to analyze@@ -58,6 +61,7 @@ class StructurePrinter:collected_types: Dictionary of collected types to analyzecollected_enums: Dictionary of collected enumsbase_class: The base class to check for inheritance (defaults to BaseModel)+"""if seen_types is None:seen_types = set()@@ -75,18 +79,19 @@ class StructurePrinter:return str(tp)args = get_args(tp)- if origin is Union:+ # was: if origin is Union:+ if origin in (Union, types.UnionType):non_none = [a for a in args if a is not type(None)]- if len(non_none) == 1:- return f"Optional[{format_type(non_none[0])}]"- return f"Union[{', '.join(format_type(a) for a in non_none)}]"+ if len(args) == 2 and len(non_none) == 1:+ return f"{format_type(non_none[0])} | None"+ return " | ".join(format_type(a) for a in args)if str(origin).endswith("Literal") and args: # Handle both typing.Literal and typing_extensions.Literal# For enum values, just get their values- values: List[str] = []+ values: list[str] = []enum_type = Nonefor arg in args:- if isinstance(arg, Enum) or isinstance(arg, StrEnum):+ if isinstance(arg, (Enum, StrEnum)):values.append(arg.value)if enum_type is None:enum_type = type(arg)@@ -97,41 +102,38 @@ class StructurePrinter:collected_enums[enum_type.__name__] = enum_type# Return multi-line format for Literal fieldsif len(values) > 1:- lines: List[str] = []- for value in values:- lines.append(f'"{value}"')+ lines = [f'"{value}"' for value in values]return "Literal[\n " + ",\n ".join(lines) + ",\n ]"return f"Literal[{', '.join(values)}]"- if origin in (list, List):+ if origin in (list, list):return f"List[{format_type(args[0])}]"- if origin in (dict, Dict):+ if origin in (dict, dict):return f"Dict[{format_type(args[0])}, {format_type(args[1])}]"return str(tp)- def collect_types(tp: Type[Any]) -> None:+ def collect_types(tp: type[Any]) -> None:"""Recursively collect types and enums"""+ args = get_args(tp)+origin = get_origin(tp)args = get_args(tp)if origin:- if origin is Union:- non_none = [a for a in args if a is not type(None)]- for arg in non_none:- if isinstance(arg, type):- collect_types(arg)- elif hasattr(arg, "__origin__"): # Handle nested generics- collect_types(arg)- elif origin in (list, List):- if isinstance(args[0], type):- collect_types(args[0])- elif hasattr(args[0], "__origin__"): # Handle nested generics- collect_types(args[0])- elif origin in (dict, Dict):+ # was: if origin is Union:+ if origin in (Union, types.UnionType):for arg in args:- if isinstance(arg, type):+ if arg is type(None):+ continue+ if isinstance(arg, type) or get_origin(arg) is not None:collect_types(arg)- elif hasattr(arg, "__origin__"): # Handle nested generics+ elif origin is list:+ inner = args[0]+ if isinstance(inner, type) or get_origin(inner) is not None:+ collect_types(inner)+ elif origin is dict:+ for arg in args:+ if isinstance(arg, type) or get_origin(arg) is not None:collect_types(arg)return@@ -150,8 +152,7 @@ class StructurePrinter:if (issubclass(base, BaseModel)and base is not BaseModel- and base.__module__ != "pydantic.main"- and base.__module__ != "abc"+ and base.__module__ not in ("pydantic.main", "abc")and not base.__module__.startswith("pipelex.core")):collect_types(base)@@ -161,11 +162,11 @@ class StructurePrinter:model_fields = getattr(tp, "model_fields", {})if model_fields:- for fname, _ in model_fields.items():+ for fname in model_fields:ftype = type_hints[fname]collect_types(ftype)elif hasattr(tp, "__annotations__"):- for fname, ftype in type_hints.items():+ for ftype in type_hints.values():collect_types(ftype)except (TypeError, AttributeError):# Handle cases where type hints cannot be retrieved@@ -175,7 +176,7 @@ class StructurePrinter:collect_types(tp)# Generate output- output: List[str] = []+ output: list[str] = []# First output the main class and its dependenciesfor class_name, class_type in collected_types.items():@@ -236,7 +237,7 @@ class StructurePrinter:fields = [(type_k, type_hints[type_k]) for type_k in sorted(type_hints.keys())]# Check if all fields are inherited- parent_fields: Set[str] = set()+ parent_fields: set[str] = set()for base in class_type.__bases__:try:parent_fields.update(get_type_hints(base).keys())@@ -257,10 +258,11 @@ class StructurePrinter:else:field_type = type_hints[fname]ftype_str = format_type(field_type)- # Check if field is Optional+field_origin = get_origin(field_type)field_args = get_args(field_type)- is_optional = field_origin is Union and type(None) in field_args++ is_optional = field_origin in (Union, types.UnionType) and type(None) in field_args# Handle default valuesfield_default = None@@ -281,9 +283,9 @@ class StructurePrinter:field_line += " = None"elif field_default is not None:if isinstance(field_default, bool):- field_line += f" = {str(field_default)}"+ field_line += f" = {field_default!s}"elif not isinstance(field_default, (BaseModel, list, dict)):- field_line += f" = {repr(field_default)}"+ field_line += f" = {field_default!r}"# Add description as a comment if available# First check if there's a direct field description from model_fields@@ -291,8 +293,8 @@ class StructurePrinter:field_line += f" # {field_description}"# Then check if the field type itself has model_fields and a description# This handles nested content types that have field descriptions- elif hasattr(ftype, "model_fields") and fname in ftype.model_fields and hasattr(ftype.model_fields[fname], "description"): # type: ignore- field_line += f" # {ftype.model_fields[fname].description}" # type: ignore+ elif hasattr(ftype, "model_fields") and fname in ftype.model_fields and hasattr(ftype.model_fields[fname], "description"): # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType]+ field_line += f" # {ftype.model_fields[fname].description}" # pyright: ignore[reportUnknownMemberType]# Split multi-line field linesif "\n" in field_line:@@ -319,7 +321,6 @@ class StructurePrinter:if enum_type.__doc__ and enum_type.__doc__.strip() != "An enumeration.":doc = enum_type.__doc__.strip()output.append(f' """{doc}"""')- for member in enum_type:- output.append(f' {member.name} = "{member.value}"')+ output += [f' {member.name} = "{member.value}"' for member in enum_type]return outputdiff --git a/pipelex/tools/typing/validation_utils.py b/pipelex/tools/typing/validation_utils.pyindex e99aaf2fa..447df6d62 100644--- a/pipelex/tools/typing/validation_utils.py+++ b/pipelex/tools/typing/validation_utils.py@@ -1,9 +1,8 @@-from typing import Any, List, Optional+from typing import Any-def has_exactly_one_among_attributes_from_list(obj: Any, attributes_list: List[str]) -> bool:- """- Checks if exactly one attribute from a list of attributes is non-None in an object.+def has_exactly_one_among_attributes_from_list(obj: Any, attributes_list: list[str]) -> bool:+ """Checks if exactly one attribute from a list of attributes is non-None in an object.This function is useful for validating mutually exclusive attributes in data modelsor configurations, ensuring that exactly one of the specified attributes has a value.@@ -15,14 +14,14 @@ def has_exactly_one_among_attributes_from_list(obj: Any, attributes_list: List[sReturns:bool: True if exactly one attribute from the list is non-None,False otherwise.+"""provided_attributes = [attribute for attribute in attributes_list if getattr(obj, attribute) is not None]return len(provided_attributes) == 1-def has_more_than_one_among_attributes_from_list(obj: Any, attributes_list: List[str]) -> bool:- """- Checks if more than one attribute from a list of attributes is non-None in an object.+def has_more_than_one_among_attributes_from_list(obj: Any, attributes_list: list[str]) -> bool:+ """Checks if more than one attribute from a list of attributes is non-None in an object.This function is useful for detecting conflicts in configurations or data modelswhere attributes are supposed to be mutually exclusive.@@ -34,14 +33,14 @@ def has_more_than_one_among_attributes_from_list(obj: Any, attributes_list: ListReturns:bool: True if more than one attribute from the list is non-None,False otherwise.+"""provided_attributes = [attribute for attribute in attributes_list if getattr(obj, attribute) is not None]return len(provided_attributes) > 1-def has_more_than_one_among_attributes_from_lists(obj: Any, attributes_lists: List[List[str]]) -> Optional[List[str]]:- """- Checks if more than one attribute from lists (plural) of attributes is non-None in an object.+def has_more_than_one_among_attributes_from_lists(obj: Any, attributes_lists: list[list[str]]) -> list[str] | None:+ """Checks if more than one attribute from lists (plural) of attributes is non-None in an object.This function is useful for detecting conflicts in configurations or data modelswhere attributes are supposed to be mutually exclusive.@@ -51,7 +50,8 @@ def has_more_than_one_among_attributes_from_lists(obj: Any, attributes_lists: Liattributes_lists (List[List[str]]): List of lists of attribute names to check.Returns:- Optional[List[str]]: The list of attributes that are non-None if more than one, None otherwise.+ List[str] | None: The list of attributes that are non-None if more than one, None otherwise.+"""for attributes_list in attributes_lists:if has_more_than_one_among_attributes_from_list(obj, attributes_list):diff --git a/pipelex/types.py b/pipelex/types.pyindex c4dacc782..1f1c2116c 100644--- a/pipelex/types.py+++ b/pipelex/types.py@@ -1,8 +1,13 @@-import sys+# ruff: noqa: UP-if sys.version_info >= (3, 11): # real std-lib- from enum import StrEnum-else: # back-port- from backports.strenum import StrEnum+try:+ from typing import Self # Python 3.11++except ImportError: # Python 3.10+ from typing_extensions import Self # type: ignore[assignment]-__all__ = ["StrEnum"]+try:+ from enum import StrEnum # Python 3.11++except ImportError: # Python 3.10+ from backports.strenum import StrEnum # type: ignore[assignment, import-not-found, no-redef]++__all__ = ["Self", "StrEnum"]diff --git a/pyproject.toml b/pyproject.tomlindex eebc2b9c8..0cad96831 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -1,6 +1,6 @@[project]name = "pipelex"-version = "0.10.3"+version = "0.10.2"description = "Pipelex is an open-source dev tool based on a simple declarative language that lets you define replicable, structured, composable LLM pipelines."authors = [{ name = "Evotis S.A.S.", email = "[email protected]" }]maintainers = [{ name = "Pipelex staff", email = "[email protected]" }]@@ -70,8 +70,9 @@ dev = ["boto3-stubs>=1.35.24","cocode==0.1.2","mypy>=1.11.2",- "pyright==1.1.398",+ "pyright>=1.1.405","pandas-stubs>=2.2.3.241126",+ "pylint>=3.3.8","pytest>=8.3.3","pytest-asyncio>=0.24.0","pytest-cov>=6.1.1",@@ -112,15 +113,15 @@ ignore_missing_imports = truemodule = ["filetype", "json2html", "pypdfium2", "pypdfium2.raw"][tool.pyright]+pythonVersion = "3.11"+include = ["pipelex", "tests"]+exclude = ["**/__pycache__", ".venv", ".git", "build", "dist", "tests"]analyzeUnannotatedFunctions = truedeprecateTypingAliases = falsedisableBytesTypePromotions = trueenableExperimentalFeatures = falseenableTypeIgnoreComments = true-exclude = ["**/node_modules", "**/__pycache__"]extraPaths = ["./tests"]-include = ["pipelex", "tests"]-pythonVersion = "3.11"reportAbstractUsage = "error"reportArgumentType = "error"reportAssertAlwaysTrue = "error"@@ -201,6 +202,7 @@ reportUnusedFunction = "error"reportUnusedImport = "none"reportUnusedVariable = "error"reportWildcardImportFromLibrary = "error"+reportUnusedParameter = "error"strictDictionaryInference = truestrictListInference = truestrictParameterNoneValue = true@@ -240,6 +242,7 @@ exclude_lines = [[tool.ruff]exclude = [+ "tests", # TO REMOVE".cursor",".git",".github",@@ -255,23 +258,104 @@ target-version = "py311"[tool.ruff.format][tool.ruff.lint]-ignore = ["F401"]-external = ["F401"]-select = [- "E4",- "E7",- "E9",- "F",- "A001",- "A002",- "A003",- "RUF008",- "RUF009",- "RUF012",- "RUF013",- "RUF100",- "E501",- "I",+select = ["ALL"]+ignore = [+ "ANN201", # Missing return type annotation for public function `my_func`+ "ANN202", # Missing return type annotation for private function `my_func`+ "ANN204", # Missing return type annotation for special method `my_func`+ "ANN206", # Missing return type annotation for classmethod `my_func`+ "ANN401", # Dynamically typed expressions (typing.Any) are disallowed in `...`+ "ASYNC230", # Async functions should not open files with blocking methods like `open`++ "C901", # Is to complex+ "COM812", # Checks for the absence of trailing commas.++ "D100", # Missing docstring in public module+ "D101", # Missing docstring in public class+ "D102", # Missing docstring in public method+ "D103", # Missing docstring in public function+ "D104", # Missing docstring in public package+ "D105", # Missing docstring in magic method+ "D107", # Missing docstring in __init__+ "D205", # 1 blank line required between summary line and description+ "D400", # First line should end with a period+ "D401", # First line of docstring should be in imperative mood: "My docstring...."+ "D404", # First word of the docstring should not be "This"+ "D415", # First line should end with a period, question mark, or exclamation point++ "DTZ005", # `datetime.datetime.now()` called without a `tz` argument++ "ERA001", # Found commented-out code++ "FBT001", # Boolean-typed positional argument in function definition+ "FBT002", # Boolean default positional argument in function definition+ "FBT003", #Boolean positional value in function call++ "FIX002", # Line contains TODO, consider resolving the issue++ "PLC0105", # `TypeVar` name "SomethingType" does not reflect its covariance; consider renaming it to "SomethingType_co"++ "PLR0911", # Too many return statements (/6)+ "PLR0912", # Too many branches (/12)+ "PLR0913", # Too many arguments in function definition (/5)+ "PLR0915", # Too many statements (/50)+ "PLR2004", # Magic value used in comparison, consider replacing `2` with a constant variable++ "PT013", # Incorrect import of `pytest`; use `import pytest` instead++ "PTH100", # `os.path.abspath()` should be replaced by `Path.resolve()`+ "PTH103", # `os.makedirs()` should be replaced by `Path.mkdir(parents=True)`+ "PTH109", # `os.getcwd()` should be replaced by `Path.cwd()`+ "PTH118", # `os.path.join()` should be replaced by `Path` with `/` operator+ "PTH120", # `os.path.dirname()` should be replaced by `Path.parent`+ "PTH110", # `os.path.exists()` should be replaced by `Path.exists()`+ "PTH112", # `os.path.isdir()` should be replaced by `Path.is_dir()`+ "PTH119", # `os.path.basename()` should be replaced by `Path.name`+ "PTH123", # `open()` should be replaced by `Path.open()`+ "PTH208", # Use `pathlib.Path.iterdir()` instead.++ "PYI051", # `Literal["auto"]` is redundant in a union with `str`++ "RET505", # superfluous-else-return++ "RUF001", # String contains ambiguous `′` (PRIME). Did you mean ``` (GRAVE ACCENT)?+ "RUF003", # Comment contains ambiguous `’` (RIGHT SINGLE QUOTATION MARK). Did you mean ``` (GRAVE ACCENT)?++ "SIM105", # Use `contextlib.suppress(ValueError)` instead of `try`-`except`-`pass`+ "SIM108", # Use ternary operator `description = func.__doc__.strip().split("\n")[0] if func.__doc__ else func.__name__` instead of `if`-`else`-block++ "S101", # Use of `assert` detected+ "S102", # Use of `exec` detected+ "S106", # Possible hardcoded password assigned to argument: "secret"+ "S105", # Possible hardcoded password assigned to: "child_secret"++ "TD002", # Missing author in TODO; try: `# TODO(<author_name>): ...` or `# TODO @<author_name>: ...`+ "TD003", # Missing issue link for this TODO++ "T201", # `print` found++ # TODO: stop ignoring these rules+ "BLE001", # Do not catch blind exception: `Exception`+ "B027", # Checks for empty methods in abstract base classes without an abstract decorator.+ "UP007", # Use `X | Y` for type annotations+ "UP036", # Version block is outdated for minimum Python version+ "SIM102", # Use a single `if` statement instead of nested `if` statements+ "S701", # Using jinja2 templates with `autoescape=False` is dangerous and can lead to XSS. Ensure `autoescape=True` or use the `select_autoescape` function.+ "N818", # Exception name `RootException` should be named with an Error suffix+ "TRY301", # Abstract `raise` to an inner function+ "PERF401", # Use a list comprehension to create a transformed list+ "PLW2901", # `for` loop variable `line` overwritten by assignment target+ "TRY300", # Consider moving this statement to an `else` block+ "UP035", # `typing.List` is deprecated, use `list` instead+ "RET503", # Missing explicit `return` at the end of function able to return non-`None` value+]++[tool.ruff.lint.pydocstyle]+convention = "google"++[tool.ruff.lint.per-file-ignores]+"tests/**/*.py" = [+ "T201", # Allow print statements in test files][tool.uv]@@ -291,3 +375,14 @@ packages = ["pipelex"][tool.hatch.metadata]allow-direct-references = true++[tool.pylint.main]+py-version = "3.11"+reports = false++[tool.pylint.messages_control]+disable = ["all"]+enable = [+ "W0101", # Unreachable code: Used when there is some code behind a "return" or "raise" statement, which will never be accessed.+]+ignore = [".venv", "__pycache__", "build", "dist", ".git"]diff --git a/tests/cases/__init__.py b/tests/cases/__init__.pyindex 2621add48..135dbd61f 100644--- a/tests/cases/__init__.py+++ b/tests/cases/__init__.py@@ -11,11 +11,11 @@ from .templates import JINJA2TestCasesfrom .urls import TestURLs__all__ = [- "PDFTestCases",- "ImageTestCases",- "TestURLs","ClassRegistryTestCases","FileHelperTestCases","Fruit",+ "ImageTestCases","JINJA2TestCases",+ "PDFTestCases",+ "TestURLs",]diff --git a/tests/cases/documents.py b/tests/cases/documents.pyindex 841a1942f..27a18dcb9 100644--- a/tests/cases/documents.py+++ b/tests/cases/documents.py@@ -1,6 +1,6 @@"""Document constants for testing."""-from typing import ClassVar, List+from typing import ClassVarclass PDFTestCases:@@ -12,10 +12,10 @@ class PDFTestCases:# Local file pathsPDF_FILE_PATH_1 = f"{TEST_DOCUMENT_DIRECTORY}/solar_system.pdf"PDF_FILE_PATH_2 = f"{TEST_DOCUMENT_DIRECTORY}/illustrated_train_article.pdf"- DOCUMENT_FILE_PATHS: ClassVar[List[str]] = [+ DOCUMENT_FILE_PATHS: ClassVar[list[str]] = [PDF_FILE_PATH_1,PDF_FILE_PATH_2,]# Remote URLs- DOCUMENT_URLS: ClassVar[List[str]] = ["https://www.w3.org/WAI/ER/tests/xhtml/testfiles/resources/pdf/dummy.pdf"]+ DOCUMENT_URLS: ClassVar[list[str]] = ["https://www.w3.org/WAI/ER/tests/xhtml/testfiles/resources/pdf/dummy.pdf"]diff --git a/tests/cases/images.py b/tests/cases/images.pyindex b4b6c4795..8bb42a77c 100644--- a/tests/cases/images.py+++ b/tests/cases/images.py@@ -1,6 +1,6 @@"""Image constants for testing."""-from typing import ClassVar, List+from typing import ClassVarclass ImageTestCases:@@ -16,13 +16,13 @@ class ImageTestCases:IMAGE_URL_PNG = "https://www.python.org/static/community_logos/python-logo-master-v3-TM.png"# File path collections- IMAGE_FILE_PATHS: ClassVar[List[str]] = [+ IMAGE_FILE_PATHS: ClassVar[list[str]] = [f"{TEST_IMAGE_DIRECTORY}/ai_lympics.png",f"{TEST_IMAGE_DIRECTORY}/animal_lympics.jpg",]# URL collections- IMAGE_URLS: ClassVar[List[str]] = [+ IMAGE_URLS: ClassVar[list[str]] = ["https://www.w3.org/People/mimasa/test/imgformat/img/w3c_home.png","https://www.google.com/images/branding/googlelogo/1x/googlelogo_color_272x92dp.png",]diff --git a/tests/cases/registry.py b/tests/cases/registry.pyindex 2b90d6852..ecde1e5e5 100644--- a/tests/cases/registry.py+++ b/tests/cases/registry.py@@ -1,6 +1,6 @@"""Registry and model constants for testing."""-from typing import ClassVar, List+from typing import ClassVarfrom pydantic import BaseModelfrom typing_extensions import override@@ -11,14 +11,14 @@ class ClassRegistryTestCases:MODEL_FOLDER_PATH = "tests/data/tools_data/mock_folder_with_classes"- CLASSES_TO_REGISTER: ClassVar[List[str]] = [+ CLASSES_TO_REGISTER: ClassVar[list[str]] = ["Class1","Class2","Class3","Class4",]- CLASSES_NOT_TO_REGISTER: ClassVar[List[str]] = [+ CLASSES_NOT_TO_REGISTER: ClassVar[list[str]] = ["ClassA","ClassB",]diff --git a/tests/cases/source_code.py b/tests/cases/source_code.pyindex 4253aa1b9..75c390e3e 100644--- a/tests/cases/source_code.py+++ b/tests/cases/source_code.py@@ -1,27 +1,26 @@"""Source code manipulation functions for testing."""-from typing import Listfrom pipelex.core.memory.working_memory import WorkingMemoryfrom pipelex.core.stuffs.stuff_content import TextContentdef wrap_lines(working_memory: WorkingMemory) -> TextContent:- """- Wraps each line of source code text with HTML span tags.+ """Wraps each line of source code text with HTML span tags.Args:working_memory: The working memory containing the source textReturns:TextContent with each line wrapped in span tags with class="line"+"""# Get the source text from working memory as stringsource_text = working_memory.get_stuff_as_str("source_text")# Split into lines and wrap each linelines = source_text.split("\n")- wrapped_lines: List[str] = []+ wrapped_lines: list[str] = []for line in lines:wrapped_line = f'<span class="line">{line}</span>'diff --git a/tests/cases/templates.py b/tests/cases/templates.pyindex de891ea42..9f1d3e719 100644--- a/tests/cases/templates.py+++ b/tests/cases/templates.py@@ -1,6 +1,6 @@"""Template constants for testing."""-from typing import Any, ClassVar, List, Tuple+from typing import Any, ClassVarfrom pipelex.tools.templating.templating_models import PromptingStyle, TagStyle, TextFormat@@ -11,12 +11,12 @@ class JINJA2TestCases:"""Jinja2 template test constants."""# Template names- JINJA2_NAME: ClassVar[List[str]] = [+ JINJA2_NAME: ClassVar[list[str]] = ["jinja2_test_template",]# Basic templates- JINJA2_FOR_ANY: ClassVar[List[str]] = [+ JINJA2_FOR_ANY: ClassVar[list[str]] = ["I want a {{ place_holder }} cocktail.",]@@ -78,7 +78,7 @@ Format filter html:"""# Template collections- JINJA2_FOR_STUFF: ClassVar[List[str]] = [+ JINJA2_FOR_STUFF: ClassVar[list[str]] = [JINJA2_FILTER_TAG,JINJA2_FILTER_FORMAT,JINJA2_FILTER_FORMAT_PLAIN,@@ -90,7 +90,7 @@ Format filter html:]# Style configurations- STYLE: ClassVar[List[PromptingStyle]] = [+ STYLE: ClassVar[list[PromptingStyle]] = [PromptingStyle(tag_style=TagStyle.NO_TAG,text_format=TextFormat.PLAIN,@@ -110,20 +110,20 @@ Format filter html:]# Test data- COLOR: ClassVar[List[str]] = [+ COLOR: ClassVar[list[str]] = ["red","blue","green",]- FRUIT: ClassVar[List[Fruit]] = [+ FRUIT: ClassVar[list[Fruit]] = [Fruit(color="red", name="cherry"),Fruit(color="blue", name="blueberry"),Fruit(color="green", name="grape"),]# Mixed object types for comprehensive testing- ANY_OBJECT: ClassVar[List[Tuple[str, Any]]] = [ # topic, any_object+ ANY_OBJECT: ClassVar[list[tuple[str, Any]]] = [ # topic, any_object("string", "test_string"),("integer", 42),("float", 3.14),diff --git a/tests/cases/urls.py b/tests/cases/urls.pyindex 173768f0b..cb707aa64 100644--- a/tests/cases/urls.py+++ b/tests/cases/urls.py@@ -1,6 +1,6 @@"""URL constants for testing."""-from typing import ClassVar, List+from typing import ClassVarclass TestURLs:@@ -9,7 +9,7 @@ class TestURLs:GCP_PUBLIC = "https://storage.googleapis.com/public_test_files_7fa6_4277_9ab/diagrams/gantt_tree_house.png"WIKIPEDIA = "https://upload.wikimedia.org/wikipedia/commons/thumb/e/ec/Olympic_rings_on_the_Eiffel_Tower_2024_%2819%29.jpg/440px-Olympic_rings_on_the_Eiffel_Tower_2024_%2819%29.jpg"- PUBLIC_URLS: ClassVar[List[str]] = [+ PUBLIC_URLS: ClassVar[list[str]] = [GCP_PUBLIC,WIKIPEDIA,]diff --git a/tests/conftest.py b/tests/conftest.pyindex e280c2d64..4e5954112 100644--- a/tests/conftest.py+++ b/tests/conftest.py@@ -40,7 +40,7 @@ def reset_pipelex_config_fixture():@pytest.fixture(scope="function", autouse=True)def pretty():# Code to run before each test- yield+ return# Code to run after each testdiff --git a/tests/e2e/pipelex/migration/test_migrate_v0_1_0_to_v0_2_0_e2e.py b/tests/e2e/pipelex/migration/test_migrate_v0_1_0_to_v0_2_0_e2e.pyindex 635c55de3..a6f726191 100644--- a/tests/e2e/pipelex/migration/test_migrate_v0_1_0_to_v0_2_0_e2e.py+++ b/tests/e2e/pipelex/migration/test_migrate_v0_1_0_to_v0_2_0_e2e.py@@ -1,7 +1,7 @@"""End-to-end tests for the migration functionality."""from pathlib import Path-from typing import Any, List, Tuple+from typing import Anyimport pytest@@ -13,7 +13,6 @@ class TestMigrationE2E:def test_real_world_migration_scenario(self, tmp_path: Path) -> None:"""Test a realistic migration scenario with multiple files and complex structures."""-# Create a realistic pipelex libraries structurepipelines_dir = tmp_path / "pipelines"pipelines_dir.mkdir()@@ -177,7 +176,6 @@ prompt_template = "Process: @input_text"def test_migration_with_complex_formatting(self, tmp_path: Path) -> None:"""Test migration preserves complex formatting and edge cases."""-test_file = tmp_path / "complex_formatting.toml"original_content = """# Complex formatting test filedomain = "formatting_test"@@ -272,7 +270,6 @@ Process this: @input_textdef test_dry_run_does_not_modify_files(self, tmp_path: Path) -> None:"""Test that dry run mode doesn't modify any files."""-test_file = tmp_path / "dry_run_test.toml"original_content = """domain = "dry_run_test"@@ -310,7 +307,6 @@ refines = "Document"def test_error_handling_with_permission_issues(self, tmp_path: Path, mocker: Any) -> None:"""Test proper error handling when file operations fail."""-test_file = tmp_path / "permission_test.toml"test_file.write_text("""[concept.Test]Concept = "Test concept"@@ -375,7 +371,7 @@ Concept = "Test concept"# Verify the challenge case is still there and unchangedchallenge_line_found = False- concept_lines_outside_multiline: List[Tuple[int, str]] = []+ concept_lines_outside_multiline: list[tuple[int, str]] = []lines = migrated_content.split("\n")for i, line in enumerate(lines, 1):diff --git a/tests/e2e/pipelex/pipes/pipe_controller/__init__.py b/tests/e2e/pipelex/pipes/pipe_controller/__init__.pynew file mode 100644index 000000000..e69de29bbdiff --git a/tests/e2e/pipelex/pipes/pipe_controller/pipe_condition/__init__.py b/tests/e2e/pipelex/pipes/pipe_controller/pipe_condition/__init__.pynew file mode 100644index 000000000..e69de29bbdiff --git a/tests/e2e/pipelex/pipes/pipe_controller/pipe_condition/test_pipe_condition_simple.py b/tests/e2e/pipelex/pipes/pipe_controller/pipe_condition/test_pipe_condition_simple.pyindex 9c6fca945..41fd7d788 100644--- a/tests/e2e/pipelex/pipes/pipe_controller/pipe_condition/test_pipe_condition_simple.py+++ b/tests/e2e/pipelex/pipes/pipe_controller/pipe_condition/test_pipe_condition_simple.py@@ -86,8 +86,8 @@ class TestPipeConditionSimple:structure_class_name="CategoryInput",),structure_class=CategoryInput,- )- ]+ ),+ ],)try:diff --git a/tests/e2e/pipelex/pipes/pipe_controller/pipe_parallel/__init__.py b/tests/e2e/pipelex/pipes/pipe_controller/pipe_parallel/__init__.pynew file mode 100644index 000000000..e69de29bbdiff --git a/tests/e2e/pipelex/pipes/pipe_controller/pipe_parallel/test_pipe_parallel_1.py b/tests/e2e/pipelex/pipes/pipe_controller/pipe_parallel/test_pipe_parallel_1.pyindex 581fd4ed4..4f2fb755a 100644--- a/tests/e2e/pipelex/pipes/pipe_controller/pipe_parallel/test_pipe_parallel_1.py+++ b/tests/e2e/pipelex/pipes/pipe_controller/pipe_parallel/test_pipe_parallel_1.py@@ -80,8 +80,8 @@ class TestPipeParallelDocumentAnalysis:assert hasattr(combined_analysis, "content_result")# Verify that both analyses contain meaningful content- length_analysis = cast(LengthAnalysis, length_result_stuff.content)- content_analysis = cast(ContentAnalysis, content_result_stuff.content)+ length_analysis = cast("LengthAnalysis", length_result_stuff.content)+ content_analysis = cast("ContentAnalysis", content_result_stuff.content)assert isinstance(length_analysis.analysis, str)assert isinstance(content_analysis.analysis, str)diff --git a/tests/e2e/pipelex/pipes/pipe_controller/pipe_sequence/__init__.py b/tests/e2e/pipelex/pipes/pipe_controller/pipe_sequence/__init__.pynew file mode 100644index 000000000..e69de29bbdiff --git a/tests/e2e/pipelex/pipes/pipe_controller/pipe_sequence/test_pipe_sequence_batching.py b/tests/e2e/pipelex/pipes/pipe_controller/pipe_sequence/test_pipe_sequence_batching.pyindex 6b5ea1fd1..14dc0cf0c 100644--- a/tests/e2e/pipelex/pipes/pipe_controller/pipe_sequence/test_pipe_sequence_batching.py+++ b/tests/e2e/pipelex/pipes/pipe_controller/pipe_sequence/test_pipe_sequence_batching.py@@ -36,7 +36,7 @@ async def test_review_analysis_sequence_with_batching(pipe_run_mode: PipeRunMode),structure_class=Document,),- ]+ ],)pipe = get_required_pipe(pipe_code="analyze_reviews_sequence")pipe_output = await pipe.run_pipe(@@ -82,7 +82,7 @@ async def test_review_analysis_sequence_with_batching(pipe_run_mode: PipeRunMode# Verify final product ratingstuff = pipe_output.working_memory.get_stuff("product_rating")# Use cast to tell the type system what we know about the object- product_rating_stuff = cast(ProductRating, stuff.content)+ product_rating_stuff = cast("ProductRating", stuff.content)assert product_rating_stuff is not None# Check that the ProductRating has meaningful valuesdiff --git a/tests/helpers/paths.py b/tests/helpers/paths.pyindex 45011de38..9fc99a60b 100644--- a/tests/helpers/paths.py+++ b/tests/helpers/paths.py@@ -16,6 +16,7 @@ def data_path(relative: str) -> Path:Example:>>> img = data_path("images/white_square.png")>>> doc = data_path("documents/solar_system.pdf")+"""data_dir = resources.files("tests.data")return Path(str(data_dir)) / relativediff --git a/tests/integration/pipelex/client/__init__.py b/tests/integration/pipelex/client/__init__.pynew file mode 100644index 000000000..e69de29bbdiff --git a/tests/integration/pipelex/client/test_client.py b/tests/integration/pipelex/client/test_client.pyindex 62777912c..f91ca2ac0 100644--- a/tests/integration/pipelex/client/test_client.py+++ b/tests/integration/pipelex/client/test_client.py@@ -1,4 +1,3 @@-from typing import Listimport pytestfrom pydantic import BaseModel@@ -17,16 +16,15 @@ from pipelex.core.stuffs.stuff_factory import StuffFactoryclass Example(BaseModel):pipe_code: str- memory: List[Stuff]+ memory: list[Stuff]@pytest.mark.pipelex_api@pytest.mark.asyncio(loop_scope="class")class TestPipelexApiClient:@pytest.fixture- def examples(self) -> List[Example]:- """- Fixture providing test example for API client tests.+ def examples(self) -> list[Example]:+ """Fixture providing test example for API client tests."""return [Example(@@ -74,12 +72,12 @@ class TestPipelexApiClient:distinctive bionic design inspired by peregrine falcons, the train uses an innovativehybrid propulsion system that combines traditional electric motors withcompressed air boosters for acceleration phases.- """+ """,),),StuffFactory.make_stuff(concept=ConceptFactory.make(- concept_code="Question", domain="answer", definition="answer.Question", structure_class_name="Question"+ concept_code="Question", domain="answer", definition="answer.Question", structure_class_name="Question",),name="question",content=TextContent(text="Aerodynamic features?"),@@ -90,13 +88,13 @@ class TestPipelexApiClient:async def test_client_execute_pipeline(self,- examples: List[Example],+ examples: list[Example],):- """- Test the execute_pipe method with the example.+ """Test the execute_pipe method with the example.Args:examples: List of test examples from the fixture+"""for example in examples:# Create working memory from example datadiff --git a/tests/integration/pipelex/cogt/__init__.py b/tests/integration/pipelex/cogt/__init__.pynew file mode 100644index 000000000..e69de29bbdiff --git a/tests/integration/pipelex/cogt/test_content_generator.py b/tests/integration/pipelex/cogt/test_content_generator.pyindex 658fb5235..e5781b45c 100644--- a/tests/integration/pipelex/cogt/test_content_generator.py+++ b/tests/integration/pipelex/cogt/test_content_generator.py@@ -1,4 +1,3 @@-from typing import Listimport pytestfrom pytest import FixtureRequest@@ -90,7 +89,7 @@ class TestContentGenerator:async def test_make_object_list_direct(self, request: FixtureRequest):llm_setting_for_object = get_model_deck().get_llm_setting(llm_choice="llm_for_testing_gen_object")- person_list_direct: List[Employee] = await get_content_generator().make_object_list_direct(+ person_list_direct: list[Employee] = await get_content_generator().make_object_list_direct(job_metadata=JobMetadata(job_name=request.node.originalname), # type: ignoreobject_class=Employee,llm_prompt_for_object_list=LLMPrompt(user_text=USER_TEXTS_FOR_PEOPLE_STR),diff --git a/tests/integration/pipelex/cogt/test_data.py b/tests/integration/pipelex/cogt/test_data.pyindex 279051250..72c75a44b 100644--- a/tests/integration/pipelex/cogt/test_data.py+++ b/tests/integration/pipelex/cogt/test_data.py@@ -1,4 +1,4 @@-from typing import Any, ClassVar, Dict, List, Tuple+from typing import Any, ClassVarfrom pydantic import BaseModel, Field, field_validator@@ -59,17 +59,17 @@ class LLMVisionTestCases:PATH_IMG_GANTT_1 = f"{TEST_IMAGE_DIRECTORY}/gantt_tree_house.png"- IMAGE_PATHS: ClassVar[List[Tuple[str, str]]] = [ # topic, image_path+ IMAGE_PATHS: ClassVar[list[tuple[str, str]]] = [ # topic, image_path("AI Lympics PNG", PATH_IMG_PNG_1),("AI Lympics JPEG", PATH_IMG_JPEG_1),("Gantt Chart", PATH_IMG_GANTT_1),("Eiffel Tower", PATH_IMG_JPEG_3),]- IMAGE_PATH_PAIRS: ClassVar[List[Tuple[str, Tuple[str, str]]]] = [ # topic, image_pair+ IMAGE_PATH_PAIRS: ClassVar[list[tuple[str, tuple[str, str]]]] = [ # topic, image_pair("AI Lympics PNG", (PATH_IMG_PNG_1, PATH_IMG_PNG_2)),]- IMAGE_URLS: ClassVar[List[Tuple[str, str]]] = [ # topic, image_uri+ IMAGE_URLS: ClassVar[list[tuple[str, str]]] = [ # topic, image_uri("Alan Turing",URL_WIKIPEDIA_ALAN_TURING,@@ -85,7 +85,7 @@ class LLMTestConstants:USER_TEXT_SHORT = "In one sentence, who is Bill Gates?"# USER_TEXT_SHORT = "What's the biggest football match tonight in Europe?"PROMPT_TEMPLATE_TEXT = "Can you give one example of flower which is {color} in color ?"- PROMPT_COLOR_EXAMPLES: ClassVar[List[str]] = [+ PROMPT_COLOR_EXAMPLES: ClassVar[list[str]] = ["red","blue","green",@@ -104,17 +104,17 @@ class LLMTestCases:When my son was 7 he was 3ft tall. When he was 8 he was 4ft tall. When he was 9 he was 5ft tall.How tall do you think he was when he was 12? and at 15?"""- SINGLE_TEXT: ClassVar[List[Tuple[str, str]]] = [ # topic, prompt_text+ SINGLE_TEXT: ClassVar[list[tuple[str, str]]] = [ # topic, prompt_text("Haiku", USER_TEXT_HAIKU),("Tricky", USER_TEXT_TRICKY),]- SINGLE_OBJECT: ClassVar[List[Tuple[str, BaseModel]]] = [+ SINGLE_OBJECT: ClassVar[list[tuple[str, BaseModel]]] = [("name: John, age: 30", Person(name="John", age=30)),("Betty Draper, 51", Person(name="Betty Draper", age=51)),("Whiskers, the cat", Pet(species=PetSpecies.CAT, name="Whiskers")),("Whiskers, the dog", Pet(species=PetSpecies.DOG, name="Whiskers")),]- MULTIPLE_OBJECTS: ClassVar[List[List[Tuple[str, BaseModel]]]] = [+ MULTIPLE_OBJECTS: ClassVar[list[list[tuple[str, BaseModel]]]] = [[("name: John, age: 30", Person(name="John", age=30)),# ("Betty Draper, 51", Person(name="Betty Draper", age=51)),@@ -155,7 +155,7 @@ class SerDeTestLLMCases:)# Dictionary representation example ------------------------------------- DICT_1: ClassVar[Dict[str, Any]] = {+ DICT_1: ClassVar[dict[str, Any]] = {"proto_prompt": LLMPrompt(system_text=None,user_text="Some user text in the template",@@ -176,13 +176,13 @@ class SerDeTestLLMCases:)# Group constants for parametrization ----------------------------------- PYDANTIC_EXAMPLES: ClassVar[List[BaseModel]] = [+ PYDANTIC_EXAMPLES: ClassVar[list[BaseModel]] = [MY_PROMPT_TEMPLATE_MODEL_1,MY_PROMPT_TEMPLATE_MODEL_2,]- PYDANTIC_EXAMPLES_USING_SUBCLASS: ClassVar[List[BaseModel]] = [+ PYDANTIC_EXAMPLES_USING_SUBCLASS: ClassVar[list[BaseModel]] = [PROMPT_WITH_IMAGE_PATH,]- PYDANTIC_EXAMPLES_DICT: ClassVar[List[Dict[str, Any]]] = [+ PYDANTIC_EXAMPLES_DICT: ClassVar[list[dict[str, Any]]] = [DICT_1,]diff --git a/tests/integration/pipelex/cogt/test_llm_gen_object.py b/tests/integration/pipelex/cogt/test_llm_gen_object.pyindex 3a609494b..321bd7782 100644--- a/tests/integration/pipelex/cogt/test_llm_gen_object.py+++ b/tests/integration/pipelex/cogt/test_llm_gen_object.py@@ -1,5 +1,4 @@import asyncio-from typing import List, Tupleimport pytestfrom pydantic import BaseModel@@ -56,12 +55,12 @@ class TestLLMGenObject:@pytest.mark.parametrize("case_tuples", LLMTestCases.MULTIPLE_OBJECTS)async def test_gen_object_async_multiple_using_handle(- self, llm_job_params: LLMJobParams, llm_handle: str, case_tuples: List[Tuple[str, BaseModel]]+ self, llm_job_params: LLMJobParams, llm_handle: str, case_tuples: list[tuple[str, BaseModel]],):llm_worker = get_llm_worker(llm_handle=llm_handle)if not llm_worker.is_gen_object_supported:pytest.skip(f"'{llm_worker.desc}' does not support object generation")- tasks: List[asyncio.Task[BaseModel]] = []+ tasks: list[asyncio.Task[BaseModel]] = []for case_tuple in case_tuples:user_text, expected_instance = case_tupleexpected_class = expected_instance.__class__@@ -81,11 +80,11 @@ class TestLLMGenObject:assert output_instance.model_dump(serialize_as_any=True) == expected_instance.model_dump(serialize_as_any=True)@pytest.mark.parametrize("case_tuples", LLMTestCases.MULTIPLE_OBJECTS)- async def test_gen_object_async_multiple_using_llm_preset(self, llm_preset_id: str, case_tuples: List[Tuple[str, BaseModel]]):+ async def test_gen_object_async_multiple_using_llm_preset(self, llm_preset_id: str, case_tuples: list[tuple[str, BaseModel]]):llm_worker, llm_job = get_async_worker_and_job(llm_preset_id=llm_preset_id, user_text=case_tuples[0][0])if not llm_worker.is_gen_object_supported:pytest.skip(f"'{llm_worker.desc}' does not support object generation")- tasks: List[asyncio.Task[BaseModel]] = []+ tasks: list[asyncio.Task[BaseModel]] = []for case_tuple in case_tuples:user_text, expected_instance = case_tupleexpected_class = expected_instance.__class__diff --git a/tests/integration/pipelex/cogt/test_llm_gen_text.py b/tests/integration/pipelex/cogt/test_llm_gen_text.pyindex b340b04b8..d0b5e90b2 100644--- a/tests/integration/pipelex/cogt/test_llm_gen_text.py+++ b/tests/integration/pipelex/cogt/test_llm_gen_text.py@@ -1,5 +1,4 @@import asyncio-from typing import List, Tupleimport pytest@@ -14,7 +13,7 @@ from pipelex.hub import get_llm_worker, get_model_deckfrom tests.integration.pipelex.cogt.test_data import LLMTestCases-def get_worker_and_job(llm_preset_id: str, user_text: str) -> Tuple[LLMWorkerAbstract, LLMJob]:+def get_worker_and_job(llm_preset_id: str, user_text: str) -> tuple[LLMWorkerAbstract, LLMJob]:llm_setting = get_model_deck().get_llm_setting(llm_choice=llm_preset_id)pretty_print(llm_setting, title=llm_preset_id)pretty_print(user_text)@@ -56,7 +55,7 @@ class TestLLMGenText:job_params_base = llm_job.job_paramsmax_tokens = 30temperature = 0.1- tasks: List[asyncio.Task[str]] = []+ tasks: list[asyncio.Task[str]] = []for _ in range(4):max_tokens += 50temperature += 0.2diff --git a/tests/integration/pipelex/cogt/test_llm_report.py b/tests/integration/pipelex/cogt/test_llm_report.pyindex 96807b22c..2ac786f5b 100644--- a/tests/integration/pipelex/cogt/test_llm_report.py+++ b/tests/integration/pipelex/cogt/test_llm_report.py@@ -1,5 +1,4 @@import asyncio-from typing import Listimport pytest@@ -44,7 +43,7 @@ class TestLLMReport:"llm_for_testing_gen_text","llm_for_testing_gen_object",]- tasks: List[asyncio.Task[str]] = []+ tasks: list[asyncio.Task[str]] = []for llm_preset_id in llm_preset_ids:llm_worker, llm_job = self._get_async_worker_and_job(llm_preset_id=llm_preset_id, prompt_text=prompt_text)job_params_base = llm_job.job_paramsdiff --git a/tests/integration/pipelex/cogt/test_llm_vision.py b/tests/integration/pipelex/cogt/test_llm_vision.pyindex da74826aa..99b63a216 100644--- a/tests/integration/pipelex/cogt/test_llm_vision.py+++ b/tests/integration/pipelex/cogt/test_llm_vision.py@@ -1,4 +1,3 @@-from typing import Tupleimport pytest@@ -74,7 +73,7 @@ class TestLLMVision:pytest.skip(f"Prompt Image format not supported for this LLM: {llm_handle_for_vision} because {exc}")@pytest.mark.parametrize("topic, image_pair", LLMVisionTestCases.IMAGE_PATH_PAIRS)- async def test_gen_text_from_vision_2_images(self, llm_handle_for_vision: str, topic: str, image_pair: Tuple[str, str]):+ async def test_gen_text_from_vision_2_images(self, llm_handle_for_vision: str, topic: str, image_pair: tuple[str, str]):prompt_image1 = PromptImagePath(file_path=image_pair[0])prompt_image2 = PromptImagePath(file_path=image_pair[1])llm_worker = get_llm_worker(llm_handle=llm_handle_for_vision)diff --git a/tests/integration/pipelex/cogt/test_serde_llm_classes.py b/tests/integration/pipelex/cogt/test_serde_llm_classes.pyindex 532830be3..caca27a2c 100644--- a/tests/integration/pipelex/cogt/test_serde_llm_classes.py+++ b/tests/integration/pipelex/cogt/test_serde_llm_classes.py@@ -1,4 +1,4 @@-from typing import Any, Dict+from typing import Anyimport pytestfrom kajson import kajson@@ -42,7 +42,7 @@ class TestSerDeLLMClasses:@pytest.mark.parametrize("test_dict", SerDeTestLLMCases.PYDANTIC_EXAMPLES_DICT)def test_serde_model_validate_from_dict(self,- test_dict: Dict[str, Any],+ test_dict: dict[str, Any],):pretty_print(test_dict, title="Original dict")# Deserialize the dictionary to a LLMPromptTemplate model@@ -63,7 +63,7 @@ class TestSerDeLLMClasses:@pytest.mark.parametrize("test_dict", SerDeTestLLMCases.PYDANTIC_EXAMPLES_DICT)def test_serde_instantiate_from_kwargs(self,- test_dict: Dict[str, Any],+ test_dict: dict[str, Any],):pretty_print(test_dict, title="Original dict")# Deserialize the dictionary to a LLMPromptTemplate modeldiff --git a/tests/integration/pipelex/conftest.py b/tests/integration/pipelex/conftest.pyindex f284d92d7..898cb0107 100644--- a/tests/integration/pipelex/conftest.py+++ b/tests/integration/pipelex/conftest.py@@ -1,4 +1,3 @@-from typing import Optionalimport pytest@@ -11,7 +10,7 @@ from pipelex.plugins.plugin_sdk_registry import Plugin# "llm_for_testing_gen_text",# "llm_for_testing_gen_object","llm_for_creative_writing",- ]+ ],)def llm_preset_id(request: pytest.FixtureRequest) -> str:assert isinstance(request.param, str)@@ -63,7 +62,7 @@ def llm_preset_id(request: pytest.FixtureRequest) -> str:"gemini-2.5-flash-lite",# "gemini-2.5-flash",# "gemini-2.5-pro",- ]+ ],)def llm_handle(request: pytest.FixtureRequest) -> str:assert isinstance(request.param, str)@@ -95,7 +94,7 @@ def llm_handle(request: pytest.FixtureRequest) -> str:# "gemini-2.5-flash",# "gemini-2.5-pro",# "qwen3:8b",- ]+ ],)def llm_handle_for_vision(request: pytest.FixtureRequest) -> str:assert isinstance(request.param, str)@@ -106,7 +105,7 @@ def llm_handle_for_vision(request: pytest.FixtureRequest) -> str:params=[Plugin(sdk="openai", backend="openai"),Plugin(sdk="azure_openai", backend="azure_openai"),- ]+ ],)def plugin_for_openai(request: pytest.FixtureRequest) -> Plugin:assert isinstance(request.param, Plugin)@@ -117,7 +116,7 @@ def plugin_for_openai(request: pytest.FixtureRequest) -> Plugin:params=[Plugin(sdk="anthropic", backend="anthropic"),Plugin(sdk="bedrock_anthropic", backend="bedrock_anthropic"),- ]+ ],)def plugin_for_anthropic(request: pytest.FixtureRequest) -> Plugin:assert isinstance(request.param, Plugin)@@ -128,9 +127,9 @@ def plugin_for_anthropic(request: pytest.FixtureRequest) -> Plugin:params=[# None,"https://inference.pipelex.com/v1",- ]+ ],)-def openai_endpoint(request: pytest.FixtureRequest) -> Optional[str]:+def openai_endpoint(request: pytest.FixtureRequest) -> str | None:assert isinstance(request.param, str) or request.param is Nonereturn request.param@@ -176,7 +175,7 @@ def openai_endpoint(request: pytest.FixtureRequest) -> Optional[str]:# "pipelex/gpt-4o-mini",# "pipelex/claude-3.7-sonnet",# "vertex_ai/gemini-2.0-flash",- ]+ ],)def llm_id(request: pytest.FixtureRequest) -> str:assert isinstance(request.param, str)@@ -190,7 +189,7 @@ def llm_id(request: pytest.FixtureRequest) -> str:max_tokens=None,seed=None,),- ]+ ],)def llm_job_params(request: pytest.FixtureRequest) -> LLMJobParams:assert isinstance(request.param, LLMJobParams)@@ -204,7 +203,7 @@ def llm_job_params(request: pytest.FixtureRequest) -> LLMJobParams:# "flux-pro/v1.1-ultra","fast-lightning-sdxl","gpt-image-1",- ]+ ],)def img_gen_handle(request: pytest.FixtureRequest) -> str:assert isinstance(request.param, str)@@ -215,7 +214,7 @@ def img_gen_handle(request: pytest.FixtureRequest) -> str:params=["pypdfium2-extract-text","mistral-ocr",- ]+ ],)def ocr_handle(request: pytest.FixtureRequest) -> str:assert isinstance(request.param, str)@@ -225,7 +224,7 @@ def ocr_handle(request: pytest.FixtureRequest) -> str:@pytest.fixture(params=["mistral-ocr",- ]+ ],)def ocr_handle_from_image(request: pytest.FixtureRequest) -> str:assert isinstance(request.param, str)@@ -235,7 +234,7 @@ def ocr_handle_from_image(request: pytest.FixtureRequest) -> str:@pytest.fixture(params=["base_ocr_mistral",- ]+ ],)def ocr_choice_for_pdf(request: pytest.FixtureRequest) -> str:assert isinstance(request.param, str)@@ -245,7 +244,7 @@ def ocr_choice_for_pdf(request: pytest.FixtureRequest) -> str:@pytest.fixture(params=["base_ocr_mistral",- ]+ ],)def ocr_choice_for_image(request: pytest.FixtureRequest) -> str:assert isinstance(request.param, str)diff --git a/tests/integration/pipelex/core/memory/__init__.py b/tests/integration/pipelex/core/memory/__init__.pynew file mode 100644index 000000000..e69de29bbdiff --git a/tests/integration/pipelex/core/memory/test_working_memory_factory.py b/tests/integration/pipelex/core/memory/test_working_memory_factory.pyindex 95ccdff34..0f39216ca 100644--- a/tests/integration/pipelex/core/memory/test_working_memory_factory.py+++ b/tests/integration/pipelex/core/memory/test_working_memory_factory.py@@ -11,7 +11,7 @@ class TestWorkingMemoryFactory:"text_item": {"concept_code": NativeConceptEnum.TEXT,"content": "Hello, world!",- }+ },}working_memory = WorkingMemoryFactory.make_from_compact_memory(compact_memory)@@ -33,7 +33,7 @@ class TestWorkingMemoryFactory:"text_and_images": {"text": {"text": "This is a complex document page with multiple images and rich text content. "- "It demonstrates nested structured content handling."+ "It demonstrates nested structured content handling.",},"images": [{@@ -51,7 +51,7 @@ class TestWorkingMemoryFactory:},"page_view": {"url": "mock_url", "caption": "Full page screenshot"},},- }+ },}working_memory = WorkingMemoryFactory.make_from_compact_memory(compact_memory)diff --git a/tests/integration/pipelex/pipes/__init__.py b/tests/integration/pipelex/pipes/__init__.pynew file mode 100644index 000000000..e69de29bbdiff --git a/tests/integration/pipelex/pipes/conftest.py b/tests/integration/pipelex/pipes/conftest.pyindex 33ff3509f..c635d5faf 100644--- a/tests/integration/pipelex/pipes/conftest.py+++ b/tests/integration/pipelex/pipes/conftest.py@@ -1,5 +1,5 @@import os-from typing import AsyncIterator, Awaitable, Callable+from collections.abc import AsyncIterator, Awaitable, Callableimport pytest_asynciofrom pytest import FixtureRequest@@ -14,8 +14,7 @@ from tests.conftest import TEST_OUTPUTS_DIR@pytest_asyncio.fixture # pyright: ignore[reportUntypedFunctionDecorator, reportUnknownMemberType]async def pipe_result_handler(request: FixtureRequest) -> AsyncIterator[tuple[str, ActivityHandlerForResultFiles]]:- """- This fixture is used to handle the result of a pipe run in unit tests.+ """This fixture is used to handle the result of a pipe run in unit tests.It creates and registers an activity handler to save the activities of the pipe into a specific directory.It returns a tuple with the result directory path and the activity handler, which enables thecalling test to save whatever it wants into that result directory.diff --git a/tests/integration/pipelex/pipes/controller/pipe_batch/__init__.py b/tests/integration/pipelex/pipes/controller/pipe_batch/__init__.pynew file mode 100644index 000000000..e69de29bbdiff --git a/tests/integration/pipelex/pipes/controller/pipe_batch/test_pipe_batch_simple.py b/tests/integration/pipelex/pipes/controller/pipe_batch/test_pipe_batch_simple.pyindex bd5c25b78..b34893d67 100644--- a/tests/integration/pipelex/pipes/controller/pipe_batch/test_pipe_batch_simple.py+++ b/tests/integration/pipelex/pipes/controller/pipe_batch/test_pipe_batch_simple.py@@ -1,6 +1,6 @@"""Simple integration test for PipeBatch controller."""-from typing import List, cast+from typing import castimport pytestfrom pytest import FixtureRequest@@ -73,7 +73,7 @@ class TestPipeBatchSimple:text_list_stuff = StuffFactory.make_stuff(concept=ConceptFactory.make_native_concept(native_concept_data=NATIVE_CONCEPTS_DATA[NativeConceptEnum.TEXT]),- content=ListContent[StuffContent](items=cast(List[StuffContent], text_items)),+ content=ListContent[StuffContent](items=cast("list[StuffContent]", text_items)),name="text_list",)@@ -95,7 +95,7 @@ class TestPipeBatchSimple:assert isinstance(text_list.content, ListContent)# Cast the content to the proper type for type checking- list_content = cast(ListContent[TextContent], text_list.content) # type: ignore+ list_content = cast("ListContent[TextContent]", text_list.content) # type: ignoreassert len(list_content.items) == 3# Verify each item in the list@@ -109,7 +109,7 @@ class TestPipeBatchSimple:# Actually run the PipeBatch pipepipe_output = await pipe_batch._run_controller_pipe( # pyright: ignore[reportPrivateUsage]- job_metadata=JobMetadata(job_name=cast(str, request.node.originalname)), # type: ignore+ job_metadata=JobMetadata(job_name=cast("str", request.node.originalname)), # type: ignoreworking_memory=working_memory,pipe_run_params=PipeRunParamsFactory.make_run_params(pipe_run_mode=pipe_run_mode),output_name="batch_result",@@ -141,7 +141,7 @@ class TestPipeBatchSimple:original_list = final_working_memory.get_stuff("text_list")assert original_list is not Noneassert isinstance(original_list.content, ListContent)- original_items = cast(ListContent[TextContent], original_list.content) # type: ignore+ original_items = cast("ListContent[TextContent]", original_list.content) # type: ignoreassert len(original_items.items) == 3assert original_items.items[0].text == "hello"assert original_items.items[1].text == "world"@@ -155,7 +155,7 @@ class TestPipeBatchSimple:# Verify the batch result content matches exactlyassert isinstance(batch_result.content, ListContent)- result_list = cast(ListContent[TextContent], batch_result.content) # type: ignore+ result_list = cast("ListContent[TextContent]", batch_result.content) # type: ignoreassert len(result_list.items) == 3if pipe_run_mode != PipeRunMode.DRY:assert result_list.items[0].text == "UPPER: HELLO"diff --git a/tests/integration/pipelex/pipes/controller/pipe_condition/__init__.py b/tests/integration/pipelex/pipes/controller/pipe_condition/__init__.pynew file mode 100644index 000000000..e69de29bbdiff --git a/tests/integration/pipelex/pipes/controller/pipe_condition/test_pipe_condition_simple.py b/tests/integration/pipelex/pipes/controller/pipe_condition/test_pipe_condition_simple.pyindex 3b2bb5dd8..9107b8acd 100644--- a/tests/integration/pipelex/pipes/controller/pipe_condition/test_pipe_condition_simple.py+++ b/tests/integration/pipelex/pipes/controller/pipe_condition/test_pipe_condition_simple.py@@ -66,7 +66,7 @@ class TestPipeConditionSimple:assert input_text.content.text == "hello world"pipe_output = await pipe_condition.run_pipe(- job_metadata=JobMetadata(job_name=cast(str, request.node.originalname)), # type: ignore+ job_metadata=JobMetadata(job_name=cast("str", request.node.originalname)), # type: ignoreworking_memory=working_memory,pipe_run_params=PipeRunParamsFactory.make_run_params(pipe_run_mode=pipe_run_mode),)@@ -135,7 +135,7 @@ class TestPipeConditionSimple:# Actually run the PipeCondition pipepipe_output = await pipe_condition.run_pipe(- job_metadata=JobMetadata(job_name=cast(str, request.node.originalname)), # type: ignore+ job_metadata=JobMetadata(job_name=cast("str", request.node.originalname)), # type: ignoreworking_memory=working_memory,pipe_run_params=PipeRunParamsFactory.make_run_params(pipe_run_mode=pipe_run_mode),)diff --git a/tests/integration/pipelex/pipes/controller/pipe_condition/test_pipe_condition_validation.py b/tests/integration/pipelex/pipes/controller/pipe_condition/test_pipe_condition_validation.pyindex cd6e2d004..bbe1c6ed4 100644--- a/tests/integration/pipelex/pipes/controller/pipe_condition/test_pipe_condition_validation.py+++ b/tests/integration/pipelex/pipes/controller/pipe_condition/test_pipe_condition_validation.py@@ -18,7 +18,7 @@ class TestPipeConditionValidation:concept_codes_from_the_same_domain=["TestConcept"],)concept_2 = ConceptFactory.make_from_blueprint(- concept_code="Result", domain=domain, blueprint=ConceptBlueprint(definition="Lorem Ipsum"), concept_codes_from_the_same_domain=["Result"]+ concept_code="Result", domain=domain, blueprint=ConceptBlueprint(definition="Lorem Ipsum"), concept_codes_from_the_same_domain=["Result"],)concept_library = get_concept_provider()concept_library.add_concepts([concept_1, concept_2])@@ -58,7 +58,7 @@ class TestPipeConditionValidation:concept_codes_from_the_same_domain=["TestConcept"],)concept_2 = ConceptFactory.make_from_blueprint(- concept_code="Result", domain=domain, blueprint=ConceptBlueprint(definition="Lorem Ipsum"), concept_codes_from_the_same_domain=["Result"]+ concept_code="Result", domain=domain, blueprint=ConceptBlueprint(definition="Lorem Ipsum"), concept_codes_from_the_same_domain=["Result"],)concept_library.add_concepts([concept_1, concept_2])diff --git a/tests/integration/pipelex/pipes/controller/pipe_parallel/__init__.py b/tests/integration/pipelex/pipes/controller/pipe_parallel/__init__.pynew file mode 100644index 000000000..e69de29bbdiff --git a/tests/integration/pipelex/pipes/controller/pipe_parallel/test_pipe_parallel_simple.py b/tests/integration/pipelex/pipes/controller/pipe_parallel/test_pipe_parallel_simple.pyindex 4529ab07b..572850e0c 100644--- a/tests/integration/pipelex/pipes/controller/pipe_parallel/test_pipe_parallel_simple.py+++ b/tests/integration/pipelex/pipes/controller/pipe_parallel/test_pipe_parallel_simple.py@@ -78,7 +78,7 @@ class TestPipeParallelSimple:# Actually run the PipeParallel pipepipe_output = await pipe_parallel.run_pipe(- job_metadata=JobMetadata(job_name=cast(str, request.node.originalname)), # type: ignore+ job_metadata=JobMetadata(job_name=cast("str", request.node.originalname)), # type: ignoreworking_memory=working_memory,output_name="parallel_results",pipe_run_params=PipeRunParamsFactory.make_run_params(pipe_run_mode=pipe_run_mode),@@ -172,7 +172,7 @@ class TestPipeParallelSimple:# Actually run the PipeParallel pipepipe_output = await pipe_parallel.run_pipe(- job_metadata=JobMetadata(job_name=cast(str, request.node.originalname)), # type: ignore+ job_metadata=JobMetadata(job_name=cast("str", request.node.originalname)), # type: ignoreworking_memory=working_memory,output_name="parallel_results",pipe_run_params=PipeRunParamsFactory.make_run_params(pipe_run_mode=pipe_run_mode),diff --git a/tests/integration/pipelex/pipes/controller/pipe_parallel/test_pipe_parallel_validation.py b/tests/integration/pipelex/pipes/controller/pipe_parallel/test_pipe_parallel_validation.pyindex ed89f5364..1c521b07d 100644--- a/tests/integration/pipelex/pipes/controller/pipe_parallel/test_pipe_parallel_validation.py+++ b/tests/integration/pipelex/pipes/controller/pipe_parallel/test_pipe_parallel_validation.py@@ -20,13 +20,13 @@ class TestPipeParallelValidation:concept_library = get_concept_provider()concept_blueprint = ConceptBlueprint(definition="A test document")concept_1 = ConceptFactory.make_from_blueprint(- domain=domain, concept_code="Document", blueprint=concept_blueprint, concept_codes_from_the_same_domain=["Document"]+ domain=domain, concept_code="Document", blueprint=concept_blueprint, concept_codes_from_the_same_domain=["Document"],)concept_2 = ConceptFactory.make_from_blueprint(- domain=domain, concept_code="Context", blueprint=concept_blueprint, concept_codes_from_the_same_domain=["Context"]+ domain=domain, concept_code="Context", blueprint=concept_blueprint, concept_codes_from_the_same_domain=["Context"],)concept_3 = ConceptFactory.make_from_blueprint(- domain=domain, concept_code="Analysis", blueprint=concept_blueprint, concept_codes_from_the_same_domain=["Analysis"]+ domain=domain, concept_code="Analysis", blueprint=concept_blueprint, concept_codes_from_the_same_domain=["Analysis"],)concept_library.add_concepts(concepts=[concept_1, concept_2, concept_3])@@ -92,13 +92,13 @@ class TestPipeParallelValidation:concept_library = get_concept_provider()concept_blueprint = ConceptBlueprint(definition="Lorem Ipsum")concept_1 = ConceptFactory.make_from_blueprint(- domain=domain, concept_code="Document", blueprint=concept_blueprint, concept_codes_from_the_same_domain=["Document"]+ domain=domain, concept_code="Document", blueprint=concept_blueprint, concept_codes_from_the_same_domain=["Document"],)concept_2 = ConceptFactory.make_from_blueprint(- domain=domain, concept_code="Context", blueprint=concept_blueprint, concept_codes_from_the_same_domain=["Context"]+ domain=domain, concept_code="Context", blueprint=concept_blueprint, concept_codes_from_the_same_domain=["Context"],)concept_3 = ConceptFactory.make_from_blueprint(- domain=domain, concept_code="ProcessedAnalysis", blueprint=concept_blueprint, concept_codes_from_the_same_domain=["ProcessedAnalysis"]+ domain=domain, concept_code="ProcessedAnalysis", blueprint=concept_blueprint, concept_codes_from_the_same_domain=["ProcessedAnalysis"],)concept_library.add_concepts(concepts=[concept_1, concept_2, concept_3])@@ -135,13 +135,13 @@ class TestPipeParallelValidation:concept_library = get_concept_provider()concept_blueprint = ConceptBlueprint(definition="A test document")concept_1 = ConceptFactory.make_from_blueprint(- domain=domain, concept_code="Document", blueprint=concept_blueprint, concept_codes_from_the_same_domain=["Document"]+ domain=domain, concept_code="Document", blueprint=concept_blueprint, concept_codes_from_the_same_domain=["Document"],)concept_2 = ConceptFactory.make_from_blueprint(- domain=domain, concept_code="Context", blueprint=concept_blueprint, concept_codes_from_the_same_domain=["Context"]+ domain=domain, concept_code="Context", blueprint=concept_blueprint, concept_codes_from_the_same_domain=["Context"],)concept_3 = ConceptFactory.make_from_blueprint(- domain=domain, concept_code="ProcessedAnalysis", blueprint=concept_blueprint, concept_codes_from_the_same_domain=["ProcessedAnalysis"]+ domain=domain, concept_code="ProcessedAnalysis", blueprint=concept_blueprint, concept_codes_from_the_same_domain=["ProcessedAnalysis"],)concept_library.add_concepts(concepts=[concept_1, concept_2, concept_3])diff --git a/tests/integration/pipelex/pipes/controller/pipe_sequence/__init__.py b/tests/integration/pipelex/pipes/controller/pipe_sequence/__init__.pynew file mode 100644index 000000000..e69de29bbdiff --git a/tests/integration/pipelex/pipes/controller/pipe_sequence/test_pipe_sequence_dry_run.py b/tests/integration/pipelex/pipes/controller/pipe_sequence/test_pipe_sequence_dry_run.pyindex 8f9aa5957..eda074a7d 100644--- a/tests/integration/pipelex/pipes/controller/pipe_sequence/test_pipe_sequence_dry_run.py+++ b/tests/integration/pipelex/pipes/controller/pipe_sequence/test_pipe_sequence_dry_run.py@@ -36,7 +36,7 @@ class TestPipeSequenceDryRun:# Create structured DiscordChannelUpdate objectsdiscord_channel_updates = ListContent[DiscordChannelUpdate](- items=[DiscordChannelUpdate.model_validate(article_data) for article_data in discord_channel_updates_data]+ items=[DiscordChannelUpdate.model_validate(article_data) for article_data in discord_channel_updates_data],)# Create Stuff object for the discord channel updates listdiff --git a/tests/integration/pipelex/pipes/controller/pipe_sequence/test_pipe_sequence_simple.py b/tests/integration/pipelex/pipes/controller/pipe_sequence/test_pipe_sequence_simple.pyindex 65c3d8b82..2d60260a1 100644--- a/tests/integration/pipelex/pipes/controller/pipe_sequence/test_pipe_sequence_simple.py+++ b/tests/integration/pipelex/pipes/controller/pipe_sequence/test_pipe_sequence_simple.py@@ -90,7 +90,7 @@ class TestPipeSequenceSimple:# Actually run the PipeSequence pipepipe_output = await pipe_sequence.run_pipe(- job_metadata=JobMetadata(job_name=cast(str, request.node.originalname)), # type: ignore+ job_metadata=JobMetadata(job_name=cast("str", request.node.originalname)), # type: ignoreworking_memory=working_memory,output_name="sequence_result",pipe_run_params=PipeRunParamsFactory.make_run_params(pipe_run_mode=pipe_run_mode),diff --git a/tests/integration/pipelex/pipes/operator/__init__.py b/tests/integration/pipelex/pipes/operator/__init__.pynew file mode 100644index 000000000..e69de29bbdiff --git a/tests/integration/pipelex/pipes/operator/pipe_imgg/__init__.py b/tests/integration/pipelex/pipes/operator/pipe_imgg/__init__.pynew file mode 100644index 000000000..e69de29bbdiff --git a/tests/integration/pipelex/pipes/operator/pipe_llm/__init__.py b/tests/integration/pipelex/pipes/operator/pipe_llm/__init__.pynew file mode 100644index 000000000..e69de29bbdiff --git a/tests/integration/pipelex/pipes/operator/pipe_llm/test_pipe_llm.py b/tests/integration/pipelex/pipes/operator/pipe_llm/test_pipe_llm.pyindex 76c0a9673..c1698885c 100644--- a/tests/integration/pipelex/pipes/operator/pipe_llm/test_pipe_llm.py+++ b/tests/integration/pipelex/pipes/operator/pipe_llm/test_pipe_llm.py@@ -1,4 +1,3 @@-from typing import Listimport pytest@@ -147,7 +146,7 @@ class TestPipeLLM:async def test_pipe_llm_attribute_image(self,stuff: Stuff,- attribute_paths: List[str],+ attribute_paths: list[str],pipe_run_mode: PipeRunMode,):for attribute_path in attribute_paths:diff --git a/tests/integration/pipelex/pipes/operator/pipe_ocr/__init__.py b/tests/integration/pipelex/pipes/operator/pipe_ocr/__init__.pynew file mode 100644index 000000000..e69de29bbdiff --git a/tests/integration/pipelex/pipes/test_image_inputs.py b/tests/integration/pipelex/pipes/test_image_inputs.pyindex df4717e6c..bf6c55e20 100644--- a/tests/integration/pipelex/pipes/test_image_inputs.py+++ b/tests/integration/pipelex/pipes/test_image_inputs.py@@ -52,8 +52,7 @@ class TestImageInputs:assert pipe_output.main_stuff is not Noneasync def test_describe_page(self, request: FixtureRequest, pipe_run_mode: PipeRunMode) -> None:- """- Test that a pipe can accept a PageContent input, give to the LLM the image via subattributes,+ """Test that a pipe can accept a PageContent input, give to the LLM the image via subattributes,But also accepts basic objects"""# Create the page contentdiff --git a/tests/integration/pipelex/pipes/test_pipe_running_variants.py b/tests/integration/pipelex/pipes/test_pipe_running_variants.pyindex 03dea72b8..076b1c201 100644--- a/tests/integration/pipelex/pipes/test_pipe_running_variants.py+++ b/tests/integration/pipelex/pipes/test_pipe_running_variants.py@@ -1,5 +1,5 @@from pathlib import Path-from typing import Any, Optional, Tuple, Type+from typing import Anyimport pytestfrom pytest import FixtureRequest@@ -28,7 +28,7 @@ class TestPipeRunningVariants:self,pipe_run_mode: PipeRunMode,request: FixtureRequest,- pipe_result_handler: Tuple[str, ActivityHandlerForResultFiles],+ pipe_result_handler: tuple[str, ActivityHandlerForResultFiles],save_working_memory: Any,topic: str,stuff: Stuff,@@ -54,7 +54,7 @@ class TestPipeRunningVariants:self,pipe_run_mode: PipeRunMode,request: FixtureRequest,- pipe_result_handler: Tuple[str, ActivityHandlerForResultFiles],+ pipe_result_handler: tuple[str, ActivityHandlerForResultFiles],save_working_memory: Any,topic: str,pipe_code: str,@@ -83,11 +83,11 @@ class TestPipeRunningVariants:self,pipe_run_mode: PipeRunMode,request: FixtureRequest,- pipe_result_handler: Tuple[str, ActivityHandlerForResultFiles],+ pipe_result_handler: tuple[str, ActivityHandlerForResultFiles],save_working_memory: Any,topic: str,pipe_code: str,- output_multiplicity: Optional[PipeOutputMultiplicity],+ output_multiplicity: PipeOutputMultiplicity | None,):log.verbose(f"{topic}: just run pipe '{pipe_code}'")pipe_output = await get_pipe_router().run(@@ -117,13 +117,13 @@ class TestPipeRunningVariants:pipe_run_mode: PipeRunMode,request: FixtureRequest,pipe_code: str,- exception: Type[Exception],+ exception: type[Exception],expected_error_message: str,):failing_pipelines_file_paths = get_config().pipelex.library_config.failing_pipelines_file_pathslibrary_manager = get_library_manager()library_manager.load_libraries(- library_file_paths=[Path(failing_pipeline_file_path) for failing_pipeline_file_path in failing_pipelines_file_paths]+ library_file_paths=[Path(failing_pipeline_file_path) for failing_pipeline_file_path in failing_pipelines_file_paths],)log.verbose(f"This pipe '{pipe_code}' is supposed to cause an error of type: {exception.__name__}")diff --git a/tests/integration/pipelex/plugins/__init__.py b/tests/integration/pipelex/plugins/__init__.pynew file mode 100644index 000000000..e69de29bbdiff --git a/tests/integration/pipelex/plugins/test_bedrock.py b/tests/integration/pipelex/plugins/test_bedrock.pyindex 57c9d4bdf..8c0dad4b3 100644--- a/tests/integration/pipelex/plugins/test_bedrock.py+++ b/tests/integration/pipelex/plugins/test_bedrock.py@@ -1,5 +1,5 @@import warnings-from typing import Any, Dict, List+from typing import Anyimport boto3import pytest@@ -36,8 +36,8 @@ class TestBedrock:if any_env_var_is_placeholder(REQUIRED_ENV_VARS):pytest.skip(f"Some key(s) among {REQUIRED_ENV_VARS} are a placeholder, can't be used to test listing models")client = boto3.client("bedrock", region_name=bedrock_region_name) # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType]- response: Dict[str, Any] = client.list_foundation_models(byProvider=bedrock_provider) # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType]- bedrock_models_list: List[Dict[str, Any]] = response["modelSummaries"] # pyright: ignore[reportUnknownVariableType]+ response: dict[str, Any] = client.list_foundation_models(byProvider=bedrock_provider) # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType]+ bedrock_models_list: list[dict[str, Any]] = response["modelSummaries"] # pyright: ignore[reportUnknownVariableType]if pytestconfig.get_verbosity() >= 2:# Create and configure the tableconsole = Console()@@ -72,8 +72,8 @@ class TestBedrock:if any_env_var_is_placeholder(REQUIRED_ENV_VARS):pytest.skip(f"Some key(s) among {REQUIRED_ENV_VARS} are a placeholder, can't be used to test listing models")client = boto3.client("bedrock", region_name=bedrock_region_name) # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType]- response: Dict[str, Any] = client.list_inference_profiles() # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType]- inference_profiles_list: List[Dict[str, Any]] = response["inferenceProfileSummaries"] # pyright: ignore[reportUnknownVariableType]+ response: dict[str, Any] = client.list_inference_profiles() # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType]+ inference_profiles_list: list[dict[str, Any]] = response["inferenceProfileSummaries"] # pyright: ignore[reportUnknownVariableType]if pytestconfig.get_verbosity() >= 2:# Create and configure the tableconsole = Console()diff --git a/tests/integration/pipelex/plugins/test_external_plugin.py b/tests/integration/pipelex/plugins/test_external_plugin.pyindex e147437c2..ca309407c 100644--- a/tests/integration/pipelex/plugins/test_external_plugin.py+++ b/tests/integration/pipelex/plugins/test_external_plugin.py@@ -1,4 +1,3 @@-from typing import Typeimport pytestfrom polyfactory.factories.pydantic_factory import ModelFactory@@ -48,7 +47,7 @@ class MockExternalLLMWorker(LLMWorkerAbstract):async def _gen_object(self,llm_job: LLMJob,- schema: Type[BaseModelTypeVar],+ schema: type[BaseModelTypeVar],) -> BaseModelTypeVar:class ObjectFactory(ModelFactory[schema]): # type: ignore__model__ = schemadiff --git a/tests/integration/pipelex/plugins/test_openai_image_gen.py b/tests/integration/pipelex/plugins/test_openai_image_gen.pyindex b8d5cd718..e66d0542f 100644--- a/tests/integration/pipelex/plugins/test_openai_image_gen.py+++ b/tests/integration/pipelex/plugins/test_openai_image_gen.py@@ -13,7 +13,7 @@ from tests.integration.pipelex.test_data import ImageGenTestCases@pytest.mark.img_gen@pytest.mark.inference@pytest.mark.asyncio(loop_scope="class")-class TestImggByOpenAIGpt:+class TestImgGenByOpenAIGpt:@pytest.mark.parametrize("topic, image_desc", ImageGenTestCases.IMAGE_DESC)async def test_gpt_image_generation(self, topic: str, image_desc: str):backend = get_models_manager().get_required_inference_backend("openai")diff --git a/tests/integration/pipelex/test_data.py b/tests/integration/pipelex/test_data.pyindex 63b3bcdbb..a4f22dc0a 100644--- a/tests/integration/pipelex/test_data.py+++ b/tests/integration/pipelex/test_data.py@@ -1,4 +1,4 @@-from typing import ClassVar, List, Optional, Tuple, Type+from typing import ClassVar, Optionalfrom pipelex.core.concepts.concept_factory import ConceptFactoryfrom pipelex.core.concepts.concept_native import NATIVE_CONCEPTS_DATA, NativeConceptEnum@@ -23,7 +23,7 @@ class SomeContentWithImageAttribute(StructuredContent):class SomeContentWithImageSubObjectAttribute(StructuredContent):image_attribute: ImageContent- sub_object: Optional["SomeContentWithImageAttribute"] = None+ sub_object: SomeContentWithImageAttribute | None = Noneclass PipeTestCases:@@ -71,7 +71,7 @@ class PipeTestCases:items=[TextContent(text="The quick brown fox jumps over the lazy dog"),ImageContent(url=URL_IMG_GANTT_1),- ]+ ],),)@@ -90,7 +90,7 @@ class PipeTestCases:content=STUFF_CONTENT_WITH_IMAGE_ATTRIBUTE_IN_SUB_OBJECT,name="stuff_with_image_in_sub_object",)- STUFFS_IMAGE_ATTRIBUTES: ClassVar[List[Tuple[Stuff, List[str]]]] = [ # stuff, attribute_paths+ STUFFS_IMAGE_ATTRIBUTES: ClassVar[list[tuple[Stuff, list[str]]]] = [ # stuff, attribute_paths(STUFF_WITH_IMAGE_ATTRIBUTE, ["stuff_with_image.image_attribute"]),(STUFF_WITH_IMAGE_ATTRIBUTE_IN_SUB_OBJECT,@@ -113,14 +113,14 @@ class PipeTestCases:concept_string="answer.Question",content=USER_TEXT_TRICKY_2,)- BLUEPRINT_AND_PIPE: ClassVar[List[Tuple[str, StuffBlueprint, str]]] = [ # topic, blueprint, pipe+ BLUEPRINT_AND_PIPE: ClassVar[list[tuple[str, StuffBlueprint, str]]] = [ # topic, blueprint, pipe("Tricky question conclude",TRICKY_QUESTION_BLUEPRINT,"conclude_tricky_question_by_steps",),]- NO_INPUT: ClassVar[List[Tuple[str, str]]] = [ # topic, pipe+ NO_INPUT: ClassVar[list[tuple[str, str]]] = [ # topic, pipe("Test with no input","test_no_input",@@ -130,7 +130,7 @@ class PipeTestCases:"test_no_input_that_could_be_long",),]- NO_INPUT_PARALLEL1: ClassVar[List[Tuple[str, str, Optional[PipeOutputMultiplicity]]]] = [ # topic, pipe, multiplicity+ NO_INPUT_PARALLEL1: ClassVar[list[tuple[str, str, PipeOutputMultiplicity | None]]] = [ # topic, pipe, multiplicity("Nature colors painting","choose_colors",@@ -148,7 +148,7 @@ class PipeTestCases:),]- STUFF_AND_PIPE: ClassVar[List[Tuple[str, Stuff, str]]] = [ # topic, stuff, pipe_code+ STUFF_AND_PIPE: ClassVar[list[tuple[str, Stuff, str]]] = [ # topic, stuff, pipe_code("Process Simple Image",SIMPLE_STUFF_IMAGE,@@ -160,7 +160,7 @@ class PipeTestCases:"ocr_page_contents_from_pdf",),]- FAILURE_PIPES: ClassVar[List[Tuple[str, Type[Exception], str]]] = [+ FAILURE_PIPES: ClassVar[list[tuple[str, type[Exception], str]]] = [("infinite_loop_1",PipeStackOverflowError,@@ -170,7 +170,7 @@ class PipeTestCases:class LibraryTestCases:- KNOWN_CONCEPTS_AND_PIPES: ClassVar[List[Tuple[str, str]]] = [ # concept, pipe+ KNOWN_CONCEPTS_AND_PIPES: ClassVar[list[tuple[str, str]]] = [ # concept, pipe("cars.CarDescription", "generate_car_description"),("animals.AnimalDescription", "generate_animal_description"),("flowers.FlowerDescription", "generate_flower_description"),@@ -178,11 +178,11 @@ class LibraryTestCases:class PipeOcrTestCases:- PIPE_OCR_IMAGE_TEST_CASES: ClassVar[List[str]] = [+ PIPE_OCR_IMAGE_TEST_CASES: ClassVar[list[str]] = [ImageTestCases.IMAGE_FILE_PATH_PNG,ImageTestCases.IMAGE_URL_PNG,]- PIPE_OCR_PDF_TEST_CASES: ClassVar[List[str]] = PDFTestCases.DOCUMENT_FILE_PATHS + PDFTestCases.DOCUMENT_URLS+ PIPE_OCR_PDF_TEST_CASES: ClassVar[list[str]] = PDFTestCases.DOCUMENT_FILE_PATHS + PDFTestCases.DOCUMENT_URLSclass ImageGenTestCases:@@ -192,7 +192,7 @@ class ImageGenTestCases:Woman typing on a laptop. On the laptop screen you see python code to generate code to write a prompt for an AI model."""- IMAGE_DESC: ClassVar[List[Tuple[str, str]]] = [ # topic, img_gen_prompt_text+ IMAGE_DESC: ClassVar[list[tuple[str, str]]] = [ # topic, img_gen_prompt_text# (IMG_GEN_PROMPT_1, IMG_GEN_PROMPT_1),# (IMG_GEN_PROMPT_2, IMG_GEN_PROMPT_2),# (IMG_GEN_PROMPT_3, IMG_GEN_PROMPT_3),@@ -305,7 +305,7 @@ Extract information from the following text:"""# Combined test cases for parametrized tests- STRUCTURE_TEST_CASES: ClassVar[List[Tuple[str, str, str]]] = [ # topic, data, concept+ STRUCTURE_TEST_CASES: ClassVar[list[tuple[str, str, str]]] = [ # topic, data, concept("Simple structure basic", SIMPLE_STRUCTURE_1, "ConceptWithSimpleStructure"),("Simple structure narrative", SIMPLE_STRUCTURE_2, "ConceptWithSimpleStructure"),("Optionals all present", OPTIONAL_ALL_PRESENT, "ConceptWithOptionals"),@@ -430,7 +430,7 @@ Extract information from the following text:"""# Combined test cases for parametrized tests- STRUCTURE_TEST_CASES: ClassVar[List[Tuple[str, str, str]]] = [ # topic, data, concept+ STRUCTURE_TEST_CASES: ClassVar[list[tuple[str, str, str]]] = [ # topic, data, concept("Dicts with data", DICTS_WITH_DATA, "ConceptWithDicts"),("Dicts empty", DICTS_EMPTY, "ConceptWithDicts"),("Unions string variant", UNIONS_STRING_VARIANT, "ConceptWithUnions"),diff --git a/tests/integration/pipelex/test_inline_structure_concepts.py b/tests/integration/pipelex/test_inline_structure_concepts.pyindex c6cbbde14..264aafd54 100644--- a/tests/integration/pipelex/test_inline_structure_concepts.py+++ b/tests/integration/pipelex/test_inline_structure_concepts.py@@ -1,6 +1,5 @@"""Integration tests for inline structure definitions in concepts."""-from typing import Dict, Unionimport pytest@@ -21,18 +20,18 @@ class TestInlineStructureConcepts:def test_inline_structure_definition_creation(self):"""Test that concepts with inline structure definitions are created correctly."""# Define inline structure with mixed syntax- inline_structure: Dict[str, Union[str, ConceptStructureBlueprint]] = {+ inline_structure: dict[str, str | ConceptStructureBlueprint] = {"dominant_feature": ConceptStructureBlueprint(- type=ConceptStructureBlueprintFieldType.TEXT, definition="The most important feature", required=False+ type=ConceptStructureBlueprintFieldType.TEXT, definition="The most important feature", required=False,),"visual_elements": ConceptStructureBlueprint(- type=ConceptStructureBlueprintFieldType.TEXT, definition="Key visual elements", required=False+ type=ConceptStructureBlueprintFieldType.TEXT, definition="Key visual elements", required=False,),"composition": ConceptStructureBlueprint(- type=ConceptStructureBlueprintFieldType.TEXT, definition="Analysis of the image composition", required=False+ type=ConceptStructureBlueprintFieldType.TEXT, definition="Analysis of the image composition", required=False,),"color_palette": ConceptStructureBlueprint(- type=ConceptStructureBlueprintFieldType.TEXT, definition="Description of the main colors", required=False+ type=ConceptStructureBlueprintFieldType.TEXT, definition="Description of the main colors", required=False,),"mood_atmosphere": "The overall mood or atmosphere",}@@ -42,7 +41,7 @@ class TestInlineStructureConcepts:# Create concept from blueprintconcept = ConceptFactory.make_from_blueprint(- domain="test_domain", concept_code="TestFeatureAnalysis", blueprint=blueprint, concept_codes_from_the_same_domain=["TestFeatureAnalysis"]+ domain="test_domain", concept_code="TestFeatureAnalysis", blueprint=blueprint, concept_codes_from_the_same_domain=["TestFeatureAnalysis"],)# Verify concept properties@@ -80,7 +79,7 @@ class TestInlineStructureConcepts:# Create concept from blueprintconcept = ConceptFactory.make_from_blueprint(- domain="test_domain", concept_code="TestStringRef", blueprint=blueprint, concept_codes_from_the_same_domain=["TestStringRef"]+ domain="test_domain", concept_code="TestStringRef", blueprint=blueprint, concept_codes_from_the_same_domain=["TestStringRef"],)# Verify concept properties@@ -96,7 +95,7 @@ class TestInlineStructureConcepts:# Create concept from blueprintconcept = ConceptFactory.make_from_blueprint(- domain="test_domain", concept_code="TestAutoDetect", blueprint=blueprint, concept_codes_from_the_same_domain=["TestAutoDetect"]+ domain="test_domain", concept_code="TestAutoDetect", blueprint=blueprint, concept_codes_from_the_same_domain=["TestAutoDetect"],)# Should default to TextContent since TestAutoDetect is not a registered class@@ -104,7 +103,7 @@ class TestInlineStructureConcepts:def test_inline_structure_with_complex_types(self):"""Test inline structure with complex field types."""- inline_structure: Dict[str, Union[str, ConceptStructureBlueprint]] = {+ inline_structure: dict[str, str | ConceptStructureBlueprint] = {"title": ConceptStructureBlueprint(type=ConceptStructureBlueprintFieldType.TEXT, definition="Document title"),"tags": ConceptStructureBlueprint(type=ConceptStructureBlueprintFieldType.LIST,@@ -122,14 +121,14 @@ class TestInlineStructureConcepts:"priority": ConceptStructureBlueprint(choices=["low", "medium", "high"], definition="Priority level", required=False),"page_count": ConceptStructureBlueprint(type=ConceptStructureBlueprintFieldType.INTEGER, definition="Number of pages", required=False),"is_active": ConceptStructureBlueprint(- type=ConceptStructureBlueprintFieldType.BOOLEAN, definition="Whether document is active", required=False+ type=ConceptStructureBlueprintFieldType.BOOLEAN, definition="Whether document is active", required=False,),}blueprint = ConceptBlueprint(definition="Complex document structure", structure=inline_structure)concept = ConceptFactory.make_from_blueprint(- domain="test_domain", concept_code="ComplexDocument", blueprint=blueprint, concept_codes_from_the_same_domain=["ComplexDocument"]+ domain="test_domain", concept_code="ComplexDocument", blueprint=blueprint, concept_codes_from_the_same_domain=["ComplexDocument"],)# Verify concept creation@@ -174,25 +173,25 @@ not a registered subclass of StuffContent",def test_multiple_inline_structures_do_not_conflict(self):"""Test that multiple inline structures with same field names don't conflict."""# First structure- structure1: Dict[str, Union[str, ConceptStructureBlueprint]] = {+ structure1: dict[str, str | ConceptStructureBlueprint] = {"name": ConceptStructureBlueprint(type=ConceptStructureBlueprintFieldType.TEXT, definition="Person name"),"age": ConceptStructureBlueprint(type=ConceptStructureBlueprintFieldType.INTEGER, definition="Person age", required=False),}blueprint1 = ConceptBlueprint(definition="Person information", structure=structure1)concept1 = ConceptFactory.make_from_blueprint(- domain="test_domain", concept_code="Person", blueprint=blueprint1, concept_codes_from_the_same_domain=["Person"]+ domain="test_domain", concept_code="Person", blueprint=blueprint1, concept_codes_from_the_same_domain=["Person"],)# Second structure with same field names but different context- structure2: Dict[str, Union[str, ConceptStructureBlueprint]] = {+ structure2: dict[str, str | ConceptStructureBlueprint] = {"name": "Product name","age": ConceptStructureBlueprint(type=ConceptStructureBlueprintFieldType.INTEGER, definition="Product age in days", required=False),}blueprint2 = ConceptBlueprint(definition="Product information", structure=structure2)concept2 = ConceptFactory.make_from_blueprint(- domain="test_domain", concept_code="Product", blueprint=blueprint2, concept_codes_from_the_same_domain=["Product"]+ domain="test_domain", concept_code="Product", blueprint=blueprint2, concept_codes_from_the_same_domain=["Product"],)# Both should be created successfullydiff --git a/tests/integration/pipelex/test_libraries.py b/tests/integration/pipelex/test_libraries.pyindex 0709d76e5..d11302bb6 100644--- a/tests/integration/pipelex/test_libraries.py+++ b/tests/integration/pipelex/test_libraries.py@@ -1,9 +1,7 @@-"""-make t TEST=TestLibraries+"""make t TEST=TestLibraries"""from pathlib import Path-from typing import Optionalimport pytestfrom rich import box@@ -20,7 +18,7 @@ from tests.integration.pipelex.test_data import LibraryTestCasesdef pretty_print_all_pipes(pipe_library: PipeLibrary,- title: Optional[str] = None,+ title: str | None = None,):console = Console()table = Table(@@ -54,7 +52,7 @@ def pretty_print_all_pipes(def pretty_print_all_concepts(concept_library: ConceptLibrary,- title: Optional[str] = None,+ title: str | None = None,):console = Console()table = Table(@@ -101,7 +99,7 @@ class TestLibraries:# Test individual concepts and pipesassert library_manager.concept_library.get_required_concept(concept_string=known_concept) is not Nonepretty_print(- f"Concept: {known_concept} is correctly loaded as {library_manager.concept_library.get_required_concept(concept_string=known_concept)}"+ f"Concept: {known_concept} is correctly loaded as {library_manager.concept_library.get_required_concept(concept_string=known_concept)}",)assert library_manager.pipe_library.get_optional_pipe(known_pipe) is not Nonepretty_print(f"Pipe: {known_pipe} is correctly loaded as {library_manager.pipe_library.get_optional_pipe(known_pipe)}")diff --git a/tests/integration/pipelex/tools/aws/__init__.py b/tests/integration/pipelex/tools/aws/__init__.pynew file mode 100644index 000000000..e69de29bbdiff --git a/tests/integration/pipelex/tools/class_registry/__init__.py b/tests/integration/pipelex/tools/class_registry/__init__.pynew file mode 100644index 000000000..e69de29bbdiff --git a/tests/integration/pipelex/tools/class_registry/test_class_registry_utils.py b/tests/integration/pipelex/tools/class_registry/test_class_registry_utils.pyindex 2e4e40af4..0fea03ef5 100644--- a/tests/integration/pipelex/tools/class_registry/test_class_registry_utils.py+++ b/tests/integration/pipelex/tools/class_registry/test_class_registry_utils.py@@ -16,7 +16,7 @@ class TestClassRegistryUtilsIntegration:"""Integration test for registering StuffContent classes recursively."""class_registry = get_class_registry()ClassRegistryUtils.register_classes_in_folder(- folder_path=ClassRegistryTestCases.MODEL_FOLDER_PATH, base_class=StuffContent, is_recursive=True+ folder_path=ClassRegistryTestCases.MODEL_FOLDER_PATH, base_class=StuffContent, is_recursive=True,)# Should register classes that inherit from StuffContent@@ -28,7 +28,7 @@ class TestClassRegistryUtilsIntegration:"""Integration test for registering StuffContent classes non-recursively."""class_registry = get_class_registry()ClassRegistryUtils.register_classes_in_folder(- folder_path=ClassRegistryTestCases.MODEL_FOLDER_PATH, base_class=StuffContent, is_recursive=False+ folder_path=ClassRegistryTestCases.MODEL_FOLDER_PATH, base_class=StuffContent, is_recursive=False,)# Should register only top-level StuffContent classesdiff --git a/tests/integration/pipelex/tools/httpx/__init__.py b/tests/integration/pipelex/tools/httpx/__init__.pynew file mode 100644index 000000000..e69de29bbdiff --git a/tests/integration/pipelex/tools/httpx/test_httpx_fetching.py b/tests/integration/pipelex/tools/httpx/test_httpx_fetching.pyindex 8374b460d..18b9b41f9 100644--- a/tests/integration/pipelex/tools/httpx/test_httpx_fetching.py+++ b/tests/integration/pipelex/tools/httpx/test_httpx_fetching.py@@ -15,7 +15,7 @@ class TestHttpxFetching:assert (await fetch_file_from_url_httpx_async(url=url,- timeout=60,+ request_timeout=60,)is not None)@@ -28,7 +28,7 @@ class TestHttpxFetching:assert (fetch_file_from_url_httpx(url=url,- timeout=60,+ request_timeout=60,)is not None)diff --git a/tests/integration/pipelex/tools/pdf/__init__.py b/tests/integration/pipelex/tools/pdf/__init__.pynew file mode 100644index 000000000..e69de29bbdiff --git a/tests/test_pipelines/__init__.py b/tests/test_pipelines/__init__.pynew file mode 100644index 000000000..e69de29bbdiff --git a/tests/test_pipelines/concept_library/__init__.py b/tests/test_pipelines/concept_library/__init__.pynew file mode 100644index 000000000..e69de29bbdiff --git a/tests/test_pipelines/concept_library/is_compatible_concept_code.py b/tests/test_pipelines/concept_library/is_compatible_concept_code.pyindex e3ddcd65d..40f1406d3 100644--- a/tests/test_pipelines/concept_library/is_compatible_concept_code.py+++ b/tests/test_pipelines/concept_library/is_compatible_concept_code.py@@ -1,4 +1,3 @@-from typing import Dict, List, Optionalfrom pydantic import Field@@ -6,17 +5,17 @@ from pipelex.core.stuffs.stuff_content import StructuredContentclass FundamentalsDoc(StructuredContent):- project_overview: Optional[str] = Field(+ project_overview: str | None = Field(None,description="Mission, key features, architecture diagram, demo links",)- core_concepts: Optional[Dict[str, str]] = Field(+ core_concepts: dict[str, str] | None = Field(None,description=("Names and definitions for project-specific terms, acronyms, data model names, background knowledge, business rules, domain entities"),)- repository_map: Optional[str] = Field(+ repository_map: str | None = Field(None,description="Directory layout explanation and purpose of each folder",)@@ -26,20 +25,20 @@ class DocumentationConcept(StructuredContent):"""A specialized documentation concept that extends FundamentalsDoc."""title: str = Field(..., description="Title of the documentation")- sections: List[str] = Field(default_factory=list, description="List of section names")- last_updated: Optional[str] = Field(None, description="Last update timestamp")+ sections: list[str] = Field(default_factory=list, description="List of section names")+ last_updated: str | None = Field(None, description="Last update timestamp")class MultiMediaConcept(StructuredContent):"""A concept that combines text and images."""text_content: str = Field(..., description="The text content")- image_urls: List[str] = Field(default_factory=list, description="List of image URLs")- caption: Optional[str] = Field(None, description="Optional caption for the content")+ image_urls: list[str] = Field(default_factory=list, description="List of image URLs")+ caption: str | None = Field(None, description="Optional caption for the content")class IndependentConcept(StructuredContent):"""An independent concept with its own structure."""unique_field: str = Field(..., description="A unique field for this concept")- metadata: Dict[str, str] = Field(default_factory=dict, description="Metadata dictionary")+ metadata: dict[str, str] = Field(default_factory=dict, description="Metadata dictionary")diff --git a/tests/test_pipelines/discord_newsletter.py b/tests/test_pipelines/discord_newsletter.pyindex 2680b2110..6de3cb373 100644--- a/tests/test_pipelines/discord_newsletter.py+++ b/tests/test_pipelines/discord_newsletter.py@@ -1,4 +1,3 @@-from typing import Listfrom pydantic import Field@@ -25,8 +24,8 @@ class DiscordMessage(StructuredContent):author: str = Field(..., description="Author of the message")content: str = Field(..., description="Content of the message")- attachments: List[Attachment] = Field(default_factory=list, description="List of message attachments")- embeds: List[Embed] = Field(default_factory=list, description="List of message embeds")+ attachments: list[Attachment] = Field(default_factory=list, description="List of message attachments")+ embeds: list[Embed] = Field(default_factory=list, description="List of message embeds")link: str = Field(..., description="Link to the message")@@ -35,20 +34,20 @@ class DiscordChannelUpdate(StructuredContent):name: str = Field(..., description="Name of the Discord channel")position: int = Field(..., description="Position of the channel")- messages: List[DiscordMessage] = Field(default_factory=list, description="List of messages in the channel")+ messages: list[DiscordMessage] = Field(default_factory=list, description="List of messages in the channel")class ChannelSummary(StructuredContent):"""Represents a summarized Discord channel for newsletter inclusion"""channel_name: str = Field(..., description="Name of the Discord channel")- summary_items: List[str] = Field(..., description="Well-written summaries of the channel's activity")+ summary_items: list[str] = Field(..., description="Well-written summaries of the channel's activity")class Newsletter(StructuredContent):"""Represents the final newsletter content"""weekly_summary: str = Field(..., description="200 character summary of weekly Share channel content")- new_members: List[str] = Field(default_factory=list, description="New member introductions in bullet points")- channel_sections: List[ChannelSummary] = Field(default_factory=list, description="Ordered channel summaries")- geographic_hubs: List[ChannelSummary] = Field(default_factory=list, description="Geographic hub channels grouped at end")+ new_members: list[str] = Field(default_factory=list, description="New member introductions in bullet points")+ channel_sections: list[ChannelSummary] = Field(default_factory=list, description="Ordered channel summaries")+ geographic_hubs: list[ChannelSummary] = Field(default_factory=list, description="Geographic hub channels grouped at end")diff --git a/tests/test_pipelines/misc_tests/subfolder_2/subfolder/__init__.py b/tests/test_pipelines/misc_tests/subfolder_2/subfolder/__init__.pynew file mode 100644index 000000000..e69de29bbdiff --git a/tests/test_pipelines/pipe_controllers/pipe_batch/__init__.py b/tests/test_pipelines/pipe_controllers/pipe_batch/__init__.pynew file mode 100644index 000000000..e69de29bbdiff --git a/tests/test_pipelines/pipe_controllers/pipe_condition/__init__.py b/tests/test_pipelines/pipe_controllers/pipe_condition/__init__.pynew file mode 100644index 000000000..e69de29bbdiff --git a/tests/test_pipelines/pipe_controllers/pipe_parallel/__init__.py b/tests/test_pipelines/pipe_controllers/pipe_parallel/__init__.pynew file mode 100644index 000000000..e69de29bbdiff --git a/tests/test_pipelines/pipe_controllers/pipe_sequence/__init__.py b/tests/test_pipelines/pipe_controllers/pipe_sequence/__init__.pynew file mode 100644index 000000000..e69de29bbdiff --git a/tests/test_pipelines/pipe_llm_prompt.py b/tests/test_pipelines/pipe_llm_prompt.pyindex 79920a4f1..3464df296 100644--- a/tests/test_pipelines/pipe_llm_prompt.py+++ b/tests/test_pipelines/pipe_llm_prompt.py@@ -1,4 +1,4 @@-from typing import List, Literal+from typing import Literalfrom pydantic import Field@@ -60,5 +60,5 @@ class PersonContent(StructuredContent):class ComplexListContent(StructuredContent):"""A content class with list fields."""- items: List[str] = Field(description="List of items")- tags: List[str] = Field(description="List of tags")+ items: list[str] = Field(description="List of items")+ tags: list[str] = Field(description="List of tags")diff --git a/tests/test_pipelines/test_file_func_registry.py b/tests/test_pipelines/test_file_func_registry.pyindex 73a317db4..7730054e9 100644--- a/tests/test_pipelines/test_file_func_registry.py+++ b/tests/test_pipelines/test_file_func_registry.py@@ -1,4 +1,3 @@-from typing import Listfrom pydantic import Field@@ -27,19 +26,19 @@ def read_file_content(working_memory: WorkingMemory) -> ListContent[CodebaseFileReturns:ListContent of CodebaseFileContent objects- """+ """file_paths_list = working_memory.get_stuff_as_list("related_file_paths", item_type=FilePath)- codebase_files: List[CodebaseFileContent] = []+ codebase_files: list[CodebaseFileContent] = []for file_path in file_paths_list.items:try:- with open(file_path.path, "r", encoding="utf-8") as file:+ with open(file_path.path, encoding="utf-8") as file:content = file.read()codebase_files.append(CodebaseFileContent(file_path=file_path.path, file_content=content))except Exception as e:codebase_files.append(- CodebaseFileContent(file_path=file_path.path, file_content=f"# File not found or unreadable: {file_path.path}\n# Error: {str(e)}")+ CodebaseFileContent(file_path=file_path.path, file_content=f"# File not found or unreadable: {file_path.path}\n# Error: {e!s}"),)return ListContent[CodebaseFileContent](items=codebase_files)diff --git a/tests/test_pipelines/test_structures_basic.py b/tests/test_pipelines/test_structures_basic.pyindex 5460a61f4..cfc6fddfe 100644--- a/tests/test_pipelines/test_structures_basic.py+++ b/tests/test_pipelines/test_structures_basic.py@@ -1,7 +1,6 @@"""Test structures for basic concepts without union types."""from datetime import datetime-from typing import List, Optionalfrom pydantic import Field@@ -20,22 +19,22 @@ class ConceptWithOptionals(StructuredContent):"""A structure with optional fields."""required_field: str = Field(..., description="A required field")- optional_string: Optional[str] = Field(None, description="An optional string field")- optional_number: Optional[int] = Field(None, description="An optional number field")- optional_date: Optional[datetime] = Field(None, description="An optional date field")+ optional_string: str | None = Field(None, description="An optional string field")+ optional_number: int | None = Field(None, description="An optional number field")+ optional_date: datetime | None = Field(None, description="An optional date field")class ConceptWithLists(StructuredContent):"""A structure with list fields."""- string_list: List[str] = Field(default_factory=list, description="A list of strings")- number_list: List[int] = Field(default_factory=list, description="A list of numbers")- optional_list: Optional[List[str]] = Field(None, description="An optional list")+ string_list: list[str] = Field(default_factory=list, description="A list of strings")+ number_list: list[int] = Field(default_factory=list, description="A list of numbers")+ optional_list: list[str] | None = Field(None, description="An optional list")class ConceptWithNestedStructures(StructuredContent):"""A structure with nested structures."""simple_nested: ConceptWithSimpleStructure = Field(..., description="A nested simple structure")- optional_nested: Optional[ConceptWithOptionals] = Field(None, description="An optional nested structure")- list_of_nested: List[ConceptWithSimpleStructure] = Field(default_factory=list, description="A list of nested structures")+ optional_nested: ConceptWithOptionals | None = Field(None, description="An optional nested structure")+ list_of_nested: list[ConceptWithSimpleStructure] = Field(default_factory=list, description="A list of nested structures")diff --git a/tests/test_pipelines/test_structures_complex.py b/tests/test_pipelines/test_structures_complex.pyindex 06e13705d..7ba682a6d 100644--- a/tests/test_pipelines/test_structures_complex.py+++ b/tests/test_pipelines/test_structures_complex.py@@ -1,6 +1,5 @@"""Test structures for complex concepts (dicts, unions, etc.)."""-from typing import Dict, List, Optional, Unionfrom pydantic import Field@@ -11,35 +10,35 @@ from tests.test_pipelines.test_structures_basic import ConceptWithOptionals, Conclass ConceptWithDicts(StructuredContent):"""A structure with dictionary fields."""- string_dict: Dict[str, str] = Field(default_factory=dict, description="A dictionary of strings")- number_dict: Dict[str, int] = Field(default_factory=dict, description="A dictionary of numbers")- optional_dict: Optional[Dict[str, str]] = Field(None, description="An optional dictionary")+ string_dict: dict[str, str] = Field(default_factory=dict, description="A dictionary of strings")+ number_dict: dict[str, int] = Field(default_factory=dict, description="A dictionary of numbers")+ optional_dict: dict[str, str] | None = Field(None, description="An optional dictionary")class ConceptWithUnions(StructuredContent):"""A structure with union types."""- string_or_int: Union[str, int] = Field(..., description="A field that can be string or int")- optional_union: Optional[Union[str, bool]] = Field(None, description="An optional union field")- list_of_unions: List[Union[str, int]] = Field(default_factory=list, description="A list of union types")+ string_or_int: str | int = Field(..., description="A field that can be string or int")+ optional_union: str | bool | None = Field(None, description="An optional union field")+ list_of_unions: list[str | int] = Field(default_factory=list, description="A list of union types")class ConceptWithComplexUnions(StructuredContent):"""A structure with more complex union combinations."""- mixed_dict: Dict[str, Union[str, int, bool]] = Field(default_factory=dict, description="A dictionary with mixed value types")- union_or_list: Union[str, List[int]] = Field(..., description="Either a string or a list of integers")- optional_complex_union: Optional[Union[Dict[str, str], List[str]]] = Field(None, description="Optional dict or list")- number_or_bool: Union[int, float, bool] = Field(..., description="Number or boolean value")+ mixed_dict: dict[str, str | int | bool] = Field(default_factory=dict, description="A dictionary with mixed value types")+ union_or_list: str | list[int] = Field(..., description="Either a string or a list of integers")+ optional_complex_union: dict[str, str] | list[str] | None = Field(None, description="Optional dict or list")+ number_or_bool: int | float | bool = Field(..., description="Number or boolean value")class ConceptWithNestedUnions(StructuredContent):"""A structure with nested unions including other structures."""- simple_or_complex: Union[ConceptWithSimpleStructure, ConceptWithOptionals] = Field(..., description="Simple or complex structure")- list_of_union_structures: List[Union[ConceptWithSimpleStructure, ConceptWithUnions]] = Field(- default_factory=list, description="List of different structure types"+ simple_or_complex: ConceptWithSimpleStructure | ConceptWithOptionals = Field(..., description="Simple or complex structure")+ list_of_union_structures: list[ConceptWithSimpleStructure | ConceptWithUnions] = Field(+ default_factory=list, description="List of different structure types",)- optional_nested_union: Optional[Union[ConceptWithUnions, ConceptWithComplexUnions]] = Field(- None, description="Optional union of union structures"+ optional_nested_union: ConceptWithUnions | ConceptWithComplexUnions | None = Field(+ None, description="Optional union of union structures",)diff --git a/tests/test_pipelines/test_tweet.py b/tests/test_pipelines/test_tweet.pyindex c4ae1cf08..b96b842dd 100644--- a/tests/test_pipelines/test_tweet.py+++ b/tests/test_pipelines/test_tweet.py@@ -5,4 +5,3 @@ class OptimizedTweet(TextContent):"""A tweet optimized for Twitter/X engagement following best practices."""# TBD- passdiff --git a/tests/unit/pipelex/cli/commands/__init__.py b/tests/unit/pipelex/cli/commands/__init__.pynew file mode 100644index 000000000..e69de29bbdiff --git a/tests/unit/pipelex/cli/commands/test_init_cmd.py b/tests/unit/pipelex/cli/commands/test_init_cmd.pyindex 69d4755fe..97501af0c 100644--- a/tests/unit/pipelex/cli/commands/test_init_cmd.py+++ b/tests/unit/pipelex/cli/commands/test_init_cmd.py@@ -10,10 +10,7 @@ from pipelex.exceptions import PipelexCLIErrorclass TestInitCmd:- """Test cases for the init command functionality."""-def test_do_init_config_copies_all_files(self, tmp_path: Path, mocker: MockerFixture) -> None:- """Test that all files from config template are copied to target directory."""# Setup directoriestemplate_dir = tmp_path / "config_template"template_dir.mkdir()@@ -38,7 +35,6 @@ class TestInitCmd:mock_echo.assert_any_call(f"✅ Copied 1 files to {target_dir}:")def test_do_init_config_skips_existing_files(self, tmp_path: Path, mocker: MockerFixture) -> None:- """Test that existing files are skipped when reset=False."""# Setup directories with existing filetemplate_dir = tmp_path / "config_template"template_dir.mkdir()@@ -73,7 +69,6 @@ class TestInitCmd:assert any("ℹ️ Skipped 1 existing files" in call for call in calls)def test_do_init_config_reset_overwrites_files(self, tmp_path: Path, mocker: MockerFixture) -> None:- """Test that reset=True overwrites existing files."""# Setup directories with existing filetemplate_dir = tmp_path / "config_template"template_dir.mkdir()@@ -104,7 +99,6 @@ class TestInitCmd:assert not any("ℹ️ Skipped" in call for call in calls)def test_do_init_config_nested_directory_structure(self, tmp_path: Path, mocker: MockerFixture) -> None:- """Test copying of nested directory structures."""# Setup complex nested structuretemplate_dir = tmp_path / "config_template"template_dir.mkdir()@@ -144,7 +138,6 @@ class TestInitCmd:assert any("✅ Copied 3 files to" in call for call in calls)def test_do_init_config_handles_permission_error(self, tmp_path: Path, mocker: MockerFixture) -> None:- """Test handling of permission errors during file copying."""# Setup directoriestemplate_dir = tmp_path / "config_template"template_dir.mkdir()@@ -168,7 +161,6 @@ class TestInitCmd:assert "Failed to initialize configuration" in str(exc_info.value)def test_do_init_config_creates_target_directory(self, tmp_path: Path, mocker: MockerFixture) -> None:- """Test that target directory is created if it doesn't exist."""# Setup template directory onlytemplate_dir = tmp_path / "config_template"template_dir.mkdir()diff --git a/tests/unit/pipelex/client/test_api_serialization.py b/tests/unit/pipelex/client/test_api_serialization.pyindex ed9b40fee..93f5f1789 100644--- a/tests/unit/pipelex/client/test_api_serialization.py+++ b/tests/unit/pipelex/client/test_api_serialization.py@@ -2,7 +2,7 @@ import jsonfrom datetime import datetimefrom decimal import Decimalfrom enum import Enum-from typing import Any, Dict, List, Optional+from typing import Anyimport pytestfrom pydantic import BaseModel@@ -26,8 +26,8 @@ class Priority(Enum):class TaskStatus(BaseModel):is_complete: bool- completion_date: Optional[datetime] = None- notes: List[str] = []+ completion_date: datetime | None = None+ notes: list[str] = []class ComplexTask(BaseModel):@@ -35,34 +35,31 @@ class ComplexTask(BaseModel):title: strpriority: Prioritystatus: TaskStatus- due_dates: List[datetime]- metadata: Dict[str, Any]- score: Optional[Decimal] = None+ due_dates: list[datetime]+ metadata: dict[str, Any]+ score: Decimal | None = Noneclass Project(BaseModel):name: strcreated_at: datetime- tasks: List[ComplexTask]- settings: Dict[str, Any]+ tasks: list[ComplexTask]+ settings: dict[str, Any]class TestApiSerialization:- """Test API-specific serialization with kajson, datetime formatting, and cleanup."""-@pytest.fixturedef datetime_content_memory(self) -> WorkingMemory:- """Create WorkingMemory with datetime content."""datetime_event = DateTimeEvent(event_name="Project Kickoff Meeting",- start_time=datetime(2024, 1, 15, 10, 0, 0),- end_time=datetime(2024, 1, 15, 11, 30, 0),- created_at=datetime(2024, 1, 1, 9, 0, 0),+ start_time=datetime(2024, 1, 15, 10, 0, 0, tzinfo=None),+ end_time=datetime(2024, 1, 15, 11, 30, 0, tzinfo=None),+ created_at=datetime(2024, 1, 1, 9, 0, 0, tzinfo=None),)stuff = StuffFactory.make_stuff(concept=ConceptFactory.make(- concept_code="DateTimeEvent", domain="event", definition="event.DateTimeEvent", structure_class_name="DateTimeEvent"+ concept_code="DateTimeEvent", domain="event", definition="event.DateTimeEvent", structure_class_name="DateTimeEvent",),name="project_meeting",content=datetime_event,@@ -71,7 +68,6 @@ class TestApiSerialization:@pytest.fixturedef text_content_memory(self) -> WorkingMemory:- """Create WorkingMemory with text content."""stuff = StuffFactory.make_stuff(concept=ConceptFactory.make_native_concept(native_concept_data=NATIVE_CONCEPTS_DATA[NativeConceptEnum.TEXT]),name="sample_text",@@ -81,7 +77,6 @@ class TestApiSerialization:@pytest.fixturedef number_content_memory(self) -> WorkingMemory:- """Create WorkingMemory with number content."""number_content = NumberContent(number=3.14159)stuff = StuffFactory.make_stuff(concept=ConceptFactory.make_native_concept(native_concept_data=NATIVE_CONCEPTS_DATA[NativeConceptEnum.NUMBER]),@@ -91,7 +86,6 @@ class TestApiSerialization:return WorkingMemoryFactory.make_from_single_stuff(stuff=stuff)def test_serialize_working_memory_with_datetime(self, datetime_content_memory: WorkingMemory):- """Test that datetime content is properly serialized to ISO format strings."""compact_memory = ApiSerializer.serialize_working_memory_for_api(datetime_content_memory)# Should have one entry for the datetime content@@ -124,7 +118,6 @@ class TestApiSerialization:assert "__class__" not in contentdef test_api_serialized_memory_is_json_serializable(self, datetime_content_memory: WorkingMemory):- """Test that API serialized memory is JSON serializable."""compact_memory = ApiSerializer.serialize_working_memory_for_api(datetime_content_memory)# This should NOT raise an exception now@@ -141,7 +134,6 @@ class TestApiSerialization:assert isinstance(content["created_at"], str)def test_serialize_text_content(self, text_content_memory: WorkingMemory):- """Test that text content is handled specially."""compact_memory = ApiSerializer.serialize_working_memory_for_api(text_content_memory)assert len(compact_memory) == 1@@ -153,7 +145,6 @@ class TestApiSerialization:assert text_blueprint["content"] == "Sample text content"def test_serialize_number_content(self, number_content_memory: WorkingMemory):- """Test that number content is properly serialized."""compact_memory = ApiSerializer.serialize_working_memory_for_api(number_content_memory)assert len(compact_memory) == 1diff --git a/tests/unit/pipelex/cogt/models/__init__.py b/tests/unit/pipelex/cogt/models/__init__.pynew file mode 100644index 000000000..e69de29bbdiff --git a/tests/unit/pipelex/cogt/models/test_model_deck.py b/tests/unit/pipelex/cogt/models/test_model_deck.pyindex 0620618ce..185f5eaa2 100644--- a/tests/unit/pipelex/cogt/models/test_model_deck.py+++ b/tests/unit/pipelex/cogt/models/test_model_deck.py@@ -1,7 +1,3 @@-"""Unit tests for ModelDeck class."""--from typing import List, Optional-import pytestfrom pytest_mock import MockerFixture@@ -13,10 +9,7 @@ from pipelex.cogt.usage.cost_category import CostCategoryclass TestModelDeckGetOptionalInferenceModel:- """Unit tests for get_optional_inference_model method."""-def _create_test_model_spec(self, name: str) -> InferenceModelSpec:- """Helper to create a test InferenceModelSpec."""return InferenceModelSpec(backend_name="test_backend",name=name,@@ -30,10 +23,9 @@ class TestModelDeckGetOptionalInferenceModel:def _create_test_model_deck(self,- inference_models: Optional[dict[str, InferenceModelSpec]] = None,- aliases: Optional[dict[str, str | List[str]]] = None,+ inference_models: dict[str, InferenceModelSpec] | None = None,+ aliases: dict[str, str | list[str]] | None = None,) -> ModelDeck:- """Helper to create a test ModelDeck."""return ModelDeck(inference_models=inference_models or {},aliases=aliases or {},@@ -49,7 +41,6 @@ class TestModelDeckGetOptionalInferenceModel:)def test_direct_model_lookup_success(self):- """Test successful direct model lookup."""# Arrangemodel_spec = self._create_test_model_spec("gpt-4")model_deck = self._create_test_model_deck(inference_models={"gpt-4": model_spec})@@ -61,7 +52,6 @@ class TestModelDeckGetOptionalInferenceModel:assert result == model_specdef test_direct_model_lookup_not_found(self, mocker: MockerFixture):- """Test direct model lookup when model doesn't exist."""# Arrangemodel_deck = self._create_test_model_deck()mock_log = mocker.patch("pipelex.cogt.models.model_deck.log")@@ -74,7 +64,6 @@ class TestModelDeckGetOptionalInferenceModel:mock_log.warning.assert_called_once_with("Skipping model handle 'nonexistent-model' because it's not found in deck")def test_simple_string_alias_resolution_success(self, mocker: MockerFixture):- """Test successful resolution of a simple string alias."""# Arrangemodel_spec = self._create_test_model_spec("gpt-4")model_deck = self._create_test_model_deck(inference_models={"gpt-4": model_spec}, aliases={"best-gpt": "gpt-4"})@@ -88,7 +77,6 @@ class TestModelDeckGetOptionalInferenceModel:mock_log.debug.assert_called_once_with("Redirection for 'best-gpt': gpt-4")def test_simple_string_alias_resolution_not_found(self, mocker: MockerFixture):- """Test string alias resolution when target model doesn't exist."""# Arrangemodel_deck = self._create_test_model_deck(aliases={"best-gpt": "nonexistent-model"})mock_log = mocker.patch("pipelex.cogt.models.model_deck.log")@@ -103,7 +91,6 @@ class TestModelDeckGetOptionalInferenceModel:mock_log.warning.assert_called_with("Skipping model handle 'best-gpt' because it's not found in deck")def test_list_alias_resolution_first_success(self, mocker: MockerFixture):- """Test list alias resolution where first model in list exists."""# Arrangemodel_spec = self._create_test_model_spec("gpt-4")model_deck = self._create_test_model_deck(inference_models={"gpt-4": model_spec}, aliases={"best-model": ["gpt-4", "claude-3"]})@@ -117,11 +104,10 @@ class TestModelDeckGetOptionalInferenceModel:mock_log.debug.assert_called_once_with("Redirection for 'best-model': ['gpt-4', 'claude-3']")def test_list_alias_resolution_second_success(self, mocker: MockerFixture):- """Test list alias resolution where second model in list exists."""# Arrangemodel_spec = self._create_test_model_spec("claude-3")model_deck = self._create_test_model_deck(- inference_models={"claude-3": model_spec}, aliases={"best-model": ["nonexistent-model", "claude-3"]}+ inference_models={"claude-3": model_spec}, aliases={"best-model": ["nonexistent-model", "claude-3"]},)mock_log = mocker.patch("pipelex.cogt.models.model_deck.log")@@ -133,7 +119,6 @@ class TestModelDeckGetOptionalInferenceModel:mock_log.debug.assert_called_once_with("Redirection for 'best-model': ['nonexistent-model', 'claude-3']")def test_list_alias_resolution_none_found(self, mocker: MockerFixture):- """Test list alias resolution where no models in list exist."""# Arrangemodel_deck = self._create_test_model_deck(aliases={"best-model": ["nonexistent-1", "nonexistent-2"]})mock_log = mocker.patch("pipelex.cogt.models.model_deck.log")@@ -148,7 +133,6 @@ class TestModelDeckGetOptionalInferenceModel:assert mock_log.warning.call_count == 3def test_recursive_alias_resolution_success(self, mocker: MockerFixture):- """Test recursive alias resolution where alias points to another alias."""# Arrangemodel_spec = self._create_test_model_spec("gpt-4")model_deck = self._create_test_model_deck(inference_models={"gpt-4": model_spec}, aliases={"best-model": "best-gpt", "best-gpt": "gpt-4"})@@ -163,11 +147,10 @@ class TestModelDeckGetOptionalInferenceModel:assert mock_log.debug.call_count == 2def test_recursive_alias_resolution_with_list(self, mocker: MockerFixture):- """Test recursive alias resolution where alias points to list containing another alias."""# Arrangemodel_spec = self._create_test_model_spec("gpt-4")model_deck = self._create_test_model_deck(- inference_models={"gpt-4": model_spec}, aliases={"best-model": ["nonexistent", "best-gpt"], "best-gpt": "gpt-4"}+ inference_models={"gpt-4": model_spec}, aliases={"best-model": ["nonexistent", "best-gpt"], "best-gpt": "gpt-4"},)mock_log = mocker.patch("pipelex.cogt.models.model_deck.log")@@ -180,7 +163,6 @@ class TestModelDeckGetOptionalInferenceModel:assert mock_log.debug.call_count >= 2def test_empty_alias_list(self, mocker: MockerFixture):- """Test behavior with empty alias list."""# Arrangemodel_deck = self._create_test_model_deck(aliases={"empty-alias": []})mock_log = mocker.patch("pipelex.cogt.models.model_deck.log")@@ -195,7 +177,6 @@ class TestModelDeckGetOptionalInferenceModel:mock_log.warning.assert_called_once_with("Skipping model handle 'empty-alias' because it's not found in deck")def test_circular_alias_prevention(self, mocker: MockerFixture):- """Test that circular aliases don't cause infinite loops."""# Note: The current implementation doesn't have explicit circular reference detection,# but Python's recursion limit should prevent infinite loops.# This test ensures the method handles it gracefully.@@ -213,7 +194,6 @@ class TestModelDeckGetOptionalInferenceModel:assert result is Nonedef test_complex_waterfall_scenario(self, mocker: MockerFixture):- """Test complex waterfall scenario with multiple levels."""# Arrangemodel_spec = self._create_test_model_spec("claude-3")model_deck = self._create_test_model_deck(@@ -231,7 +211,6 @@ class TestModelDeckGetOptionalInferenceModel:assert mock_log.debug.call_count >= 2def test_mixed_string_and_list_aliases(self, mocker: MockerFixture):- """Test scenario with mix of string and list aliases."""# Arrangemodel_spec1 = self._create_test_model_spec("gpt-4")model_spec2 = self._create_test_model_spec("claude-3")@@ -263,7 +242,6 @@ class TestModelDeckGetOptionalInferenceModel:],)def test_edge_case_llm_handles(self, llm_handle: str, mocker: MockerFixture):- """Test edge cases for llm_handle parameter."""# Arrangemodel_deck = self._create_test_model_deck()mock_log = mocker.patch("pipelex.cogt.models.model_deck.log")@@ -276,7 +254,6 @@ class TestModelDeckGetOptionalInferenceModel:mock_log.warning.assert_called_once_with(f"Skipping model handle '{llm_handle}' because it's not found in deck")def test_logging_behavior(self, mocker: MockerFixture):- """Test that logging is called with correct messages."""# Arrangemodel_deck = self._create_test_model_deck(aliases={"test-alias": "target-model"})mock_log = mocker.patch("pipelex.cogt.models.model_deck.log")diff --git a/tests/unit/pipelex/core/concepts/__init__.py b/tests/unit/pipelex/core/concepts/__init__.pynew file mode 100644index 000000000..e69de29bbdiff --git a/tests/unit/pipelex/core/concepts/concept/__init__.py b/tests/unit/pipelex/core/concepts/concept/__init__.pynew file mode 100644index 000000000..e69de29bbdiff --git a/tests/unit/pipelex/core/concepts/concept/test_concept.py b/tests/unit/pipelex/core/concepts/concept/test_concept.pyindex 792702e72..de683c378 100644--- a/tests/unit/pipelex/core/concepts/concept/test_concept.py+++ b/tests/unit/pipelex/core/concepts/concept/test_concept.py@@ -98,7 +98,7 @@ class TestConcept:valid_domain = "valid_domain"valid_definition = "Lorem Ipsum"- for native_concept in [native_concept for native_concept in NativeConceptEnum]:+ for native_concept in NativeConceptEnum.values_list():assert Concept.is_native_concept(ConceptFactory.make_native_concept(native_concept_data=NATIVE_CONCEPTS_DATA[native_concept])) is Trueassert (@@ -108,7 +108,7 @@ class TestConcept:domain=valid_domain,blueprint=ConceptBlueprint(definition=valid_definition),concept_codes_from_the_same_domain=["RandomConcept"],- )+ ),)is True)@@ -118,7 +118,7 @@ class TestConcept:concept_code=NativeConceptEnum.TEXT,domain=SpecialDomain.NATIVE,blueprint=ConceptBlueprint(definition=valid_definition),- )+ ),)is True)@@ -129,7 +129,7 @@ class TestConcept:domain=valid_domain,blueprint=ConceptBlueprint(definition=valid_definition),concept_codes_from_the_same_domain=["RandomConcept"],- )+ ),)is True)@@ -140,7 +140,7 @@ class TestConcept:domain=valid_domain,blueprint=ConceptBlueprint(definition=valid_definition),concept_codes_from_the_same_domain=["RandomConcept"],- )+ ),)is True)@@ -151,7 +151,7 @@ class TestConcept:domain=valid_domain,blueprint=ConceptBlueprint(definition=valid_definition),concept_codes_from_the_same_domain=["RandomConcept"],- )+ ),)is True)@@ -162,7 +162,7 @@ class TestConcept:domain=valid_domain,blueprint=ConceptBlueprint(definition=valid_definition),concept_codes_from_the_same_domain=["RandomConcept"],- )+ ),)is True)@@ -173,7 +173,7 @@ class TestConcept:domain=valid_domain,blueprint=ConceptBlueprint(definition=valid_definition),concept_codes_from_the_same_domain=["RandomConcept"],- )+ ),)is True)@@ -184,7 +184,7 @@ class TestConcept:domain=valid_domain,blueprint=ConceptBlueprint(definition=valid_definition),concept_codes_from_the_same_domain=["RandomConcept"],- )+ ),)is True)@@ -195,7 +195,7 @@ class TestConcept:domain=valid_domain,blueprint=ConceptBlueprint(definition=valid_definition),concept_codes_from_the_same_domain=["RandomConcept"],- )+ ),)is False)@@ -260,7 +260,6 @@ class TestConcept:ConceptBlueprint.validate_concept_string(f"{SpecialDomain.NATIVE}.InvalidNativeConcept")def test_are_concept_compatible(self):- """Test are_concept_compatible method."""concept1 = ConceptFactory.make_from_blueprint(concept_code="Code1",domain="domain1",diff --git a/tests/unit/pipelex/core/concepts/concept_factory.py/data.py b/tests/unit/pipelex/core/concepts/concept_factory.py/data.pyindex 85ea77036..5f1c22cc8 100644--- a/tests/unit/pipelex/core/concepts/concept_factory.py/data.py+++ b/tests/unit/pipelex/core/concepts/concept_factory.py/data.py@@ -1,6 +1,4 @@-"""Test data for ConceptFactory tests."""--from typing import ClassVar, List, Optional, Tuple+from typing import ClassVar, Optional, List, Tuplefrom pipelex.core.concepts.concept import Conceptfrom pipelex.core.concepts.concept_blueprint import (@@ -15,7 +13,7 @@ from pipelex.core.domains.domain import SpecialDomainclass TestCases:# Test cases for make_refines method - only native concepts can be refined- MAKE_REFINES_TEST_CASES: ClassVar[List[Tuple[str, ConceptBlueprint, str]]] = [+ MAKE_REFINES_TEST_CASES: ClassVar[list[tuple[str, ConceptBlueprint, str]]] = [("native_concept_string",ConceptBlueprint(definition="A concept that refines a native text concept", refines=NativeConceptEnum.TEXT),@@ -32,7 +30,7 @@ class TestCases:]# Test cases for make_domain_and_concept_code_from_concept_string_or_code method- MAKE_DOMAIN_AND_CONCEPT_CODE_TEST_CASES: ClassVar[List[Tuple[str, str, Optional[List[str]], DomainAndConceptCode]]] = [+ MAKE_DOMAIN_AND_CONCEPT_CODE_TEST_CASES: ClassVar[List[Tuple[str, str, List[str] | None, DomainAndConceptCode]]] = [# Test case 1: Concept string with dot notation("my_domain", "other_domain.ConceptName", None, DomainAndConceptCode(domain="other_domain", concept_code="ConceptName")),# Test case 2: Concept string with dot notation (ignores same domain codes)@@ -72,7 +70,7 @@ class TestCases:]# Test cases for make_from_blueprint method- MAKE_FROM_BLUEPRINT_TEST_CASES: ClassVar[List[Tuple[str, str, str, ConceptBlueprint, Optional[List[str]], Concept]]] = [+ MAKE_FROM_BLUEPRINT_TEST_CASES: ClassVar[list[tuple[str, str, str, ConceptBlueprint, list[str] | None, Concept]]] = [# Test case 1: Simple blueprint with no structure or refines (goes to implicit)("simple_blueprint",@@ -189,7 +187,7 @@ class TestCases:"name": "The person's name","age": ConceptStructureBlueprint(definition="The person's age", type=ConceptStructureBlueprintFieldType.NUMBER, required=True),"active": ConceptStructureBlueprint(- definition="Whether the person is active", type=ConceptStructureBlueprintFieldType.BOOLEAN, required=False, default_value=True+ definition="Whether the person is active", type=ConceptStructureBlueprintFieldType.BOOLEAN, required=False, default_value=True,),},),@@ -213,7 +211,7 @@ class TestCases:"name": ConceptStructureBlueprint(definition="The person's name", type=ConceptStructureBlueprintFieldType.TEXT, required=True),"age": ConceptStructureBlueprint(definition="The person's age", type=ConceptStructureBlueprintFieldType.NUMBER, required=True),"active": ConceptStructureBlueprint(- definition="Whether the person is active", type=ConceptStructureBlueprintFieldType.BOOLEAN, required=False, default_value=True+ definition="Whether the person is active", type=ConceptStructureBlueprintFieldType.BOOLEAN, required=False, default_value=True,),},),diff --git a/tests/unit/pipelex/core/concepts/concept_factory.py/test_concept_factory.py b/tests/unit/pipelex/core/concepts/concept_factory.py/test_concept_factory.pyindex d07208de3..8ea032084 100644--- a/tests/unit/pipelex/core/concepts/concept_factory.py/test_concept_factory.py+++ b/tests/unit/pipelex/core/concepts/concept_factory.py/test_concept_factory.py@@ -1,6 +1,5 @@-from typing import Dict, List, Optional, Union-import pytest+from typing import Optional, Listfrom pipelex.core.concepts.concept import Conceptfrom pipelex.core.concepts.concept_blueprint import (@@ -32,37 +31,37 @@ class TestConceptFactory:def test_normalize_structure_blueprint(self):"""Test that mixed structure blueprints are properly normalized."""- mixed_structure_blueprint: Dict[str, Union[str, ConceptStructureBlueprint]] = {+ mixed_structure_blueprint: dict[str, str | ConceptStructureBlueprint] = {"name": "The name of the person","age": ConceptStructureBlueprint(definition="The age of the person", type=ConceptStructureBlueprintFieldType.NUMBER, required=True),"active": ConceptStructureBlueprint(- definition="Whether the person is active", type=ConceptStructureBlueprintFieldType.BOOLEAN, required=False, default_value=True+ definition="Whether the person is active", type=ConceptStructureBlueprintFieldType.BOOLEAN, required=False, default_value=True,),}- expected_structure: Dict[str, ConceptStructureBlueprint] = {+ expected_structure: dict[str, ConceptStructureBlueprint] = {"name": ConceptStructureBlueprint(definition="The name of the person", type=ConceptStructureBlueprintFieldType.TEXT, required=True),"age": ConceptStructureBlueprint(definition="The age of the person", type=ConceptStructureBlueprintFieldType.NUMBER, required=True),"active": ConceptStructureBlueprint(- definition="Whether the person is active", type=ConceptStructureBlueprintFieldType.BOOLEAN, required=False, default_value=True+ definition="Whether the person is active", type=ConceptStructureBlueprintFieldType.BOOLEAN, required=False, default_value=True,),}assert ConceptFactory.normalize_structure_blueprint(mixed_structure_blueprint) == expected_structure- mixed_structure_blueprint2: Dict[str, Union[str, ConceptStructureBlueprint]] = {+ mixed_structure_blueprint2: dict[str, str | ConceptStructureBlueprint] = {"name": ConceptStructureBlueprint(definition="The name of the person", type=ConceptStructureBlueprintFieldType.TEXT, required=True),"age": ConceptStructureBlueprint(definition="The age of the person", type=ConceptStructureBlueprintFieldType.NUMBER, required=True),"active": ConceptStructureBlueprint(- definition="Whether the person is active", type=ConceptStructureBlueprintFieldType.BOOLEAN, required=False, default_value=True+ definition="Whether the person is active", type=ConceptStructureBlueprintFieldType.BOOLEAN, required=False, default_value=True,),}- expected_structure2: Dict[str, ConceptStructureBlueprint] = {+ expected_structure2: dict[str, ConceptStructureBlueprint] = {"name": ConceptStructureBlueprint(definition="The name of the person", type=ConceptStructureBlueprintFieldType.TEXT, required=True),"age": ConceptStructureBlueprint(definition="The age of the person", type=ConceptStructureBlueprintFieldType.NUMBER, required=True),"active": ConceptStructureBlueprint(- definition="Whether the person is active", type=ConceptStructureBlueprintFieldType.BOOLEAN, required=False, default_value=True+ definition="Whether the person is active", type=ConceptStructureBlueprintFieldType.BOOLEAN, required=False, default_value=True,),}@@ -76,7 +75,7 @@ class TestConceptFactory:self,domain: str,concept_string_or_code: str,- concept_codes_from_the_same_domain: Optional[List[str]],+ concept_codes_from_the_same_domain: List[str] | None,expected_result: List[str],):"""Test make_domain_and_concept_code_from_concept_string_or_code method with various inputs."""@@ -97,7 +96,7 @@ class TestConceptFactory:domain: str,concept_code: str,blueprint: ConceptBlueprint,- concept_codes_from_the_same_domain: Optional[List[str]],+ concept_codes_from_the_same_domain: list[str] | None,expected_concept: Concept,):"""Test make_from_blueprint method with various blueprint configurations."""diff --git a/tests/unit/pipelex/core/concepts/concept_factory/__init__.py b/tests/unit/pipelex/core/concepts/concept_factory/__init__.pynew file mode 100644index 000000000..e69de29bbdiff --git a/tests/unit/pipelex/core/concepts/concept_factory/data.py b/tests/unit/pipelex/core/concepts/concept_factory/data.pynew file mode 100644index 000000000..930b8696a--- /dev/null+++ b/tests/unit/pipelex/core/concepts/concept_factory/data.py@@ -0,0 +1,264 @@+"""Test data for ConceptFactory tests."""++from typing import ClassVar++from pipelex.core.concepts.concept import Concept+from pipelex.core.concepts.concept_blueprint import (+ ConceptBlueprint,+ ConceptStructureBlueprint,+ ConceptStructureBlueprintFieldType,+)+from pipelex.core.concepts.concept_factory import DomainAndConceptCode+from pipelex.core.concepts.concept_native import NATIVE_CONCEPTS_DATA, NativeConceptEnum+from pipelex.core.domains.domain import SpecialDomain+++class TestCases:+ """Test cases for ConceptFactory methods."""++ # Test cases for make_refines method - only native concepts can be refined+ MAKE_REFINES_TEST_CASES: ClassVar[list[tuple[str, ConceptBlueprint, str]]] = [+ (+ "native_concept_string",+ ConceptBlueprint(definition="A concept that refines a native text concept", refines=NativeConceptEnum.TEXT.value),+ f"{SpecialDomain.NATIVE.value}.{NativeConceptEnum.TEXT.value}",+ ),+ (+ "fully_qualified_native_string",+ ConceptBlueprint(+ definition="A concept that refines a fully qualified native concept",+ refines=f"{SpecialDomain.NATIVE.value}.{NativeConceptEnum.TEXT.value}",+ ),+ f"{SpecialDomain.NATIVE.value}.{NativeConceptEnum.TEXT.value}",+ ),+ ]++ # Test cases for make_domain_and_concept_code_from_concept_string_or_concept_code method+ MAKE_DOMAIN_AND_CONCEPT_CODE_TEST_CASES: ClassVar[list[tuple[str, str, list[str] | None, DomainAndConceptCode]]] = [+ # Test case 1: Concept string with dot notation+ ("my_domain", "other_domain.ConceptName", None, DomainAndConceptCode(domain="other_domain", concept_code="ConceptName")),+ # Test case 2: Concept string with dot notation (ignores same domain codes)+ ("my_domain", "other_domain.ConceptName", ["ConceptName"], DomainAndConceptCode(domain="other_domain", concept_code="ConceptName")),+ # Test case 3: Native concept code (Text)+ ("my_domain", "Text", None, DomainAndConceptCode(domain=SpecialDomain.NATIVE, concept_code="Text")),+ # Test case 4: Native concept code (Image)+ ("my_domain", "Image", None, DomainAndConceptCode(domain=SpecialDomain.NATIVE, concept_code="Image")),+ # Test case 5: Native concept code (PDF)+ ("my_domain", "PDF", None, DomainAndConceptCode(domain=SpecialDomain.NATIVE, concept_code="PDF")),+ # Test case 6: Native concept code with same domain codes provided (native takes precedence)+ ("my_domain", "Text", ["Text", "OtherConcept"], DomainAndConceptCode(domain=SpecialDomain.NATIVE, concept_code="Text")),+ # Test case 7: Concept code from same domain+ ("my_domain", "MyConcept", ["MyConcept", "OtherConcept"], DomainAndConceptCode(domain="my_domain", concept_code="MyConcept")),+ # Test case 8: Concept code from same domain (case sensitive)+ ("my_domain", "MyConcept", ["MyCon", "OtherConcept"], DomainAndConceptCode(domain=SpecialDomain.IMPLICIT, concept_code="MyConcept")),+ # Test case 9: Unknown concept code (no same domain codes)+ ("my_domain", "UnknownConcept", None, DomainAndConceptCode(domain=SpecialDomain.IMPLICIT, concept_code="UnknownConcept")),+ # Test case 10: Unknown concept code (not in same domain codes)+ (+ "my_domain",+ "UnknownConcept",+ ["KnownConcept", "OtherConcept"],+ DomainAndConceptCode(domain=SpecialDomain.IMPLICIT, concept_code="UnknownConcept"),+ ),+ # Test case 11: Empty same domain codes list+ ("my_domain", "SomeConcept", [], DomainAndConceptCode(domain=SpecialDomain.IMPLICIT, concept_code="SomeConcept")),+ # Test case 12: Different domain in concept string+ ("my_domain", "another_domain.SomeConcept", ["SomeConcept"], DomainAndConceptCode(domain="another_domain", concept_code="SomeConcept")),+ # Test case 13: All native concept codes+ ("my_domain", "Dynamic", None, DomainAndConceptCode(domain=SpecialDomain.NATIVE, concept_code="Dynamic")),+ ("my_domain", "TextAndImages", None, DomainAndConceptCode(domain=SpecialDomain.NATIVE, concept_code="TextAndImages")),+ ("my_domain", "Number", None, DomainAndConceptCode(domain=SpecialDomain.NATIVE, concept_code="Number")),+ ("my_domain", "LlmPrompt", None, DomainAndConceptCode(domain=SpecialDomain.NATIVE, concept_code="LlmPrompt")),+ ("my_domain", "Page", None, DomainAndConceptCode(domain=SpecialDomain.NATIVE, concept_code="Page")),+ ("my_domain", "Anything", None, DomainAndConceptCode(domain=SpecialDomain.NATIVE, concept_code="Anything")),+ ]++ # Test cases for make_from_blueprint method+ MAKE_FROM_BLUEPRINT_TEST_CASES: ClassVar[list[tuple[str, str, str, ConceptBlueprint, list[str] | None, Concept]]] = [+ # Test case 1: Simple blueprint with no structure or refines (goes to implicit)+ (+ "simple_blueprint",+ "my_domain",+ "SimpleConcept",+ ConceptBlueprint(definition="A simple concept"),+ None,+ Concept(+ domain=SpecialDomain.IMPLICIT,+ code="SimpleConcept",+ definition="A simple concept",+ structure_class_name=NATIVE_CONCEPTS_DATA[NativeConceptEnum.TEXT].content_class_name,+ refines=None,+ ),+ ),+ # Test case 2: Blueprint with string structure (existing class, goes to implicit)+ (+ "string_structure_existing_class",+ "my_domain",+ "ConceptWithStructure",+ ConceptBlueprint(definition="A concept with string structure", structure="TextContent"),+ None,+ Concept(+ domain=SpecialDomain.IMPLICIT,+ code="ConceptWithStructure",+ definition="A concept with string structure",+ structure_class_name=NATIVE_CONCEPTS_DATA[NativeConceptEnum.TEXT].content_class_name,+ refines=None,+ ),+ ),+ # Test case 3: Blueprint with refines (native concept, goes to implicit)+ (+ "blueprint_with_refines_text",+ "my_domain",+ "MyTextConcept",+ ConceptBlueprint(definition="A concept that refines Text", refines="Text"),+ None,+ Concept(+ domain=SpecialDomain.IMPLICIT,+ code="MyTextConcept",+ definition="A concept that refines Text",+ structure_class_name=NATIVE_CONCEPTS_DATA[NativeConceptEnum.TEXT].content_class_name,+ refines="native.Text",+ ),+ ),+ # Test case 4: Blueprint with refines (fully qualified native concept, goes to implicit)+ (+ "blueprint_with_refines_native_image",+ "my_domain",+ "MyImageConcept",+ ConceptBlueprint(definition="A concept that refines native.Image", refines="native.Image"),+ None,+ Concept(+ domain=SpecialDomain.IMPLICIT,+ code="MyImageConcept",+ definition="A concept that refines native.Image",+ structure_class_name=NATIVE_CONCEPTS_DATA[NativeConceptEnum.IMAGE].content_class_name,+ refines="native.Image",+ ),+ ),+ # Test case 5: Native concept code (should go to native domain)+ (+ "native_concept_code",+ "my_domain",+ "Text",+ ConceptBlueprint(definition="Native text concept"),+ None,+ Concept(+ domain=SpecialDomain.NATIVE,+ code="Text",+ definition="Native text concept",+ structure_class_name=NATIVE_CONCEPTS_DATA[NativeConceptEnum.TEXT].content_class_name,+ refines=None,+ ),+ ),+ # Test case 6: Concept code from same domain+ (+ "same_domain_concept",+ "my_domain",+ "DomainConcept",+ ConceptBlueprint(definition="A concept from same domain"),+ ["DomainConcept", "OtherConcept"],+ Concept(+ domain="my_domain",+ code="DomainConcept",+ definition="A concept from same domain",+ structure_class_name=NATIVE_CONCEPTS_DATA[NativeConceptEnum.TEXT].content_class_name,+ refines=None,+ ),+ ),+ # Test case 7: Unknown concept code (should go to implicit domain)+ (+ "implicit_concept",+ "my_domain",+ "UnknownConcept",+ ConceptBlueprint(definition="An unknown concept"),+ ["KnownConcept"],+ Concept(+ domain=SpecialDomain.IMPLICIT,+ code="UnknownConcept",+ definition="An unknown concept",+ structure_class_name=NATIVE_CONCEPTS_DATA[NativeConceptEnum.TEXT].content_class_name,+ refines=None,+ ),+ ),+ # Test case 8: Blueprint with dict structure (implicit domain)+ (+ "dict_structure_implicit",+ SpecialDomain.IMPLICIT,+ "PersonConcept",+ ConceptBlueprint(+ definition="A person with structured data",+ structure={+ "name": "The person's name",+ "age": ConceptStructureBlueprint(definition="The person's age", type=ConceptStructureBlueprintFieldType.NUMBER, required=True),+ "active": ConceptStructureBlueprint(+ definition="Whether the person is active", type=ConceptStructureBlueprintFieldType.BOOLEAN, required=False, default_value=True,+ ),+ },+ ),+ None,+ Concept(+ domain=SpecialDomain.IMPLICIT,+ code="PersonConcept",+ definition="A person with structured data",+ structure_class_name="PersonConcept",+ refines=None,+ ),+ ),+ # Test case 9: Blueprint with dict structure (same domain)+ (+ "dict_structure_same_domain",+ "my_domain",+ "PersonConcept",+ ConceptBlueprint(+ definition="A person with structured data",+ structure={+ "name": ConceptStructureBlueprint(definition="The person's name", type=ConceptStructureBlueprintFieldType.TEXT, required=True),+ "age": ConceptStructureBlueprint(definition="The person's age", type=ConceptStructureBlueprintFieldType.NUMBER, required=True),+ "active": ConceptStructureBlueprint(+ definition="Whether the person is active", type=ConceptStructureBlueprintFieldType.BOOLEAN, required=False, default_value=True,+ ),+ },+ ),+ ["PersonConcept", "OtherConcept"],+ Concept(+ domain="my_domain",+ code="PersonConcept",+ definition="A person with structured data",+ structure_class_name="PersonConcept",+ refines=None,+ ),+ ),+ # Test case 10: Blueprint with refines (implicit domain)+ (+ "refines_implicit_domain",+ "my_domain",+ "ConceptWithRefines",+ ConceptBlueprint(definition="A concept with refines", refines="Text"),+ None,+ Concept(+ domain=SpecialDomain.IMPLICIT,+ code="ConceptWithRefines",+ definition="A concept with refines",+ structure_class_name=NATIVE_CONCEPTS_DATA[NativeConceptEnum.TEXT].content_class_name,+ refines="native.Text",+ ),+ ),+ # Test case 11: Blueprint with refines (same domain)+ (+ "refines_same_domain",+ "my_domain",+ "ConceptWithRefines",+ ConceptBlueprint(definition="A concept with refines", refines="native.Image"),+ ["ConceptWithRefines", "OtherConcept"],+ Concept(+ domain="my_domain",+ code="ConceptWithRefines",+ definition="A concept with refines",+ structure_class_name=NATIVE_CONCEPTS_DATA[NativeConceptEnum.IMAGE].content_class_name,+ refines="native.Image",+ ),+ ),+ ]++ # Legacy alias for backward compatibility+ TEST_CASES = MAKE_REFINES_TEST_CASESdiff --git a/tests/unit/pipelex/core/concepts/concept_factory/test_concept_factory.py b/tests/unit/pipelex/core/concepts/concept_factory/test_concept_factory.pynew file mode 100644index 000000000..195a43805--- /dev/null+++ b/tests/unit/pipelex/core/concepts/concept_factory/test_concept_factory.py@@ -0,0 +1,125 @@+"""Tests for ConceptFactory methods."""+++import pytest++from pipelex.core.concepts.concept import Concept+from pipelex.core.concepts.concept_blueprint import (+ ConceptBlueprint,+ ConceptStructureBlueprint,+ ConceptStructureBlueprintFieldType,+)+from pipelex.core.concepts.concept_factory import ConceptFactory++from .data import TestCases+++class TestConceptFactory:+ """Test ConceptFactory methods with various configurations."""++ @pytest.mark.parametrize(+ "test_name,blueprint,expected_result",+ TestCases.MAKE_REFINES_TEST_CASES,+ )+ def test_make_refines(+ self,+ test_name: str,+ blueprint: ConceptBlueprint,+ expected_result: str,+ ):+ """Test make_refines method with different blueprint configurations."""+ result = ConceptFactory.make_refines(blueprint=blueprint)+ assert result == expected_result, f"Failed for test case: {test_name}"++ def test_normalize_structure_blueprint(self):+ """Test that mixed structure blueprints are properly normalized."""+ mixed_structure_blueprint: dict[str, str | ConceptStructureBlueprint] = {+ "name": "The name of the person",+ "age": ConceptStructureBlueprint(definition="The age of the person", type=ConceptStructureBlueprintFieldType.NUMBER, required=True),+ "active": ConceptStructureBlueprint(+ definition="Whether the person is active", type=ConceptStructureBlueprintFieldType.BOOLEAN, required=False, default_value=True,+ ),+ }++ expected_structure: dict[str, ConceptStructureBlueprint] = {+ "name": ConceptStructureBlueprint(definition="The name of the person", type=ConceptStructureBlueprintFieldType.TEXT, required=True),+ "age": ConceptStructureBlueprint(definition="The age of the person", type=ConceptStructureBlueprintFieldType.NUMBER, required=True),+ "active": ConceptStructureBlueprint(+ definition="Whether the person is active", type=ConceptStructureBlueprintFieldType.BOOLEAN, required=False, default_value=True,+ ),+ }++ assert ConceptFactory.normalize_structure_blueprint(mixed_structure_blueprint) == expected_structure++ mixed_structure_blueprint2: dict[str, str | ConceptStructureBlueprint] = {+ "name": ConceptStructureBlueprint(definition="The name of the person", type=ConceptStructureBlueprintFieldType.TEXT, required=True),+ "age": ConceptStructureBlueprint(definition="The age of the person", type=ConceptStructureBlueprintFieldType.NUMBER, required=True),+ "active": ConceptStructureBlueprint(+ definition="Whether the person is active", type=ConceptStructureBlueprintFieldType.BOOLEAN, required=False, default_value=True,+ ),+ }++ expected_structure2: dict[str, ConceptStructureBlueprint] = {+ "name": ConceptStructureBlueprint(definition="The name of the person", type=ConceptStructureBlueprintFieldType.TEXT, required=True),+ "age": ConceptStructureBlueprint(definition="The age of the person", type=ConceptStructureBlueprintFieldType.NUMBER, required=True),+ "active": ConceptStructureBlueprint(+ definition="Whether the person is active", type=ConceptStructureBlueprintFieldType.BOOLEAN, required=False, default_value=True,+ ),+ }++ assert ConceptFactory.normalize_structure_blueprint(mixed_structure_blueprint2) == expected_structure2++ @pytest.mark.parametrize(+ "domain,concept_string_or_code,concept_codes_from_the_same_domain,expected_result",+ TestCases.MAKE_DOMAIN_AND_CONCEPT_CODE_TEST_CASES,+ )+ def test_make_domain_and_concept_code_from_concept_string_orcode(+ self,+ domain: str,+ concept_string_or_code: str,+ concept_codes_from_the_same_domain: list[str] | None,+ expected_result: list[str],+ ):+ """Test make_domain_and_concept_code_from_concept_string_or_code method with various inputs."""+ result = ConceptFactory.make_domain_and_concept_code_from_concept_string_or_code(+ domain=domain,+ concept_string_or_code=concept_string_or_code,+ concept_codes_from_the_same_domain=concept_codes_from_the_same_domain,+ )+ assert result == expected_result++ @pytest.mark.parametrize(+ "test_name,domain,concept_code,blueprint,concept_codes_from_the_same_domain,expected_concept",+ TestCases.MAKE_FROM_BLUEPRINT_TEST_CASES,+ )+ def test_make_from_blueprint(+ self,+ test_name: str,+ domain: str,+ concept_code: str,+ blueprint: ConceptBlueprint,+ concept_codes_from_the_same_domain: list[str] | None,+ expected_concept: Concept,+ ):+ """Test make_from_blueprint method with various blueprint configurations."""+ result = ConceptFactory.make_from_blueprint(+ domain=domain,+ concept_code=concept_code,+ blueprint=blueprint,+ concept_codes_from_the_same_domain=concept_codes_from_the_same_domain,+ )++ assert result == expected_concept, f"Concept mismatch for {test_name}"++ def test_make_from_blueprint_with_invalid_structure_class(self):+ """Test that make_from_blueprint raises StructureClassError for invalid structure class."""+ blueprint = ConceptBlueprint(definition="A concept with invalid structure", structure="NonExistentStructureClass")++ from pipelex.exceptions import StructureClassError++ with pytest.raises(StructureClassError, match="is not a registered subclass of StuffContent"):+ ConceptFactory.make_from_blueprint(+ domain="my_domain",+ concept_code="TestConcept",+ blueprint=blueprint,+ )diff --git a/tests/unit/pipelex/core/concepts/concept_library/__init__.py b/tests/unit/pipelex/core/concepts/concept_library/__init__.pynew file mode 100644index 000000000..e69de29bbdiff --git a/tests/unit/pipelex/core/concepts/test_concept_blueprint.py b/tests/unit/pipelex/core/concepts/test_concept_blueprint.pyindex 49e486db5..149d1f5a2 100644--- a/tests/unit/pipelex/core/concepts/test_concept_blueprint.py+++ b/tests/unit/pipelex/core/concepts/test_concept_blueprint.py@@ -16,7 +16,7 @@ class TestConceptStructureBlueprintValidation:"""Test that valid structure blueprints are accepted."""# Valid text field with defaulttext_blueprint = ConceptStructureBlueprint(- definition="A text field", type=ConceptStructureBlueprintFieldType.TEXT, default_value="default text"+ definition="A text field", type=ConceptStructureBlueprintFieldType.TEXT, default_value="default text",)assert text_blueprint.default_value == "default text"@@ -38,7 +38,7 @@ class TestConceptStructureBlueprintValidation:# Valid list field with defaultlist_blueprint = ConceptStructureBlueprint(- definition="A list field", type=ConceptStructureBlueprintFieldType.LIST, item_type="text", default_value=["item1", "item2"]+ definition="A list field", type=ConceptStructureBlueprintFieldType.LIST, item_type="text", default_value=["item1", "item2"],)assert list_blueprint.default_value == ["item1", "item2"]@@ -77,7 +77,7 @@ class TestConceptStructureBlueprintValidation:# List field with non-list defaultwith pytest.raises(ConceptStructureBlueprintError, match="default_value type mismatch: expected list"):ConceptStructureBlueprint(- definition="A list field", type=ConceptStructureBlueprintFieldType.LIST, item_type="text", default_value="not a list"+ definition="A list field", type=ConceptStructureBlueprintFieldType.LIST, item_type="text", default_value="not a list",)# Dict field with non-dict default@@ -136,21 +136,21 @@ class TestConceptStructureBlueprintValidation:# Boolean field with False default (should be valid)bool_false_blueprint = ConceptStructureBlueprint(- definition="Boolean field with False default", type=ConceptStructureBlueprintFieldType.BOOLEAN, default_value=False+ definition="Boolean field with False default", type=ConceptStructureBlueprintFieldType.BOOLEAN, default_value=False,)assert bool_false_blueprint.default_value is False# Integer field with 0 default (should be valid)int_zero_blueprint = ConceptStructureBlueprint(- definition="Integer field with 0 default", type=ConceptStructureBlueprintFieldType.INTEGER, default_value=0+ definition="Integer field with 0 default", type=ConceptStructureBlueprintFieldType.INTEGER, default_value=0,)assert int_zero_blueprint.default_value == 0# Empty list as default (should be valid)empty_list_blueprint = ConceptStructureBlueprint(- definition="List field with empty default", type=ConceptStructureBlueprintFieldType.LIST, item_type="text", default_value=[]+ definition="List field with empty default", type=ConceptStructureBlueprintFieldType.LIST, item_type="text", default_value=[],)- assert empty_list_blueprint.default_value == []+ assert not empty_list_blueprint.default_value# Empty dict as default (should be valid)empty_dict_blueprint = ConceptStructureBlueprint(@@ -160,4 +160,4 @@ class TestConceptStructureBlueprintValidation:value_type="text",default_value={},)- assert empty_dict_blueprint.default_value == {}+ assert not empty_dict_blueprint.default_valuediff --git a/tests/unit/pipelex/core/interpreter/__init__.py b/tests/unit/pipelex/core/interpreter/__init__.pynew file mode 100644index 000000000..e69de29bbdiff --git a/tests/unit/pipelex/core/interpreter/test_interpreter.py b/tests/unit/pipelex/core/interpreter/test_interpreter.pyindex fb2c2be72..12bf546de 100644--- a/tests/unit/pipelex/core/interpreter/test_interpreter.py+++ b/tests/unit/pipelex/core/interpreter/test_interpreter.py@@ -1,8 +1,8 @@from pathlib import Path-from typing import Typeimport pytest+from pipelex import pretty_printfrom pipelex.core.bundles.pipelex_bundle_blueprint import PipelexBundleBlueprintfrom pipelex.core.interpreter import PipelexInterpreterfrom tests.unit.pipelex.core.test_data import InterpreterTestCases@@ -10,7 +10,6 @@ from tests.unit.pipelex.core.test_data import InterpreterTestCasesclass TestPipelexInterpreter:def test_init_with_both_file_path_and_content(self, tmp_path: Path):- """Test initialization with both file_path and file_content."""test_file = tmp_path / "test.plx"test_file.write_text("domain = 'test'\n[concept]\n")content = "domain = 'other'\n[concept]\n"@@ -21,9 +20,8 @@ class TestPipelexInterpreter:@pytest.mark.parametrize("test_name,plx_content,expected_blueprint", InterpreterTestCases.VALID_TEST_CASES)def test_make_pipelex_bundle_blueprint(self, test_name: str, plx_content: str, expected_blueprint: PipelexBundleBlueprint):- """Test making blueprint from various valid PLX content."""converter = PipelexInterpreter(file_content=plx_content)- from pipelex import pretty_print+blueprint = converter.make_pipelex_bundle_blueprint()pretty_print(blueprint, title=f"Blueprint {test_name}")@@ -32,17 +30,14 @@ class TestPipelexInterpreter:@pytest.mark.parametrize("test_name,plx_content,expected_blueprint", InterpreterTestCases.VALID_TEST_CASES)def test_make_plx_content(self, test_name: str, plx_content: str, expected_blueprint: PipelexBundleBlueprint):- """Test making blueprint from various valid PLX content."""get_plx_content = PipelexInterpreter.make_plx_content(blueprint=expected_blueprint)- from pipelex import pretty_printpretty_print(get_plx_content, title=f"PLx content {test_name}")pretty_print(plx_content, title=f"Expected PLX content {test_name}")assert get_plx_content == plx_content@pytest.mark.parametrize("test_name,invalid_plx_content,expected_exception", InterpreterTestCases.ERROR_TEST_CASES)- def test_invalid_plx_should_raise_exception(self, test_name: str, invalid_plx_content: str, expected_exception: Type[Exception]):- """Test that invalid PLX content raises appropriate exceptions."""+ def test_invalid_plx_should_raise_exception(self, test_name: str, invalid_plx_content: str, expected_exception: type[Exception]):converter = PipelexInterpreter(file_content=invalid_plx_content)with pytest.raises(expected_exception):diff --git a/tests/unit/pipelex/core/interpreter/test_interpreter_serialize.py b/tests/unit/pipelex/core/interpreter/test_interpreter_serialize.pyindex 923b80aa1..97145efd8 100644--- a/tests/unit/pipelex/core/interpreter/test_interpreter_serialize.py+++ b/tests/unit/pipelex/core/interpreter/test_interpreter_serialize.py@@ -1,6 +1,4 @@-"""Unit tests for PipelexInterpreter serialize functions."""--from typing import Any, Dict+from typing import Anyfrom pipelex.core.concepts.concept_blueprint import ConceptBlueprint, ConceptStructureBlueprint, ConceptStructureBlueprintFieldTypefrom pipelex.core.interpreter import PipelexInterpreter@@ -12,10 +10,7 @@ from pipelex.pipe_operators.ocr.pipe_ocr_blueprint import PipeOcrBlueprintclass TestSerializeInputRequirement:- """Test _serialize_input_requirement function."""-def test_serialize_input_requirement_basic(self):- """Test serializing basic InputRequirementBlueprint."""input_req = InputRequirementBlueprint(concept="Text")result = PipelexInterpreter.serialize_input_requirement(input_req)@@ -23,7 +18,6 @@ class TestSerializeInputRequirement:assert result == expecteddef test_serialize_input_requirement_with_multiplicity_int(self):- """Test serializing InputRequirementBlueprint with integer multiplicity."""input_req = InputRequirementBlueprint(concept="Text", multiplicity=3)result = PipelexInterpreter.serialize_input_requirement(input_req)@@ -31,7 +25,6 @@ class TestSerializeInputRequirement:assert result == expecteddef test_serialize_input_requirement_with_multiplicity_bool(self):- """Test serializing InputRequirementBlueprint with boolean multiplicity."""input_req = InputRequirementBlueprint(concept="Text", multiplicity=True)result = PipelexInterpreter.serialize_input_requirement(input_req)@@ -40,11 +33,8 @@ class TestSerializeInputRequirement:class TestSerializeInputs:- """Test _serialize_inputs function."""-def test_serialize_inputs_mixed_types(self):- """Test serializing mixed input types."""- inputs: Dict[str, str | InputRequirementBlueprint] = {+ inputs: dict[str, str | InputRequirementBlueprint] = {"text": "Text","data": InputRequirementBlueprint(concept="Data", multiplicity=2),"query": InputRequirementBlueprint(concept="Text"),@@ -55,25 +45,21 @@ class TestSerializeInputs:assert result == expecteddef test_serialize_inputs_empty(self):- """Test serializing empty inputs."""- inputs: Dict[str, Any] = {}+ inputs: dict[str, Any] = {}result = PipelexInterpreter.serialize_inputs(inputs)assert result == {}class TestSerializeConceptStructure:- """Test concept structure serialization functions."""def test_serialize_concept_structure_field_string(self):- """Test serializing string concept structure field."""field_value = "Simple description"result = PipelexInterpreter.serialize_concept_structure_field(field_value)assert result == "Simple description"def test_serialize_concept_structure_field_blueprint(self):- """Test serializing ConceptStructureBlueprint field."""field_value = ConceptStructureBlueprint(definition="A text field", type=ConceptStructureBlueprintFieldType.TEXT, required=True)result = PipelexInterpreter.serialize_concept_structure_field(field_value)@@ -81,7 +67,6 @@ class TestSerializeConceptStructure:assert result == expecteddef test_serialize_concept_structure_field_blueprint_with_all_fields(self):- """Test serializing ConceptStructureBlueprint with all optional fields."""field_value = ConceptStructureBlueprint(definition="A complex field",type=ConceptStructureBlueprintFieldType.DICT,@@ -105,14 +90,12 @@ class TestSerializeConceptStructure:assert result == expecteddef test_serialize_concept_structure_string(self):- """Test serializing string concept structure."""structure = "Simple structure"result = PipelexInterpreter.serialize_concept_structure(structure)assert result == "Simple structure"def test_serialize_concept_structure_dict(self):- """Test serializing dict concept structure."""structure = {"name": "Person's name","age": ConceptStructureBlueprint(definition="Person's age", type=ConceptStructureBlueprintFieldType.INTEGER, required=True),@@ -127,24 +110,20 @@ class TestSerializeConceptStructure:class TestSerializeSingleConcept:- """Test _serialize_single_concept function."""def test_serialize_single_concept_string(self):- """Test serializing string concept."""concept = "Simple concept definition"result = PipelexInterpreter.serialize_single_concept(concept)assert result == "Simple concept definition"def test_serialize_single_concept_simple_definition(self):- """Test serializing ConceptBlueprint with just definition."""concept = ConceptBlueprint(definition="A simple concept")result = PipelexInterpreter.serialize_single_concept(concept)assert result == "A simple concept"def test_serialize_single_concept_with_refines(self):- """Test serializing ConceptBlueprint with refines."""concept = ConceptBlueprint(definition="A specialized text", refines="Text")result = PipelexInterpreter.serialize_single_concept(concept)@@ -152,7 +131,6 @@ class TestSerializeSingleConcept:assert result == expecteddef test_serialize_single_concept_with_structure(self):- """Test serializing ConceptBlueprint with structure."""concept = ConceptBlueprint(definition="A person",structure={@@ -173,16 +151,12 @@ class TestSerializeSingleConcept:class TestSerializeConcepts:- """Test _serialize_concepts function."""-def test_serialize_concepts_empty(self):- """Test serializing empty concepts."""- result = PipelexInterpreter.serialize_concepts(None, "test_domain")+ result = PipelexInterpreter.serialize_concepts(None)assert result == {}def test_serialize_concepts_mixed(self):- """Test serializing mixed concept types."""- concepts: Dict[str, ConceptBlueprint | str] = {+ concepts: dict[str, ConceptBlueprint | str] = {"SimpleText": "A simple text concept","PersonInfo": ConceptBlueprint(definition="Information about a person", refines="Text"),"StructuredData": ConceptBlueprint(@@ -193,7 +167,7 @@ class TestSerializeConcepts:},),}- result = PipelexInterpreter.serialize_concepts(concepts, "test_domain")+ result = PipelexInterpreter.serialize_concepts(concepts)expected = {"SimpleText": "A simple text concept",@@ -210,10 +184,8 @@ class TestSerializeConcepts:class TestSerializeSubPipe:- """Test _serialize_sub_pipe function."""def test_serialize_sub_pipe_basic(self):- """Test serializing basic sub pipe."""sub_pipe = SubPipeBlueprint(pipe="test_pipe", result="test_result")result = PipelexInterpreter.serialize_sub_pipe(sub_pipe)@@ -221,7 +193,6 @@ class TestSerializeSubPipe:assert result == expecteddef test_serialize_sub_pipe_with_nb_output_and_batch(self):- """Test serializing sub pipe with nb_output and batch fields."""sub_pipe = SubPipeBlueprint(pipe="test_pipe", result="test_result", nb_output=3, batch_over="input_list", batch_as="current_item")result = PipelexInterpreter.serialize_sub_pipe(sub_pipe)@@ -229,7 +200,6 @@ class TestSerializeSubPipe:assert result == expecteddef test_serialize_sub_pipe_with_multiple_output_and_batch(self):- """Test serializing sub pipe with multiple_output and batch fields."""sub_pipe = SubPipeBlueprint(pipe="test_pipe", result="test_result", multiple_output=True, batch_over="input_list", batch_as="current_item")result = PipelexInterpreter.serialize_sub_pipe(sub_pipe)@@ -237,7 +207,6 @@ class TestSerializeSubPipe:assert result == expecteddef test_serialize_sub_pipe_skip_defaults(self):- """Test that default values are not included in serialization."""sub_pipe = SubPipeBlueprint(pipe="test_pipe",result="test_result",@@ -251,18 +220,14 @@ class TestSerializeSubPipe:class TestSerializePipeBlueprints:- """Test individual pipe blueprint serialization functions."""-def test_serialize_llm_pipe_basic(self):- """Test serializing basic PipeLLM blueprint."""pipe = PipeLLMBlueprint(type="PipeLLM", definition="Generate text using LLM", output="Text", prompt_template="Generate a story")- result = PipelexInterpreter.serialize_llm_pipe(pipe, "test_domain")+ result = PipelexInterpreter.serialize_llm_pipe(pipe)expected = {"type": "PipeLLM", "definition": "Generate text using LLM", "output": "Text", "prompt_template": "Generate a story"}assert result == expecteddef test_serialize_llm_pipe_with_inputs_and_options(self):- """Test serializing PipeLLM blueprint with inputs and optional fields."""pipe = PipeLLMBlueprint(type="PipeLLM",definition="Extract information",@@ -272,7 +237,7 @@ class TestSerializePipeBlueprints:prompt_template="Extract info about $topic from @text",nb_output=2,)- result = PipelexInterpreter.serialize_llm_pipe(pipe, "test_domain")+ result = PipelexInterpreter.serialize_llm_pipe(pipe)expected = {"type": "PipeLLM",@@ -286,22 +251,20 @@ class TestSerializePipeBlueprints:assert result == expecteddef test_serialize_ocr_pipe_basic(self):- """Test serializing basic PipeOcr blueprint."""pipe = PipeOcrBlueprint(type="PipeOcr", definition="Extract text from PDF", output="Page", ocr="base_ocr_pypdfium2")- result = PipelexInterpreter.serialize_ocr_pipe(pipe, "test_domain")+ result = PipelexInterpreter.serialize_ocr_pipe(pipe)expected = {"type": "PipeOcr", "definition": "Extract text from PDF", "output": "Page", "ocr": "base_ocr_pypdfium2"}assert result == expecteddef test_serialize_sequence_pipe_basic(self):- """Test serializing basic PipeSequence blueprint."""pipe = PipeSequenceBlueprint(type="PipeSequence",definition="Process in sequence",output="Result",steps=[SubPipeBlueprint(pipe="step1", result="result1"), SubPipeBlueprint(pipe="step2", result="result2")],)- result = PipelexInterpreter.serialize_sequence_pipe(pipe, "test_domain")+ result = PipelexInterpreter.serialize_sequence_pipe(pipe)expected = {"type": "PipeSequence",@@ -313,11 +276,8 @@ class TestSerializePipeBlueprints:class TestAddCommonPipeFields:- """Test _add_common_pipe_fields helper function."""-def test_add_common_pipe_fields_with_inputs(self):- """Test adding common pipe fields when inputs exist."""- result: Dict[str, Any] = {"type": "TestPipe"}+ result: dict[str, Any] = {"type": "TestPipe"}# Mock pipe with inputsclass MockPipe:@@ -331,8 +291,7 @@ class TestAddCommonPipeFields:assert result == expecteddef test_add_common_pipe_fields_without_inputs(self):- """Test adding common pipe fields when no inputs exist."""- result: Dict[str, Any] = {"type": "TestPipe"}+ result: dict[str, Any] = {"type": "TestPipe"}# Mock pipe without inputsclass MockPipe:@@ -346,8 +305,7 @@ class TestAddCommonPipeFields:assert result == expecteddef test_add_common_pipe_fields_no_inputs_attribute(self):- """Test adding common pipe fields when pipe has no inputs attribute."""- result: Dict[str, Any] = {"type": "TestPipe"}+ result: dict[str, Any] = {"type": "TestPipe"}# Mock pipe without inputs attributeclass MockPipe:diff --git a/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_batch_plx.py b/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_batch_plx.pyindex f8c27a00c..7e55a2efd 100644--- a/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_batch_plx.py+++ b/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_batch_plx.py@@ -5,8 +5,6 @@ from pipelex.pipe_controllers.batch.pipe_batch_blueprint import PipeBatchBluepriclass TestPipelexInterpreterBatchPLX:- """Test Batch pipe to PLX string conversion."""-@pytest.mark.parametrize("pipe_name,blueprint,expected_plx",[@@ -50,5 +48,5 @@ branch_pipe_code = "transform_item\"""",)def test_batch_pipe_to_plx_string(self, pipe_name: str, blueprint: PipeBatchBlueprint, expected_plx: str):"""Test converting Batch pipe blueprint to PLX string."""- result = PipelexInterpreter.batch_pipe_to_plx_string(pipe_name, blueprint, "test_domain")+ result = PipelexInterpreter.batch_pipe_to_plx_string(pipe_name, blueprint)assert result == expected_plxdiff --git a/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_concept_plx.py b/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_concept_plx.pyindex 5aec6a2a3..3e319da41 100644--- a/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_concept_plx.py+++ b/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_concept_plx.py@@ -5,8 +5,6 @@ from pipelex.core.interpreter import PipelexInterpreterclass TestPipelexInterpreterConceptPLX:- """Test concept to PLX string conversion."""-@pytest.mark.parametrize("concepts,expected_plx",[@@ -93,5 +91,5 @@ structure = "ProductStructure\"""",)def test_concepts_to_plx_string(self, concepts: dict[str, ConceptBlueprint | str], expected_plx: str):"""Test converting concepts dict to PLX string."""- result = PipelexInterpreter.concepts_to_plx_string(concepts, "test_domain")+ result = PipelexInterpreter.concepts_to_plx_string(concepts)assert result == expected_plxdiff --git a/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_condition_plx.py b/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_condition_plx.pyindex 216a8de5c..fd7098489 100644--- a/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_condition_plx.py+++ b/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_condition_plx.py@@ -5,8 +5,6 @@ from pipelex.pipe_controllers.condition.pipe_condition_blueprint import PipeCondclass TestPipelexInterpreterConditionPLX:- """Test Condition pipe to PLX string conversion."""-@pytest.mark.parametrize("pipe_name,blueprint,expected_plx",[@@ -50,5 +48,5 @@ expression_template = "{{ data.size if data.size > threshold else 'low' }}\"""",)def test_condition_pipe_to_plx_string(self, pipe_name: str, blueprint: PipeConditionBlueprint, expected_plx: str):"""Test converting Condition pipe blueprint to PLX string."""- result = PipelexInterpreter.condition_pipe_to_plx_string(pipe_name, blueprint, "test_domain")+ result = PipelexInterpreter.condition_pipe_to_plx_string(pipe_name, blueprint)assert result == expected_plxdiff --git a/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_func_plx.py b/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_func_plx.pyindex 38e2cdb8f..80d6d53fb 100644--- a/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_func_plx.py+++ b/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_func_plx.py@@ -5,8 +5,6 @@ from pipelex.pipe_operators.func.pipe_func_blueprint import PipeFuncBlueprintclass TestPipelexInterpreterFuncPLX:- """Test Func pipe to PLX string conversion."""-@pytest.mark.parametrize("pipe_name,blueprint,expected_plx",[@@ -45,6 +43,5 @@ function_name = "transform_function\"""",],)def test_func_pipe_to_plx_string(self, pipe_name: str, blueprint: PipeFuncBlueprint, expected_plx: str):- """Test converting Func pipe blueprint to PLX string."""- result = PipelexInterpreter.func_pipe_to_plx_string(pipe_name, blueprint, "test_domain")+ result = PipelexInterpreter.func_pipe_to_plx_string(pipe_name, blueprint)assert result == expected_plxdiff --git a/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_img_gen_plx.py b/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_img_gen_plx.pyindex 663f44187..5b02034f4 100644--- a/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_img_gen_plx.py+++ b/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_img_gen_plx.py@@ -6,8 +6,6 @@ from pipelex.pipe_operators.img_gen.pipe_img_gen_blueprint import PipeImgGenBlueclass TestPipelexInterpreterImgGenPLX:- """Test ImgGen pipe to PLX string conversion."""-@pytest.mark.parametrize("pipe_name,blueprint,expected_plx",[@@ -48,6 +46,5 @@ seed = 12345""",],)def test_img_gen_pipe_to_plx_string(self, pipe_name: str, blueprint: PipeImgGenBlueprint, expected_plx: str):- """Test converting ImgGen pipe blueprint to PLX string."""- result = PipelexInterpreter.img_gen_pipe_to_plx_string(pipe_name, blueprint, "test_domain")+ result = PipelexInterpreter.img_gen_pipe_to_plx_string(pipe_name, blueprint)assert result == expected_plxdiff --git a/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_input_spec.py b/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_input_spec.pyindex 4a9f30432..ff135bbe1 100644--- a/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_input_spec.py+++ b/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_input_spec.py@@ -7,10 +7,7 @@ from pipelex.core.pipes.pipe_input_blueprint import InputRequirementBlueprintclass TestPipelexInterpreterInputSpec:- """Test input serialization for both simple strings and InputRequirementBlueprint objects."""-def test_inputs_to_plx_string_simple_strings(self):- """Test converting simple string inputs to PLX format like { text = "Text", topic = "Text" }."""# Simple string inputs (like "Text")simple_inputs = {"text": "Text", "topic": "Text"}@@ -20,7 +17,6 @@ class TestPipelexInterpreterInputSpec:assert result == expecteddef test_inputs_to_plx_string_input_requirement_blueprints(self):- """Test converting InputRequirementBlueprint inputs to PLX format like { text = { concept = "Text", multiplicity = 1 } }."""# InputRequirementBlueprint inputscomplex_inputs = {"text": InputRequirementBlueprint(concept="Text", multiplicity=1),@@ -53,6 +49,5 @@ class TestPipelexInterpreterInputSpec:],)def test_serialize_input_requirement(self, input_req: InputRequirementBlueprint, expected: dict[str, Any]):- """Test serializing a single InputRequirementBlueprint with various multiplicity values."""result = PipelexInterpreter.serialize_input_requirement(input_req)assert result == expecteddiff --git a/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_jinja2_plx.py b/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_jinja2_plx.pyindex 290efeb5f..525a17a9c 100644--- a/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_jinja2_plx.py+++ b/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_jinja2_plx.py@@ -5,8 +5,6 @@ from pipelex.pipe_operators.jinja2.pipe_jinja2_blueprint import PipeJinja2Blueprclass TestPipelexInterpreterJinja2PLX:- """Test Jinja2 pipe to PLX string conversion."""-@pytest.mark.parametrize("pipe_name,blueprint,expected_plx",[@@ -45,6 +43,5 @@ jinja2_name = "data_formatter\"""",],)def test_jinja2_pipe_to_plx_string(self, pipe_name: str, blueprint: PipeJinja2Blueprint, expected_plx: str):- """Test converting Jinja2 pipe blueprint to PLX string."""- result = PipelexInterpreter.jinja2_pipe_to_plx_string(pipe_name, blueprint, "test_domain")+ result = PipelexInterpreter.jinja2_pipe_to_plx_string(pipe_name, blueprint)assert result == expected_plxdiff --git a/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_jinja2_spacing.py b/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_jinja2_spacing.pyindex 15e1b9d8c..41b8ec9f7 100644--- a/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_jinja2_spacing.py+++ b/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_jinja2_spacing.py@@ -5,8 +5,6 @@ from pipelex.pipe_operators.jinja2.pipe_jinja2_blueprint import PipeJinja2Blueprclass TestPipelexInterpreterJinja2PipePLX:- """Test the jinja2_pipe_to_plx_string function directly."""-@pytest.mark.parametrize("pipe_name,blueprint,expected_plx",[@@ -75,6 +73,5 @@ jinja2_name = "greeting_template\"""",],)def test_jinja2_pipe_to_plx_string(self, pipe_name: str, blueprint: PipeJinja2Blueprint, expected_plx: str):- """Test converting PipeJinja2Blueprint to PLX string with various configurations."""- result = PipelexInterpreter.jinja2_pipe_to_plx_string(pipe_name, blueprint, "test_domain")+ result = PipelexInterpreter.jinja2_pipe_to_plx_string(pipe_name, blueprint)assert result == expected_plxdiff --git a/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_llm_plx.py b/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_llm_plx.pyindex 08d6641a7..5b9b0e6f4 100644--- a/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_llm_plx.py+++ b/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_llm_plx.py@@ -6,8 +6,6 @@ from pipelex.pipe_operators.llm.pipe_llm_blueprint import PipeLLMBlueprintclass TestPipelexInterpreterLLMPLX:- """Test LLM pipe to PLX string conversion."""-@pytest.mark.parametrize("pipe_name,blueprint,expected_plx",[@@ -89,6 +87,5 @@ prompt_template = "Generate ideas about: $topic\"""",],)def test_llm_pipe_to_plx_string(self, pipe_name: str, blueprint: PipeLLMBlueprint, expected_plx: str):- """Test converting LLM pipe blueprint to PLX string."""- result = PipelexInterpreter.llm_pipe_to_plx_string(pipe_name, blueprint, "test_domain")+ result = PipelexInterpreter.llm_pipe_to_plx_string(pipe_name, blueprint)assert result == expected_plxdiff --git a/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_ocr_plx.py b/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_ocr_plx.pyindex 54d83bf8d..c8f6b3fc9 100644--- a/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_ocr_plx.py+++ b/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_ocr_plx.py@@ -5,8 +5,6 @@ from pipelex.pipe_operators.ocr.pipe_ocr_blueprint import PipeOcrBlueprintclass TestPipelexInterpreterOcrPLX:- """Test OCR pipe to PLX string conversion."""-@pytest.mark.parametrize("pipe_name,blueprint,expected_plx",[@@ -45,6 +43,5 @@ ocr = "base_ocr_pypdfium2\"""",],)def test_ocr_pipe_to_plx_string(self, pipe_name: str, blueprint: PipeOcrBlueprint, expected_plx: str):- """Test converting OCR pipe blueprint to PLX string."""- result = PipelexInterpreter.ocr_pipe_to_plx_string(pipe_name, blueprint, "test_domain")+ result = PipelexInterpreter.ocr_pipe_to_plx_string(pipe_name, blueprint)assert result == expected_plxdiff --git a/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_parallel_plx.py b/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_parallel_plx.pyindex e2a086d3c..4ddbac40d 100644--- a/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_parallel_plx.py+++ b/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_parallel_plx.py@@ -6,8 +6,6 @@ from pipelex.pipe_controllers.sub_pipe_blueprint import SubPipeBlueprintclass TestPipelexInterpreterParallelPLX:- """Test Parallel pipe to PLX string conversion."""-@pytest.mark.parametrize("pipe_name,blueprint,expected_plx",[@@ -64,6 +62,5 @@ combined_output = "MergedResults\"""",],)def test_parallel_pipe_to_plx_string(self, pipe_name: str, blueprint: PipeParallelBlueprint, expected_plx: str):- """Test converting Parallel pipe blueprint to PLX string."""- result = PipelexInterpreter.parallel_pipe_to_plx_string(pipe_name, blueprint, "test_domain")+ result = PipelexInterpreter.parallel_pipe_to_plx_string(pipe_name, blueprint)assert result == expected_plxdiff --git a/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_sequence_plx.py b/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_sequence_plx.pyindex 1c0f95bdf..8c2e15b65 100644--- a/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_sequence_plx.py+++ b/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_sequence_plx.py@@ -1,5 +1,3 @@-"""Test PipelexInterpreter Sequence pipe to PLX string conversion."""-import pytestfrom pipelex.core.interpreter import PipelexInterpreter@@ -9,8 +7,6 @@ from pipelex.pipe_controllers.sub_pipe_blueprint import SubPipeBlueprintclass TestPipelexInterpreterSequencePLX:- """Test Sequence pipe to PLX string conversion."""-@pytest.mark.parametrize("pipe_name,blueprint,expected_plx",[@@ -84,6 +80,5 @@ steps = [],)def test_sequence_pipe_to_plx_string(self, pipe_name: str, blueprint: PipeSequenceBlueprint, expected_plx: str):- """Test converting Sequence pipe blueprint to PLX string."""- result = PipelexInterpreter.sequence_pipe_to_plx_string(pipe_name, blueprint, "test_domain")+ result = PipelexInterpreter.sequence_pipe_to_plx_string(pipe_name, blueprint)assert result == expected_plxdiff --git a/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_serialize_sub_pipe.py b/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_serialize_sub_pipe.pyindex dac9c022e..95d72fa7c 100644--- a/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_serialize_sub_pipe.py+++ b/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_serialize_sub_pipe.py@@ -1,6 +1,4 @@-"""Test PipelexInterpreter serialize_sub_pipe function."""--from typing import Any, Dict+from typing import Anyimport pytest@@ -9,8 +7,6 @@ from pipelex.pipe_controllers.sub_pipe_blueprint import SubPipeBlueprintclass TestPipelexInterpreterSerializeSubPipe:- """Test the serialize_sub_pipe function directly."""-@pytest.mark.parametrize("sub_pipe_blueprint,expected_result",[@@ -98,8 +94,7 @@ class TestPipelexInterpreterSerializeSubPipe:),],)- def test_serialize_sub_pipe(self, sub_pipe_blueprint: SubPipeBlueprint, expected_result: Dict[str, Any]):- """Test serializing SubPipeBlueprint with various configurations."""+ def test_serialize_sub_pipe(self, sub_pipe_blueprint: SubPipeBlueprint, expected_result: dict[str, Any]):result = PipelexInterpreter.serialize_sub_pipe(sub_pipe_blueprint)assert result == expected_result@@ -154,6 +149,5 @@ class TestPipelexInterpreterSerializeSubPipe:],)def test_sub_pipe_to_plx_string(self, sub_pipe_blueprint: SubPipeBlueprint, expected_plx_string: str):- """Test converting SubPipeBlueprint directly to PLX inline table string."""result = PipelexInterpreter.sub_pipe_to_plx_string(sub_pipe_blueprint)assert result == expected_plx_stringdiff --git a/tests/unit/pipelex/core/memory/__init__.py b/tests/unit/pipelex/core/memory/__init__.pynew file mode 100644index 000000000..e69de29bbdiff --git a/tests/unit/pipelex/core/memory/test_dry_working_memory.py b/tests/unit/pipelex/core/memory/test_dry_working_memory.pyindex 83a6b5f53..89f923921 100644--- a/tests/unit/pipelex/core/memory/test_dry_working_memory.py+++ b/tests/unit/pipelex/core/memory/test_dry_working_memory.py@@ -1,7 +1,3 @@-"""Test dry working memory functionality."""--from typing import List-import pytestfrom pytest import FixtureRequest@@ -17,13 +13,10 @@ from tests.test_pipelines.tricky_questions import ThoughtfulAnswer@pytest.mark.dry_runnable@pytest.mark.asyncio(loop_scope="class")class TestDryWorkingMemory:- """Test WorkingMemory dry run functionality."""-async def test_make_for_dry_run_with_page_content(self,request: FixtureRequest,):- """Test that make_for_dry_run creates appropriate mocks for PageContent."""log.info("Testing dry run with PageContent")# Test the specific inputs requested by the user@@ -65,7 +58,6 @@ class TestDryWorkingMemory:self,request: FixtureRequest,):- """Test that make_for_dry_run creates appropriate mocks for structured content."""log.info("Testing dry run with structured content (ThoughtfulAnswer)")# Use ThoughtfulAnswer from tricky questions domain@@ -88,9 +80,9 @@ class TestDryWorkingMemory:]dry_memory = WorkingMemoryFactory.make_for_dry_run(needed_inputs=needed_inputs)+ expected_number_of_elements_in_memory = 2- # Verify mocks were created- assert len(dry_memory.root) == 2+ assert len(dry_memory.root) == expected_number_of_elements_in_memoryassert dry_memory.get_optional_stuff("thoughtful_answer") is not Noneassert dry_memory.get_optional_stuff("question") is not None@@ -124,7 +116,6 @@ class TestDryWorkingMemory:self,request: FixtureRequest,):- """Test that make_for_dry_run falls back to TextContent when needed."""log.info("Testing dry run with TextContent fallback")needed_inputs = [@@ -152,7 +143,6 @@ class TestDryWorkingMemory:dry_memory = WorkingMemoryFactory.make_for_dry_run(needed_inputs=needed_inputs)- # Verify mocks were createdassert len(dry_memory.root) == 2assert dry_memory.get_optional_stuff("question_analysis") is not Noneassert dry_memory.get_optional_stuff("conclusion") is not None@@ -174,7 +164,6 @@ class TestDryWorkingMemory:self,request: FixtureRequest,):- """Test that make_for_dry_run handles mixed content types correctly."""log.info("Testing dry run with mixed content types")needed_inputs = [@@ -207,7 +196,6 @@ class TestDryWorkingMemory:dry_memory = WorkingMemoryFactory.make_for_dry_run(needed_inputs=needed_inputs)- # Verify all mocks were createdassert len(dry_memory.root) == 3# Verify structured content@@ -234,10 +222,9 @@ class TestDryWorkingMemory:self,request: FixtureRequest,):- """Test that make_for_dry_run handles empty inputs gracefully."""log.info("Testing dry run with empty inputs")- needed_inputs: List[TypedNamedInputRequirement] = []+ needed_inputs: list[TypedNamedInputRequirement] = []dry_memory = WorkingMemoryFactory.make_for_dry_run(needed_inputs=needed_inputs)@@ -251,7 +238,6 @@ class TestDryWorkingMemory:self,request: FixtureRequest,):- """Test make_for_dry_run with a realistic pipeline scenario from tricky questions."""log.info("Testing dry run with realistic tricky questions pipeline scenario")# Simulate the conclude_tricky_question_by_steps pipeline needsdiff --git a/tests/unit/pipelex/core/memory/test_working_memory.py b/tests/unit/pipelex/core/memory/test_working_memory.pyindex 8158b9017..85e081f8b 100644--- a/tests/unit/pipelex/core/memory/test_working_memory.py+++ b/tests/unit/pipelex/core/memory/test_working_memory.py@@ -1,4 +1,4 @@-from typing import ClassVar, List, Tuple+from typing import ClassVarimport pytest@@ -12,8 +12,6 @@ from pipelex.core.stuffs.stuff_factory import StuffFactoryclass TestWorkingMemoryData:- """Test data for WorkingMemory tests."""-# Sample text contentSAMPLE_TEXT = """The Dawn of Ultra-Rapid Transit: NextGen High-Speed Trains Redefine Travel@@ -40,7 +38,7 @@ class TestWorkingMemoryData:HTML_CONTENT_CASE = "html_content"NUMBER_CONTENT_CASE = "number_content"- TEST_CASES: ClassVar[List[Tuple[str, str]]] = [+ TEST_CASES: ClassVar[list[tuple[str, str]]] = [("Single text content", SINGLE_TEXT_CASE),("Single image content", SINGLE_IMAGE_CASE),("Single PDF content", SINGLE_PDF_CASE),@@ -67,7 +65,7 @@ class TestWorkingMemory:def single_image_memory(self) -> WorkingMemory:"""Create WorkingMemory with single image content."""return WorkingMemoryFactory.make_from_image(- image_url=TestWorkingMemoryData.SAMPLE_IMAGE_URL, concept_string="gantt.GanttImage", name="gantt_chart_image"+ image_url=TestWorkingMemoryData.SAMPLE_IMAGE_URL, concept_string="gantt.GanttImage", name="gantt_chart_image",)@pytest.fixture@@ -129,7 +127,7 @@ class TestWorkingMemory:TextContent(text="The quick brown fox jumps over the lazy dog"),ImageContent(url=TestWorkingMemoryData.SAMPLE_IMAGE_URL),NumberContent(number=42.5),- ]+ ],)complex_stuff = StuffFactory.make_stuff(diff --git a/tests/unit/pipelex/core/pipes/data.py b/tests/unit/pipelex/core/pipes/data.pyindex 27f4f22c1..145267a20 100644--- a/tests/unit/pipelex/core/pipes/data.py+++ b/tests/unit/pipelex/core/pipes/data.py@@ -1,11 +1,7 @@-"""Test data for make_output_multiplicity function tests."""--from typing import Optional, Tuple-from pipelex.core.pipes.pipe_run_params import OutputMultiplicityResolution, PipeOutputMultiplicity# Test cases format: (nb_output, multiple_output, expected_result, test_description)-MAKE_OUTPUT_MULTIPLICITY_TEST_CASES: list[Tuple[Optional[int], Optional[bool], Optional[PipeOutputMultiplicity], str]] = [+MAKE_OUTPUT_MULTIPLICITY_TEST_CASES: list[tuple[int | None, bool | None, PipeOutputMultiplicity | None, str]] = [# nb_output, multiple_output, expected_result, description# Test cases where nb_output takes precedence(3, None, 3, "nb_output=3, multiple_output=None -> returns 3"),@@ -31,7 +27,7 @@ MAKE_OUTPUT_MULTIPLICITY_TEST_CASES: list[Tuple[Optional[int], Optional[bool], O# Test cases format: (base, override, expected_result, description)OUTPUT_MULTIPLICITY_TO_APPLY_TEST_CASES: list[- Tuple[Optional[PipeOutputMultiplicity], Optional[PipeOutputMultiplicity], OutputMultiplicityResolution, str]+ tuple[PipeOutputMultiplicity | None, PipeOutputMultiplicity | None, OutputMultiplicityResolution, str]] = [# base, override, expected_result, description# Override is None - use base value as-isdiff --git a/tests/unit/pipelex/core/pipes/test_pipe_run_params.py b/tests/unit/pipelex/core/pipes/test_pipe_run_params.pyindex 0da380d08..3e96e7027 100644--- a/tests/unit/pipelex/core/pipes/test_pipe_run_params.py+++ b/tests/unit/pipelex/core/pipes/test_pipe_run_params.py@@ -1,7 +1,3 @@-"""Unit tests for pipe_run_params module."""--from typing import Optional-import pytestfrom pipelex.core.pipes.pipe_run_params import (@@ -14,15 +10,13 @@ from tests.unit.pipelex.core.pipes.data import MAKE_OUTPUT_MULTIPLICITY_TEST_CASclass TestMakeOutputMultiplicity:- """Test cases for make_output_multiplicity function."""-@pytest.mark.parametrize("nb_output,multiple_output,expected_result,description",MAKE_OUTPUT_MULTIPLICITY_TEST_CASES,ids=[case[3] for case in MAKE_OUTPUT_MULTIPLICITY_TEST_CASES],)def test_make_output_multiplicity_all_cases(- self, nb_output: Optional[int], multiple_output: Optional[bool], expected_result: Optional[PipeOutputMultiplicity], description: str+ self, nb_output: int | None, multiple_output: bool | None, expected_result: PipeOutputMultiplicity | None, description: str,):"""Test make_output_multiplicity with all parameter combinations."""result = make_output_multiplicity(nb_output=nb_output, multiple_output=multiple_output)@@ -118,8 +112,8 @@ class TestOutputMultiplicityToApply:)def test_output_multiplicity_to_apply_all_cases(self,- base: Optional[PipeOutputMultiplicity],- override: Optional[PipeOutputMultiplicity],+ base: PipeOutputMultiplicity | None,+ override: PipeOutputMultiplicity | None,expected_result: OutputMultiplicityResolution,description: str,):@@ -138,19 +132,19 @@ class TestOutputMultiplicityToApply:assert result.specific_output_count is None# Base True -> multiple output, LLM decides- result = output_multiplicity_to_apply(True, None)+ result = output_multiplicity_to_apply(base_multiplicity=True, override_multiplicity=None)assert result.resolved_multiplicity is Trueassert result.enable_multiple_outputs is Trueassert result.specific_output_count is None# Base int -> multiple output with specific count- result = output_multiplicity_to_apply(3, None)+ result = output_multiplicity_to_apply(base_multiplicity=3, override_multiplicity=None)assert result.resolved_multiplicity == 3assert result.enable_multiple_outputs is Trueassert result.specific_output_count == 3# Base False -> single output- result = output_multiplicity_to_apply(False, None)+ result = output_multiplicity_to_apply(base_multiplicity=False, override_multiplicity=None)assert result.resolved_multiplicity is Falseassert result.enable_multiple_outputs is Falseassert result.specific_output_count is None@@ -161,7 +155,7 @@ class TestOutputMultiplicityToApply:test_bases = [None, True, False, 1, 5, 10, -1]for base in test_bases:- result = output_multiplicity_to_apply(base, False)+ result = output_multiplicity_to_apply(base_multiplicity=base, override_multiplicity=False)assert result.resolved_multiplicity is False, f"Failed for base={base}: resolved_multiplicity should be False"assert result.enable_multiple_outputs is False, f"Failed for base={base}: enable_multiple_outputs should be False"assert result.specific_output_count is None, f"Failed for base={base}: specific_output_count should be None"@@ -169,26 +163,26 @@ class TestOutputMultiplicityToApply:def test_override_true_enables_multiple_outputs(self):"""Test that override=True enables multiple outputs, preserving int base values."""# Base None -> preserve None but enable multiple- result = output_multiplicity_to_apply(None, True)+ result = output_multiplicity_to_apply(base_multiplicity=None, override_multiplicity=True)assert result.resolved_multiplicity is Noneassert result.enable_multiple_outputs is Trueassert result.specific_output_count is None# Base False -> enable multiple, LLM decides count- result = output_multiplicity_to_apply(False, True)+ result = output_multiplicity_to_apply(base_multiplicity=False, override_multiplicity=True)assert result.resolved_multiplicity is Trueassert result.enable_multiple_outputs is Trueassert result.specific_output_count is None# Base True -> keep True- result = output_multiplicity_to_apply(True, True)+ result = output_multiplicity_to_apply(base_multiplicity=True, override_multiplicity=True)assert result.resolved_multiplicity is Trueassert result.enable_multiple_outputs is Trueassert result.specific_output_count is None# Base int -> preserve count, enable multiplefor base_int in [1, 3, 5, 10]:- result = output_multiplicity_to_apply(base_int, True)+ result = output_multiplicity_to_apply(base_multiplicity=base_int, override_multiplicity=True)assert result.resolved_multiplicity == base_intassert result.enable_multiple_outputs is Trueassert result.specific_output_count == base_int@@ -199,14 +193,14 @@ class TestOutputMultiplicityToApply:override_int = 5for base in test_bases:- result = output_multiplicity_to_apply(base, override_int)+ result = output_multiplicity_to_apply(base_multiplicity=base, override_multiplicity=override_int)assert result.resolved_multiplicity == override_int, f"Failed for base={base}: resolved_multiplicity should be {override_int}"assert result.enable_multiple_outputs is True, f"Failed for base={base}: enable_multiple_outputs should be True"assert result.specific_output_count == override_int, f"Failed for base={base}: specific_output_count should be {override_int}"def test_return_basemodel_structure(self):"""Test that the function always returns an OutputMultiplicityResolution with correct types."""- result = output_multiplicity_to_apply(None, None)+ result = output_multiplicity_to_apply(base_multiplicity=None, override_multiplicity=None)assert isinstance(result, OutputMultiplicityResolution)# Test field types@@ -217,29 +211,29 @@ class TestOutputMultiplicityToApply:def test_edge_cases(self):"""Test edge cases and boundary conditions."""# Negative numbers- result = output_multiplicity_to_apply(-1, None)+ result = output_multiplicity_to_apply(base_multiplicity=-1, override_multiplicity=None)assert result.resolved_multiplicity == -1assert result.enable_multiple_outputs is Trueassert result.specific_output_count == -1- result = output_multiplicity_to_apply(3, -2)+ result = output_multiplicity_to_apply(base_multiplicity=3, override_multiplicity=-2)assert result.resolved_multiplicity == -2assert result.enable_multiple_outputs is Trueassert result.specific_output_count == -2# Zero values- result = output_multiplicity_to_apply(0, None)+ result = output_multiplicity_to_apply(base_multiplicity=0, override_multiplicity=None)assert result.resolved_multiplicity == 0assert result.enable_multiple_outputs is Trueassert result.specific_output_count == 0- result = output_multiplicity_to_apply(True, 0)+ result = output_multiplicity_to_apply(base_multiplicity=True, override_multiplicity=0)assert result.resolved_multiplicity == 0assert result.enable_multiple_outputs is Trueassert result.specific_output_count == 0# Large numbers- result = output_multiplicity_to_apply(1000, None)+ result = output_multiplicity_to_apply(base_multiplicity=1000, override_multiplicity=None)assert result.resolved_multiplicity == 1000assert result.enable_multiple_outputs is Trueassert result.specific_output_count == 1000@@ -247,40 +241,40 @@ class TestOutputMultiplicityToApply:def test_override_precedence_logic(self):"""Test the precedence logic where override always takes precedence when provided."""# Override False beats any base- result = output_multiplicity_to_apply(True, False)+ result = output_multiplicity_to_apply(base_multiplicity=True, override_multiplicity=False)assert result.resolved_multiplicity is Falseassert result.enable_multiple_outputs is Falseassert result.specific_output_count is None- result = output_multiplicity_to_apply(5, False)+ result = output_multiplicity_to_apply(base_multiplicity=5, override_multiplicity=False)assert result.resolved_multiplicity is Falseassert result.enable_multiple_outputs is Falseassert result.specific_output_count is None# Override int beats any base- result = output_multiplicity_to_apply(True, 3)+ result = output_multiplicity_to_apply(base_multiplicity=True, override_multiplicity=3)assert result.resolved_multiplicity == 3assert result.enable_multiple_outputs is Trueassert result.specific_output_count == 3- result = output_multiplicity_to_apply(7, 3)+ result = output_multiplicity_to_apply(base_multiplicity=7, override_multiplicity=3)assert result.resolved_multiplicity == 3assert result.enable_multiple_outputs is Trueassert result.specific_output_count == 3# Override True with int base preserves int- result = output_multiplicity_to_apply(5, True)+ result = output_multiplicity_to_apply(base_multiplicity=5, override_multiplicity=True)assert result.resolved_multiplicity == 5assert result.enable_multiple_outputs is Trueassert result.specific_output_count == 5# Override True with bool base becomes True- result = output_multiplicity_to_apply(True, True)+ result = output_multiplicity_to_apply(base_multiplicity=True, override_multiplicity=True)assert result.resolved_multiplicity is Trueassert result.enable_multiple_outputs is Trueassert result.specific_output_count is None- result = output_multiplicity_to_apply(False, True)+ result = output_multiplicity_to_apply(base_multiplicity=False, override_multiplicity=True)assert result.resolved_multiplicity is Trueassert result.enable_multiple_outputs is Trueassert result.specific_output_count is None@@ -288,25 +282,25 @@ class TestOutputMultiplicityToApply:def test_complex_scenarios(self):"""Test complex real-world scenarios."""# Scenario: Pipe has base multiplicity=3, runtime wants to disable it- result = output_multiplicity_to_apply(3, False)+ result = output_multiplicity_to_apply(base_multiplicity=3, override_multiplicity=False)assert result.resolved_multiplicity is Falseassert result.enable_multiple_outputs is Falseassert result.specific_output_count is None# Scenario: Pipe has no base multiplicity, runtime wants 5 outputs- result = output_multiplicity_to_apply(None, 5)+ result = output_multiplicity_to_apply(base_multiplicity=None, override_multiplicity=5)assert result.resolved_multiplicity == 5assert result.enable_multiple_outputs is Trueassert result.specific_output_count == 5# Scenario: Pipe has base True, runtime wants specific count- result = output_multiplicity_to_apply(True, 2)+ result = output_multiplicity_to_apply(base_multiplicity=True, override_multiplicity=2)assert result.resolved_multiplicity == 2assert result.enable_multiple_outputs is Trueassert result.specific_output_count == 2# Scenario: Pipe has base count, runtime wants to enable but let LLM decide- result = output_multiplicity_to_apply(4, True)+ result = output_multiplicity_to_apply(base_multiplicity=4, override_multiplicity=True)assert result.resolved_multiplicity == 4 # Preserves base countassert result.enable_multiple_outputs is Trueassert result.specific_output_count == 4diff --git a/tests/unit/pipelex/core/stuffs/__init__.py b/tests/unit/pipelex/core/stuffs/__init__.pynew file mode 100644index 000000000..e69de29bbdiff --git a/tests/unit/pipelex/core/stuffs/test_stuff_content_factory.py b/tests/unit/pipelex/core/stuffs/test_stuff_content_factory.pyindex 8e6574b9f..860d9968a 100644--- a/tests/unit/pipelex/core/stuffs/test_stuff_content_factory.py+++ b/tests/unit/pipelex/core/stuffs/test_stuff_content_factory.py@@ -1,4 +1,4 @@-from typing import Any, ClassVar, Dict, List, Tuple+from typing import Any, ClassVar, Dictimport pytest@@ -29,24 +29,24 @@ class TestCases:}# Test cases for registered class (using actual registered class)- REGISTERED_CLASS_BLUEPRINT: ClassVar[Dict[str, Any]] = {+ REGISTERED_CLASS_BLUEPRINT: ClassVar[dict[str, Any]] = {"concept_string": "test.MockRegisteredContent","content": {"title": "Test Question", "description": "What are aerodynamic features?"},}# Test cases for unregistered class (creates implicit concept, returns TextContent)- UNREGISTERED_STRING_BLUEPRINT: ClassVar[Dict[str, Any]] = {+ UNREGISTERED_STRING_BLUEPRINT: ClassVar[dict[str, Any]] = {"concept_string": "unknown.NonExistentConcept","content": "This should create implicit concept and return TextContent",}# Test cases for unregistered class with dict content- UNREGISTERED_DICT_BLUEPRINT: ClassVar[Dict[str, Any]] = {+ UNREGISTERED_DICT_BLUEPRINT: ClassVar[dict[str, Any]] = {"concept_string": "unknown.NonExistentConcept","content": {"text": "Dict content for implicit concept"},}- TEST_BLUEPRINTS: ClassVar[List[Tuple[str, Dict[str, Any]]]] = [+ TEST_BLUEPRINTS: ClassVar[list[tuple[str, dict[str, Any]]]] = [("text_string", TEXT_STRING_BLUEPRINT),("text_dict", TEXT_DICT_BLUEPRINT),("text_no_prefix", TEXT_NO_PREFIX_BLUEPRINT),@@ -79,7 +79,7 @@ class TestStuffContentFactory:description: strresult = StuffContentFactory.make_content_from_value(- stuff_content_subclass=MockStructuredContent, value={"title": "Test Title", "description": "Test Description"}+ stuff_content_subclass=MockStructuredContent, value={"title": "Test Title", "description": "Test Description"},)assert isinstance(result, MockStructuredContent)@@ -89,7 +89,7 @@ class TestStuffContentFactory:def test_make_stuffcontent_from_concept_code_required_text_content(self):"""Test required method with native.Text concept (should work)."""result = StuffContentFactory.make_stuff_content_from_concept_required(- concept=ConceptFactory.make_native_concept(native_concept_data=NATIVE_CONCEPTS_DATA[NativeConceptEnum.TEXT]), value="Test text content"+ concept=ConceptFactory.make_native_concept(native_concept_data=NATIVE_CONCEPTS_DATA[NativeConceptEnum.TEXT]), value="Test text content",)assert isinstance(result, TextContent)@@ -108,7 +108,7 @@ class TestStuffContentFactory:def test_make_stuffcontent_from_concept_code_with_fallback_text_success(self):"""Test fallback method with native.Text concept."""result = StuffContentFactory.make_stuff_content_from_concept_with_fallback(- concept=ConceptFactory.make_native_concept(native_concept_data=NATIVE_CONCEPTS_DATA[NativeConceptEnum.TEXT]), value="Test text content"+ concept=ConceptFactory.make_native_concept(native_concept_data=NATIVE_CONCEPTS_DATA[NativeConceptEnum.TEXT]), value="Test text content",)assert isinstance(result, TextContent)@@ -138,14 +138,14 @@ class TestStuffContentFactory:"test_name, blueprint",TestCases.TEST_BLUEPRINTS,)- def test_blueprint_scenarios(self, test_name: str, blueprint: Dict[str, Any]):+ def test_blueprint_scenarios(self, test_name: str, blueprint: dict[str, Any]):"""Test various blueprint scenarios with parametrized test cases."""content = blueprint["content"]if test_name.startswith("text_"):# Test native.Text concept with required methodresult = StuffContentFactory.make_stuff_content_from_concept_required(- concept=ConceptFactory.make_native_concept(native_concept_data=NATIVE_CONCEPTS_DATA[NativeConceptEnum.TEXT]), value=content+ concept=ConceptFactory.make_native_concept(native_concept_data=NATIVE_CONCEPTS_DATA[NativeConceptEnum.TEXT]), value=content,)assert isinstance(result, TextContent)if isinstance(content, str):@@ -162,7 +162,7 @@ class TestStuffContentFactory:# The dict will be passed through model_validate which should fail for TextContenttry:result_required = StuffContentFactory.make_stuff_content_from_concept_required(- concept=ConceptFactory.make_native_concept(native_concept_data=NATIVE_CONCEPTS_DATA[NativeConceptEnum.TEXT]), value=content+ concept=ConceptFactory.make_native_concept(native_concept_data=NATIVE_CONCEPTS_DATA[NativeConceptEnum.TEXT]), value=content,)# If it succeeds, it should be TextContent (due to implicit concept)assert isinstance(result_required, TextContent)@@ -173,7 +173,7 @@ class TestStuffContentFactory:# Test fallback method - same behavior expectedtry:result_fallback = StuffContentFactory.make_stuff_content_from_concept_with_fallback(- concept=ConceptFactory.make_native_concept(native_concept_data=NATIVE_CONCEPTS_DATA[NativeConceptEnum.TEXT]), value=content+ concept=ConceptFactory.make_native_concept(native_concept_data=NATIVE_CONCEPTS_DATA[NativeConceptEnum.TEXT]), value=content,)# If it succeeds, it should be TextContent (due to implicit concept)assert isinstance(result_fallback, TextContent)@@ -186,12 +186,12 @@ class TestStuffContentFactory:# Both methods should work and return TextContent due to implicit concept creationresult_required = StuffContentFactory.make_stuff_content_from_concept_required(- concept=ConceptFactory.make_native_concept(native_concept_data=NATIVE_CONCEPTS_DATA[NativeConceptEnum.TEXT]), value=content+ concept=ConceptFactory.make_native_concept(native_concept_data=NATIVE_CONCEPTS_DATA[NativeConceptEnum.TEXT]), value=content,)assert isinstance(result_required, TextContent)result_fallback = StuffContentFactory.make_stuff_content_from_concept_with_fallback(- concept=ConceptFactory.make_native_concept(native_concept_data=NATIVE_CONCEPTS_DATA[NativeConceptEnum.TEXT]), value=content+ concept=ConceptFactory.make_native_concept(native_concept_data=NATIVE_CONCEPTS_DATA[NativeConceptEnum.TEXT]), value=content,)assert isinstance(result_fallback, TextContent)diff --git a/tests/unit/pipelex/core/stuffs/test_stuff_factory_make_stuff_from_content.py b/tests/unit/pipelex/core/stuffs/test_stuff_factory_make_stuff_from_content.pyindex 5bbe352fd..623b7eaf7 100644--- a/tests/unit/pipelex/core/stuffs/test_stuff_factory_make_stuff_from_content.py+++ b/tests/unit/pipelex/core/stuffs/test_stuff_factory_make_stuff_from_content.py@@ -1,4 +1,4 @@-from typing import Any, ClassVar, Dict, List+from typing import Any, ClassVar, Dictimport pytestfrom pytest_mock import MockerFixture@@ -15,24 +15,19 @@ from pipelex.core.stuffs.stuff_factory import StuffFactory, StuffFactoryErrorclass MockCustomContent(StructuredContent):- """Mock content class for testing custom concepts."""-title: strdescription: strclass TestData:- """Test data for various input scenarios."""-- # Basic test dataTEXT_STRING: ClassVar[str] = "This is a test text"CONCEPT_NAME: ClassVar[str] = "TestConcept"- SEARCH_DOMAINS: ClassVar[List[str]] = ["test", "native"]+ SEARCH_DOMAINS: ClassVar[list[str]] = ["test", "native"]STUFF_NAME: ClassVar[str] = "test_stuff"STUFF_CODE: ClassVar[str] = "test123"# ListContent test data- TEXT_LIST_ITEMS: ClassVar[List[TextContent]] = [+ TEXT_LIST_ITEMS: ClassVar[list[TextContent]] = [TextContent(text="Item 1"),TextContent(text="Item 2"),]@@ -42,24 +37,22 @@ class TestData:NATIVE_TEXT_DICT: ClassVar[Dict[str, Any]] = {"concept": NativeConceptEnum.TEXT, "content": {"text": "Native text content"}}# Dictionary test data - custom concept with concept field- CUSTOM_CONCEPT_DICT: ClassVar[Dict[str, Any]] = {+ CUSTOM_CONCEPT_DICT: ClassVar[dict[str, Any]] = {"concept": "test.CustomConcept","content": {"title": "Test Title", "description": "Test Description"},}# Dictionary test data - custom concept with concept_code field- CUSTOM_CONCEPT_CODE_DICT: ClassVar[Dict[str, Any]] = {"concept_code": "test.AnotherConcept", "content": {"data": "test data"}}+ CUSTOM_CONCEPT_CODE_DICT: ClassVar[dict[str, Any]] = {"concept_code": "test.AnotherConcept", "content": {"data": "test data"}}# Dictionary test data - missing concept- MISSING_CONCEPT_DICT: ClassVar[Dict[str, Any]] = {"content": {"data": "test data"}}+ MISSING_CONCEPT_DICT: ClassVar[dict[str, Any]] = {"content": {"data": "test data"}}# Dictionary test data - missing content- MISSING_CONTENT_DICT: ClassVar[Dict[str, Any]] = {"concept": "test.SomeConcept"}+ MISSING_CONTENT_DICT: ClassVar[dict[str, Any]] = {"concept": "test.SomeConcept"}class TestMakeStuffFromStuffContentUsingSearchDomains:- """Unit tests for make_stuff_from_stuff_content_using_search_domains method."""-def test_listcontent_with_valid_items(self, mocker: MockerFixture):"""Test ListContent with valid items."""list_content: ListContent[TextContent] = ListContent(items=TestData.TEXT_LIST_ITEMS)@@ -280,7 +273,7 @@ class TestMakeStuffFromStuffContentUsingSearchDomains:assert result == mock_stuffmock_provider.get_required_concept.assert_called_with(concept_string="test.CustomConcept")mock_content_factory.make_stuff_content_from_concept_with_fallback.assert_called_once_with(- concept=mock_concept, value={"title": "Test Title", "description": "Test Description"}+ concept=mock_concept, value={"title": "Test Title", "description": "Test Description"},)def test_dict_with_concept_code_field(self, mocker: MockerFixture):diff --git a/tests/unit/pipelex/core/test_data/__init__.py b/tests/unit/pipelex/core/test_data/__init__.pyindex fcbd717bb..6c1d15983 100644--- a/tests/unit/pipelex/core/test_data/__init__.py+++ b/tests/unit/pipelex/core/test_data/__init__.py@@ -1,10 +1,7 @@-"""Organized test data for PipelexInterpreter tests."""-from typing import ClassVar, List, Tuple, Type, Unionfrom pipelex.core.bundles.pipelex_bundle_blueprint import PipelexBundleBlueprint-# Import all test data from organized modulesfrom .complex.multi_feature import COMPLEX_TEST_CASESfrom .concepts.refining_concepts import REFINING_CONCEPT_TEST_CASESfrom .concepts.simple_concepts import SIMPLE_CONCEPT_TEST_CASES@@ -26,7 +23,7 @@ class InterpreterTestCases:"""Test cases for PipelexInterpreter with PLX content and expected blueprints."""# Aggregate all valid test cases from organized modules- VALID_TEST_CASES: ClassVar[List[Tuple[str, str, PipelexBundleBlueprint]]] = [+ VALID_TEST_CASES: ClassVar[list[tuple[str, str, PipelexBundleBlueprint]]] = [# Domain tests*DOMAIN_TEST_CASES,# Concept tests@@ -49,4 +46,4 @@ class InterpreterTestCases:]# Error test cases- ERROR_TEST_CASES: ClassVar[List[Tuple[str, str, Union[Type[Exception], Tuple[Type[Exception], ...]]]]] = ERROR_TEST_CASES+ ERROR_TEST_CASES: ClassVar[list[tuple[str, str, type[Exception] | tuple[type[Exception], ...]]]] = ERROR_TEST_CASESdiff --git a/tests/unit/pipelex/core/test_data/complex/multi_feature.py b/tests/unit/pipelex/core/test_data/complex/multi_feature.pyindex 85f8225d9..04641cb16 100644--- a/tests/unit/pipelex/core/test_data/complex/multi_feature.py+++ b/tests/unit/pipelex/core/test_data/complex/multi_feature.py@@ -1,5 +1,3 @@-"""Complex test cases combining multiple features."""-from pipelex.core.bundles.pipelex_bundle_blueprint import PipelexBundleBlueprintfrom pipelex.pipe_controllers.sequence.pipe_sequence_blueprint import PipeSequenceBlueprintfrom pipelex.pipe_controllers.sub_pipe_blueprint import SubPipeBlueprintdiff --git a/tests/unit/pipelex/core/test_data/concepts/refining_concepts.py b/tests/unit/pipelex/core/test_data/concepts/refining_concepts.pyindex da3d9170c..fc23dd0dc 100644--- a/tests/unit/pipelex/core/test_data/concepts/refining_concepts.py+++ b/tests/unit/pipelex/core/test_data/concepts/refining_concepts.py@@ -1,5 +1,3 @@-"""Refining concept test cases."""-from pipelex.core.bundles.pipelex_bundle_blueprint import PipelexBundleBlueprintfrom pipelex.core.concepts.concept_blueprint import ConceptBlueprintdiff --git a/tests/unit/pipelex/core/test_data/concepts/simple_concepts.py b/tests/unit/pipelex/core/test_data/concepts/simple_concepts.pyindex ff4f54df1..41fe3202b 100644--- a/tests/unit/pipelex/core/test_data/concepts/simple_concepts.py+++ b/tests/unit/pipelex/core/test_data/concepts/simple_concepts.py@@ -1,5 +1,3 @@-"""Simple concept test cases."""-from pipelex.core.bundles.pipelex_bundle_blueprint import PipelexBundleBlueprintSIMPLE_CONCEPTS = (diff --git a/tests/unit/pipelex/core/test_data/concepts/structured_concepts.py b/tests/unit/pipelex/core/test_data/concepts/structured_concepts.pyindex 61c1f6fc4..b52a65e39 100644--- a/tests/unit/pipelex/core/test_data/concepts/structured_concepts.py+++ b/tests/unit/pipelex/core/test_data/concepts/structured_concepts.py@@ -1,5 +1,3 @@-"""Structured concept test cases."""-from pipelex.core.bundles.pipelex_bundle_blueprint import PipelexBundleBlueprintfrom pipelex.core.concepts.concept_blueprint import ConceptBlueprint, ConceptStructureBlueprint, ConceptStructureBlueprintFieldType@@ -29,10 +27,10 @@ birthdate = { type = "date", definition = "The birthdate of the person", requirestructure={"name": "The name of the person","age": ConceptStructureBlueprint(- type=ConceptStructureBlueprintFieldType.NUMBER, definition="The age of the person", required=True+ type=ConceptStructureBlueprintFieldType.NUMBER, definition="The age of the person", required=True,),"birthdate": ConceptStructureBlueprint(- type=ConceptStructureBlueprintFieldType.DATE, definition="The birthdate of the person", required=True+ type=ConceptStructureBlueprintFieldType.DATE, definition="The birthdate of the person", required=True,),},),diff --git a/tests/unit/pipelex/core/test_data/domain/simple_domains.py b/tests/unit/pipelex/core/test_data/domain/simple_domains.pyindex 3dc8c2fd5..9e391cc6b 100644--- a/tests/unit/pipelex/core/test_data/domain/simple_domains.py+++ b/tests/unit/pipelex/core/test_data/domain/simple_domains.py@@ -1,5 +1,3 @@-"""Simple domain test cases."""-from pipelex.core.bundles.pipelex_bundle_blueprint import PipelexBundleBlueprintSIMPLE_DOMAIN = (diff --git a/tests/unit/pipelex/core/test_data/errors/invalid_plx.py b/tests/unit/pipelex/core/test_data/errors/invalid_plx.pyindex 4d71aab81..ec70c8d7a 100644--- a/tests/unit/pipelex/core/test_data/errors/invalid_plx.py+++ b/tests/unit/pipelex/core/test_data/errors/invalid_plx.py@@ -1,13 +1,8 @@-"""Invalid PLX test cases focused on PLX structure and PipelexBundleBlueprint validation."""--from typing import List, Tuple, Type, Union-from pydantic import ValidationErrorfrom pipelex.core.domains.exceptions import DomainErrorfrom pipelex.core.interpreter import PLXDecodeError-# PLX Syntax ErrorsINVALID_PLX_SYNTAX = ("invalid_plx_syntax","""domain = "test_domain"@@ -207,7 +202,7 @@ definition = "Domain with invalid array syntax")# Export all error test cases-ERROR_TEST_CASES: List[Tuple[str, str, Union[Type[Exception], Tuple[Type[Exception], ...]]]] = [+ERROR_TEST_CASES: list[tuple[str, str, type[Exception] | tuple[type[Exception], ...]]] = [# PLX Syntax ErrorsINVALID_PLX_SYNTAX,MALFORMED_SECTION,diff --git a/tests/unit/pipelex/core/test_data/pipes/controllers/batch/pipe_batch.py b/tests/unit/pipelex/core/test_data/pipes/controllers/batch/pipe_batch.pyindex 5aa459c10..0cc0c1226 100644--- a/tests/unit/pipelex/core/test_data/pipes/controllers/batch/pipe_batch.py+++ b/tests/unit/pipelex/core/test_data/pipes/controllers/batch/pipe_batch.py@@ -1,5 +1,3 @@-"""PipeBatch test cases."""-from pipelex.core.bundles.pipelex_bundle_blueprint import PipelexBundleBlueprintfrom pipelex.pipe_controllers.batch.pipe_batch_blueprint import PipeBatchBlueprintdiff --git a/tests/unit/pipelex/core/test_data/pipes/controllers/condition/pipe_condition.py b/tests/unit/pipelex/core/test_data/pipes/controllers/condition/pipe_condition.pyindex 150f9d445..d83deab06 100644--- a/tests/unit/pipelex/core/test_data/pipes/controllers/condition/pipe_condition.py+++ b/tests/unit/pipelex/core/test_data/pipes/controllers/condition/pipe_condition.py@@ -1,5 +1,3 @@-"""PipeCondition test cases."""-from pipelex.core.bundles.pipelex_bundle_blueprint import PipelexBundleBlueprintfrom pipelex.pipe_controllers.condition.pipe_condition_blueprint import PipeConditionBlueprint, PipeConditionPipeMapBlueprint@@ -28,7 +26,7 @@ expression = "input_data.category"{"small": "process_small","large": "process_large",- }+ },),),},diff --git a/tests/unit/pipelex/core/test_data/pipes/controllers/parallel/pipe_parallel.py b/tests/unit/pipelex/core/test_data/pipes/controllers/parallel/pipe_parallel.pyindex 1c4382957..1056a0abc 100644--- a/tests/unit/pipelex/core/test_data/pipes/controllers/parallel/pipe_parallel.py+++ b/tests/unit/pipelex/core/test_data/pipes/controllers/parallel/pipe_parallel.py@@ -1,5 +1,3 @@-"""PipeParallel test cases."""-from pipelex.core.bundles.pipelex_bundle_blueprint import PipelexBundleBlueprintfrom pipelex.pipe_controllers.parallel.pipe_parallel_blueprint import PipeParallelBlueprintfrom pipelex.pipe_controllers.sub_pipe_blueprint import SubPipeBlueprintdiff --git a/tests/unit/pipelex/core/test_data/pipes/controllers/sequence/pipe_sequence.py b/tests/unit/pipelex/core/test_data/pipes/controllers/sequence/pipe_sequence.pyindex 0267d74b7..0875c8edb 100644--- a/tests/unit/pipelex/core/test_data/pipes/controllers/sequence/pipe_sequence.py+++ b/tests/unit/pipelex/core/test_data/pipes/controllers/sequence/pipe_sequence.py@@ -1,5 +1,3 @@-"""PipeSequence test cases."""-from pipelex.core.bundles.pipelex_bundle_blueprint import PipelexBundleBlueprintfrom pipelex.pipe_controllers.sequence.pipe_sequence_blueprint import PipeSequenceBlueprintfrom pipelex.pipe_controllers.sub_pipe_blueprint import SubPipeBlueprintdiff --git a/tests/unit/pipelex/core/test_data/pipes/operators/func/pipe_func.py b/tests/unit/pipelex/core/test_data/pipes/operators/func/pipe_func.pyindex 9130e6a26..2a17137a4 100644--- a/tests/unit/pipelex/core/test_data/pipes/operators/func/pipe_func.py+++ b/tests/unit/pipelex/core/test_data/pipes/operators/func/pipe_func.py@@ -1,5 +1,3 @@-"""PipeFunc test cases."""-from pipelex.core.bundles.pipelex_bundle_blueprint import PipelexBundleBlueprintfrom pipelex.pipe_operators.func.pipe_func_blueprint import PipeFuncBlueprintdiff --git a/tests/unit/pipelex/core/test_data/pipes/operators/img_gen/pipe_img_gen.py b/tests/unit/pipelex/core/test_data/pipes/operators/img_gen/pipe_img_gen.pyindex 51c194749..7247803fb 100644--- a/tests/unit/pipelex/core/test_data/pipes/operators/img_gen/pipe_img_gen.py+++ b/tests/unit/pipelex/core/test_data/pipes/operators/img_gen/pipe_img_gen.py@@ -1,5 +1,3 @@-"""PipeImgGen test cases."""-from pipelex.core.bundles.pipelex_bundle_blueprint import PipelexBundleBlueprintfrom pipelex.core.concepts.concept_native import NativeConceptEnumfrom pipelex.pipe_operators.img_gen.pipe_img_gen_blueprint import PipeImgGenBlueprintdiff --git a/tests/unit/pipelex/core/test_data/pipes/operators/jinja2/pipe_jinja2.py b/tests/unit/pipelex/core/test_data/pipes/operators/jinja2/pipe_jinja2.pyindex 8e5b3f984..4781fe0e8 100644--- a/tests/unit/pipelex/core/test_data/pipes/operators/jinja2/pipe_jinja2.py+++ b/tests/unit/pipelex/core/test_data/pipes/operators/jinja2/pipe_jinja2.py@@ -1,5 +1,3 @@-"""PipeJinja2 test cases."""-from pipelex.core.bundles.pipelex_bundle_blueprint import PipelexBundleBlueprintfrom pipelex.core.concepts.concept_native import NativeConceptEnumfrom pipelex.pipe_operators.jinja2.pipe_jinja2_blueprint import PipeJinja2Blueprintdiff --git a/tests/unit/pipelex/core/test_data/pipes/operators/llm/pipe_llm.py b/tests/unit/pipelex/core/test_data/pipes/operators/llm/pipe_llm.pyindex 9e5eb9816..112a341ac 100644--- a/tests/unit/pipelex/core/test_data/pipes/operators/llm/pipe_llm.py+++ b/tests/unit/pipelex/core/test_data/pipes/operators/llm/pipe_llm.py@@ -1,5 +1,3 @@-"""PipeLLM test cases."""-from pipelex.core.bundles.pipelex_bundle_blueprint import PipelexBundleBlueprintfrom pipelex.core.concepts.concept_native import NativeConceptEnumfrom pipelex.core.pipes.pipe_input_blueprint import InputRequirementBlueprintdiff --git a/tests/unit/pipelex/core/test_data/pipes/operators/ocr/pipe_ocr.py b/tests/unit/pipelex/core/test_data/pipes/operators/ocr/pipe_ocr.pyindex 72d099884..5a860f26e 100644--- a/tests/unit/pipelex/core/test_data/pipes/operators/ocr/pipe_ocr.py+++ b/tests/unit/pipelex/core/test_data/pipes/operators/ocr/pipe_ocr.py@@ -1,5 +1,3 @@-"""PipeOcr test cases."""-from pipelex.core.bundles.pipelex_bundle_blueprint import PipelexBundleBlueprintfrom pipelex.core.concepts.concept_native import NativeConceptEnumfrom pipelex.pipe_operators.ocr.pipe_ocr_blueprint import PipeOcrBlueprintdiff --git a/tests/unit/pipelex/create/__init__.py b/tests/unit/pipelex/create/__init__.pynew file mode 100644index 000000000..e69de29bbdiff --git a/tests/unit/pipelex/create/test_structured_output_generator.py b/tests/unit/pipelex/create/test_structured_output_generator.pyindex 38cce78f7..1204c0f7a 100644--- a/tests/unit/pipelex/create/test_structured_output_generator.py+++ b/tests/unit/pipelex/create/test_structured_output_generator.py@@ -1,7 +1,3 @@-"""Tests for structured output generator."""--from typing import Dict, Union-from pipelex import pretty_printfrom pipelex.core.concepts.concept_blueprint import ConceptStructureBlueprint, ConceptStructureBlueprintFieldTypefrom pipelex.core.concepts.concept_factory import ConceptFactory@@ -15,7 +11,7 @@ class TestStructureGenerator:"name": ConceptStructureBlueprint(definition="Name field", type=ConceptStructureBlueprintFieldType.TEXT, required=True),"age": ConceptStructureBlueprint(definition="Age field", type=ConceptStructureBlueprintFieldType.INTEGER, required=False),"active": ConceptStructureBlueprint(- definition="Active status", type=ConceptStructureBlueprintFieldType.BOOLEAN, required=False, default_value=True+ definition="Active status", type=ConceptStructureBlueprintFieldType.BOOLEAN, required=False, default_value=True,),}@@ -44,13 +40,13 @@ class TestStructureGenerator:"""Test generation with complex types like lists and dicts."""structure_blueprint = {"tags": ConceptStructureBlueprint(- definition="List of tags", type=ConceptStructureBlueprintFieldType.LIST, item_type="text", required=False+ definition="List of tags", type=ConceptStructureBlueprintFieldType.LIST, item_type="text", required=False,),"metadata": ConceptStructureBlueprint(- definition="Metadata dictionary", type=ConceptStructureBlueprintFieldType.DICT, key_type="text", value_type="text", required=False+ definition="Metadata dictionary", type=ConceptStructureBlueprintFieldType.DICT, key_type="text", value_type="text", required=False,),"scores": ConceptStructureBlueprint(- definition="List of scores", type=ConceptStructureBlueprintFieldType.LIST, item_type="number", required=True+ definition="List of scores", type=ConceptStructureBlueprintFieldType.LIST, item_type="number", required=True,),}@@ -106,7 +102,7 @@ class TestStructureGenerator:def test_empty_structure(self):"""Test generation of structure with no fields."""- structure_blueprint: Dict[str, ConceptStructureBlueprint] = {}+ structure_blueprint: dict[str, ConceptStructureBlueprint] = {}result = StructureGenerator().generate_from_structure_blueprint("EmptyModel", structure_blueprint)@@ -157,7 +153,7 @@ class TestStructureGenerator:def test_generate_from_blueprint_dict_function(self):"""Test the convenience function for generating from blueprint dict."""structure_blueprint = {- "value": ConceptStructureBlueprint(definition="Test value", type=ConceptStructureBlueprintFieldType.TEXT, required=True)+ "value": ConceptStructureBlueprint(definition="Test value", type=ConceptStructureBlueprintFieldType.TEXT, required=True),}result = StructureGenerator().generate_from_structure_blueprint("ConvenienceTest", structure_blueprint)@@ -187,10 +183,10 @@ class TestStructureGenerator:"integer_field": ConceptStructureBlueprint(definition="Integer field", type=ConceptStructureBlueprintFieldType.INTEGER, required=False),"boolean_field": ConceptStructureBlueprint(definition="Boolean field", type=ConceptStructureBlueprintFieldType.BOOLEAN, required=False),"list_field": ConceptStructureBlueprint(- definition="List field", type=ConceptStructureBlueprintFieldType.LIST, item_type="text", required=False+ definition="List field", type=ConceptStructureBlueprintFieldType.LIST, item_type="text", required=False,),"dict_field": ConceptStructureBlueprint(- definition="Dict field", type=ConceptStructureBlueprintFieldType.DICT, key_type="text", value_type="integer", required=False+ definition="Dict field", type=ConceptStructureBlueprintFieldType.DICT, key_type="text", value_type="integer", required=False,),}@@ -249,13 +245,13 @@ class TestStructureGenerator:"""Test fields with default values."""structure_blueprint = {"name": ConceptStructureBlueprint(- definition="Person name", type=ConceptStructureBlueprintFieldType.TEXT, required=False, default_value="Anonymous"+ definition="Person name", type=ConceptStructureBlueprintFieldType.TEXT, required=False, default_value="Anonymous",),"age": ConceptStructureBlueprint(- definition="Person age", type=ConceptStructureBlueprintFieldType.INTEGER, required=False, default_value=0+ definition="Person age", type=ConceptStructureBlueprintFieldType.INTEGER, required=False, default_value=0,),"active": ConceptStructureBlueprint(- definition="Is active", type=ConceptStructureBlueprintFieldType.BOOLEAN, required=False, default_value=True+ definition="Is active", type=ConceptStructureBlueprintFieldType.BOOLEAN, required=False, default_value=True,),}@@ -284,13 +280,13 @@ class TestStructureGenerator:"""Test nested list types with different item types."""structure_blueprint = {"text_list": ConceptStructureBlueprint(- definition="List of text items", type=ConceptStructureBlueprintFieldType.LIST, item_type="text", required=False+ definition="List of text items", type=ConceptStructureBlueprintFieldType.LIST, item_type="text", required=False,),"number_list": ConceptStructureBlueprint(- definition="List of numbers", type=ConceptStructureBlueprintFieldType.LIST, item_type="number", required=True+ definition="List of numbers", type=ConceptStructureBlueprintFieldType.LIST, item_type="number", required=True,),"integer_list": ConceptStructureBlueprint(- definition="List of integers", type=ConceptStructureBlueprintFieldType.LIST, item_type="integer", required=False+ definition="List of integers", type=ConceptStructureBlueprintFieldType.LIST, item_type="integer", required=False,),}@@ -368,16 +364,16 @@ class TestStructureGenerator:"id": ConceptStructureBlueprint(definition="Unique identifier", type=ConceptStructureBlueprintFieldType.INTEGER, required=True),"name": ConceptStructureBlueprint(definition="Display name", type=ConceptStructureBlueprintFieldType.TEXT, required=True),"tags": ConceptStructureBlueprint(- definition="Associated tags", type=ConceptStructureBlueprintFieldType.LIST, item_type="text", required=False+ definition="Associated tags", type=ConceptStructureBlueprintFieldType.LIST, item_type="text", required=False,),"metadata": ConceptStructureBlueprint(- definition="Additional metadata", type=ConceptStructureBlueprintFieldType.DICT, key_type="text", value_type="text", required=False+ definition="Additional metadata", type=ConceptStructureBlueprintFieldType.DICT, key_type="text", value_type="text", required=False,),"active": ConceptStructureBlueprint(- definition="Whether item is active", type=ConceptStructureBlueprintFieldType.BOOLEAN, required=False, default_value=True+ definition="Whether item is active", type=ConceptStructureBlueprintFieldType.BOOLEAN, required=False, default_value=True,),"priority": ConceptStructureBlueprint(- definition="Priority level", choices=["low", "medium", "high", "urgent"], required=False, default_value="medium"+ definition="Priority level", choices=["low", "medium", "high", "urgent"], required=False, default_value="medium",),}@@ -408,11 +404,11 @@ class TestStructureGenerator:def test_mixed_structure_blueprint_normalization(self):"""Test that mixed structure blueprints (strings and ConceptStructureBlueprint objects) are properly normalized."""# Create a mixed structure blueprint similar to what would come from TOML parsing- mixed_structure_blueprint: Dict[str, Union[str, ConceptStructureBlueprint]] = {+ mixed_structure_blueprint: dict[str, str | ConceptStructureBlueprint] = {"name": "The name of the person", # Simple string definition"age": ConceptStructureBlueprint(definition="The age of the person", type=ConceptStructureBlueprintFieldType.NUMBER, required=True),"birthdate": ConceptStructureBlueprint(- definition="The birthdate of the person", type=ConceptStructureBlueprintFieldType.DATE, required=True+ definition="The birthdate of the person", type=ConceptStructureBlueprintFieldType.DATE, required=True,),}diff --git a/tests/unit/pipelex/libraries/pipelines/builder/__init__.py b/tests/unit/pipelex/libraries/pipelines/builder/__init__.pynew file mode 100644index 000000000..e69de29bbdiff --git a/tests/unit/pipelex/libraries/pipelines/builder/concept/__init__.py b/tests/unit/pipelex/libraries/pipelines/builder/concept/__init__.pynew file mode 100644index 000000000..e69de29bbdiff --git a/tests/unit/pipelex/libraries/pipelines/builder/concept/test_data.py b/tests/unit/pipelex/libraries/pipelines/builder/concept/test_data.pyindex 1bc960393..27152474f 100644--- a/tests/unit/pipelex/libraries/pipelines/builder/concept/test_data.py+++ b/tests/unit/pipelex/libraries/pipelines/builder/concept/test_data.py@@ -1,12 +1,10 @@-from typing import ClassVar, List, Tuple+from typing import ClassVarfrom pipelex.core.concepts.concept_blueprint import ConceptBlueprint, ConceptStructureBlueprint, ConceptStructureBlueprintFieldTypefrom pipelex.libraries.pipelines.builder.concept.concept_spec import ConceptSpec, ConceptStructureSpec, ConceptStructureSpecFieldTypeclass ConceptBlueprintTestCases:- """Test cases for ConceptBlueprint.to_blueprint conversion."""-SIMPLE_CONCEPT = ("simple_concept",ConceptSpec(@@ -148,7 +146,7 @@ class ConceptBlueprintTestCases:),)- TEST_CASES: ClassVar[List[Tuple[str, ConceptSpec, ConceptBlueprint]]] = [+ TEST_CASES: ClassVar[list[tuple[str, ConceptSpec, ConceptBlueprint]]] = [SIMPLE_CONCEPT,CONCEPT_WITH_REFINES,CONCEPT_WITH_TEXT_FIELD,diff --git a/tests/unit/pipelex/libraries/pipelines/builder/pipe/__init__.py b/tests/unit/pipelex/libraries/pipelines/builder/pipe/__init__.pynew file mode 100644index 000000000..e69de29bbdiff --git a/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_controllers/pipe_batch/__init__.py b/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_controllers/pipe_batch/__init__.pynew file mode 100644index 000000000..e69de29bbdiff --git a/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_controllers/pipe_batch/test_data.py b/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_controllers/pipe_batch/test_data.pyindex 98b0a8016..eee197946 100644--- a/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_controllers/pipe_batch/test_data.py+++ b/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_controllers/pipe_batch/test_data.py@@ -1,4 +1,4 @@-from typing import ClassVar, List, Tuple+from typing import ClassVarfrom pipelex.core.pipes.pipe_input_blueprint import InputRequirementBlueprintfrom pipelex.libraries.pipelines.builder.pipe.inputs_spec import InputRequirementSpec@@ -16,7 +16,6 @@ class PipeBatchTestCases:output="ProcessedItems",branch_pipe_code="process_item",),- "test_domain",PipeBatchBlueprint(definition="Process items in batch",inputs={"items": InputRequirementBlueprint(concept="ItemList")},@@ -40,7 +39,6 @@ class PipeBatchTestCases:input_list_name="data_list",input_item_name="current_data",),- "test_domain",PipeBatchBlueprint(definition="Batch with custom names",inputs={"data": InputRequirementBlueprint(concept="DataList")},@@ -53,7 +51,7 @@ class PipeBatchTestCases:),)- TEST_CASES: ClassVar[List[Tuple[str, PipeBatchSpec, str, PipeBatchBlueprint]]] = [+ TEST_CASES: ClassVar[list[tuple[str, PipeBatchSpec, PipeBatchBlueprint]]] = [SIMPLE_BATCH,BATCH_WITH_NAMES,]diff --git a/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_controllers/pipe_batch/test_pipe_batch.py b/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_controllers/pipe_batch/test_pipe_batch.pyindex b469ef32e..eec8f555d 100644--- a/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_controllers/pipe_batch/test_pipe_batch.py+++ b/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_controllers/pipe_batch/test_pipe_batch.py@@ -8,14 +8,13 @@ from .test_data import PipeBatchTestCasesclass TestPipeBatchBlueprintConversion:@pytest.mark.parametrize(- "test_name,pipe_spec,domain,expected_blueprint",+ "test_name,pipe_spec,expected_blueprint",PipeBatchTestCases.TEST_CASES,)def test_pipe_batch_spec_to_blueprint(self,test_name: str,pipe_spec: PipeBatchSpec,- domain: str,expected_blueprint: PipeBatchBlueprint,):result = pipe_spec.to_blueprint()diff --git a/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_controllers/pipe_condition/__init__.py b/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_controllers/pipe_condition/__init__.pynew file mode 100644index 000000000..e69de29bbdiff --git a/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_controllers/pipe_condition/test_data.py b/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_controllers/pipe_condition/test_data.pyindex 09f189046..fa7f84917 100644--- a/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_controllers/pipe_condition/test_data.py+++ b/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_controllers/pipe_condition/test_data.py@@ -1,4 +1,4 @@-from typing import ClassVar, List, Tuple+from typing import ClassVarfrom pipelex.core.pipes.pipe_input_blueprint import InputRequirementBlueprintfrom pipelex.libraries.pipelines.builder.pipe.inputs_spec import InputRequirementSpec@@ -19,10 +19,9 @@ class PipeConditionTestCases:root={"active": "process_active","inactive": "process_inactive",- }+ },),),- "test_domain",PipeConditionBlueprint(definition="Choose pipe based on condition",inputs={"data": InputRequirementBlueprint(concept="Data")},@@ -35,7 +34,7 @@ class PipeConditionTestCases:root={"active": "process_active","inactive": "process_inactive",- }+ },),default_pipe_code=None,add_alias_from_expression_to=None,@@ -55,12 +54,11 @@ class PipeConditionTestCases:"A": "process_a","B": "process_b","C": "process_c",- }+ },),default_pipe_code="process_default",add_alias_from_expression_to="category_result",),- "test_domain",PipeConditionBlueprint(definition="Conditional with template",inputs={"item": InputRequirementBlueprint(concept="Item")},@@ -74,14 +72,14 @@ class PipeConditionTestCases:"A": "process_a","B": "process_b","C": "process_c",- }+ },),default_pipe_code="process_default",add_alias_from_expression_to="category_result",),)- TEST_CASES: ClassVar[List[Tuple[str, PipeConditionSpec, str, PipeConditionBlueprint]]] = [+ TEST_CASES: ClassVar[list[tuple[str, PipeConditionSpec, PipeConditionBlueprint]]] = [SIMPLE_CONDITION,CONDITION_WITH_TEMPLATE,]diff --git a/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_controllers/pipe_condition/test_pipe_condition.py b/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_controllers/pipe_condition/test_pipe_condition.pyindex ba32686fb..6820e806b 100644--- a/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_controllers/pipe_condition/test_pipe_condition.py+++ b/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_controllers/pipe_condition/test_pipe_condition.py@@ -8,14 +8,13 @@ from .test_data import PipeConditionTestCasesclass TestPipeConditionBlueprintConversion:@pytest.mark.parametrize(- "test_name,pipe_spec,domain,expected_blueprint",+ "test_name,pipe_spec,expected_blueprint",PipeConditionTestCases.TEST_CASES,)def test_pipe_condition_spec_to_blueprint(self,test_name: str,pipe_spec: PipeConditionSpec,- domain: str,expected_blueprint: PipeConditionBlueprint,):result = pipe_spec.to_blueprint()diff --git a/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_controllers/pipe_parallel/__init__.py b/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_controllers/pipe_parallel/__init__.pynew file mode 100644index 000000000..e69de29bbdiff --git a/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_controllers/pipe_parallel/test_data.py b/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_controllers/pipe_parallel/test_data.pyindex ed5579415..25023d6eb 100644--- a/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_controllers/pipe_parallel/test_data.py+++ b/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_controllers/pipe_parallel/test_data.py@@ -1,4 +1,4 @@-from typing import ClassVar, List, Tuple+from typing import ClassVarfrom pipelex.core.pipes.pipe_input_blueprint import InputRequirementBlueprintfrom pipelex.libraries.pipelines.builder.pipe.inputs_spec import InputRequirementSpec@@ -22,7 +22,6 @@ class PipeParallelTestCases:SubPipeSpec(the_pipe_code="validate_data", result="validation"),],),- "test_domain",PipeParallelBlueprint(definition="Run pipes in parallel",inputs={"data": InputRequirementBlueprint(concept="Data")},@@ -53,7 +52,6 @@ class PipeParallelTestCases:add_each_output=False,combined_output="MergedData",),- "test_domain",PipeParallelBlueprint(definition="Parallel with combined output",inputs={"input": InputRequirementBlueprint(concept="Input")},@@ -69,7 +67,7 @@ class PipeParallelTestCases:),)- TEST_CASES: ClassVar[List[Tuple[str, PipeParallelSpec, str, PipeParallelBlueprint]]] = [+ TEST_CASES: ClassVar[list[tuple[str, PipeParallelSpec, PipeParallelBlueprint]]] = [SIMPLE_PARALLEL,PARALLEL_WITH_COMBINED,]diff --git a/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_controllers/pipe_parallel/test_pipe_parallel.py b/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_controllers/pipe_parallel/test_pipe_parallel.pyindex cf3315971..04d999a7f 100644--- a/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_controllers/pipe_parallel/test_pipe_parallel.py+++ b/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_controllers/pipe_parallel/test_pipe_parallel.py@@ -8,14 +8,13 @@ from .test_data import PipeParallelTestCasesclass TestPipeParallelBlueprintConversion:@pytest.mark.parametrize(- "test_name,pipe_spec,domain,expected_blueprint",+ "test_name,pipe_spec,expected_blueprint",PipeParallelTestCases.TEST_CASES,)def test_pipe_parallel_spec_to_blueprint(self,test_name: str,pipe_spec: PipeParallelSpec,- domain: str,expected_blueprint: PipeParallelBlueprint,):result = pipe_spec.to_blueprint()diff --git a/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_controllers/pipe_sequence/__init__.py b/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_controllers/pipe_sequence/__init__.pynew file mode 100644index 000000000..e69de29bbdiff --git a/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_controllers/pipe_sequence/test_data.py b/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_controllers/pipe_sequence/test_data.pyindex 3448c51d9..75ee231b8 100644--- a/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_controllers/pipe_sequence/test_data.py+++ b/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_controllers/pipe_sequence/test_data.py@@ -1,4 +1,4 @@-from typing import ClassVar, List, Tuple+from typing import ClassVarfrom pipelex.core.pipes.pipe_input_blueprint import InputRequirementBlueprintfrom pipelex.libraries.pipelines.builder.pipe.inputs_spec import InputRequirementSpec@@ -22,7 +22,6 @@ class PipeSequenceTestCases:SubPipeSpec(the_pipe_code="step3", result="final_result"),],),- "test_domain",PipeSequenceBlueprint(definition="A sequence of operations",inputs={"input_data": InputRequirementBlueprint(concept="Text")},@@ -54,7 +53,6 @@ class PipeSequenceTestCases:),],),- "test_domain",PipeSequenceBlueprint(definition="Sequence with batch",inputs={"items": InputRequirementBlueprint(concept="ItemList")},@@ -73,7 +71,7 @@ class PipeSequenceTestCases:),)- TEST_CASES: ClassVar[List[Tuple[str, PipeSequenceSpec, str, PipeSequenceBlueprint]]] = [+ TEST_CASES: ClassVar[list[tuple[str, PipeSequenceSpec, PipeSequenceBlueprint]]] = [SIMPLE_SEQUENCE,SEQUENCE_WITH_BATCH,]diff --git a/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_controllers/pipe_sequence/test_pipe_sequence.py b/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_controllers/pipe_sequence/test_pipe_sequence.pyindex 4d57af598..dbd99537c 100644--- a/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_controllers/pipe_sequence/test_pipe_sequence.py+++ b/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_controllers/pipe_sequence/test_pipe_sequence.py@@ -8,14 +8,13 @@ from .test_data import PipeSequenceTestCasesclass TestPipeSequenceBlueprintConversion:@pytest.mark.parametrize(- "test_name,pipe_spec,domain,expected_blueprint",+ "test_name,pipe_spec,expected_blueprint",PipeSequenceTestCases.TEST_CASES,)def test_pipe_sequence_spec_to_blueprint(self,test_name: str,pipe_spec: PipeSequenceSpec,- domain: str,expected_blueprint: PipeSequenceBlueprint,):result = pipe_spec.to_blueprint()diff --git a/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_operators/pipe_func/__init__.py b/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_operators/pipe_func/__init__.pynew file mode 100644index 000000000..e69de29bbdiff --git a/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_operators/pipe_func/test_data.py b/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_operators/pipe_func/test_data.pyindex b184452fa..4bfdf9bc6 100644--- a/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_operators/pipe_func/test_data.py+++ b/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_operators/pipe_func/test_data.py@@ -1,4 +1,4 @@-from typing import ClassVar, List, Tuple+from typing import ClassVarfrom pipelex.core.pipes.pipe_input_blueprint import InputRequirementBlueprintfrom pipelex.libraries.pipelines.builder.pipe.inputs_spec import InputRequirementSpec@@ -16,7 +16,6 @@ class PipeFuncTestCases:output="ProcessedData",function_name="process_data",),- "test_domain",PipeFuncBlueprint(definition="Execute a function",inputs={"data": InputRequirementBlueprint(concept="Data")},@@ -36,7 +35,6 @@ class PipeFuncTestCases:output="GeneratedData",function_name="generate_data",),- "test_domain",PipeFuncBlueprint(definition="Generate data",inputs=None,@@ -47,7 +45,7 @@ class PipeFuncTestCases:),)- TEST_CASES: ClassVar[List[Tuple[str, PipeFuncSpec, str, PipeFuncBlueprint]]] = [+ TEST_CASES: ClassVar[list[tuple[str, PipeFuncSpec, PipeFuncBlueprint]]] = [SIMPLE_FUNC,FUNC_NO_INPUTS,]diff --git a/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_operators/pipe_func/test_pipe_func.py b/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_operators/pipe_func/test_pipe_func.pyindex b77d93727..b735e19c8 100644--- a/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_operators/pipe_func/test_pipe_func.py+++ b/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_operators/pipe_func/test_pipe_func.py@@ -8,14 +8,13 @@ from .test_data import PipeFuncTestCasesclass TestPipeFuncBlueprintConversion:@pytest.mark.parametrize(- "test_name,pipe_spec,domain,expected_blueprint",+ "test_name,pipe_spec,expected_blueprint",PipeFuncTestCases.TEST_CASES,)def test_pipe_func_spec_to_blueprint(self,test_name: str,pipe_spec: PipeFuncSpec,- domain: str,expected_blueprint: PipeFuncBlueprint,):result = pipe_spec.to_blueprint()diff --git a/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_operators/pipe_img/__init__.py b/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_operators/pipe_img/__init__.pynew file mode 100644index 000000000..e69de29bbdiff --git a/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_operators/pipe_img/test_data.py b/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_operators/pipe_img/test_data.pyindex 6920cc151..e239f230b 100644--- a/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_operators/pipe_img/test_data.py+++ b/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_operators/pipe_img/test_data.py@@ -1,4 +1,4 @@-from typing import ClassVar, List, Tuple+from typing import ClassVarfrom pipelex.cogt.img_gen.img_gen_job_components import AspectRatiofrom pipelex.core.pipes.pipe_input_blueprint import InputRequirementBlueprint@@ -65,7 +65,7 @@ class PipeImgGenTestCases:),)- TEST_CASES: ClassVar[List[Tuple[str, PipeImgGenSpec, PipeImgGenBlueprint]]] = [+ TEST_CASES: ClassVar[list[tuple[str, PipeImgGenSpec, PipeImgGenBlueprint]]] = [SIMPLE_IMG_GEN,IMG_GEN_WITH_OPTIONS,]diff --git a/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_operators/pipe_img/test_pipe_img.py b/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_operators/pipe_img/test_pipe_img.pyindex e54b5360f..09f8c6aa9 100644--- a/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_operators/pipe_img/test_pipe_img.py+++ b/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_operators/pipe_img/test_pipe_img.py@@ -18,8 +18,4 @@ class TestPipeImgGenBlueprintConversion:expected_blueprint: PipeImgGenBlueprint,):result = pipe_spec.to_blueprint()- from pipelex import pretty_print-- pretty_print(result, title=f"Result {test_name}")- pretty_print(expected_blueprint, title=f"Expected {test_name}")assert result == expected_blueprintdiff --git a/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_operators/pipe_jinja2/__init__.py b/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_operators/pipe_jinja2/__init__.pynew file mode 100644index 000000000..e69de29bbdiff --git a/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_operators/pipe_jinja2/test_data.py b/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_operators/pipe_jinja2/test_data.pyindex df261e843..32a2a4d96 100644--- a/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_operators/pipe_jinja2/test_data.py+++ b/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_operators/pipe_jinja2/test_data.py@@ -1,4 +1,4 @@-from typing import ClassVar, List, Tuple+from typing import ClassVarfrom pipelex.core.pipes.pipe_input_blueprint import InputRequirementBlueprintfrom pipelex.libraries.pipelines.builder.pipe.inputs_spec import InputRequirementSpec@@ -18,7 +18,6 @@ class PipeJinja2TestCases:output="RenderedText",jinja2="Hello {{ data.name }}!",),- "test_domain",PipeJinja2Blueprint(definition="Render a template",inputs={"data": InputRequirementBlueprint(concept="Data")},@@ -48,7 +47,6 @@ class PipeJinja2TestCases:template_category=Jinja2TemplateCategory.MARKDOWN,extra_context={"version": "1.0"},),- "test_domain",PipeJinja2Blueprint(definition="Template with prompting style",inputs={"input": InputRequirementBlueprint(concept="Input")},@@ -66,7 +64,7 @@ class PipeJinja2TestCases:),)- TEST_CASES: ClassVar[List[Tuple[str, PipeJinja2Spec, str, PipeJinja2Blueprint]]] = [+ TEST_CASES: ClassVar[list[tuple[str, PipeJinja2Spec, PipeJinja2Blueprint]]] = [SIMPLE_JINJA2,JINJA2_WITH_STYLE,]diff --git a/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_operators/pipe_jinja2/test_pipe_jinja2.py b/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_operators/pipe_jinja2/test_pipe_jinja2.pyindex d471ae3e4..a0afd4e26 100644--- a/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_operators/pipe_jinja2/test_pipe_jinja2.py+++ b/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_operators/pipe_jinja2/test_pipe_jinja2.py@@ -8,14 +8,13 @@ from .test_data import PipeJinja2TestCasesclass TestPipeJinja2BlueprintConversion:@pytest.mark.parametrize(- "test_name,pipe_spec,domain,expected_blueprint",+ "test_name,pipe_spec,expected_blueprint",PipeJinja2TestCases.TEST_CASES,)def test_pipe_jinja2_spec_to_blueprint(self,test_name: str,pipe_spec: PipeJinja2Spec,- domain: str,expected_blueprint: PipeJinja2Blueprint,):result = pipe_spec.to_blueprint()diff --git a/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_operators/pipe_llm/__init__.py b/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_operators/pipe_llm/__init__.pynew file mode 100644index 000000000..e69de29bbdiff --git a/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_operators/pipe_llm/test_data.py b/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_operators/pipe_llm/test_data.pyindex d0a1e5441..e3a6d25a7 100644--- a/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_operators/pipe_llm/test_data.py+++ b/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_operators/pipe_llm/test_data.py@@ -1,4 +1,4 @@-from typing import ClassVar, List, Tuple+from typing import ClassVarfrom pipelex.cogt.llm.llm_setting import LLMSettingfrom pipelex.core.pipes.pipe_input_blueprint import InputRequirementBlueprint@@ -17,7 +17,6 @@ class PipeLLMTestCases:output="Text",prompt_template="Write about $topic",),- "test_domain",PipeLLMBlueprint(type="PipeLLM",definition="Generate text",@@ -36,7 +35,6 @@ class PipeLLMTestCases:output="Text",prompt_template="Generate something interesting",),- "test_domain",PipeLLMBlueprint(type="PipeLLM",definition="Generate without inputs",@@ -55,7 +53,6 @@ class PipeLLMTestCases:prompt_template="Generate text",llm="llm_to_reason",),- "test_domain",PipeLLMBlueprint(type="PipeLLM",definition="Generate with preset",@@ -79,7 +76,6 @@ class PipeLLMTestCases:max_tokens=None, # "auto" is handled at conversion to core),),- "test_domain",PipeLLMBlueprint(type="PipeLLM",definition="Generate with settings",@@ -103,7 +99,6 @@ class PipeLLMTestCases:system_prompt="You are a data analyst",prompt_template="Analyze: @data",),- "test_domain",PipeLLMBlueprint(type="PipeLLM",definition="Generate with system prompt",@@ -124,7 +119,6 @@ class PipeLLMTestCases:prompt_template="Generate items",multiple_output=True,),- "test_domain",PipeLLMBlueprint(type="PipeLLM",definition="Generate multiple items",@@ -145,7 +139,6 @@ class PipeLLMTestCases:prompt_template="Generate items",nb_output=5,),- "test_domain",PipeLLMBlueprint(type="PipeLLM",definition="Generate exactly 5 items",@@ -174,7 +167,6 @@ class PipeLLMTestCases:prompt_template_to_structure="Structure the output",system_prompt_to_structure="You are a data structurer",),- "test_domain",PipeLLMBlueprint(type="PipeLLM",definition="Extract structured data",@@ -192,7 +184,7 @@ class PipeLLMTestCases:),)- TEST_CASES: ClassVar[List[Tuple[str, PipeLLMSpec, str, PipeLLMBlueprint]]] = [+ TEST_CASES: ClassVar[list[tuple[str, PipeLLMSpec, PipeLLMBlueprint]]] = [SIMPLE_LLM,LLM_NO_INPUTS,LLM_WITH_PRESET,diff --git a/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_operators/pipe_llm/test_pipe_llm.py b/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_operators/pipe_llm/test_pipe_llm.pyindex 48f075d7a..88288672e 100644--- a/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_operators/pipe_llm/test_pipe_llm.py+++ b/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_operators/pipe_llm/test_pipe_llm.py@@ -8,9 +8,9 @@ from .test_data import PipeLLMTestCasesclass TestPipeLLMBlueprintConversion:@pytest.mark.parametrize(- "test_name,pipe_spec,domain,expected_blueprint",+ "test_name,pipe_spec,expected_blueprint",PipeLLMTestCases.TEST_CASES,)- def test_pipe_llm_spec_to_blueprint(self, test_name: str, pipe_spec: PipeLLMSpec, domain: str, expected_blueprint: PipeLLMBlueprint):+ def test_pipe_llm_spec_to_blueprint(self, test_name: str, pipe_spec: PipeLLMSpec, expected_blueprint: PipeLLMBlueprint):result = pipe_spec.to_blueprint()assert result == expected_blueprintdiff --git a/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_operators/pipe_ocr/__init__.py b/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_operators/pipe_ocr/__init__.pynew file mode 100644index 000000000..e69de29bbdiff --git a/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_operators/pipe_ocr/test_data.py b/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_operators/pipe_ocr/test_data.pyindex 2e5fc5b05..06786ebdc 100644--- a/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_operators/pipe_ocr/test_data.py+++ b/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_operators/pipe_ocr/test_data.py@@ -1,4 +1,4 @@-from typing import ClassVar, List, Tuple+from typing import ClassVarfrom pipelex.core.pipes.pipe_input_blueprint import InputRequirementBlueprintfrom pipelex.libraries.pipelines.builder.pipe.inputs_spec import InputRequirementSpec@@ -16,7 +16,6 @@ class PipeOcrTestCases:output="ExtractedText",ocr="mistral-pixtral",),- "test_domain",PipeOcrBlueprint(definition="Extract text from image",inputs={"ocr_input": InputRequirementBlueprint(concept="Image")},@@ -40,7 +39,6 @@ class PipeOcrTestCases:page_views=True,page_views_dpi=300,),- "test_domain",PipeOcrBlueprint(definition="OCR with page options",inputs={"ocr_input": InputRequirementBlueprint(concept="PDF")},@@ -51,7 +49,7 @@ class PipeOcrTestCases:),)- TEST_CASES: ClassVar[List[Tuple[str, PipeOcrSpec, str, PipeOcrBlueprint]]] = [+ TEST_CASES: ClassVar[list[tuple[str, PipeOcrSpec, PipeOcrBlueprint]]] = [SIMPLE_OCR,OCR_WITH_OPTIONS,]diff --git a/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_operators/pipe_ocr/test_pipe_ocr.py b/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_operators/pipe_ocr/test_pipe_ocr.pyindex b33043418..f4cdfdafb 100644--- a/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_operators/pipe_ocr/test_pipe_ocr.py+++ b/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_operators/pipe_ocr/test_pipe_ocr.py@@ -8,14 +8,13 @@ from .test_data import PipeOcrTestCasesclass TestPipeOcrBlueprintConversion:@pytest.mark.parametrize(- "test_name,pipe_spec,domain,expected_blueprint",+ "test_name,pipe_spec,expected_blueprint",PipeOcrTestCases.TEST_CASES,)def test_pipe_ocr_spec_to_blueprint(self,test_name: str,pipe_spec: PipeOcrSpec,- domain: str,expected_blueprint: PipeOcrBlueprint,):result = pipe_spec.to_blueprint()diff --git a/tests/unit/pipelex/libraries/pipelines/builder/pipe/test_data_inputs.py b/tests/unit/pipelex/libraries/pipelines/builder/pipe/test_data_inputs.pyindex dd84d0c7b..77f1856e8 100644--- a/tests/unit/pipelex/libraries/pipelines/builder/pipe/test_data_inputs.py+++ b/tests/unit/pipelex/libraries/pipelines/builder/pipe/test_data_inputs.py@@ -1,4 +1,4 @@-from typing import ClassVar, List, Tuple+from typing import ClassVarfrom pipelex.core.pipes.pipe_input_blueprint import InputRequirementBlueprintfrom pipelex.libraries.pipelines.builder.pipe.inputs_spec import InputRequirementSpec@@ -33,7 +33,7 @@ class InputRequirementTestCases:InputRequirementBlueprint(concept="domain.ConceptName"),)- TEST_CASES: ClassVar[List[Tuple[str, InputRequirementSpec, str, InputRequirementBlueprint]]] = [+ TEST_CASES: ClassVar[list[tuple[str, InputRequirementSpec, str, InputRequirementBlueprint]]] = [SIMPLE_TEXT_INPUT,IMAGE_INPUT,CUSTOM_CONCEPT_INPUT,diff --git a/tests/unit/pipelex/libraries/pipelines/builder/pipe/test_data_pipe.py b/tests/unit/pipelex/libraries/pipelines/builder/pipe/test_data_pipe.pyindex 0769cb741..f52961f78 100644--- a/tests/unit/pipelex/libraries/pipelines/builder/pipe/test_data_pipe.py+++ b/tests/unit/pipelex/libraries/pipelines/builder/pipe/test_data_pipe.py@@ -1,4 +1,4 @@-from typing import ClassVar, List, Tuple+from typing import ClassVarfrom pipelex.core.pipes.pipe_blueprint import PipeBlueprintfrom pipelex.core.pipes.pipe_input_blueprint import InputRequirementBlueprint@@ -16,7 +16,6 @@ class PipeBlueprintTestCases:inputs={"input": "Text"},output="ProcessedText",),- "test_domain",PipeBlueprint(type="PipeLLM",category="PipeOperator",@@ -38,7 +37,6 @@ class PipeBlueprintTestCases:},output="Result",),- "test_domain",PipeBlueprint(type="PipeFunc",category="PipeOperator",@@ -60,7 +58,6 @@ class PipeBlueprintTestCases:inputs=None,output="GeneratedData",),- "test_domain",PipeBlueprint(type="PipeFunc",category="PipeOperator",@@ -70,7 +67,7 @@ class PipeBlueprintTestCases:),)- TEST_CASES: ClassVar[List[Tuple[str, PipeSpec, str, PipeBlueprint]]] = [+ TEST_CASES: ClassVar[list[tuple[str, PipeSpec, PipeBlueprint]]] = [SIMPLE_PIPE,PIPE_WITH_INPUT_REQUIREMENTS,PIPE_NO_INPUTS,diff --git a/tests/unit/pipelex/libraries/pipelines/builder/pipe/test_data_sub_pipe.py b/tests/unit/pipelex/libraries/pipelines/builder/pipe/test_data_sub_pipe.pyindex 46bfdd378..0938419e1 100644--- a/tests/unit/pipelex/libraries/pipelines/builder/pipe/test_data_sub_pipe.py+++ b/tests/unit/pipelex/libraries/pipelines/builder/pipe/test_data_sub_pipe.py@@ -1,4 +1,4 @@-from typing import ClassVar, List, Tuple+from typing import ClassVarfrom pipelex.libraries.pipelines.builder.pipe.sub_pipe_spec import SubPipeSpecfrom pipelex.pipe_controllers.sub_pipe_blueprint import SubPipeBlueprint@@ -29,7 +29,7 @@ class SubPipeTestCases:SubPipeBlueprint(pipe="process_item", result="processed_items", batch_over="input_list", batch_as="current_item"),)- TEST_CASES: ClassVar[List[Tuple[str, SubPipeSpec, SubPipeBlueprint]]] = [+ TEST_CASES: ClassVar[list[tuple[str, SubPipeSpec, SubPipeBlueprint]]] = [SIMPLE_SUB_PIPE,SUB_PIPE_WITH_MULTIPLE_OUTPUT,SUB_PIPE_WITH_FIXED_OUTPUT,diff --git a/tests/unit/pipelex/libraries/pipelines/builder/pipe/test_inputs_blueprint.py b/tests/unit/pipelex/libraries/pipelines/builder/pipe/test_inputs_blueprint.pyindex 4657f1d67..53cfdfb3e 100644--- a/tests/unit/pipelex/libraries/pipelines/builder/pipe/test_inputs_blueprint.py+++ b/tests/unit/pipelex/libraries/pipelines/builder/pipe/test_inputs_blueprint.py@@ -12,7 +12,7 @@ class TestInputRequirementBlueprintConversion:InputRequirementTestCases.TEST_CASES,)def test_input_requirement_spec_to_blueprint(- self, test_name: str, input_spec: InputRequirementSpec, domain: str, expected_blueprint: InputRequirementBlueprint+ self, test_name: str, input_spec: InputRequirementSpec, domain: str, expected_blueprint: InputRequirementBlueprint,):result = input_spec.to_blueprint()assert result == expected_blueprintdiff --git a/tests/unit/pipelex/libraries/pipelines/builder/pipe/test_pipe_blueprint.py b/tests/unit/pipelex/libraries/pipelines/builder/pipe/test_pipe_blueprint.pyindex 191b33820..ab0d3b66e 100644--- a/tests/unit/pipelex/libraries/pipelines/builder/pipe/test_pipe_blueprint.py+++ b/tests/unit/pipelex/libraries/pipelines/builder/pipe/test_pipe_blueprint.py@@ -8,14 +8,13 @@ from .test_data_pipe import PipeBlueprintTestCasesclass TestPipeBlueprintConversion:@pytest.mark.parametrize(- "test_name,pipe_spec,domain,expected_blueprint",+ "test_name,pipe_spec,expected_blueprint",PipeBlueprintTestCases.TEST_CASES,)def test_pipe_blueprint_to_core(self,test_name: str,pipe_spec: PipeSpec,- domain: str,expected_blueprint: PipeBlueprint,):result = pipe_spec.to_blueprint()diff --git a/tests/unit/pipelex/migration/test_migrate_v0_1_0_to_v0_2_0.py b/tests/unit/pipelex/migration/test_migrate_v0_1_0_to_v0_2_0.pyindex 086ae324c..51309e346 100644--- a/tests/unit/pipelex/migration/test_migrate_v0_1_0_to_v0_2_0.py+++ b/tests/unit/pipelex/migration/test_migrate_v0_1_0_to_v0_2_0.py@@ -226,7 +226,7 @@ refines = "Text"assert changes[1]["new_line"] == 'definition = "Another concept"'def test_migrate_file_success(- self, migrator: TOMLMigrator, tmp_path: Path, sample_old_syntax_content: str, sample_new_syntax_content: str+ self, migrator: TOMLMigrator, tmp_path: Path, sample_old_syntax_content: str, sample_new_syntax_content: str,) -> None:"""Test successful file migration."""test_file = tmp_path / "test.toml"@@ -266,7 +266,7 @@ refines = "Text"assert changes_count == 0def test_migrate_file_without_backup(- self, migrator: TOMLMigrator, tmp_path: Path, sample_old_syntax_content: str, sample_new_syntax_content: str+ self, migrator: TOMLMigrator, tmp_path: Path, sample_old_syntax_content: str, sample_new_syntax_content: str,) -> None:"""Test file migration without creating backup."""test_file = tmp_path / "test.toml"diff --git a/tests/unit/pipelex/tools/config/__init__.py b/tests/unit/pipelex/tools/config/__init__.pynew file mode 100644index 000000000..e69de29bbdiff --git a/tests/unit/pipelex/tools/misc/__init__.py b/tests/unit/pipelex/tools/misc/__init__.pynew file mode 100644index 000000000..e69de29bbdiff --git a/tests/unit/pipelex/tools/misc/test_dict_utils.py b/tests/unit/pipelex/tools/misc/test_dict_utils.pyindex f24765c01..0f0c2c44a 100644--- a/tests/unit/pipelex/tools/misc/test_dict_utils.py+++ b/tests/unit/pipelex/tools/misc/test_dict_utils.py@@ -1,14 +1,10 @@-"""Unit tests for dict_utils module."""--from typing import Any, Dict+from typing import Anyfrom pipelex.core.concepts.concept_native import NativeConceptEnumfrom pipelex.tools.misc.dict_utils import apply_to_strings_in_list, apply_to_strings_recursive, insert_beforeclass TestDictUtils:- """Test class for dict_utils functions."""-def test_insert_before_basic(self) -> None:"""Test basic insert_before functionality."""original = {"a": 1, "c": 3}@@ -94,7 +90,7 @@ class TestDictUtils:def test_apply_to_strings_recursive_empty_structures(self) -> None:"""Test apply_to_strings_recursive with empty dictionaries and lists."""- data: Dict[str, Any] = {"empty_dict": {}, "empty_list": [], "mixed": {"inner_empty": {}, "inner_list": []}}+ data: dict[str, Any] = {"empty_dict": {}, "empty_list": [], "mixed": {"inner_empty": {}, "inner_list": []}}def dummy_transform(s: str) -> str:return s.upper()diff --git a/tests/unit/pipelex/tools/misc/test_filetype_utils.py b/tests/unit/pipelex/tools/misc/test_filetype_utils.pyindex 8ef4afadb..15799d2c2 100644--- a/tests/unit/pipelex/tools/misc/test_filetype_utils.py+++ b/tests/unit/pipelex/tools/misc/test_filetype_utils.py@@ -131,7 +131,7 @@ class TestDetectFileTypeFromBase64:def test_detect_file_type_from_base64_string_success(self, mocker: MockerFixture):# Mock detect_file_type_from_bytesmock_detect_bytes = mocker.patch(- "pipelex.tools.misc.filetype_utils.detect_file_type_from_bytes", return_value=FileType(extension="gif", mime="image/gif")+ "pipelex.tools.misc.filetype_utils.detect_file_type_from_bytes", return_value=FileType(extension="gif", mime="image/gif"),)# GIF header encoded in base64@@ -148,7 +148,7 @@ class TestDetectFileTypeFromBase64:def test_detect_file_type_from_base64_bytes_success(self, mocker: MockerFixture):# Mock detect_file_type_from_bytesmock_detect_bytes = mocker.patch(- "pipelex.tools.misc.filetype_utils.detect_file_type_from_bytes", return_value=FileType(extension="jpg", mime="image/jpeg")+ "pipelex.tools.misc.filetype_utils.detect_file_type_from_bytes", return_value=FileType(extension="jpg", mime="image/jpeg"),)# JPEG header encoded in base64@@ -165,7 +165,7 @@ class TestDetectFileTypeFromBase64:def test_detect_file_type_from_base64_data_url_success(self, mocker: MockerFixture):# Mock detect_file_type_from_bytesmock_detect_bytes = mocker.patch(- "pipelex.tools.misc.filetype_utils.detect_file_type_from_bytes", return_value=FileType(extension="png", mime="image/png")+ "pipelex.tools.misc.filetype_utils.detect_file_type_from_bytes", return_value=FileType(extension="png", mime="image/png"),)# PNG header encoded in base64@@ -183,7 +183,7 @@ class TestDetectFileTypeFromBase64:def test_detect_file_type_from_base64_data_url_with_whitespace(self, mocker: MockerFixture):# Mock detect_file_type_from_bytesmock_detect_bytes = mocker.patch(- "pipelex.tools.misc.filetype_utils.detect_file_type_from_bytes", return_value=FileType(extension="txt", mime="text/plain")+ "pipelex.tools.misc.filetype_utils.detect_file_type_from_bytes", return_value=FileType(extension="txt", mime="text/plain"),)test_bytes = b"hello world"@@ -215,7 +215,7 @@ class TestDetectFileTypeFromBase64:def test_detect_file_type_from_base64_data_url_no_comma(self, mocker: MockerFixture):# Test data URL without comma (should be treated as regular base64)mocker.patch(- "pipelex.tools.misc.filetype_utils.detect_file_type_from_bytes", return_value=FileType(extension="bin", mime="application/octet-stream")+ "pipelex.tools.misc.filetype_utils.detect_file_type_from_bytes", return_value=FileType(extension="bin", mime="application/octet-stream"),)# This will be treated as a base64 string since there's no commadiff --git a/tests/unit/pipelex/tools/misc/test_find_files_in_dir.py b/tests/unit/pipelex/tools/misc/test_find_files_in_dir.pyindex 5473b60f5..eee007216 100644--- a/tests/unit/pipelex/tools/misc/test_find_files_in_dir.py+++ b/tests/unit/pipelex/tools/misc/test_find_files_in_dir.py@@ -5,8 +5,6 @@ from pipelex.tools.misc.file_utils import find_files_in_dirclass TestFindFilesInDir:- """Test the find_files_in_dir helper function."""-def test_find_files_non_recursive(self):"""Test finding files non-recursively."""with tempfile.TemporaryDirectory() as temp_dir:@@ -45,7 +43,8 @@ class TestFindFilesInDir:# Find Python files recursivelyfiles = find_files_in_dir(temp_dir, "*.py", is_recursive=True)- assert len(files) == 3+ expected_files_length = 3+ assert len(files) == expected_files_lengthfile_names = [f.name for f in files]assert "file1.py" in file_namesassert "file2.py" in file_namesdiff --git a/tests/unit/pipelex/tools/misc/test_markdown.py b/tests/unit/pipelex/tools/misc/test_markdown.pyindex ff04d563b..943269d56 100644--- a/tests/unit/pipelex/tools/misc/test_markdown.py+++ b/tests/unit/pipelex/tools/misc/test_markdown.py@@ -1,4 +1,4 @@-from typing import Any, Dict, List+from typing import Anyfrom pydantic import BaseModel@@ -8,9 +8,9 @@ from pipelex.tools.misc.markdown_utils import convert_to_markdownclass Article(BaseModel):title: strauthor: str- tags: List[str]+ tags: list[str]content: str- metadata: Dict[str, Any]+ metadata: dict[str, Any]def test_convert_md():diff --git a/tests/unit/pipelex/tools/misc/test_path_utils.py b/tests/unit/pipelex/tools/misc/test_path_utils.pyindex 4c280bc91..9741b0f73 100644--- a/tests/unit/pipelex/tools/misc/test_path_utils.py+++ b/tests/unit/pipelex/tools/misc/test_path_utils.py@@ -1,3 +1,5 @@+import os+import pytestfrom pytest_mock import MockerFixture@@ -71,7 +73,6 @@ class TestInterpretPathOrUrl:def test_interpret_path_with_current_os_separator(self, mocker: MockerFixture):# Test with the current OS path separator- import ospath_with_sep = f"user{os.sep}file.txt"result = interpret_path_or_url(path_with_sep)@@ -109,7 +110,7 @@ class TestClarifyPathOrUrl:assert file_path == "/C:/Users/user/file.txt"assert url is None- def test_clarify_file_uri_with_spaces(self):+ def test_clarify_file_uri_with_spaces(self) -> None:file_path, url = clarify_path_or_url("file:///home/user/my%20file.txt")assert file_path == "/home/user/my file.txt"assert url is None@@ -150,12 +151,12 @@ class TestClarifyPathOrUrl:assert url is Nonedef test_clarify_base_64_raises_not_implemented(self, mocker: MockerFixture):+ # Mock interpret_path_or_url to return BASE_64+ mocker.patch(+ "pipelex.tools.misc.path_utils.interpret_path_or_url",+ return_value=InterpretedPathOrUrl.BASE_64,+ )with pytest.raises(NotImplementedError, match="Base 64 is not supported yet by clarify_path_or_url"):- # Mock interpret_path_or_url to return BASE_64- mocker.patch(- "pipelex.tools.misc.path_utils.interpret_path_or_url",- return_value=InterpretedPathOrUrl.BASE_64,- )clarify_path_or_url("some_base64_data")def test_clarify_uses_interpret_path_or_url(self, mocker: MockerFixture):diff --git a/tests/unit/pipelex/tools/misc/test_pretty.py b/tests/unit/pipelex/tools/misc/test_pretty.pyindex 12a269093..31c91172f 100644--- a/tests/unit/pipelex/tools/misc/test_pretty.py+++ b/tests/unit/pipelex/tools/misc/test_pretty.py@@ -1,6 +1,5 @@import reimport textwrap-from typing import Dict, List, Optional, Unionimport pytestfrom pydantic import BaseModel, Field@@ -13,22 +12,22 @@ class Address(BaseModel):street: strcity: strcountry: str- postal_code: Optional[str] = None+ postal_code: str | None = Noneclass UserPreferences(BaseModel):theme: str = "dark"notifications: bool = True- tags: List[str] = Field(default_factory=list)+ tags: list[str] = Field(default_factory=list)class ComplexUser(BaseModel):name: strage: int- email: Optional[str]- addresses: List[Address]+ email: str | None+ addresses: list[Address]preferences: UserPreferences- metadata: Dict[str, Union[str, int, bool]] = Field(default_factory=dict)+ metadata: dict[str, str | int | bool] = Field(default_factory=dict)def remove_ansi_escape_codes(text: str) -> str:diff --git a/tests/unit/pipelex/tools/misc/test_string_utils.py b/tests/unit/pipelex/tools/misc/test_string_utils.pyindex 80c3e7366..9d76d99ff 100644--- a/tests/unit/pipelex/tools/misc/test_string_utils.py+++ b/tests/unit/pipelex/tools/misc/test_string_utils.py@@ -17,7 +17,8 @@ from pipelex.tools.misc.string_utils import (class BadStr:def __str__(self) -> str: # pyright: ignore[reportImplicitOverride] pragma: no cover - used only for raising- raise RuntimeError("boom")+ msg = "boom"+ raise RuntimeError(msg)@pytest.mark.parametrize(@@ -262,7 +263,6 @@ class TestMatchesWildcardPattern:],)def test_matches_wildcard_pattern(self, text: str, pattern: str, expected: bool) -> None:- """Test wildcard pattern matching with comprehensive test cases."""assert matches_wildcard_pattern(text, pattern) is expecteddef test_matches_wildcard_pattern_model_routing_examples(self) -> None:diff --git a/tests/unit/pipelex/tools/secrets/__init__.py b/tests/unit/pipelex/tools/secrets/__init__.pynew file mode 100644index 000000000..e69de29bbdiff --git a/tests/unit/pipelex/tools/secrets/test_secrets_utils.py b/tests/unit/pipelex/tools/secrets/test_secrets_utils.pyindex d344f1337..86218178f 100644--- a/tests/unit/pipelex/tools/secrets/test_secrets_utils.py+++ b/tests/unit/pipelex/tools/secrets/test_secrets_utils.py@@ -3,7 +3,7 @@ from typing import Anyimport pytestfrom pytest_mock import MockerFixture-+from pipelex.tools.secrets.secrets_errors import SecretNotFoundErrorfrom pipelex.tools.secrets.secrets_provider_abstract import SecretsProviderAbstractfrom pipelex.tools.secrets.secrets_utils import UnknownVarPrefixError, VarFallbackPatternError, VarNotFoundError, substitute_vars@@ -16,10 +16,7 @@ def mock_secrets_provider(mocker: MockerFixture) -> Any:class TestSubstituteVars:- """Test cases for substitute_vars function."""-def test_simple_secret_substitution(self, mocker: MockerFixture, mock_secrets_provider: Any) -> None:- """Test basic secret substitution using default behavior."""mock_secrets_provider.get_secret.return_value = "secret_value"mocker.patch("pipelex.tools.secrets.secrets_utils.get_secrets_provider", return_value=mock_secrets_provider)@@ -29,14 +26,12 @@ class TestSubstituteVars:mock_secrets_provider.get_secret.assert_called_once_with(secret_id="API_KEY")def test_explicit_env_substitution(self, mocker: MockerFixture) -> None:- """Test explicit environment variable substitution."""mocker.patch.dict(os.environ, {"TEST_VAR": "env_value"})result = substitute_vars("Value: ${env:TEST_VAR}")assert result == "Value: env_value"def test_explicit_secret_substitution(self, mocker: MockerFixture, mock_secrets_provider: Any) -> None:- """Test explicit secret substitution."""mock_secrets_provider.get_secret.return_value = "secret_value"mocker.patch("pipelex.tools.secrets.secrets_utils.get_secrets_provider", return_value=mock_secrets_provider)@@ -46,7 +41,6 @@ class TestSubstituteVars:mock_secrets_provider.get_secret.assert_called_once_with(secret_id="API_KEY")def test_fallback_env_to_secret(self, mocker: MockerFixture, mock_secrets_provider: Any) -> None:- """Test fallback from env to secret when env var doesn't exist."""mock_secrets_provider.get_secret.return_value = "secret_fallback"mocker.patch("pipelex.tools.secrets.secrets_utils.get_secrets_provider", return_value=mock_secrets_provider)@@ -56,7 +50,6 @@ class TestSubstituteVars:mock_secrets_provider.get_secret.assert_called_once_with(secret_id="FALLBACK_KEY")def test_env_takes_precedence_in_fallback(self, mocker: MockerFixture, mock_secrets_provider: Any) -> None:- """Test that env var takes precedence when both env and secret are available."""mocker.patch.dict(os.environ, {"EXISTING_VAR": "env_value"})mocker.patch("pipelex.tools.secrets.secrets_utils.get_secrets_provider", return_value=mock_secrets_provider)@@ -133,9 +126,6 @@ class TestSubstituteVars:substitute_vars("Value: ${MISSING_SECRET}")def test_fallback_both_missing_raises_error(self, mocker: MockerFixture, mock_secrets_provider: Any) -> None:- """Test that when both env and secret are missing, error is raised."""- from pipelex.tools.secrets.secrets_errors import SecretNotFoundError-mock_secrets_provider.get_secret.side_effect = SecretNotFoundError("Secret not found")mocker.patch("pipelex.tools.secrets.secrets_utils.get_secrets_provider", return_value=mock_secrets_provider)diff --git a/tests/unit/pipelex/tools/templating/__init__.py b/tests/unit/pipelex/tools/templating/__init__.pynew file mode 100644index 000000000..e69de29bbdiff --git a/tests/unit/pipelex/tools/templating/test_template_preprocessor.py b/tests/unit/pipelex/tools/templating/test_template_preprocessor.pyindex 16be6635c..76f56f99b 100644--- a/tests/unit/pipelex/tools/templating/test_template_preprocessor.py+++ b/tests/unit/pipelex/tools/templating/test_template_preprocessor.py@@ -2,8 +2,6 @@ from pipelex.tools.templating.template_preprocessor import preprocess_templateclass TestTemplatePreprocessor:- """Test the template preprocessing functionality."""-def test_at_variable_pattern(self):"""Test basic @variable pattern replacement."""template = "@expense\n@invoices"diff --git a/tests/unit/pipelex/tools/test_func_registry.py b/tests/unit/pipelex/tools/test_func_registry.pyindex a49068dab..17f308cbb 100644--- a/tests/unit/pipelex/tools/test_func_registry.py+++ b/tests/unit/pipelex/tools/test_func_registry.py@@ -1,5 +1,6 @@import logging-from typing import Any, Callable, Dict, List, Tuple+from collections.abc import Callable+from typing import Anyimport pytestfrom pytest import LogCaptureFixture@@ -53,10 +54,10 @@ def no_type_hints(working_memory): # type: ignore# Type alias for test parameters-TestParams = Tuple[str, Callable[..., Any], bool]+TestParams = tuple[str, Callable[..., Any], bool]# Test data-TEST_CASES: List[TestParams] = [+TEST_CASES: list[TestParams] = [# Valid cases("valid_function", valid_function, True),("valid_async_function", valid_async_function, True),@@ -84,8 +85,6 @@ def registry():class TestFuncRegistry:@pytest.mark.parametrize("test_name, func, is_eligible", TEST_CASES)def test_function_eligibility_and_registration(self, registry: FuncRegistry, test_name: str, func: Callable[..., Any], is_eligible: bool):- """Test function eligibility check and registration behavior."""-# Test eligibility check directlyactual_eligibility = registry.is_eligible_function(func)assert actual_eligibility == is_eligible, f"Eligibility check failed for {test_name}: expected {is_eligible}, got {actual_eligibility}"@@ -131,7 +130,7 @@ class TestFuncRegistry:registry.unregister_function_by_name("non_existent")def test_register_functions_dict_with_ineligible_functions(self, registry: FuncRegistry):- functions: Dict[str, Callable[..., Any]] = {"func1": sample_function, "func2": another_function}+ functions: dict[str, Callable[..., Any]] = {"func1": sample_function, "func2": another_function}registry.register_functions_dict(functions)# Ineligible functions should be silently skippedassert not registry.has_function("func1")@@ -139,7 +138,7 @@ class TestFuncRegistry:assert len(registry.root) == 0def test_register_functions_list_with_ineligible_functions(self, registry: FuncRegistry):- functions: List[Callable[..., Any]] = [sample_function, another_function]+ functions: list[Callable[..., Any]] = [sample_function, another_function]registry.register_functions(functions)# Ineligible functions should be silently skippedassert not registry.has_function("sample_function")@@ -164,17 +163,17 @@ class TestFuncRegistry:def test_get_required_function_with_signature(self, registry: FuncRegistry):registry.register_function(valid_function)- func = registry.get_required_function_with_signature("valid_function", valid_function)+ func = registry.get_required_function_with_signature("valid_function")assert func is valid_functiondef test_get_required_function_with_signature_not_found(self, registry: FuncRegistry):with pytest.raises(FuncRegistryError, match="not found in registry"):- registry.get_required_function_with_signature("non_existent", sample_function)+ registry.get_required_function_with_signature("non_existent")def test_get_required_function_with_signature_not_callable(self, registry: FuncRegistry):- registry.root["not_a_function"] = "a string" # type: ignore+ registry.root["not_a_function"] = "a string" # type: ignorewith pytest.raises(FuncRegistryError, match="is not a callable function"):- registry.get_required_function_with_signature("not_a_function", sample_function)+ registry.get_required_function_with_signature("not_a_function")def test_set_logger(self, registry: FuncRegistry, caplog: LogCaptureFixture):"""Test setting a custom logger"""diff --git a/tests/unit/pipelex/tools/test_func_registry_utils.py b/tests/unit/pipelex/tools/test_func_registry_utils.pyindex b1f55469d..bb0696603 100644--- a/tests/unit/pipelex/tools/test_func_registry_utils.py+++ b/tests/unit/pipelex/tools/test_func_registry_utils.py@@ -1,38 +1,30 @@-#!/usr/bin/env python3-"""Test script for FuncRegistryUtils"""-import tempfilefrom pathlib import Path-from typing import ClassVar, List, Tuple+from typing import ClassVarimport pytestfrom pydantic import Field-from pipelex.core.stuffs.stuff_content import StructuredContent+from pipelex.core.memory.working_memory import WorkingMemory+from pipelex.core.stuffs.stuff_content import StructuredContent, TextContentfrom pipelex.tools.func_registry import func_registryfrom pipelex.tools.func_registry_utils import FuncRegistryUtilsclass FilePath(StructuredContent):- """A path to a file in the codebase."""-path: str = Field(description="Path to the file")class CodebaseFileContent(StructuredContent):- """Content of a codebase file."""-file_path: str = Field(description="Path to the codebase file")file_content: str = Field(description="Content of the codebase file")class TestCases:- """Test cases for function eligibility."""-VALID_ASYNC_FUNCTION = """async def read_file_content(working_memory: WorkingMemory) -> ListContent[CodebaseFileContent]:'''Read the content of related codebase files.'''-+file_paths_list = working_memory.get_stuff_as_list("related_file_paths", item_type=FilePath)codebase_files: List[CodebaseFileContent] = []@@ -88,7 +80,7 @@ def invalid_function_no_type_hints(working_memory):pass"""- TEST_CASES: ClassVar[List[Tuple[str, str, List[str], List[str]]]] = [+ TEST_CASES: ClassVar[list[tuple[str, str, list[str], list[str]]]] = [("valid_async_function", VALID_ASYNC_FUNCTION, ["read_file_content"], []),("valid_sync_function", VALID_SYNC_FUNCTION, ["sync_function"], []),(@@ -108,8 +100,6 @@ def invalid_function_no_type_hints(working_memory):class TestFuncRegistryUtils:- """Test class for FuncRegistryUtils."""-@pytest.mark.parametrize("test_name, function_code, expected_registered, expected_not_registered",TestCases.TEST_CASES,@@ -118,11 +108,9 @@ class TestFuncRegistryUtils:self,test_name: str,function_code: str,- expected_registered: List[str],- expected_not_registered: List[str],+ expected_registered: list[str],+ expected_not_registered: list[str],):- """Test that only eligible functions are registered (both sync and async are now eligible)."""-# Create a temporary directory and file with test functionswith tempfile.TemporaryDirectory() as temp_dir:test_file = Path(temp_dir) / "test_functions.py"@@ -171,7 +159,6 @@ class CodebaseFileContent(StructuredContent):def test_recursive_folder_search(self):"""Test that recursive folder search works correctly."""-with tempfile.TemporaryDirectory() as temp_dir:# Create nested directory structurenested_dir = Path(temp_dir) / "subdir"@@ -216,18 +203,12 @@ async def nested_function(working_memory: WorkingMemory) -> TextContent:assert not func_registry.has_function("nested_function"), "nested_function should NOT be registered"def test_eligibility_check_directly(self):- """Test the eligibility check method directly."""-- # Create test functions dynamically for direct testing- from pipelex.core.memory.working_memory import WorkingMemory- from pipelex.core.stuffs.stuff_content import TextContent-# Valid async function- async def valid_async_function(working_memory: WorkingMemory) -> TextContent:+ async def valid_async_function(working_memory: WorkingMemory) -> TextContent: # noqa: ARG001return TextContent(text="test")# Valid sync function- def valid_sync_function(working_memory: WorkingMemory) -> TextContent:+ def valid_sync_function(working_memory: WorkingMemory) -> TextContent: # noqa: ARG001return TextContent(text="test")# Test eligibility@@ -235,20 +216,14 @@ async def nested_function(working_memory: WorkingMemory) -> TextContent:assert func_registry.is_eligible_function(valid_sync_function), "Valid sync function should be eligible"def test_register_function_checks_eligibility(self):- """Test that register_function checks eligibility before registering."""-- from pipelex.core.memory.working_memory import WorkingMemory- from pipelex.core.stuffs.stuff_content import TextContent-- # Clear registryfunc_registry.teardown()# Valid async function- async def valid_async_function(working_memory: WorkingMemory) -> TextContent:+ async def valid_async_function(working_memory: WorkingMemory) -> TextContent: # noqa: ARG001return TextContent(text="valid")# Valid sync function- def valid_sync_function(working_memory: WorkingMemory) -> TextContent:+ def valid_sync_function(working_memory: WorkingMemory) -> TextContent: # noqa: ARG001return TextContent(text="valid")# Try to register both functions@@ -262,14 +237,14 @@ async def nested_function(working_memory: WorkingMemory) -> TextContent:# Test register_functions method as wellfunc_registry.teardown()- async def another_valid_async_function(working_memory: WorkingMemory) -> TextContent:+ async def another_valid_async_function(working_memory: WorkingMemory) -> TextContent: # noqa: ARG001return TextContent(text="another_valid_async")- def another_valid_sync_function(working_memory: WorkingMemory) -> TextContent:+ def another_valid_sync_function(working_memory: WorkingMemory) -> TextContent: # noqa: ARG001return TextContent(text="another_valid_sync")# Invalid function (wrong parameter name)- def invalid_function(other_param: WorkingMemory) -> TextContent:+ def invalid_function(other_param: WorkingMemory) -> TextContent: # noqa: ARG001return TextContent(text="invalid")# Register multiple functions at once - invalid functions should be silently skippeddiff --git a/tests/unit/pipelex/tools/test_testing.py b/tests/unit/pipelex/tools/test_testing.pyindex 8866810d9..a0632b494 100644--- a/tests/unit/pipelex/tools/test_testing.py+++ b/tests/unit/pipelex/tools/test_testing.py@@ -9,4 +9,5 @@ def test_testing():case RunMode.UNIT_TEST:assert not is_env_var_set(key="GITHUB_ACTIONS") and not is_env_var_set(key="CI")case _:- raise RuntimeError(f"Invalid run mode: {runtime_manager.run_mode}")+ msg = f"Invalid run mode: {runtime_manager.run_mode}"+ raise RuntimeError(msg)diff --git a/tests/unit/pipelex/tools/typing/__init__.py b/tests/unit/pipelex/tools/typing/__init__.pynew file mode 100644index 000000000..e69de29bbdiff --git a/tests/unit/pipelex/tools/typing/test_module_inspector.py b/tests/unit/pipelex/tools/typing/test_module_inspector.pyindex 686affe16..a075e7dee 100644--- a/tests/unit/pipelex/tools/typing/test_module_inspector.py+++ b/tests/unit/pipelex/tools/typing/test_module_inspector.py@@ -1,9 +1,7 @@-# SPDX-FileCopyrightText: © 2025 Evotis S.A.S.-# SPDX-License-Identifier: Apache-2.0-import osimport shutilimport sys+import typesfrom pathlib import Pathimport pytest@@ -33,11 +31,11 @@ class TestImportModuleFromFile:# Clean up nested directories and __pycache__ directoriesdirectories_to_remove = ["nested/subdir", "nested", "__pycache__", "nested/__pycache__", "nested/subdir/__pycache__"]for dir_path in directories_to_remove:- if os.path.exists(dir_path):+ if Path(dir_path).exists():shutil.rmtree(dir_path)# Clean up sys.modules entries for test modules- modules_to_remove = [name for name in sys.modules.keys() if name.startswith("test_module")]+ modules_to_remove = [name for name in sys.modules if name.startswith("test_module")]for module_name in modules_to_remove:del sys.modules[module_name]@@ -57,7 +55,7 @@ class TestClass:assert hasattr(module, "TestClass")assert module.test_function() == "test_value"- def test_import_non_python_file_raises_error(self, tmp_path: Path):+ def test_import_non_python_file_raises_error(self):"""Test that importing a non-Python file raises ModuleFileError."""test_file_path = "test_file.txt"with open(test_file_path, "w") as test_file:@@ -66,13 +64,13 @@ class TestClass:import_module_from_file(test_file_path)assert "is not a Python file" in str(excinfo.value)- def test_import_nonexistent_file_raises_error(self, tmp_path: Path):+ def test_import_nonexistent_file_raises_error(self):"""Test that importing a nonexistent file raises FileNotFoundError."""nonexistent_file_path = "nonexistent.py"with pytest.raises(FileNotFoundError):import_module_from_file(nonexistent_file_path)- def test_import_file_with_syntax_error_raises_error(self, tmp_path: Path):+ def test_import_file_with_syntax_error_raises_error(self):"""Test that importing a file with syntax errors raises SyntaxError."""test_file_path = "syntax_error.py"with open(test_file_path, "w") as test_file:@@ -83,7 +81,7 @@ def test_function(with pytest.raises(SyntaxError):import_module_from_file(test_file_path)- def test_import_file_with_import_error_raises_error(self, tmp_path: Path):+ def test_import_file_with_import_error_raises_error(self):"""Test that importing a file with import errors raises ImportError."""test_file_path = "import_error.py"with open(test_file_path, "w") as test_file:@@ -93,7 +91,7 @@ import nonexistent_modulewith pytest.raises(ImportError):import_module_from_file(test_file_path)- def test_module_added_to_sys_modules(self, tmp_path: Path):+ def test_module_added_to_sys_modules(self):"""Test that imported module is added to sys.modules."""test_file_path = "test_module.py"with open(test_file_path, "w") as test_file:@@ -108,7 +106,7 @@ def test_function():assert expected_module_name in sys.modulesassert sys.modules[expected_module_name] is module- def test_module_name_with_path_separators(self, tmp_path: Path):+ def test_module_name_with_path_separators(self):"""Test that module name is correctly formatted with path separators (should use file basename)."""test_file_path = "nested/subdir/test_module.py"# Create nested directories@@ -126,8 +124,6 @@ def test_function():class TestFindClassesInModule:def test_find_all_classes_no_base_class(self):"""Test finding all classes when no base class is specified."""- import types-test_module = types.ModuleType("test_module")class ClassA:@@ -143,14 +139,13 @@ class TestFindClassesInModule:setattr(test_module, "ClassB", ClassB)setattr(test_module, "some_function", lambda: None)classes = find_classes_in_module(test_module, base_class=None, include_imported=False)- assert len(classes) == 2+ expected_number_of_classes = 2+ assert len(classes) == expected_number_of_classesassert ClassA in classesassert ClassB in classesdef test_find_classes_with_base_class(self):"""Test finding classes that inherit from a specific base class."""- import types-test_module = types.ModuleType("test_module")class BaseClass:@@ -170,15 +165,14 @@ class TestFindClassesInModule:setattr(test_module, "SubClass", SubClass)setattr(test_module, "UnrelatedClass", UnrelatedClass)classes = find_classes_in_module(test_module, base_class=BaseClass, include_imported=False)- assert len(classes) == 2 # BaseClass and SubClass+ expected_number_of_classes = 2+ assert len(classes) == expected_number_of_classesassert BaseClass in classesassert SubClass in classesassert UnrelatedClass not in classesdef test_find_classes_exclude_imported(self):"""Test that imported classes are excluded when include_imported=False."""- import types-test_module = types.ModuleType("test_module")class LocalClass:@@ -192,14 +186,13 @@ class TestFindClassesInModule:setattr(test_module, "LocalClass", LocalClass)setattr(test_module, "ImportedClass", ImportedClass)classes = find_classes_in_module(test_module, base_class=None, include_imported=False)- assert len(classes) == 1+ expected_number_of_classes = 1+ assert len(classes) == expected_number_of_classesassert LocalClass in classesassert ImportedClass not in classesdef test_find_classes_include_imported(self):"""Test that imported classes are included when include_imported=True."""- import types-test_module = types.ModuleType("test_module")class LocalClass:@@ -213,22 +206,20 @@ class TestFindClassesInModule:setattr(test_module, "LocalClass", LocalClass)setattr(test_module, "ImportedClass", ImportedClass)classes = find_classes_in_module(test_module, base_class=None, include_imported=True)- assert len(classes) == 2+ expected_number_of_classes = 2+ assert len(classes) == expected_number_of_classesassert LocalClass in classesassert ImportedClass in classesdef test_find_classes_empty_module(self):"""Test finding classes in an empty module."""- import types-test_module = types.ModuleType("test_module")classes = find_classes_in_module(test_module, base_class=None, include_imported=False)- assert len(classes) == 0+ expected_number_of_classes = 0+ assert len(classes) == expected_number_of_classesdef test_find_classes_with_functions_and_variables(self):"""Test that functions and variables are not included in class search."""- import types-test_module = types.ModuleType("test_module")class TestClass:@@ -244,13 +235,12 @@ class TestFindClassesInModule:setattr(test_module, "some_variable", 42)setattr(test_module, "some_string", "hello")classes = find_classes_in_module(test_module, base_class=None, include_imported=False)- assert len(classes) == 1+ expected_number_of_classes = 1+ assert len(classes) == expected_number_of_classesassert TestClass in classesdef test_find_classes_with_nested_classes(self):"""Test finding classes including nested classes."""- import types-test_module = types.ModuleType("test_module")class OuterClass:@@ -260,13 +250,12 @@ class TestFindClassesInModule:OuterClass.__module__ = test_module.__name__setattr(test_module, "OuterClass", OuterClass)classes = find_classes_in_module(test_module, base_class=None, include_imported=False)- assert len(classes) == 1+ expected_number_of_classes = 1+ assert len(classes) == expected_number_of_classesassert OuterClass in classesdef test_find_classes_with_builtin_types(self):"""Test finding classes including user-defined types (not builtins, which can't be assigned __module__)."""- import types-test_module = types.ModuleType("test_module")class MyClassA:@@ -280,14 +269,12 @@ class TestFindClassesInModule:setattr(test_module, "MyClassA", MyClassA)setattr(test_module, "MyClassB", MyClassB)classes = find_classes_in_module(test_module, base_class=None, include_imported=False)- assert len(classes) == 2+ expected_number_of_classes = 2+ assert len(classes) == expected_number_of_classesassert MyClassA in classesassert MyClassB in classesdef test_find_classes_with_base_class_and_imported(self):- """Test finding classes with base class filter and imported classes."""- import types-test_module = types.ModuleType("test_module")class BaseClass:@@ -306,12 +293,14 @@ class TestFindClassesInModule:setattr(test_module, "LocalSubClass", LocalSubClass)setattr(test_module, "ImportedSubClass", ImportedSubClass)classes = find_classes_in_module(test_module, base_class=BaseClass, include_imported=False)- assert len(classes) == 2 # BaseClass and LocalSubClass+ expected_number_of_classes = 2+ assert len(classes) == expected_number_of_classesassert BaseClass in classesassert LocalSubClass in classesassert ImportedSubClass not in classesclasses = find_classes_in_module(test_module, base_class=BaseClass, include_imported=True)- assert len(classes) == 3 # BaseClass, LocalSubClass, and ImportedSubClass+ expected_number_of_classes = 3+ assert len(classes) == expected_number_of_classesassert BaseClass in classesassert LocalSubClass in classesassert ImportedSubClass in classesdiff --git a/tests/unit/pipelex/tools/typing/test_pydantic_utils.py b/tests/unit/pipelex/tools/typing/test_pydantic_utils.pyindex 0ed7b8d90..26b010083 100644--- a/tests/unit/pipelex/tools/typing/test_pydantic_utils.py+++ b/tests/unit/pipelex/tools/typing/test_pydantic_utils.py@@ -1,4 +1,4 @@-from typing import Any, Dict, cast+from typing import Any, castimport pytestfrom pydantic import BaseModel, ConfigDict, Field, ValidationError@@ -34,14 +34,14 @@ def test_serialize_model_with_hidden_fields():parent = ParentModel(parent_name="bar", child=child)# Test default model_dump() shows everything- model_dump_result: Dict[str, Any] = parent.model_dump()- child_dict = cast(Dict[str, Any], model_dump_result["child"])+ model_dump_result: dict[str, Any] = parent.model_dump()+ child_dict = cast("dict[str, Any]", model_dump_result["child"])assert "child_secret" in child_dictassert child_dict["child_secret"] == "secret_foo"# Test our custom serializer omits hidden fields (child_secret)- serialized_result = cast(Dict[str, Any], serialize_model(parent))- child_serialized = cast(Dict[str, Any], serialized_result["child"])+ serialized_result = cast("dict[str, Any]", serialize_model(parent))+ child_serialized = cast("dict[str, Any]", serialized_result["child"])assert "child_secret" not in child_serializedassert serialized_result == {"parent_name": "bar", "child": {"child_name": "foo"}}diff --git a/tests/unit/pipelex/tools/typing/test_structure_printer.py b/tests/unit/pipelex/tools/typing/test_structure_printer.pyindex de79872b0..233c230bf 100644--- a/tests/unit/pipelex/tools/typing/test_structure_printer.py+++ b/tests/unit/pipelex/tools/typing/test_structure_printer.py@@ -1,8 +1,8 @@from datetime import datetime-from typing import List, Literal, Optional+from typing import Literal, Optional+import pytestfrom pydantic import BaseModel, Field, field_validator-from pytest import FixtureRequestfrom pipelex.core.stuffs.stuff_content import ListContent, StructuredContent, TextContentfrom pipelex.tools.typing.structure_printer import StructurePrinter@@ -34,7 +34,6 @@ class MusicGenre(StrEnum):class SimpleTextContent(TextContent):"""A simple text content class"""- passclass MusicCategoryContent(StructuredContent):@@ -79,26 +78,26 @@ class PersonContent(StructuredContent):name: strage: intaddress: AddressContent = Field(description="Address of the person")- documents: List[DocumentTypeContent]- priority: Optional[Priority] = None+ documents: list[DocumentTypeContent]+ priority: Priority | None = Noneclass ComplexListContent(ListContent[PersonContent]):"""List content with complex items"""- items: List[PersonContent]+ items: list[PersonContent]class GanttTaskDetails(StructuredContent):"""Do not include timezone in the dates."""name: str- start_date: Optional[datetime] = None- end_date: Optional[datetime] = None+ start_date: datetime | None = None+ end_date: datetime | None = None@field_validator("start_date", "end_date")@classmethod- def remove_tzinfo(cls, v: Optional[datetime]) -> Optional[datetime]:+ def remove_tzinfo(cls, v: datetime | None) -> datetime | None:if v is not None:return v.replace(tzinfo=None)return v@@ -106,25 +105,24 @@ class GanttTaskDetails(StructuredContent):class Milestone(StructuredContent):name: str- date: Optional[datetime]+ date: datetime | None@field_validator("date")@classmethod- def remove_tzinfo(cls, v: Optional[datetime]) -> Optional[datetime]:+ def remove_tzinfo(cls, v: datetime | None) -> datetime | None:if v is not None:return v.replace(tzinfo=None)return vclass GanttChart(StructuredContent):- tasks: Optional[List[GanttTaskDetails]] = None- milestones: Optional[List[Milestone]] = None+ tasks: list[GanttTaskDetails] | None = None+ milestones: list[Milestone] | None = Noneclass TestStructurePrinter:- """Tests for the type inspector functionality"""-- def test_simple_text_content(self, request: FixtureRequest):+ @pytest.mark.usefixtures("request")+ def test_simple_text_content(self):"""Test structure of simple text content"""result = StructurePrinter().get_type_structure(SimpleTextContent)expected = [@@ -134,7 +132,8 @@ class TestStructurePrinter:]assert result == expected, f"Expected:\n{''.join(expected)}\n\nGot:\n{''.join(result)}"- def test_simple_structured_content(self, request: FixtureRequest):+ @pytest.mark.usefixtures("request")+ def test_simple_structured_content(self):"""Test structure of simple structured content"""result = StructurePrinter().get_type_structure(SimpleStructuredContent)expected = [@@ -146,7 +145,8 @@ class TestStructurePrinter:]assert result == expected, f"Expected:\n{''.join(expected)}\n\nGot:\n{''.join(result)}"- def test_enum_content(self, request: FixtureRequest):+ @pytest.mark.usefixtures("request")+ def test_enum_content(self):"""Test structure of content with enum"""result = StructurePrinter().get_type_structure(DocumentTypeContent)expected = [@@ -160,7 +160,8 @@ class TestStructurePrinter:]assert result == expected, f"Expected:\n{''.join(expected)}\n\nGot:\n{''.join(result)}"- def test_nested_content(self, request: FixtureRequest):+ @pytest.mark.usefixtures("request")+ def test_nested_content(self):"""Test structure of nested content"""result = StructurePrinter().get_type_structure(PersonContent)expected = [@@ -170,7 +171,7 @@ class TestStructurePrinter:" age: int"," address: AddressContent # Address of the person"," documents: List[DocumentTypeContent]",- " priority: Optional[Priority] = None",+ " priority: Priority | None = None","","class AddressContent(StructuredContent):",' """Nested address content"""',@@ -190,9 +191,13 @@ class TestStructurePrinter:' HIGH = "HIGH"',' LOW = "LOW"',]+ from pipelex import pretty_print+ pretty_print(result, "results")+ pretty_print(expected, "expected")assert result == expected, f"Expected:\n{''.join(expected)}\n\nGot:\n{''.join(result)}"- def test_list_content(self, request: FixtureRequest):+ @pytest.mark.usefixtures("request")+ def test_list_content(self):"""Test structure of list content"""result = StructurePrinter().get_type_structure(ComplexListContent)expected = [@@ -206,7 +211,7 @@ class TestStructurePrinter:" age: int"," address: AddressContent # Address of the person"," documents: List[DocumentTypeContent]",- " priority: Optional[Priority] = None",+ " priority: Priority | None = None","","class AddressContent(StructuredContent):",' """Nested address content"""',@@ -228,7 +233,8 @@ class TestStructurePrinter:]assert result == expected, f"Expected:\n{''.join(expected)}\n\nGot:\n{''.join(result)}"- def test_model_with_field_description(self, request: FixtureRequest):+ @pytest.mark.usefixtures("request")+ def test_model_with_field_description(self):"""Test structure of a model with field descriptions"""class Person(BaseModel):@@ -249,7 +255,8 @@ class TestStructurePrinter:]assert result == expected, f"Expected:\n{''.join(expected)}\n\nGot:\n{''.join(result)}"- def test_model_with_docstring_and_field_description(self, request: FixtureRequest):+ @pytest.mark.usefixtures("request")+ def test_model_with_docstring_and_field_description(self):"""Test structure of a model with both docstring and field descriptions"""class TaskContent(StructuredContent):@@ -261,7 +268,7 @@ class TestStructurePrinter:title: str = Field(description="The title of the task")description: str = Field(description="Detailed description of what needs to be done")- is_completed: bool = Field(False, description="Whether the task is completed")+ is_completed: bool = Field(default=False, description="Whether the task is completed")result = StructurePrinter().get_type_structure(TaskContent)expected = [@@ -277,7 +284,8 @@ class TestStructurePrinter:]assert result == expected, f"Expected:\n{''.join(expected)}\n\nGot:\n{''.join(result)}"- def test_literal_field_content(self, request: FixtureRequest):+ @pytest.mark.usefixtures("request")+ def test_literal_field_content(self) -> None:"""Test structure of content with Literal field"""result = StructurePrinter().get_type_structure(MusicCategoryContent)expected = [@@ -301,23 +309,22 @@ class TestStructurePrinter:]assert result == expected, f"Expected:\n{''.join(expected)}\n\nGot:\n{''.join(result)}"- def test_gantt_chart_content(self, request: FixtureRequest):- """Test structure of Gantt chart content with datetime validators"""-+ @pytest.mark.usefixtures("request")+ def test_gantt_chart_content(self):result = StructurePrinter().get_type_structure(GanttChart, base_class=StructuredContent)expected = ["class GanttChart(StructuredContent):",- " tasks: Optional[List[GanttTaskDetails]] = None",- " milestones: Optional[List[Milestone]] = None",+ " tasks: List[GanttTaskDetails] | None = None",+ " milestones: List[Milestone] | None = None","","class GanttTaskDetails(StructuredContent):",' """Do not include timezone in the dates."""'," name: str",- " start_date: Optional[datetime] = None",- " end_date: Optional[datetime] = None",+ " start_date: datetime | None = None",+ " end_date: datetime | None = None","","class Milestone(StructuredContent):"," name: str",- " date: Optional[datetime] = None",+ " date: datetime | None = None",]- assert result == expected, f"Expected:\n{''.join(expected)}\n\nGot:\n{''.join(result)}"+ assert result == expecteddiff --git a/uv.lock b/uv.lockindex 6b7cb9cb5..f24620d7f 100644--- a/uv.lock+++ b/uv.lock@@ -182,7 +182,7 @@ wheels = [[[package]]name = "anthropic"-version = "0.68.0"+version = "0.68.1"source = { registry = "https://pypi.org/simple" }dependencies = [{ name = "anyio" },@@ -194,9 +194,9 @@ dependencies = [{ name = "sniffio" },{ name = "typing-extensions" },]-sdist = { url = "https://files.pythonhosted.org/packages/64/46/da44bf087ddaf3f7dbe4808c00c7cde466fe68c4fc9fbebdfc231f4ea205/anthropic-0.68.0.tar.gz", hash = "sha256:507e9b5f627d1b249128ff15b21855e718fa4ed8dabc787d0e68860a4b32a7a8", size = 471584, upload-time = "2025-09-17T15:20:19.509Z" }+sdist = { url = "https://files.pythonhosted.org/packages/43/b9/5e736a924dabe0473b212597ae83a124e09818fac672998bb5960a64e9b1/anthropic-0.68.1.tar.gz", hash = "sha256:5fc01019dfffbc39e87a24f4d7cfe34e83dda4fff7b5371f33639f982f0c8dba", size = 471965, upload-time = "2025-09-26T16:27:30.547Z" }wheels = [- { url = "https://files.pythonhosted.org/packages/60/32/2d7553184b05bdbec61dd600014a55b9028408aee6128b25cb6f20e3002c/anthropic-0.68.0-py3-none-any.whl", hash = "sha256:ac579ea5eca22a7165b1042e6af57c4bf556e51afae3ca80e24768d4756b78c0", size = 325199, upload-time = "2025-09-17T15:20:17.452Z" },+ { url = "https://files.pythonhosted.org/packages/c9/5c/e871e40ba11338502569a3f5cb176650f99f7b4ecf915f4d38fe2b8cff37/anthropic-0.68.1-py3-none-any.whl", hash = "sha256:40fc545c4781ebc45d8e7849240c4001f40d61d18a17e5363c0acb10afefc33b", size = 325214, upload-time = "2025-09-26T16:27:29.049Z" },][[package]]@@ -214,6 +214,18 @@ wheels = [{ url = "https://files.pythonhosted.org/packages/15/b3/9b1a8074496371342ec1e796a96f99c82c945a339cd81a8e73de28b4cf9e/anyio-4.11.0-py3-none-any.whl", hash = "sha256:0287e96f4d26d4149305414d4e3bc32f0dcd0862365a4bddea19d7a1ec38c4fc", size = 109097, upload-time = "2025-09-23T09:19:10.601Z" },]+[[package]]+name = "astroid"+version = "3.3.11"+source = { registry = "https://pypi.org/simple" }+dependencies = [+ { name = "typing-extensions", marker = "python_full_version < '3.11'" },+]+sdist = { url = "https://files.pythonhosted.org/packages/18/74/dfb75f9ccd592bbedb175d4a32fc643cf569d7c218508bfbd6ea7ef9c091/astroid-3.3.11.tar.gz", hash = "sha256:1e5a5011af2920c7c67a53f65d536d65bfa7116feeaf2354d8b94f29573bb0ce", size = 400439, upload-time = "2025-07-13T18:04:23.177Z" }+wheels = [+ { url = "https://files.pythonhosted.org/packages/af/0f/3b8fdc946b4d9cc8cc1e8af42c4e409468c84441b933d037e101b3d72d86/astroid-3.3.11-py3-none-any.whl", hash = "sha256:54c760ae8322ece1abd213057c4b5bba7c49818853fc901ef09719a60dbf9dec", size = 275612, upload-time = "2025-07-13T18:04:21.07Z" },+]+[[package]]name = "async-timeout"version = "5.0.1"@@ -289,16 +301,16 @@ wheels = [[[package]]name = "boto3-stubs"-version = "1.40.39"+version = "1.40.40"source = { registry = "https://pypi.org/simple" }dependencies = [{ name = "botocore-stubs" },{ name = "types-s3transfer" },{ name = "typing-extensions", marker = "python_full_version < '3.12'" },]-sdist = { url = "https://files.pythonhosted.org/packages/ad/17/ec5698d04ea4d4a285d581c0d9b0152c6f1cd89e335b211a372bef659b0e/boto3_stubs-1.40.39.tar.gz", hash = "sha256:320315400d9a1e717014b30362b8453f0957a09dc331c5aec6128c32313ca8e2", size = 100837, upload-time = "2025-09-25T19:26:03.075Z" }+sdist = { url = "https://files.pythonhosted.org/packages/d5/09/598565add9282520403e74bcc385301fa3dd3eacbc6e14a5b5b43d641fb6/boto3_stubs-1.40.40.tar.gz", hash = "sha256:68a49cb1d11280b9b0443766af32c5aa7f71d1ef4ddff8dfdbf6fd64c12283bf", size = 100830, upload-time = "2025-09-26T19:29:55.021Z" }wheels = [- { url = "https://files.pythonhosted.org/packages/2e/36/620718640c7e28645da3059348800e1185485597661d10095f6624dad836/boto3_stubs-1.40.39-py3-none-any.whl", hash = "sha256:1f5319c486f792ff869efb255cd43b61c072c7d478e9025e08b8f98e6e16c1e4", size = 69687, upload-time = "2025-09-25T19:25:53.38Z" },+ { url = "https://files.pythonhosted.org/packages/6a/d7/f263878305e43adaa5de36f7cc49a9b4cf3c1d2142613651f3d7e3023f41/boto3_stubs-1.40.40-py3-none-any.whl", hash = "sha256:b52d9491a0f701c60d6d54ba310019d4293afc5929710e4c9d69c11d02a01344", size = 69690, upload-time = "2025-09-26T19:29:49.296Z" },][[package]]@@ -706,6 +718,15 @@ wheels = [{ url = "https://files.pythonhosted.org/packages/6e/c6/ac0b6c1e2d138f1002bcf799d330bd6d85084fece321e662a14223794041/Deprecated-1.2.18-py2.py3-none-any.whl", hash = "sha256:bd5011788200372a32418f888e326a09ff80d0214bd961147cfed01b5c018eec", size = 9998, upload-time = "2025-01-27T10:46:09.186Z" },]+[[package]]+name = "dill"+version = "0.4.0"+source = { registry = "https://pypi.org/simple" }+sdist = { url = "https://files.pythonhosted.org/packages/12/80/630b4b88364e9a8c8c5797f4602d0f76ef820909ee32f0bacb9f90654042/dill-0.4.0.tar.gz", hash = "sha256:0633f1d2df477324f53a895b02c901fb961bdbf65a17122586ea7019292cbcf0", size = 186976, upload-time = "2025-04-16T00:41:48.867Z" }+wheels = [+ { url = "https://files.pythonhosted.org/packages/50/3d/9373ad9c56321fdab5b41197068e1d8c25883b3fea29dd361f9b55116869/dill-0.4.0-py3-none-any.whl", hash = "sha256:44f54bf6412c2c8464c14e8243eb163690a9800dbe2c367330883b19c7561049", size = 119668, upload-time = "2025-04-16T00:41:47.671Z" },+]+[[package]]name = "diskcache"version = "5.6.3"@@ -941,7 +962,7 @@ wheels = [[[package]]name = "google-genai"-version = "1.39.0"+version = "1.39.1"source = { registry = "https://pypi.org/simple" }dependencies = [{ name = "anyio" },@@ -953,9 +974,9 @@ dependencies = [{ name = "typing-extensions" },{ name = "websockets" },]-sdist = { url = "https://files.pythonhosted.org/packages/ee/30/eda6ec8d47946ddf25fc193d8a9be6f29296f6659ab4a482607a2cf32552/google_genai-1.39.0.tar.gz", hash = "sha256:995fbe76f3f094ed3a4122f5e5f34e3601b774aa025110a2b013a9a493eb2f81", size = 244435, upload-time = "2025-09-25T21:18:49.286Z" }+sdist = { url = "https://files.pythonhosted.org/packages/f4/3e/25b88bda07ca237043f1be45d13c49ffbc73f9edf45d3232345802f67197/google_genai-1.39.1.tar.gz", hash = "sha256:4721704b43d170fc3f1b1cb5494bee1a7f7aae20de3a5383cdf6a129139df80b", size = 244631, upload-time = "2025-09-26T20:56:19.5Z" }wheels = [- { url = "https://files.pythonhosted.org/packages/c8/93/781ea98fd1dbf8ffaf78c494e59b6364b086e6cbc7dfbc1b750d92fbbe9e/google_genai-1.39.0-py3-none-any.whl", hash = "sha256:eaba325728ea8b90d6111ff009954f526ee10091badfabc31b84dbfcecdf2d02", size = 244544, upload-time = "2025-09-25T21:18:47.508Z" },+ { url = "https://files.pythonhosted.org/packages/cb/c3/12c1f386184d2fcd694b73adeabc3714a5ed65c01cc183b4e3727a26b9d1/google_genai-1.39.1-py3-none-any.whl", hash = "sha256:6ca36c7e40db6fcba7049dfdd102c86da326804f34403bd7d90fa613a45e5a78", size = 244681, upload-time = "2025-09-26T20:56:17.527Z" },][[package]]@@ -1051,6 +1072,15 @@ google-genai = [{ name = "jsonref" },]+[[package]]+name = "isort"+version = "6.0.1"+source = { registry = "https://pypi.org/simple" }+sdist = { url = "https://files.pythonhosted.org/packages/b8/21/1e2a441f74a653a144224d7d21afe8f4169e6c7c20bb13aec3a2dc3815e0/isort-6.0.1.tar.gz", hash = "sha256:1cb5df28dfbc742e490c5e41bad6da41b805b0a8be7bc93cd0fb2a8a890ac450", size = 821955, upload-time = "2025-02-26T21:13:16.955Z" }+wheels = [+ { url = "https://files.pythonhosted.org/packages/c1/11/114d0a5f4dabbdcedc1125dee0888514c3c3b16d3e9facad87ed96fad97c/isort-6.0.1-py3-none-any.whl", hash = "sha256:2dc5d7f65c9678d94c88dfc29161a320eec67328bc97aad576874cb4be1e9615", size = 94186, upload-time = "2025-02-26T21:13:14.911Z" },+]+[[package]]name = "jinja2"version = "3.1.6"@@ -1259,6 +1289,15 @@ wheels = [{ url = "https://files.pythonhosted.org/packages/4f/65/6079a46068dfceaeabb5dcad6d674f5f5c61a6fa5673746f42a9f4c233b3/MarkupSafe-3.0.2-cp313-cp313t-win_amd64.whl", hash = "sha256:e444a31f8db13eb18ada366ab3cf45fd4b31e4db1236a4448f68778c1d1a5a2f", size = 15739, upload-time = "2024-10-18T15:21:42.784Z" },]+[[package]]+name = "mccabe"+version = "0.7.0"+source = { registry = "https://pypi.org/simple" }+sdist = { url = "https://files.pythonhosted.org/packages/e7/ff/0ffefdcac38932a54d2b5eed4e0ba8a408f215002cd178ad1df0f2806ff8/mccabe-0.7.0.tar.gz", hash = "sha256:348e0240c33b60bbdf4e523192ef919f28cb2c3d7d5c7794f74009290f236325", size = 9658, upload-time = "2022-01-24T01:14:51.113Z" }+wheels = [+ { url = "https://files.pythonhosted.org/packages/27/1a/1f68f9ba0c207934b35b86a8ca3aad8395a3d6dd7921c0686e23853ff5a9/mccabe-0.7.0-py2.py3-none-any.whl", hash = "sha256:6c2d30ab6be0e4a46919781807b4f0d834ebdd6c6e3dca0bda5a15f863427b6e", size = 7350, upload-time = "2022-01-24T01:14:49.62Z" },+]+[[package]]name = "mdurl"version = "0.1.2"@@ -1838,16 +1877,16 @@ wheels = [[[package]]name = "pandas-stubs"-version = "2.3.2.250827"+version = "2.3.2.250926"source = { registry = "https://pypi.org/simple" }dependencies = [{ name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" },{ name = "numpy", version = "2.3.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" },{ name = "types-pytz" },]-sdist = { url = "https://files.pythonhosted.org/packages/03/7b/8d2076a76ddf35806319798037056e4bbdcacdc832fb7c95b517f4c03fb2/pandas_stubs-2.3.2.250827.tar.gz", hash = "sha256:bcc2d49a2766325e4a1a492c3eeda879e9521bb5e26e69e2bbf13e80e7ef569a", size = 100032, upload-time = "2025-08-27T23:18:12.802Z" }+sdist = { url = "https://files.pythonhosted.org/packages/1b/3b/32be58a125db39d0b5f62cc93795f32b5bb2915bd5c4a46f0e35171985e2/pandas_stubs-2.3.2.250926.tar.gz", hash = "sha256:c64b9932760ceefb96a3222b953e6a251321a9832a28548be6506df473a66406", size = 102147, upload-time = "2025-09-26T19:50:39.522Z" }wheels = [- { url = "https://files.pythonhosted.org/packages/a1/b8/dc820157be5aa9527f1f7ffe81737ee4d1cf0924081e1bfbd680530dde41/pandas_stubs-2.3.2.250827-py3-none-any.whl", hash = "sha256:3d613013b4189147a9a6bb18d8bec1e5b137de091496e9b9ff9f137ec3e223a9", size = 157775, upload-time = "2025-08-27T23:18:11.083Z" },+ { url = "https://files.pythonhosted.org/packages/40/96/1e4a035eaf4dce9610aac6e43026d0c6baa05773daf6d21e635a4fe19e21/pandas_stubs-2.3.2.250926-py3-none-any.whl", hash = "sha256:81121818453dcfe00f45c852f4dceee043640b813830f6e7bd084a4ef7ff7270", size = 159995, upload-time = "2025-09-26T19:50:38.241Z" },][[package]]@@ -1963,7 +2002,7 @@ wheels = [[[package]]name = "pipelex"-version = "0.10.3"+version = "0.10.2"source = { editable = "." }dependencies = [{ name = "aiofiles" },@@ -2008,6 +2047,7 @@ dev = [{ name = "cocode" },{ name = "mypy" },{ name = "pandas-stubs" },+ { name = "pylint" },{ name = "pyright" },{ name = "pytest" },{ name = "pytest-asyncio" },@@ -2080,8 +2120,9 @@ requires-dist = [{ name = "pillow", specifier = ">=11.2.1" },{ name = "polyfactory", specifier = ">=2.21.0" },{ name = "pydantic", specifier = "==2.10.6" },+ { name = "pylint", marker = "extra == 'dev'", specifier = ">=3.3.8" },{ name = "pypdfium2", specifier = ">=4.30.0,!=4.30.1" },- { name = "pyright", marker = "extra == 'dev'", specifier = "==1.1.398" },+ { name = "pyright", marker = "extra == 'dev'", specifier = ">=1.1.405" },{ name = "pytest", marker = "extra == 'dev'", specifier = ">=8.3.3" },{ name = "pytest-asyncio", marker = "extra == 'dev'", specifier = ">=0.24.0" },{ name = "pytest-cov", marker = "extra == 'dev'", specifier = ">=6.1.1" },@@ -2389,6 +2430,25 @@ crypto = [{ name = "cryptography" },]+[[package]]+name = "pylint"+version = "3.3.8"+source = { registry = "https://pypi.org/simple" }+dependencies = [+ { name = "astroid" },+ { name = "colorama", marker = "sys_platform == 'win32'" },+ { name = "dill" },+ { name = "isort" },+ { name = "mccabe" },+ { name = "platformdirs" },+ { name = "tomli", marker = "python_full_version < '3.11'" },+ { name = "tomlkit" },+]+sdist = { url = "https://files.pythonhosted.org/packages/9d/58/1f614a84d3295c542e9f6e2c764533eea3f318f4592dc1ea06c797114767/pylint-3.3.8.tar.gz", hash = "sha256:26698de19941363037e2937d3db9ed94fb3303fdadf7d98847875345a8bb6b05", size = 1523947, upload-time = "2025-08-09T09:12:57.234Z" }+wheels = [+ { url = "https://files.pythonhosted.org/packages/2d/1a/711e93a7ab6c392e349428ea56e794a3902bb4e0284c1997cff2d7efdbc1/pylint-3.3.8-py3-none-any.whl", hash = "sha256:7ef94aa692a600e82fabdd17102b73fc226758218c97473c7ad67bd4cb905d83", size = 523153, upload-time = "2025-08-09T09:12:54.836Z" },+]+[[package]]name = "pymdown-extensions"version = "10.16.1"@@ -2461,15 +2521,15 @@ wheels = [[[package]]name = "pyright"-version = "1.1.398"+version = "1.1.405"source = { registry = "https://pypi.org/simple" }dependencies = [{ name = "nodeenv" },{ name = "typing-extensions" },]-sdist = { url = "https://files.pythonhosted.org/packages/24/d6/48740f1d029e9fc4194880d1ad03dcf0ba3a8f802e0e166b8f63350b3584/pyright-1.1.398.tar.gz", hash = "sha256:357a13edd9be8082dc73be51190913e475fa41a6efb6ec0d4b7aab3bc11638d8", size = 3892675, upload-time = "2025-03-26T10:06:06.063Z" }+sdist = { url = "https://files.pythonhosted.org/packages/fb/6c/ba4bbee22e76af700ea593a1d8701e3225080956753bee9750dcc25e2649/pyright-1.1.405.tar.gz", hash = "sha256:5c2a30e1037af27eb463a1cc0b9f6d65fec48478ccf092c1ac28385a15c55763", size = 4068319, upload-time = "2025-09-04T03:37:06.776Z" }wheels = [- { url = "https://files.pythonhosted.org/packages/58/e0/5283593f61b3c525d6d7e94cfb6b3ded20b3df66e953acaf7bb4f23b3f6e/pyright-1.1.398-py3-none-any.whl", hash = "sha256:0a70bfd007d9ea7de1cf9740e1ad1a40a122592cfe22a3f6791b06162ad08753", size = 5780235, upload-time = "2025-03-26T10:06:03.994Z" },+ { url = "https://files.pythonhosted.org/packages/d5/1a/524f832e1ff1962a22a1accc775ca7b143ba2e9f5924bb6749dce566784a/pyright-1.1.405-py3-none-any.whl", hash = "sha256:a2cb13700b5508ce8e5d4546034cb7ea4aedb60215c6c33f56cec7f53996035a", size = 5905038, upload-time = "2025-09-04T03:37:04.913Z" },][[package]]