Skip to content

Build AI methods with your coding agent, then run them anywhere: as an MCP for chatbots, as a webapp for people, or via API for your software.

Fix/type checkers (#293)

Thomas Hebrardd3bd43f50263
diff --git a/.badges/tests.json b/.badges/tests.json
index 00505132c..d43b5371e 100644
--- a/.badges/tests.json
+++ b/.badges/tests.json
@@ -1,7 +1,7 @@
{
"schemaVersion": 1,
"label": "tests",
- "message": "1088",
+ "message": "1121",
"color": "blue",
"cacheSeconds": 300
}
\ No newline at end of file
diff --git a/.env.example b/.env.example
index 76865ecf5..a82594c92 100644
--- a/.env.example
+++ b/.env.example
@@ -1,6 +1,5 @@
-PYRIGHT_PYTHON_FORCE_VERSION=1.1.398
-
-PIPELEX_INFERENCE_API_KEY=
+PIPELEX_API_TOKEN=
+PIPELEX_API_BASE_URL=http://localhost:8081/api
OPENAI_API_KEY=
@@ -12,10 +11,6 @@ AZURE_API_BASE=
AZURE_API_VERSION=
AZURE_API_KEY=
-GCP_PROJECT_ID=
-GCP_LOCATION=
-GCP_CREDENTIALS_FILE_PATH=path/to/credentials.json
-
ANTHROPIC_API_KEY=
MISTRAL_API_KEY=
@@ -23,9 +18,20 @@ MISTRAL_API_KEY=
PERPLEXITY_API_KEY=
PERPLEXITY_API_ENDPOINT=https://api.perplexity.ai
+FAL_API_KEY=
+
+GCP_PROJECT_ID=
+GCP_LOCATION=
+GCP_CREDENTIALS_FILE_PATH=gcp_credentials.json
+
+CUSTOM_ENDPOINT_BASE_URL=http://localhost:11434/v1
+CUSTOM_ENDPOINT_API_KEY=
+
XAI_API_KEY=
XAI_API_ENDPOINT=https://api.x.ai/v1/
-FAL_API_KEY=
+PIPELEX_INFERENCE_API_KEY=
BLACKBOX_API_KEY=
+
+GOOGLE_API_KEY=
\ No newline at end of file
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 120b3334b..c51115498 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,6 +1,6 @@
# Changelog
-## [v0.10.3] - 2025-09-26
+## [Unreleased] - 2025-09-26
### Highlights
@@ -633,7 +633,7 @@ Simplified input memory:
### Changed
-- **Refactored Cognitive Workers**: The abstraction for `LLM`, `Imgg`, and `Ocr` workers has been elegantly simplified. The old decorator-based approach (`..._job_func`) has been replaced with a more robust pattern: a public base method now handles pre- and post-execution logic while calling a private abstract method that each worker implements.
+- **Refactored Cognitive Workers**: The abstraction for `LLM`, `ImgGen`, and `Ocr` workers has been elegantly simplified. The old decorator-based approach (`..._job_func`) has been replaced with a more robust pattern: a public base method now handles pre- and post-execution logic while calling a private abstract method that each worker implements.
- The `b64_image_bytes` field in `PromptImageBytes` was renamed to `base_64` for better consistency.
### Fixed
@@ -729,15 +729,15 @@ is_reporting_enabled = true
### Tests
- TestTemplatePreprocessor
-- TestImggByOpenAIGpt
+- TestImgGenByOpenAIGpt
- TestImageGeneration
-- TestPipeImgg
+- TestPipeImgGen
## [v0.2.9] - 2025-05-30
- Include `pyproject.toml` inside the project build.
-- Fix `ImggEngineFactory`: image generation (imgg) handle required format is `platform/model_name`
+- Fix `ImgGenEngineFactory`: image generation (imgg) handle required format is `platform/model_name`
- pipelex cli: Added `list-pipes` method that can list all the available pipes along with their descriptions.
- Use a minimum version for `uv` instead of a fixed version
- Implement `AGENTS.md` for Codex
diff --git a/Makefile b/Makefile
index 15f51017b..d9b38e0e3 100644
--- a/Makefile
+++ b/Makefile
@@ -14,6 +14,7 @@ VENV_PYRIGHT := $(VIRTUAL_ENV)/bin/pyright
VENV_MYPY := $(VIRTUAL_ENV)/bin/mypy
VENV_PIPELEX := $(VIRTUAL_ENV)/bin/pipelex
VENV_MKDOCS := $(VIRTUAL_ENV)/bin/mkdocs
+VENV_PYLINT := $(VIRTUAL_ENV)/bin/pylint
UV_MIN_VERSION = $(shell grep -m1 'required-version' pyproject.toml | sed -E 's/.*= *"([^<>=, ]+).*/\1/')
@@ -101,7 +102,7 @@ export HELP
.PHONY: \
all help env lock install update build \
- format lint pyright mypy \
+ format lint pyright mypy pylint \
cleanderived cleanenv cleanlibraries cleanall \
test test-xdist t test-quiet tq test-with-prints tp test-inference ti \
test-img-gen tg test-ocr to codex-tests gha-tests \
@@ -380,13 +381,16 @@ lint: env
pyright: env
$(call PRINT_TITLE,"Typechecking with pyright")
- @$(VENV_PYRIGHT) --pythonpath $(VIRTUAL_ENV)/bin/python3 && \
- echo "Done typechecking with pyright — disregard warning about latest version, it's giving us false positives"
+ @$(VENV_PYRIGHT) . --project pyproject.toml
mypy: env
$(call PRINT_TITLE,"Typechecking with mypy")
$(VENV_MYPY)
+pylint: env
+ $(call PRINT_TITLE,"Linting with pylint")
+ $(VENV_PYLINT) --rcfile pyproject.toml pipelex tests
+
##########################################################################################
### MERGE CHECKS
@@ -408,6 +412,10 @@ merge-check-mypy: env
$(call PRINT_TITLE,"Typechecking with mypy")
$(VENV_MYPY) --config-file pyproject.toml
+merge-check-pylint: env
+ $(call PRINT_TITLE,"Linting with pylint")
+ $(VENV_PYLINT) --rcfile pyproject.toml .
+
##########################################################################################
### MISCELLANEOUS
##########################################################################################
@@ -447,11 +455,11 @@ docs-deploy: env
### SHORTHANDS
##########################################################################################
-c: format lint pyright mypy
+c: format lint pyright pylint mypy
@echo "> done: c = check"
cc: cleanderived c
- @echo "> done: cc = cleanderived format lint pyright mypy"
+ @echo "> done: cc = cleanderived format lint pyright pylint mypy"
check: cc check-unused-imports
@echo "> done: check"
diff --git a/pipelex/cli/_cli.py b/pipelex/cli/_cli.py
index 8ca16df2e..b7dc649dd 100644
--- a/pipelex/cli/_cli.py
+++ b/pipelex/cli/_cli.py
@@ -1,5 +1,3 @@
-from typing import Optional
-
import typer
from click import Command, Context
from typer.core import TyperGroup
@@ -11,7 +9,7 @@ from pipelex.cli.commands.build_cmd import build_app
class PipelexCLI(TyperGroup):
@override
- def get_command(self, ctx: Context, cmd_name: str) -> Optional[Command]:
+ def get_command(self, ctx: Context, cmd_name: str) -> Command | None:
cmd = super().get_command(ctx, cmd_name)
if cmd is None:
typer.echo(f"Unknown command: {cmd_name}")
diff --git a/pipelex/cli/commands/build_cmd.py b/pipelex/cli/commands/build_cmd.py
index cdd4ea3bf..fb64deadb 100644
--- a/pipelex/cli/commands/build_cmd.py
+++ b/pipelex/cli/commands/build_cmd.py
@@ -1,6 +1,6 @@
import asyncio
import time
-from typing import Annotated, Optional
+from typing import Annotated
import typer
@@ -30,7 +30,7 @@ def build_pipe_cmd(
typer.Argument(help="Brief description of what the pipeline should do"),
],
output_path: Annotated[
- Optional[str],
+ str | None,
typer.Option("--output", "-o", help="Path to save the generated PLX file (use --output='' to skip saving)"),
] = "./generated_pipeline.plx",
) -> None:
diff --git a/pipelex/cli/commands/init_cmd.py b/pipelex/cli/commands/init_cmd.py
index b1b8b68f0..60861e604 100644
--- a/pipelex/cli/commands/init_cmd.py
+++ b/pipelex/cli/commands/init_cmd.py
@@ -1,7 +1,7 @@
import os
import shutil
from importlib.metadata import metadata
-from typing import Annotated, List
+from typing import Annotated
import typer
@@ -26,7 +26,8 @@ def do_init_libraries(directory: str = ".", overwrite: bool = False) -> None:
else:
typer.echo(f"✅ Successfully initialized pipelex libraries at '{target_dir}' (only created non-existing files)")
except Exception as exc:
- raise PipelexCLIError(f"Failed to initialize libraries at '{directory}': {exc}") from exc
+ msg = f"Failed to initialize libraries at '{directory}': {exc}"
+ raise PipelexCLIError(msg) from exc
def do_init_config(reset: bool = False) -> None:
@@ -37,8 +38,8 @@ def do_init_config(reset: bool = False) -> None:
os.makedirs(target_config_dir, exist_ok=True)
try:
- copied_files: List[str] = []
- existing_files: List[str] = []
+ copied_files: list[str] = []
+ existing_files: list[str] = []
def copy_directory_structure(src_dir: str, dst_dir: str, relative_path: str = "") -> None:
"""Recursively copy directory structure, handling existing files."""
@@ -50,12 +51,11 @@ def do_init_config(reset: bool = False) -> None:
if os.path.isdir(src_item):
os.makedirs(dst_item, exist_ok=True)
copy_directory_structure(src_item, dst_item, relative_item)
+ elif os.path.exists(dst_item) and not reset:
+ existing_files.append(relative_item)
else:
- if os.path.exists(dst_item) and not reset:
- existing_files.append(relative_item)
- else:
- shutil.copy2(src_item, dst_item)
- copied_files.append(relative_item)
+ shutil.copy2(src_item, dst_item)
+ copied_files.append(relative_item)
copy_directory_structure(config_template_dir, target_config_dir)
@@ -74,7 +74,8 @@ def do_init_config(reset: bool = False) -> None:
typer.echo(f"✅ Configuration directory {target_config_dir} is already up to date")
except Exception as exc:
- raise PipelexCLIError(f"Failed to initialize configuration: {exc}") from exc
+ msg = f"Failed to initialize configuration: {exc}"
+ raise PipelexCLIError(msg) from exc
# Typer group for init commands
diff --git a/pipelex/cli/commands/migrate_cmd.py b/pipelex/cli/commands/migrate_cmd.py
index c6cd93cb9..28482710d 100644
--- a/pipelex/cli/commands/migrate_cmd.py
+++ b/pipelex/cli/commands/migrate_cmd.py
@@ -1,7 +1,7 @@
from __future__ import annotations
from pathlib import Path
-from typing import Annotated
+from typing import TYPE_CHECKING, Annotated
import typer
@@ -9,6 +9,9 @@ from pipelex.cli.commands.common import is_pipelex_libraries_folder
from pipelex.migration.migrate_v0_1_0_to_v0_2_0 import TOMLMigrator, migrate_concept_syntax
from pipelex.migration.migration_result import MigrationResult
+if TYPE_CHECKING:
+ from pipelex.migration.migration_result import MigrationResult
+
def do_migrate(
target_path: str = "./pipelex_libraries",
@@ -59,7 +62,7 @@ def do_migrate(
base_for_print = pipelines_dir_or_file if pipelines_dir_or_file.is_dir() else pipelines_dir_or_file.parent
for file_path in result.modified_files:
try:
- with open(file_path, "r", encoding="utf-8") as f:
+ with open(file_path, encoding="utf-8") as f:
content = f.read()
changes = migrator.get_migration_preview(content)
@@ -94,10 +97,10 @@ def do_migrate(
typer.echo(" Run 'pipelex validate all -c pipelex/libraries' to verify the migration")
except FileNotFoundError as exc:
typer.echo(f"❌ {exc}")
- raise typer.Exit(1)
+ raise typer.Exit(1) from exc
except Exception as exc:
typer.echo(f"❌ Migration failed: {exc}")
- raise typer.Exit(1)
+ raise typer.Exit(1) from exc
# Typer group for migration commands
diff --git a/pipelex/cli/commands/show_cmd.py b/pipelex/cli/commands/show_cmd.py
index 4c23ec10d..e74f3ffc2 100644
--- a/pipelex/cli/commands/show_cmd.py
+++ b/pipelex/cli/commands/show_cmd.py
@@ -20,7 +20,8 @@ def do_show_config() -> None:
title=f"Pipelex configuration for project: {config_manager.get_project_name()}",
)
except Exception as exc:
- raise PipelexConfigError(f"Error loading configuration: {exc}") from exc
+ msg = f"Error loading configuration: {exc}"
+ raise PipelexConfigError(msg) from exc
def do_list_pipes(relative_config_folder_path: str = "pipelex_libraries") -> None:
@@ -30,7 +31,8 @@ def do_list_pipes(relative_config_folder_path: str = "pipelex_libraries") -> Non
try:
get_pipe_provider().pretty_list_pipes()
except Exception as exc:
- raise PipelexCLIError(f"Failed to list pipes: {exc}") from exc
+ msg = f"Failed to list pipes: {exc}"
+ raise PipelexCLIError(msg) from exc
def do_show_pipe(pipe_code: str, relative_config_folder_path: str = "./pipelex_libraries") -> None:
@@ -52,7 +54,8 @@ def show_config_cmd() -> None:
@show_app.command("pipes")
def list_pipes_cmd(
relative_config_folder_path: Annotated[
- str, typer.Option("--config-folder-path", "-c", help="Relative path to the config folder path")
+ str,
+ typer.Option("--config-folder-path", "-c", help="Relative path to the config folder path"),
] = "pipelex_libraries",
) -> None:
do_list_pipes(relative_config_folder_path=relative_config_folder_path)
@@ -62,7 +65,8 @@ def list_pipes_cmd(
def show_pipe_cmd(
pipe_code: Annotated[str, typer.Argument(help="Pipeline code to show definition for")],
relative_config_folder_path: Annotated[
- str, typer.Option("--config-folder-path", "-c", help="Relative path to the config folder path")
+ str,
+ typer.Option("--config-folder-path", "-c", help="Relative path to the config folder path"),
] = "./pipelex_libraries",
) -> None:
do_show_pipe(pipe_code=pipe_code, relative_config_folder_path=relative_config_folder_path)
diff --git a/pipelex/cli/commands/validate_cmd.py b/pipelex/cli/commands/validate_cmd.py
index bf99797ce..fcab31d02 100644
--- a/pipelex/cli/commands/validate_cmd.py
+++ b/pipelex/cli/commands/validate_cmd.py
@@ -39,7 +39,7 @@ def do_dry_run_pipe(pipe_code: str, relative_config_folder_path: str = "./pipele
dry_run_pipe(
get_pipe_provider().get_required_pipe(pipe_code=pipe_code),
raise_on_failure=True,
- )
+ ),
)
get_pipeline_tracker().output_flowchart()
@@ -51,7 +51,8 @@ validate_app = typer.Typer(help="Validation and dry-run commands", no_args_is_he
@validate_app.command("all")
def validate_all_cmd(
relative_config_folder_path: Annotated[
- str, typer.Option("--config-folder-path", "-c", help="Relative path to the config folder path")
+ str,
+ typer.Option("--config-folder-path", "-c", help="Relative path to the config folder path"),
] = "./pipelex_libraries",
) -> None:
do_validate_all_libraries_and_dry_run(relative_config_folder_path=relative_config_folder_path)
@@ -61,7 +62,8 @@ def validate_all_cmd(
def dry_run_pipe_cmd(
pipe_code: Annotated[str, typer.Argument(help="The pipe code to dry run")],
relative_config_folder_path: Annotated[
- str, typer.Option("--config-folder-path", "-c", help="Relative path to the config folder path")
+ str,
+ typer.Option("--config-folder-path", "-c", help="Relative path to the config folder path"),
] = "./pipelex_libraries",
) -> None:
do_dry_run_pipe(pipe_code=pipe_code, relative_config_folder_path=relative_config_folder_path)
diff --git a/tests/e2e/pipelex/tools/__init__.py b/pipelex/client/__init__.py
similarity index 100%
rename from tests/e2e/pipelex/tools/__init__.py
rename to pipelex/client/__init__.py
diff --git a/pipelex/client/api_serializer.py b/pipelex/client/api_serializer.py
index 7d8ef22ee..9cec0aa91 100644
--- a/pipelex/client/api_serializer.py
+++ b/pipelex/client/api_serializer.py
@@ -2,13 +2,15 @@ from datetime import datetime
from decimal import Decimal
from enum import Enum
from pathlib import Path
-from typing import Any, Dict, List, Optional, cast
+from typing import TYPE_CHECKING, Any, cast
from pipelex.client.protocol import CompactMemory
from pipelex.core.concepts.concept_native import NativeConceptEnum
from pipelex.core.memory.working_memory import WorkingMemory
from pipelex.core.pipes.pipe_output import PipeOutput
-from pipelex.core.stuffs.stuff_content import TextContent
+
+if TYPE_CHECKING:
+ from pipelex.core.stuffs.stuff_content import TextContent
class ApiSerializer:
@@ -19,15 +21,15 @@ class ApiSerializer:
FIELDS_TO_SKIP = ("__class__", "__module__")
@classmethod
- def serialize_working_memory_for_api(cls, working_memory: Optional[WorkingMemory] = None) -> CompactMemory:
- """
- Convert WorkingMemory to API-ready format using kajson with proper datetime handling.
+ def serialize_working_memory_for_api(cls, working_memory: WorkingMemory | None = None) -> CompactMemory:
+ """Convert WorkingMemory to API-ready format using kajson with proper datetime handling.
Args:
working_memory: The WorkingMemory to serialize
Returns:
Dict ready for API transmission with datetime strings and no __class__/__module__
+
"""
compact_memory: CompactMemory = {}
if working_memory is None:
@@ -35,8 +37,8 @@ class ApiSerializer:
for stuff_name, stuff in working_memory.root.items():
if NativeConceptEnum.is_text(concept_code=stuff.concept.code):
- stuff_content = cast(TextContent, stuff.content)
- item_dict: Dict[str, Any] = {
+ stuff_content = cast("TextContent", stuff.content)
+ item_dict: dict[str, Any] = {
"concept_code": stuff.concept.code,
"content": stuff_content.text,
}
@@ -55,41 +57,40 @@ class ApiSerializer:
@classmethod
def serialize_pipe_output_for_api(cls, pipe_output: PipeOutput) -> CompactMemory:
- """
- Convert PipeOutput to API-ready format.
+ """Convert PipeOutput to API-ready format.
Args:
pipe_output: The PipeOutput to serialize
Returns:
Dict ready for API transmission
+
"""
return {"compact_memory": cls.serialize_working_memory_for_api(pipe_output.working_memory)}
@classmethod
def _clean_and_format_content(cls, content: Any) -> Any:
- """
- Recursively clean content by removing the fields in FIELDS_TO_SKIP and formatting datetimes.
+ """Recursively clean content by removing the fields in FIELDS_TO_SKIP and formatting datetimes.
Args:
content: Content to clean
Returns:
Cleaned content with formatted datetimes
+
"""
if isinstance(content, dict):
- cleaned: Dict[str, Any] = {}
- content_dict = cast(Dict[str, Any], content)
+ cleaned: dict[str, Any] = {}
+ content_dict = cast("dict[str, Any]", content)
for key in content_dict:
if key in cls.FIELDS_TO_SKIP:
continue
cleaned[key] = cls._clean_and_format_content(content_dict[key])
return cleaned
elif isinstance(content, list):
- cleaned_list: List[Any] = []
- content_list = cast(List[Any], content)
- for idx in range(len(content_list)):
- cleaned_list.append(cls._clean_and_format_content(content_list[idx]))
+ cleaned_list: list[Any] = []
+ content_list = cast("list[Any]", content)
+ cleaned_list.extend(cls._clean_and_format_content(content_list[idx]) for idx in range(len(content_list)))
return cleaned_list
elif isinstance(content, datetime):
return content.strftime(cls.API_DATETIME_FORMAT)
diff --git a/pipelex/client/client.py b/pipelex/client/client.py
index 7da3480ed..99f133027 100644
--- a/pipelex/client/client.py
+++ b/pipelex/client/client.py
@@ -1,4 +1,4 @@
-from typing import Any, Optional
+from typing import Any
import httpx
from typing_extensions import override
@@ -14,31 +14,34 @@ from pipelex.tools.environment import get_required_env
class PipelexClient(PipelexProtocol):
- """
- A client for interacting with Pipelex pipelines through the API.
+ """A client for interacting with Pipelex pipelines through the API.
This client provides a user-friendly interface for executing pipelines through
the remote API.
+
Args:
api_token: The API token to use for authentication. If not provided, it will be loaded from the PIPELEX_API_TOKEN environment variable.
If the environment variable is not set, an error will be raised.
+
"""
def __init__(
self,
- api_token: Optional[str] = None,
- api_base_url: Optional[str] = None,
+ api_token: str | None = None,
+ api_base_url: str | None = None,
):
self.api_token = api_token or get_required_env("PIPELEX_API_TOKEN")
if not self.api_token:
- raise ClientAuthenticationError("API token is required for API execution")
+ msg = "API token is required for API execution"
+ raise ClientAuthenticationError(msg)
self.api_base_url = api_base_url or get_required_env("PIPELEX_API_BASE_URL")
if not self.api_base_url:
- raise ClientAuthenticationError("API base URL is required for API execution")
+ msg = "API base URL is required for API execution"
+ raise ClientAuthenticationError(msg)
- self.client: Optional[httpx.AsyncClient] = None
+ self.client: httpx.AsyncClient | None = None
def start_client(self) -> "PipelexClient":
"""Initialize the HTTP client for API calls."""
@@ -51,8 +54,9 @@ class PipelexClient(PipelexProtocol):
await self.client.aclose()
self.client = None
- async def _make_api_call(self, endpoint: str, request: Optional[str] = None) -> Any:
+ async def _make_api_call(self, endpoint: str, request: str | None = None) -> Any:
"""Make an API call to the Pipelex server.
+
Args:
endpoint: The API endpoint to call, relative to the base URL
request: A JSON-formatted string to send as the request body, or None if no body is needed
@@ -60,6 +64,7 @@ class PipelexClient(PipelexProtocol):
Any: The JSON-decoded response from the server
Raises:
httpx.HTTPError: If the request fails or returns a non-200 status code
+
"""
if not self.client:
self.start_client()
@@ -75,14 +80,15 @@ class PipelexClient(PipelexProtocol):
async def execute_pipeline(
self,
pipe_code: str,
- working_memory: Optional[WorkingMemory] = None,
- input_memory: Optional[CompactMemory] = None,
- output_name: Optional[str] = None,
- output_multiplicity: Optional[PipeOutputMultiplicity] = None,
- dynamic_output_concept_code: Optional[str] = None,
+ working_memory: WorkingMemory | None = None,
+ input_memory: CompactMemory | None = None,
+ output_name: str | None = None,
+ output_multiplicity: PipeOutputMultiplicity | None = None,
+ dynamic_output_concept_code: str | None = None,
) -> PipelineResponse:
if working_memory and input_memory:
- raise ValueError(f"working_memory and input_memory cannot be provided together to the API execute_pipeline {pipe_code=}")
+ msg = f"working_memory and input_memory cannot be provided together to the API execute_pipeline {pipe_code=}"
+ raise ValueError(msg)
if input_memory is not None:
working_memory = WorkingMemoryFactory.make_from_compact_memory(input_memory)
@@ -99,14 +105,15 @@ class PipelexClient(PipelexProtocol):
async def start_pipeline(
self,
pipe_code: str,
- working_memory: Optional[WorkingMemory] = None,
- input_memory: Optional[CompactMemory] = None,
- output_name: Optional[str] = None,
- output_multiplicity: Optional[PipeOutputMultiplicity] = None,
- dynamic_output_concept_code: Optional[str] = None,
+ working_memory: WorkingMemory | None = None,
+ input_memory: CompactMemory | None = None,
+ output_name: str | None = None,
+ output_multiplicity: PipeOutputMultiplicity | None = None,
+ dynamic_output_concept_code: str | None = None,
) -> PipelineResponse:
if working_memory and input_memory:
- raise ValueError(f"working_memory and input_memory cannot be provided together to the API start_pipeline {pipe_code=}")
+ msg = f"working_memory and input_memory cannot be provided together to the API start_pipeline {pipe_code=}"
+ raise ValueError(msg)
if input_memory is not None:
working_memory = WorkingMemoryFactory.make_from_compact_memory(input_memory)
diff --git a/pipelex/client/pipeline_request_factory.py b/pipelex/client/pipeline_request_factory.py
index 51b732806..9cc57c2a2 100644
--- a/pipelex/client/pipeline_request_factory.py
+++ b/pipelex/client/pipeline_request_factory.py
@@ -1,4 +1,4 @@
-from typing import Any, Dict, Optional
+from typing import Any
from pipelex.client.api_serializer import ApiSerializer
from pipelex.client.protocol import COMPACT_MEMORY_KEY, PipelineRequest
@@ -11,13 +11,12 @@ class PipelineRequestFactory:
@staticmethod
def make_from_working_memory(
- working_memory: Optional[WorkingMemory] = None,
- output_name: Optional[str] = None,
- output_multiplicity: Optional[PipeOutputMultiplicity] = None,
- dynamic_output_concept_code: Optional[str] = None,
+ working_memory: WorkingMemory | None = None,
+ output_name: str | None = None,
+ output_multiplicity: PipeOutputMultiplicity | None = None,
+ dynamic_output_concept_code: str | None = None,
) -> PipelineRequest:
- """
- Create a PipelineRequest from a WorkingMemory object.
+ """Create a PipelineRequest from a WorkingMemory object.
Args:
working_memory: The WorkingMemory to convert
@@ -27,8 +26,8 @@ class PipelineRequestFactory:
Returns:
PipelineRequest with the working memory serialized to reduced format
- """
+ """
return PipelineRequest(
input_memory=ApiSerializer.serialize_working_memory_for_api(working_memory),
output_name=output_name,
@@ -37,15 +36,15 @@ class PipelineRequestFactory:
)
@staticmethod
- def make_from_body(request_body: Dict[str, Any]) -> PipelineRequest:
- """
- Create a PipelineRequest from raw request body dictionary.
+ def make_from_body(request_body: dict[str, Any]) -> PipelineRequest:
+ """Create a PipelineRequest from raw request body dictionary.
Args:
request_body: Raw dictionary from API request body
Returns:
PipelineRequest object with dictionary working_memory
+
"""
return PipelineRequest(
input_memory=request_body.get(COMPACT_MEMORY_KEY),
diff --git a/pipelex/client/pipeline_response_factory.py b/pipelex/client/pipeline_response_factory.py
index f0e70db84..1d1d947fa 100644
--- a/pipelex/client/pipeline_response_factory.py
+++ b/pipelex/client/pipeline_response_factory.py
@@ -1,4 +1,4 @@
-from typing import Any, Dict, Optional
+from typing import Any
from pipelex.client.api_serializer import ApiSerializer
from pipelex.client.protocol import PipelineResponse, PipelineState
@@ -10,17 +10,16 @@ class PipelineResponseFactory:
@staticmethod
def make_from_pipe_output(
- pipe_output: Optional[PipeOutput] = None,
+ pipe_output: PipeOutput | None = None,
pipeline_run_id: str = "",
created_at: str = "",
pipeline_state: PipelineState = PipelineState.COMPLETED,
- finished_at: Optional[str] = None,
- status: Optional[str] = "success",
- message: Optional[str] = None,
- error: Optional[str] = None,
+ finished_at: str | None = None,
+ status: str | None = "success",
+ message: str | None = None,
+ error: str | None = None,
) -> PipelineResponse:
- """
- Create a PipelineResponse from a PipeOutput object.
+ """Create a PipelineResponse from a PipeOutput object.
Args:
pipe_output: The PipeOutput to convert
@@ -34,6 +33,7 @@ class PipelineResponseFactory:
Returns:
PipelineResponse with the pipe output serialized to reduced format
+
"""
compact_output = None
if pipe_output is not None:
@@ -51,14 +51,14 @@ class PipelineResponseFactory:
)
@staticmethod
- def make_from_api_response(response: Dict[str, Any]) -> PipelineResponse:
- """
- Create a PipelineResponse from an API response dictionary.
+ def make_from_api_response(response: dict[str, Any]) -> PipelineResponse:
+ """Create a PipelineResponse from an API response dictionary.
Args:
response: Dictionary containing the API response data
Returns:
PipelineResponse instance created from the response data
+
"""
- return PipelineResponse(**response)
+ return PipelineResponse.model_validate(response)
diff --git a/pipelex/client/protocol.py b/pipelex/client/protocol.py
index cddce3300..f47172cf5 100644
--- a/pipelex/client/protocol.py
+++ b/pipelex/client/protocol.py
@@ -1,5 +1,5 @@
from abc import abstractmethod
-from typing import Any, Dict, List, Optional, Protocol
+from typing import Any, Protocol
from pydantic import BaseModel
from typing_extensions import runtime_checkable
@@ -9,16 +9,14 @@ from pipelex.core.pipes.pipe_run_params import PipeOutputMultiplicity
from pipelex.core.stuffs.stuff_content import StuffContent
from pipelex.types import StrEnum
-StuffContentOrData = Dict[str, Any] | StuffContent | List[Any] | str
-ImplicitMemory = Dict[str, StuffContentOrData]
-CompactMemory = Dict[str, Dict[str, Any]]
+StuffContentOrData = dict[str, Any] | StuffContent | list[Any] | str
+ImplicitMemory = dict[str, StuffContentOrData]
+CompactMemory = dict[str, dict[str, Any]]
COMPACT_MEMORY_KEY = "compact_memory"
class PipelineState(StrEnum):
- """
- Enum representing the possible states of a pipe execution.
- """
+ """Enum representing the possible states of a pipe execution."""
RUNNING = "RUNNING"
COMPLETED = "COMPLETED"
@@ -29,47 +27,46 @@ class PipelineState(StrEnum):
class ApiResponse(BaseModel):
- """
- Base response class for Pipelex API calls.
+ """Base response class for Pipelex API calls.
Attributes:
- status (Optional[str]): Status of the API call ("success", "error", etc.)
- message (Optional[str]): Optional message providing additional information
- error (Optional[str]): Optional error message when status is not "success"
+ status (str | None): Status of the API call ("success", "error", etc.)
+ message (str | None): Optional message providing additional information
+ error (str | None): Optional error message when status is not "success"
+
"""
- status: Optional[str] = None
- message: Optional[str] = None
- error: Optional[str] = None
+ status: str | None = None
+ message: str | None = None
+ error: str | None = None
class PipelineRequest(BaseModel):
- """
- Request for executing a pipeline.
+ """Request for executing a pipeline.
Attributes:
- input_memory (Optional[CompactMemory]): In the format of WorkingMemory.to_compact_memory()
- output_name (Optional[str]): Name of the output slot to write to
- output_multiplicity (Optional[PipeOutputMultiplicity]): Output multiplicity setting
- dynamic_output_concept_code (Optional[str]): Override for the dynamic output concept code
+ input_memory (CompactMemory | None): In the format of WorkingMemory.to_compact_memory()
+ output_name (str | None): Name of the output slot to write to
+ output_multiplicity (PipeOutputMultiplicity | None): Output multiplicity setting
+ dynamic_output_concept_code (str | None): Override for the dynamic output concept code
+
"""
- input_memory: Optional[CompactMemory] = None
- output_name: Optional[str] = None
- output_multiplicity: Optional[PipeOutputMultiplicity] = None
- dynamic_output_concept_code: Optional[str] = None
+ input_memory: CompactMemory | None = None
+ output_name: str | None = None
+ output_multiplicity: PipeOutputMultiplicity | None = None
+ dynamic_output_concept_code: str | None = None
class PipelineResponse(ApiResponse):
- """
- Response for pipeline execution requests.
+ """Response for pipeline execution requests.
Attributes:
pipeline_run_id (str): Unique identifier for the pipeline run
created_at (str): Timestamp when the pipeline was created
pipeline_state (PipelineState): Current state of the pipeline
- finished_at (Optional[str]): Timestamp when the pipeline finished, if completed
- pipe_output (Optional[CompactMemory]): Output data from the pipeline execution as raw dict, if available
+ finished_at (str | None): Timestamp when the pipeline finished, if completed
+ pipe_output (CompactMemory | None): Output data from the pipeline execution as raw dict, if available
Example of pipe_output:
"pipe_output": {
@@ -98,26 +95,27 @@ class PipelineResponse(ApiResponse):
}
}
}
+
"""
pipeline_run_id: str
created_at: str
pipeline_state: PipelineState
- finished_at: Optional[str] = None
- pipe_output: Optional[CompactMemory] = None
+ finished_at: str | None = None
+ pipe_output: CompactMemory | None = None
@runtime_checkable
class PipelexProtocol(Protocol):
- """
- Protocol defining the contract for the Pipelex API.
+ """Protocol defining the contract for the Pipelex API.
This protocol specifies the interface that any Pipelex API implementation must adhere to.
All methods are asynchronous and handle pipeline execution, monitoring, and control.
Attributes:
- api_token (Optional[str]): Authentication token for API access
- api_base_url (Optional[str]): Base URL for the API
+ api_token (str): Authentication token for API access
+ api_base_url (str): Base URL for the API
+
"""
api_token: str
@@ -127,28 +125,28 @@ class PipelexProtocol(Protocol):
async def execute_pipeline(
self,
pipe_code: str,
- working_memory: Optional[WorkingMemory] = None,
- input_memory: Optional[CompactMemory] = None,
- output_name: Optional[str] = None,
- output_multiplicity: Optional[PipeOutputMultiplicity] = None,
- dynamic_output_concept_code: Optional[str] = None,
+ working_memory: WorkingMemory | None = None,
+ input_memory: CompactMemory | None = None,
+ output_name: str | None = None,
+ output_multiplicity: PipeOutputMultiplicity | None = None,
+ dynamic_output_concept_code: str | None = None,
) -> PipelineResponse:
- """
- Execute a pipeline synchronously and wait for its completion.
+ """Execute a pipeline synchronously and wait for its completion.
Args:
pipe_code (str): The code identifying the pipeline to execute
- working_memory (Optional[WorkingMemory]): Memory context passed to the pipeline
- input_memory (Optional[CompactMemory]): Input memory passed to the pipeline
- output_name (Optional[str]): Target output slot name
- output_multiplicity (Optional[PipeOutputMultiplicity]): Output multiplicity setting
- dynamic_output_concept_code (Optional[str]): Override for dynamic output concept
+ working_memory (WorkingMemory | None): Memory context passed to the pipeline
+ input_memory (CompactMemory | None): Input memory passed to the pipeline
+ output_name (str | None): Target output slot name
+ output_multiplicity (PipeOutputMultiplicity | None): Output multiplicity setting
+ dynamic_output_concept_code (str | None): Override for dynamic output concept
Returns:
PipelineResponse: Complete execution results including pipeline state and output
Raises:
HTTPException: On execution failure or error
ClientAuthenticationError: If API token is missing for API execution
+
"""
...
@@ -156,22 +154,21 @@ class PipelexProtocol(Protocol):
async def start_pipeline(
self,
pipe_code: str,
- working_memory: Optional[WorkingMemory] = None,
- input_memory: Optional[CompactMemory] = None,
- output_name: Optional[str] = None,
- output_multiplicity: Optional[PipeOutputMultiplicity] = None,
- dynamic_output_concept_code: Optional[str] = None,
+ working_memory: WorkingMemory | None = None,
+ input_memory: CompactMemory | None = None,
+ output_name: str | None = None,
+ output_multiplicity: PipeOutputMultiplicity | None = None,
+ dynamic_output_concept_code: str | None = None,
) -> PipelineResponse:
- """
- Start a pipeline execution asynchronously without waiting for completion.
+ """Start a pipeline execution asynchronously without waiting for completion.
Args:
pipe_code (str): The code identifying the pipeline to execute
- working_memory (Optional[WorkingMemory]): Memory context passed to the pipeline
- input_memory (Optional[CompactMemory]): Input memory passed to the pipeline
- output_name (Optional[str]): Target output slot name
- output_multiplicity (Optional[PipeOutputMultiplicity]): Output multiplicity setting
- dynamic_output_concept_code (Optional[str]): Override for dynamic output concept
+ working_memory (WorkingMemory | None): Memory context passed to the pipeline
+ input_memory (CompactMemory | None): Input memory passed to the pipeline
+ output_name (str | None): Target output slot name
+ output_multiplicity (PipeOutputMultiplicity | None): Output multiplicity setting
+ dynamic_output_concept_code (str | None): Override for dynamic output concept
Returns:
PipelineResponse: Initial response with pipeline_run_id and created_at timestamp
@@ -179,5 +176,6 @@ class PipelexProtocol(Protocol):
Raises:
HTTPException: On pipeline start failure
ClientAuthenticationError: If API token is missing for API execution
+
"""
...
diff --git a/pipelex/cogt/config_cogt.py b/pipelex/cogt/config_cogt.py
index b29539574..478e108e3 100644
--- a/pipelex/cogt/config_cogt.py
+++ b/pipelex/cogt/config_cogt.py
@@ -1,5 +1,3 @@
-from typing import List
-
from pipelex.cogt.img_gen.img_gen_job_components import ImgGenJobConfig, ImgGenJobParams, ImgGenJobParamsDefaults
from pipelex.cogt.llm.llm_job_components import LLMJobConfig
from pipelex.plugins.fal.fal_config import FalConfig
@@ -52,7 +50,7 @@ class InferenceConfig(ConfigModel):
def model_specs_path(self, backend_name: str) -> str:
return f"{self.inference_config_path}/backends/{backend_name}.toml"
- def get_model_deck_paths(self) -> List[str]:
+ def get_model_deck_paths(self) -> list[str]:
"""Get all LLM deck TOML file paths sorted alphabetically."""
model_deck_paths = [
str(path)
diff --git a/pipelex/cogt/content_generation/assignment_models.py b/pipelex/cogt/content_generation/assignment_models.py
index 3a3e5e0e7..efd3da823 100644
--- a/pipelex/cogt/content_generation/assignment_models.py
+++ b/pipelex/cogt/content_generation/assignment_models.py
@@ -1,4 +1,4 @@
-from typing import Any, Dict, Optional, Type
+from typing import Any
from pydantic import BaseModel
from typing_extensions import override
@@ -25,8 +25,8 @@ class LLMAssignmentFactory(BaseModel):
async def make_llm_assignment(
self,
- job_metadata: Optional[JobMetadata] = None,
- llm_setting: Optional[LLMSetting] = None,
+ job_metadata: JobMetadata | None = None,
+ llm_setting: LLMSetting | None = None,
**prompt_arguments: Any,
) -> "LLMAssignment":
llm_prompt = await self.llm_prompt_factory.make_llm_prompt_from_args(**prompt_arguments)
@@ -95,7 +95,7 @@ class ObjectAssignment(BaseModel):
@staticmethod
def make_for_class(
- object_class: Type[BaseModel],
+ object_class: type[BaseModel],
llm_assignment: LLMAssignment,
) -> "ObjectAssignment":
object_class_name = object_class.__name__
@@ -127,10 +127,10 @@ class ImgGenAssignment(BaseModel):
class Jinja2Assignment(BaseModel):
- context: Dict[str, Any]
- jinja2_name: Optional[str] = None
- jinja2: Optional[str] = None
- prompting_style: Optional[PromptingStyle] = None
+ context: dict[str, Any]
+ jinja2_name: str | None = None
+ jinja2: str | None = None
+ prompting_style: PromptingStyle | None = None
template_category: Jinja2TemplateCategory = Jinja2TemplateCategory.LLM_PROMPT
diff --git a/pipelex/cogt/content_generation/content_generator.py b/pipelex/cogt/content_generation/content_generator.py
index 5da23a8e2..0280a4ef8 100644
--- a/pipelex/cogt/content_generation/content_generator.py
+++ b/pipelex/cogt/content_generation/content_generator.py
@@ -1,4 +1,4 @@
-from typing import Any, Dict, List, Optional, Type, cast
+from typing import Any, cast
from typing_extensions import override
@@ -60,7 +60,7 @@ class ContentGenerator(ContentGeneratorProtocol):
async def make_object_direct( # pyright: ignore[reportIncompatibleMethodOverride]
self,
job_metadata: JobMetadata,
- object_class: Type[BaseModelTypeVar],
+ object_class: type[BaseModelTypeVar],
llm_setting_for_object: LLMSetting,
llm_prompt_for_object: LLMPrompt,
) -> BaseModelTypeVar:
@@ -76,18 +76,18 @@ class ContentGenerator(ContentGeneratorProtocol):
)
obj = await llm_gen_object(object_assignment=object_assignment)
log.verbose(f"{self.__class__.__name__} generated object direct: {obj}")
- return cast(BaseModelTypeVar, obj)
+ return cast("BaseModelTypeVar", obj)
@override
@update_job_metadata
async def make_text_then_object( # pyright: ignore[reportIncompatibleMethodOverride]
self,
job_metadata: JobMetadata,
- object_class: Type[BaseModelTypeVar],
+ object_class: type[BaseModelTypeVar],
llm_setting_main: LLMSetting,
llm_setting_for_object: LLMSetting,
llm_prompt_for_text: LLMPrompt,
- llm_prompt_factory_for_object: Optional[LLMPromptFactoryAbstract] = None,
+ llm_prompt_factory_for_object: LLMPromptFactoryAbstract | None = None,
) -> BaseModelTypeVar:
log.verbose(llm_prompt_for_text.user_text, title="llm_prompt_for_text")
llm_assignment_for_text = LLMAssignment.make_from_prompt(
@@ -123,18 +123,18 @@ class ContentGenerator(ContentGeneratorProtocol):
obj = await llm_gen_object(object_assignment=fup_obj_assignment)
log.verbose(f"{self.__class__.__name__} generated object after text: {obj}")
- return cast(BaseModelTypeVar, obj)
+ return cast("BaseModelTypeVar", obj)
@override
@update_job_metadata
async def make_object_list_direct( # pyright: ignore[reportIncompatibleMethodOverride]
self,
job_metadata: JobMetadata,
- object_class: Type[BaseModelTypeVar],
+ object_class: type[BaseModelTypeVar],
llm_setting_for_object_list: LLMSetting,
llm_prompt_for_object_list: LLMPrompt,
- nb_items: Optional[int] = None,
- ) -> List[BaseModelTypeVar]:
+ nb_items: int | None = None,
+ ) -> list[BaseModelTypeVar]:
llm_assignment_for_object = LLMAssignment.make_from_prompt(
job_metadata=job_metadata,
llm_setting=llm_setting_for_object_list,
@@ -146,20 +146,20 @@ class ContentGenerator(ContentGeneratorProtocol):
)
obj_list = await llm_gen_object_list(object_assignment=object_assignment)
log.verbose(f"{self.__class__.__name__} generated object list direct: {obj_list}")
- return cast(List[BaseModelTypeVar], obj_list)
+ return cast("list[BaseModelTypeVar]", obj_list)
@override
@update_job_metadata
async def make_text_then_object_list( # pyright: ignore[reportIncompatibleMethodOverride]
self,
job_metadata: JobMetadata,
- object_class: Type[BaseModelTypeVar],
+ object_class: type[BaseModelTypeVar],
llm_setting_main: LLMSetting,
llm_setting_for_object_list: LLMSetting,
llm_prompt_for_text: LLMPrompt,
- llm_prompt_factory_for_object_list: Optional[LLMPromptFactoryAbstract] = None,
- nb_items: Optional[int] = None,
- ) -> List[BaseModelTypeVar]:
+ llm_prompt_factory_for_object_list: LLMPromptFactoryAbstract | None = None,
+ nb_items: int | None = None,
+ ) -> list[BaseModelTypeVar]:
llm_assignment_for_text = LLMAssignment.make_from_prompt(
job_metadata=job_metadata,
llm_setting=llm_setting_main,
@@ -192,7 +192,7 @@ class ContentGenerator(ContentGeneratorProtocol):
obj_list = await llm_gen_object_list(object_assignment=fup_obj_assignment)
log.verbose(f"{self.__class__.__name__} generated object list after text: {obj_list}")
- return cast(List[BaseModelTypeVar], obj_list)
+ return cast("list[BaseModelTypeVar]", obj_list)
@override
@update_job_metadata
@@ -201,8 +201,8 @@ class ContentGenerator(ContentGeneratorProtocol):
job_metadata: JobMetadata,
img_gen_handle: str,
img_gen_prompt: ImgGenPrompt,
- img_gen_job_params: Optional[ImgGenJobParams] = None,
- img_gen_job_config: Optional[ImgGenJobConfig] = None,
+ img_gen_job_params: ImgGenJobParams | None = None,
+ img_gen_job_config: ImgGenJobConfig | None = None,
) -> GeneratedImage:
img_gen_config = get_config().cogt.img_gen_config
img_gen_assignment = ImgGenAssignment(
@@ -225,9 +225,9 @@ class ContentGenerator(ContentGeneratorProtocol):
img_gen_handle: str,
img_gen_prompt: ImgGenPrompt,
nb_images: int,
- img_gen_job_params: Optional[ImgGenJobParams] = None,
- img_gen_job_config: Optional[ImgGenJobConfig] = None,
- ) -> List[GeneratedImage]:
+ img_gen_job_params: ImgGenJobParams | None = None,
+ img_gen_job_config: ImgGenJobConfig | None = None,
+ ) -> list[GeneratedImage]:
img_gen_config = get_config().cogt.img_gen_config
img_gen_assignment = ImgGenAssignment(
job_metadata=job_metadata,
@@ -244,10 +244,10 @@ class ContentGenerator(ContentGeneratorProtocol):
@override
async def make_jinja2_text(
self,
- context: Dict[str, Any],
- jinja2_name: Optional[str] = None,
- jinja2: Optional[str] = None,
- prompting_style: Optional[PromptingStyle] = None,
+ context: dict[str, Any],
+ jinja2_name: str | None = None,
+ jinja2: str | None = None,
+ prompting_style: PromptingStyle | None = None,
template_category: Jinja2TemplateCategory = Jinja2TemplateCategory.LLM_PROMPT,
) -> str:
jinja2_assignment = Jinja2Assignment(
@@ -267,8 +267,8 @@ class ContentGenerator(ContentGeneratorProtocol):
job_metadata: JobMetadata,
ocr_input: OcrInput,
ocr_handle: str,
- ocr_job_params: Optional[OcrJobParams] = None,
- ocr_job_config: Optional[OcrJobConfig] = None,
+ ocr_job_params: OcrJobParams | None = None,
+ ocr_job_config: OcrJobConfig | None = None,
) -> OcrOutput:
ocr_assignment = OcrAssignment(
job_metadata=job_metadata,
@@ -277,5 +277,4 @@ class ContentGenerator(ContentGeneratorProtocol):
ocr_job_params=ocr_job_params or OcrJobParams.make_default_ocr_job_params(),
ocr_job_config=ocr_job_config or OcrJobConfig(),
)
- ocr_output = await ocr_gen_extract_pages(ocr_assignment=ocr_assignment)
- return ocr_output
+ return await ocr_gen_extract_pages(ocr_assignment=ocr_assignment)
diff --git a/pipelex/cogt/content_generation/content_generator_dry.py b/pipelex/cogt/content_generation/content_generator_dry.py
index d7df93572..baa3fb44b 100644
--- a/pipelex/cogt/content_generation/content_generator_dry.py
+++ b/pipelex/cogt/content_generation/content_generator_dry.py
@@ -1,4 +1,4 @@
-from typing import Any, Dict, List, Optional, Type
+from typing import Any
from polyfactory.factories.pydantic_factory import ModelFactory
from typing_extensions import override
@@ -22,8 +22,7 @@ from pipelex.tools.typing.pydantic_utils import BaseModelTypeVar
class ContentGeneratorDry(ContentGeneratorProtocol):
- """
- This class is used to generate mock content for testing purposes.
+ """This class is used to generate mock content for testing purposes.
It does not use any inference.
"""
@@ -42,22 +41,21 @@ class ContentGeneratorDry(ContentGeneratorProtocol):
func_name = "make_llm_text"
log.dev(f"🤡 DRY RUN: {self.__class__.__name__}.{func_name}")
prompt_truncated = llm_prompt_for_text.desc(truncate_text_length=self._text_gen_truncate_length)
- generated_text = f"DRY RUN: {func_name} • llm_setting={llm_setting_main.desc()} • prompt={prompt_truncated}"
- return generated_text
+ return f"DRY RUN: {func_name} • llm_setting={llm_setting_main.desc()} • prompt={prompt_truncated}"
@override
@update_job_metadata
async def make_object_direct( # pyright: ignore[reportIncompatibleMethodOverride]
self,
job_metadata: JobMetadata,
- object_class: Type[BaseModelTypeVar],
+ object_class: type[BaseModelTypeVar],
llm_setting_for_object: LLMSetting,
llm_prompt_for_object: LLMPrompt,
) -> BaseModelTypeVar:
func_name = "make_object_direct"
log.dev(f"🤡 DRY RUN: {self.__class__.__name__}.{func_name}")
- class ObjectFactory(ModelFactory[object_class]): # type: ignore
+ class ObjectFactory(ModelFactory[object_class]): # type: ignore[valid-type]
__model__ = object_class
__check_model__ = True
__use_examples__ = True
@@ -67,19 +65,18 @@ class ContentGeneratorDry(ContentGeneratorProtocol):
# It is that way because the dry run was failing a lot of pipes that had validation test on the
# field values. For example, if a string requires to be a snake_case, the ObjectFactory would
# generate something like `DOIJZjoDoIJDZOjDZJo` which is... not a snake_case.
- obj = ObjectFactory.build(factory_use_construct=True)
- return obj
+ return ObjectFactory.build(factory_use_construct=True)
@override
@update_job_metadata
async def make_text_then_object( # pyright: ignore[reportIncompatibleMethodOverride]
self,
job_metadata: JobMetadata,
- object_class: Type[BaseModelTypeVar],
+ object_class: type[BaseModelTypeVar],
llm_setting_main: LLMSetting,
llm_setting_for_object: LLMSetting,
llm_prompt_for_text: LLMPrompt,
- llm_prompt_factory_for_object: Optional[LLMPromptFactoryAbstract] = None,
+ llm_prompt_factory_for_object: LLMPromptFactoryAbstract | None = None,
) -> BaseModelTypeVar:
func_name = "make_text_then_object"
log.dev(f"🤡 DRY RUN: {self.__class__.__name__}.{func_name}")
@@ -96,15 +93,15 @@ class ContentGeneratorDry(ContentGeneratorProtocol):
async def make_object_list_direct( # pyright: ignore[reportIncompatibleMethodOverride]
self,
job_metadata: JobMetadata,
- object_class: Type[BaseModelTypeVar],
+ object_class: type[BaseModelTypeVar],
llm_setting_for_object_list: LLMSetting,
llm_prompt_for_object_list: LLMPrompt,
- nb_items: Optional[int] = None,
- ) -> List[BaseModelTypeVar]:
+ nb_items: int | None = None,
+ ) -> list[BaseModelTypeVar]:
func_name = "make_object_list_direct"
log.dev(f"🤡 DRY RUN: {self.__class__.__name__}.{func_name}")
nb_list_items = nb_items or get_config().pipelex.dry_run_config.nb_list_items
- objects = [
+ return [
await self.make_object_direct(
job_metadata=job_metadata,
object_class=object_class,
@@ -113,20 +110,19 @@ class ContentGeneratorDry(ContentGeneratorProtocol):
)
for _ in range(nb_list_items)
]
- return objects
@override
@update_job_metadata
async def make_text_then_object_list( # pyright: ignore[reportIncompatibleMethodOverride]
self,
job_metadata: JobMetadata,
- object_class: Type[BaseModelTypeVar],
+ object_class: type[BaseModelTypeVar],
llm_setting_main: LLMSetting,
llm_setting_for_object_list: LLMSetting,
llm_prompt_for_text: LLMPrompt,
- llm_prompt_factory_for_object_list: Optional[LLMPromptFactoryAbstract] = None,
- nb_items: Optional[int] = None,
- ) -> List[BaseModelTypeVar]:
+ llm_prompt_factory_for_object_list: LLMPromptFactoryAbstract | None = None,
+ nb_items: int | None = None,
+ ) -> list[BaseModelTypeVar]:
func_name = "make_text_then_object_list"
log.dev(f"🤡 DRY RUN: {self.__class__.__name__}.{func_name}")
return await self.make_object_list_direct(
@@ -144,19 +140,18 @@ class ContentGeneratorDry(ContentGeneratorProtocol):
job_metadata: JobMetadata,
img_gen_handle: str,
img_gen_prompt: ImgGenPrompt,
- img_gen_job_params: Optional[ImgGenJobParams] = None,
- img_gen_job_config: Optional[ImgGenJobConfig] = None,
+ img_gen_job_params: ImgGenJobParams | None = None,
+ img_gen_job_config: ImgGenJobConfig | None = None,
) -> GeneratedImage:
func_name = "make_single_image"
log.dev(f"🤡 DRY RUN: {self.__class__.__name__}.{func_name}")
image_urls = get_config().pipelex.dry_run_config.image_urls
image_url = image_urls[0]
- generated_image = GeneratedImage(
+ return GeneratedImage(
url=image_url,
width=1536,
height=2752,
)
- return generated_image
@override
@update_job_metadata
@@ -166,13 +161,13 @@ class ContentGeneratorDry(ContentGeneratorProtocol):
img_gen_handle: str,
img_gen_prompt: ImgGenPrompt,
nb_images: int,
- img_gen_job_params: Optional[ImgGenJobParams] = None,
- img_gen_job_config: Optional[ImgGenJobConfig] = None,
- ) -> List[GeneratedImage]:
+ img_gen_job_params: ImgGenJobParams | None = None,
+ img_gen_job_config: ImgGenJobConfig | None = None,
+ ) -> list[GeneratedImage]:
func_name = "make_image_list"
log.dev(f"🤡 DRY RUN: {self.__class__.__name__}.{func_name}")
image_urls = get_config().pipelex.dry_run_config.image_urls
- generated_image_list = [
+ return [
GeneratedImage(
url=image_urls[image_index % len(image_urls)],
width=1536,
@@ -180,26 +175,24 @@ class ContentGeneratorDry(ContentGeneratorProtocol):
)
for image_index in range(nb_images)
]
- return generated_image_list
@override
async def make_jinja2_text(
self,
- context: Dict[str, Any],
- jinja2_name: Optional[str] = None,
- jinja2: Optional[str] = None,
- prompting_style: Optional[PromptingStyle] = None,
+ context: dict[str, Any],
+ jinja2_name: str | None = None,
+ jinja2: str | None = None,
+ prompting_style: PromptingStyle | None = None,
template_category: Jinja2TemplateCategory = Jinja2TemplateCategory.LLM_PROMPT,
) -> str:
# TODO: Use the code that checks if the jinja2 is a valid template
func_name = "make_jinja2_text"
log.dev(f"🤡 DRY RUN: {self.__class__.__name__}.{func_name}")
jinja2_truncated = jinja2[: self._text_gen_truncate_length] if jinja2 else None
- jinja2_text = (
+ return (
f"DRY RUN: {func_name} • context={context} • jinja2_name={jinja2_name} • "
f"jinja2={jinja2_truncated} • prompting_style={prompting_style} • template_category={template_category}"
)
- return jinja2_text
@override
async def make_ocr_extract_pages(
@@ -207,8 +200,8 @@ class ContentGeneratorDry(ContentGeneratorProtocol):
job_metadata: JobMetadata,
ocr_input: OcrInput,
ocr_handle: str,
- ocr_job_params: Optional[OcrJobParams] = None,
- ocr_job_config: Optional[OcrJobConfig] = None,
+ ocr_job_params: OcrJobParams | None = None,
+ ocr_job_config: OcrJobConfig | None = None,
) -> OcrOutput:
func_name = "make_ocr_extract_pages"
log.dev(f"🤡 DRY RUN: {self.__class__.__name__}.{func_name}")
diff --git a/pipelex/cogt/content_generation/content_generator_protocol.py b/pipelex/cogt/content_generation/content_generator_protocol.py
index c52c139a7..936d3a3ac 100644
--- a/pipelex/cogt/content_generation/content_generator_protocol.py
+++ b/pipelex/cogt/content_generation/content_generator_protocol.py
@@ -1,5 +1,6 @@
+from collections.abc import Awaitable, Callable, Coroutine
from functools import wraps
-from typing import Any, Awaitable, Callable, Coroutine, Dict, List, Optional, ParamSpec, Protocol, Type, TypeVar
+from typing import Any, ParamSpec, Protocol, TypeVar
from pipelex.cogt.image.generated_image import GeneratedImage
from pipelex.cogt.img_gen.img_gen_job_components import ImgGenJobConfig, ImgGenJobParams
@@ -26,10 +27,12 @@ def update_job_metadata(func: Callable[P, Coroutine[Any, Any, R]]) -> Callable[P
# Attempt to get job_metadata from kwargs or from args
job_metadata = kwargs.get("job_metadata")
if job_metadata is None:
- raise RuntimeError("job_metadata argument is required for this decorated function.")
+ msg = "job_metadata argument is required for this decorated function."
+ raise RuntimeError(msg)
if not isinstance(job_metadata, JobMetadata):
- raise TypeError("The job_metadata argument must be of type JobMetadata.")
+ msg = "The job_metadata argument must be of type JobMetadata."
+ raise TypeError(msg)
updated_metadata = JobMetadata(
content_generation_job_id=func.__name__,
@@ -52,7 +55,7 @@ class ContentGeneratorProtocol(Protocol):
async def make_object_direct(
self,
job_metadata: JobMetadata,
- object_class: Type[BaseModelTypeVar],
+ object_class: type[BaseModelTypeVar],
llm_setting_for_object: LLMSetting,
llm_prompt_for_object: LLMPrompt,
) -> BaseModelTypeVar: ...
@@ -60,40 +63,40 @@ class ContentGeneratorProtocol(Protocol):
async def make_text_then_object(
self,
job_metadata: JobMetadata,
- object_class: Type[BaseModelTypeVar],
+ object_class: type[BaseModelTypeVar],
llm_setting_main: LLMSetting,
llm_setting_for_object: LLMSetting,
llm_prompt_for_text: LLMPrompt,
- llm_prompt_factory_for_object: Optional[LLMPromptFactoryAbstract] = None,
+ llm_prompt_factory_for_object: LLMPromptFactoryAbstract | None = None,
) -> BaseModelTypeVar: ...
async def make_object_list_direct(
self,
job_metadata: JobMetadata,
- object_class: Type[BaseModelTypeVar],
+ object_class: type[BaseModelTypeVar],
llm_setting_for_object_list: LLMSetting,
llm_prompt_for_object_list: LLMPrompt,
- nb_items: Optional[int] = None,
- ) -> List[BaseModelTypeVar]: ...
+ nb_items: int | None = None,
+ ) -> list[BaseModelTypeVar]: ...
async def make_text_then_object_list(
self,
job_metadata: JobMetadata,
- object_class: Type[BaseModelTypeVar],
+ object_class: type[BaseModelTypeVar],
llm_setting_main: LLMSetting,
llm_setting_for_object_list: LLMSetting,
llm_prompt_for_text: LLMPrompt,
- llm_prompt_factory_for_object_list: Optional[LLMPromptFactoryAbstract] = None,
- nb_items: Optional[int] = None,
- ) -> List[BaseModelTypeVar]: ...
+ llm_prompt_factory_for_object_list: LLMPromptFactoryAbstract | None = None,
+ nb_items: int | None = None,
+ ) -> list[BaseModelTypeVar]: ...
async def make_single_image(
self,
job_metadata: JobMetadata,
img_gen_handle: str,
img_gen_prompt: ImgGenPrompt,
- img_gen_job_params: Optional[ImgGenJobParams] = None,
- img_gen_job_config: Optional[ImgGenJobConfig] = None,
+ img_gen_job_params: ImgGenJobParams | None = None,
+ img_gen_job_config: ImgGenJobConfig | None = None,
) -> GeneratedImage: ...
async def make_image_list(
@@ -102,16 +105,16 @@ class ContentGeneratorProtocol(Protocol):
img_gen_handle: str,
img_gen_prompt: ImgGenPrompt,
nb_images: int,
- img_gen_job_params: Optional[ImgGenJobParams] = None,
- img_gen_job_config: Optional[ImgGenJobConfig] = None,
- ) -> List[GeneratedImage]: ...
+ img_gen_job_params: ImgGenJobParams | None = None,
+ img_gen_job_config: ImgGenJobConfig | None = None,
+ ) -> list[GeneratedImage]: ...
async def make_jinja2_text(
self,
- context: Dict[str, Any],
- jinja2_name: Optional[str] = None,
- jinja2: Optional[str] = None,
- prompting_style: Optional[PromptingStyle] = None,
+ context: dict[str, Any],
+ jinja2_name: str | None = None,
+ jinja2: str | None = None,
+ prompting_style: PromptingStyle | None = None,
template_category: Jinja2TemplateCategory = Jinja2TemplateCategory.LLM_PROMPT,
) -> str: ...
diff --git a/pipelex/cogt/content_generation/img_gen_generate.py b/pipelex/cogt/content_generation/img_gen_generate.py
index b7ae6a7e5..66dec1ba7 100644
--- a/pipelex/cogt/content_generation/img_gen_generate.py
+++ b/pipelex/cogt/content_generation/img_gen_generate.py
@@ -1,5 +1,3 @@
-from typing import List, Union
-
from pipelex import log
from pipelex.cogt.content_generation.assignment_models import ImgGenAssignment
from pipelex.cogt.image.generated_image import GeneratedImage
@@ -20,7 +18,7 @@ async def img_gen_single_image(img_gen_assignment: ImgGenAssignment) -> Generate
return generated_image
-async def img_gen_image_list(img_gen_assignment: ImgGenAssignment) -> List[GeneratedImage]:
+async def img_gen_image_list(img_gen_assignment: ImgGenAssignment) -> list[GeneratedImage]:
img_gen_worker = get_img_gen_worker(img_gen_handle=img_gen_assignment.img_gen_handle)
img_gen_job = ImgGenJobFactory.make_img_gen_job_from_prompt(
img_gen_prompt=img_gen_assignment.img_gen_prompt,
@@ -36,8 +34,7 @@ async def img_gen_image_list(img_gen_assignment: ImgGenAssignment) -> List[Gener
return generated_image_list
-async def img_gen_image(img_gen_assignment: ImgGenAssignment) -> Union[GeneratedImage, List[GeneratedImage]]:
+async def img_gen_image(img_gen_assignment: ImgGenAssignment) -> GeneratedImage | list[GeneratedImage]:
if img_gen_assignment.nb_images > 1:
return await img_gen_image_list(img_gen_assignment)
- else:
- return await img_gen_single_image(img_gen_assignment)
+ return await img_gen_single_image(img_gen_assignment)
diff --git a/pipelex/cogt/content_generation/llm_generate.py b/pipelex/cogt/content_generation/llm_generate.py
index 21ddbce2c..424597d38 100644
--- a/pipelex/cogt/content_generation/llm_generate.py
+++ b/pipelex/cogt/content_generation/llm_generate.py
@@ -1,4 +1,4 @@
-from typing import List, cast
+from typing import cast
from pydantic import BaseModel
@@ -37,7 +37,7 @@ async def llm_gen_object(object_assignment: ObjectAssignment) -> BaseModel:
return generated_object
-async def llm_gen_object_list(object_assignment: ObjectAssignment) -> List[BaseModel]:
+async def llm_gen_object_list(object_assignment: ObjectAssignment) -> list[BaseModel]:
llm_assignment = object_assignment.llm_assignment_for_object
log.verbose(f"llm_gen_object_list to generate a list of '{object_assignment.object_class_name}'")
llm_worker = get_llm_worker(llm_handle=llm_assignment.llm_handle)
@@ -50,11 +50,11 @@ async def llm_gen_object_list(object_assignment: ObjectAssignment) -> List[BaseM
item_class = get_class_registry().get_required_class(name=item_class_name)
class ListSchema(BaseModel):
- items: List[item_class] # type: ignore
+ items: list[item_class] # type: ignore[valid-type] # pyright: ignore[reportInvalidTypeForm]
wrapped_list: ListSchema = await llm_worker.gen_object(
llm_job=llm_job,
schema=ListSchema,
)
- generated_list: List[BaseModel] = cast(List[BaseModel], wrapped_list.items) # pyright: ignore[reportUnknownMemberType]
+ generated_list: list[BaseModel] = cast("list[BaseModel]", wrapped_list.items) # pyright: ignore[reportUnknownMemberType]
return generated_list
diff --git a/pipelex/cogt/content_generation/ocr_generate.py b/pipelex/cogt/content_generation/ocr_generate.py
index 91c3205ee..66254b7fd 100644
--- a/pipelex/cogt/content_generation/ocr_generate.py
+++ b/pipelex/cogt/content_generation/ocr_generate.py
@@ -12,5 +12,4 @@ async def ocr_gen_extract_pages(ocr_assignment: OcrAssignment) -> OcrOutput:
ocr_job_config=ocr_assignment.ocr_job_config,
job_metadata=ocr_assignment.job_metadata,
)
- ocr_output = await ocr_worker.ocr_extract_pages(ocr_job=ocr_job)
- return ocr_output
+ return await ocr_worker.ocr_extract_pages(ocr_job=ocr_job)
diff --git a/pipelex/cogt/exceptions.py b/pipelex/cogt/exceptions.py
index a8c893aa6..075930c73 100644
--- a/pipelex/cogt/exceptions.py
+++ b/pipelex/cogt/exceptions.py
@@ -1,5 +1,3 @@
-from typing import Optional
-
from pipelex.tools.exceptions import FatalError, RootException
from pipelex.types import StrEnum
@@ -127,7 +125,7 @@ class ImgGenGeneratedTypeError(ImgGenGenerationError):
class MissingDependencyError(CogtError):
"""Raised when a required dependency is not installed."""
- def __init__(self, dependency_name: str, extra_name: str, message: Optional[str] = None):
+ def __init__(self, dependency_name: str, extra_name: str, message: str | None = None):
self.dependency_name = dependency_name
self.extra_name = extra_name
error_msg = f"Required dependency '{dependency_name}' is not installed."
diff --git a/pipelex/cogt/image/__init__.py b/pipelex/cogt/image/__init__.py
index 8b1378917..e69de29bb 100644
--- a/pipelex/cogt/image/__init__.py
+++ b/pipelex/cogt/image/__init__.py
@@ -1 +0,0 @@
-
diff --git a/pipelex/cogt/image/prompt_image_factory.py b/pipelex/cogt/image/prompt_image_factory.py
index e5b4a4df9..b0b608042 100644
--- a/pipelex/cogt/image/prompt_image_factory.py
+++ b/pipelex/cogt/image/prompt_image_factory.py
@@ -1,5 +1,3 @@
-from typing import Optional
-
from pipelex.cogt.exceptions import PromptImageFactoryError
from pipelex.cogt.image.prompt_image import PromptImage, PromptImageBase64, PromptImageBinary, PromptImagePath, PromptImageUrl
from pipelex.tools.misc.base_64_utils import (
@@ -15,9 +13,9 @@ class PromptImageFactory:
@classmethod
def make_prompt_image(
cls,
- file_path: Optional[str] = None,
- url: Optional[str] = None,
- base_64: Optional[bytes] = None,
+ file_path: str | None = None,
+ url: str | None = None,
+ base_64: bytes | None = None,
) -> PromptImage:
if file_path:
return PromptImagePath(file_path=file_path)
@@ -26,7 +24,8 @@ class PromptImageFactory:
elif base_64:
return PromptImageBase64(base_64=base_64)
else:
- raise PromptImageFactoryError("PromptImageFactory requires one of file_path, url, or image_bytes")
+ msg = "PromptImageFactory requires one of file_path, url, or image_bytes"
+ raise PromptImageFactoryError(msg)
@classmethod
def make_prompt_image_from_uri(
@@ -58,24 +57,28 @@ class PromptImageFactory:
@classmethod
async def promptimage_to_b64_async(cls, prompt_image: PromptImage) -> bytes:
- if isinstance(prompt_image, PromptImagePath):
- return await load_binary_as_base64_async(prompt_image.file_path)
- elif isinstance(prompt_image, PromptImageBase64):
- return prompt_image.base_64
- elif isinstance(prompt_image, PromptImageUrl):
- image_bytes = await cls.make_promptimagebase64_from_url_async(prompt_image)
- return image_bytes.base_64
- else:
- raise PromptImageFactoryError(f"Unknown PromptImage type: {prompt_image}")
+ match prompt_image:
+ case PromptImagePath():
+ return await load_binary_as_base64_async(prompt_image.file_path)
+ case PromptImageBase64():
+ return prompt_image.base_64
+ case PromptImageUrl():
+ image_bytes = await cls.make_promptimagebase64_from_url_async(prompt_image)
+ return image_bytes.base_64
+ case _:
+ msg = f"Unknown PromptImage type: {prompt_image}"
+ raise PromptImageFactoryError(msg)
@classmethod
async def promptimage_to_bytes_async(cls, prompt_image: PromptImage) -> bytes:
- if isinstance(prompt_image, PromptImagePath):
- return await load_binary_async(prompt_image.file_path)
- elif isinstance(prompt_image, PromptImageBase64):
- return prompt_image.get_decoded_bytes()
- elif isinstance(prompt_image, PromptImageUrl):
- image_bytes = await cls.make_promptimagebinary_from_url_async(prompt_image)
- return image_bytes.binary
- else:
- raise PromptImageFactoryError(f"Unknown PromptImage type: {prompt_image}")
+ match prompt_image:
+ case PromptImagePath():
+ return await load_binary_async(prompt_image.file_path)
+ case PromptImageBase64():
+ return prompt_image.get_decoded_bytes()
+ case PromptImageUrl():
+ image_bytes = await cls.make_promptimagebinary_from_url_async(prompt_image)
+ return image_bytes.binary
+ case _:
+ msg = f"Unknown PromptImage type: {prompt_image}"
+ raise PromptImageFactoryError(msg)
diff --git a/pipelex/cogt/img_gen/__init__.py b/pipelex/cogt/img_gen/__init__.py
index 8b1378917..e69de29bb 100644
--- a/pipelex/cogt/img_gen/__init__.py
+++ b/pipelex/cogt/img_gen/__init__.py
@@ -1 +0,0 @@
-
diff --git a/pipelex/cogt/img_gen/img_gen_job_components.py b/pipelex/cogt/img_gen/img_gen_job_components.py
index a272e8aef..53ed07cc6 100644
--- a/pipelex/cogt/img_gen/img_gen_job_components.py
+++ b/pipelex/cogt/img_gen/img_gen_job_components.py
@@ -1,4 +1,4 @@
-from typing import Literal, Optional, Union
+from typing import Literal
from pydantic import BaseModel, Field
@@ -39,30 +39,30 @@ class Background(StrEnum):
class ImgGenJobParams(BaseModel):
aspect_ratio: AspectRatio = Field(strict=False)
background: Background = Field(strict=False)
- quality: Optional[Quality] = Field(default=None, strict=False)
- nb_steps: Optional[int] = Field(default=None, gt=0)
- guidance_scale: Optional[float] = Field(default=None, gt=0)
+ quality: Quality | None = Field(default=None, strict=False)
+ nb_steps: int | None = Field(default=None, gt=0)
+ guidance_scale: float | None = Field(default=None, gt=0)
is_moderated: bool = False
- safety_tolerance: Optional[int] = Field(default=None, ge=1, le=6)
+ safety_tolerance: int | None = Field(default=None, ge=1, le=6)
is_raw: bool
output_format: OutputFormat = Field(strict=False)
- seed: Optional[int] = Field(None, ge=0)
+ seed: int | None = Field(None, ge=0)
class ImgGenJobParamsDefaults(ConfigModel):
aspect_ratio: AspectRatio = Field(strict=False)
background: Background = Field(strict=False)
- quality: Optional[Quality] = Field(default=None, strict=False)
- nb_steps: Optional[int] = Field(default=None, gt=0)
+ quality: Quality | None = Field(default=None, strict=False)
+ nb_steps: int | None = Field(default=None, gt=0)
guidance_scale: float = Field(..., gt=0)
is_moderated: bool
safety_tolerance: int = Field(..., ge=1, le=6)
is_raw: bool
output_format: OutputFormat = Field(strict=False)
- seed: Union[int, Literal["auto"]]
+ seed: int | Literal["auto"]
def make_img_gen_job_params(self) -> ImgGenJobParams:
- seed: Optional[int]
+ seed: int | None
if isinstance(self.seed, str) and self.seed == "auto":
seed = None
else:
diff --git a/pipelex/cogt/img_gen/img_gen_job_factory.py b/pipelex/cogt/img_gen/img_gen_job_factory.py
index 166c55521..e358c887f 100644
--- a/pipelex/cogt/img_gen/img_gen_job_factory.py
+++ b/pipelex/cogt/img_gen/img_gen_job_factory.py
@@ -1,5 +1,3 @@
-from typing import Optional
-
from pipelex.cogt.img_gen.img_gen_job import ImgGenJob
from pipelex.cogt.img_gen.img_gen_job_components import ImgGenJobConfig, ImgGenJobParams, ImgGenJobReport
from pipelex.cogt.img_gen.img_gen_prompt import ImgGenPrompt
@@ -12,9 +10,9 @@ class ImgGenJobFactory:
def make_img_gen_job_from_prompt(
cls,
img_gen_prompt: ImgGenPrompt,
- img_gen_job_params: Optional[ImgGenJobParams] = None,
- img_gen_job_config: Optional[ImgGenJobConfig] = None,
- job_metadata: Optional[JobMetadata] = None,
+ img_gen_job_params: ImgGenJobParams | None = None,
+ img_gen_job_config: ImgGenJobConfig | None = None,
+ job_metadata: JobMetadata | None = None,
) -> ImgGenJob:
img_gen_config = get_config().cogt.img_gen_config
job_metadata = job_metadata or JobMetadata(
@@ -36,9 +34,9 @@ class ImgGenJobFactory:
def make_img_gen_job_from_prompt_contents(
cls,
positive_text: str,
- img_gen_job_params: Optional[ImgGenJobParams] = None,
- img_gen_job_config: Optional[ImgGenJobConfig] = None,
- job_metadata: Optional[JobMetadata] = None,
+ img_gen_job_params: ImgGenJobParams | None = None,
+ img_gen_job_config: ImgGenJobConfig | None = None,
+ job_metadata: JobMetadata | None = None,
) -> ImgGenJob:
img_gen_config = get_config().cogt.img_gen_config
job_metadata = job_metadata or JobMetadata(
diff --git a/pipelex/cogt/img_gen/img_gen_prompt.py b/pipelex/cogt/img_gen/img_gen_prompt.py
index 83e3558e5..ea7859f65 100644
--- a/pipelex/cogt/img_gen/img_gen_prompt.py
+++ b/pipelex/cogt/img_gen/img_gen_prompt.py
@@ -17,7 +17,8 @@ class ImgGenPrompt(BaseModel):
pass
case ProblemReaction.RAISE:
if self.positive_text == "":
- raise ImgGenPromptError("ImgGen prompt positive_text must not be an empty string")
+ msg = "ImgGen prompt positive_text must not be an empty string"
+ raise ImgGenPromptError(msg)
case ProblemReaction.LOG:
if self.positive_text == "":
log.warning("ImgGen prompt positive_text should not be an empty string")
diff --git a/pipelex/cogt/img_gen/img_gen_setting.py b/pipelex/cogt/img_gen/img_gen_setting.py
index 5245dc66c..ad62e6962 100644
--- a/pipelex/cogt/img_gen/img_gen_setting.py
+++ b/pipelex/cogt/img_gen/img_gen_setting.py
@@ -1,25 +1,26 @@
-from typing import Optional, Union
+from typing import Union
from pydantic import Field, model_validator
-from typing_extensions import Self
from pipelex.cogt.exceptions import ImgGenSettingsValidationError
from pipelex.cogt.img_gen.img_gen_job_components import Quality
from pipelex.tools.config.config_model import ConfigModel
+from pipelex.types import Self
class ImgGenSetting(ConfigModel):
img_gen_handle: str
- quality: Optional[Quality] = Field(default=None, strict=False)
- nb_steps: Optional[int] = Field(default=None, gt=0)
- guidance_scale: Optional[float] = Field(default=None, gt=0)
+ quality: Quality | None = Field(default=None, strict=False)
+ nb_steps: int | None = Field(default=None, gt=0)
+ guidance_scale: float | None = Field(default=None, gt=0)
is_moderated: bool = False
- safety_tolerance: Optional[int] = Field(default=None, ge=1, le=6)
+ safety_tolerance: int | None = Field(default=None, ge=1, le=6)
@model_validator(mode="after")
def validate_quality_or_nb_steps(self) -> Self:
if self.quality is not None and self.nb_steps is not None:
- raise ImgGenSettingsValidationError("ImgGenSetting cannot have both 'quality' and 'nb_steps' specified. Use one or the other.")
+ msg = "ImgGenSetting cannot have both 'quality' and 'nb_steps' specified. Use one or the other."
+ raise ImgGenSettingsValidationError(msg)
return self
def desc(self) -> str:
diff --git a/pipelex/cogt/img_gen/img_gen_worker_abstract.py b/pipelex/cogt/img_gen/img_gen_worker_abstract.py
index 1e4432ae9..d55630df5 100644
--- a/pipelex/cogt/img_gen/img_gen_worker_abstract.py
+++ b/pipelex/cogt/img_gen/img_gen_worker_abstract.py
@@ -1,5 +1,4 @@
from abc import abstractmethod
-from typing import List, Optional
from typing_extensions import override
@@ -16,7 +15,7 @@ class ImgGenWorkerAbstract(InferenceWorkerAbstract):
def __init__(
self,
inference_model: InferenceModelSpec,
- reporting_delegate: Optional[ReportingProtocol] = None,
+ reporting_delegate: ReportingProtocol | None = None,
):
InferenceWorkerAbstract.__init__(self, reporting_delegate=reporting_delegate)
self.inference_model = inference_model
@@ -73,7 +72,7 @@ class ImgGenWorkerAbstract(InferenceWorkerAbstract):
self,
img_gen_job: ImgGenJob,
nb_images: int,
- ) -> List[GeneratedImage]:
+ ) -> list[GeneratedImage]:
log.debug(f"Image gen worker gen_image_list using {self.desc}")
# Verify that the job is valid
@@ -103,5 +102,5 @@ class ImgGenWorkerAbstract(InferenceWorkerAbstract):
self,
img_gen_job: ImgGenJob,
nb_images: int,
- ) -> List[GeneratedImage]:
+ ) -> list[GeneratedImage]:
pass
diff --git a/pipelex/cogt/img_gen/img_gen_worker_factory.py b/pipelex/cogt/img_gen/img_gen_worker_factory.py
index 97e7c9ed9..f04584cdc 100644
--- a/pipelex/cogt/img_gen/img_gen_worker_factory.py
+++ b/pipelex/cogt/img_gen/img_gen_worker_factory.py
@@ -1,5 +1,3 @@
-from typing import Optional
-
from pipelex.cogt.exceptions import CogtError, MissingDependencyError
from pipelex.cogt.img_gen.img_gen_worker_abstract import ImgGenWorkerAbstract
from pipelex.cogt.model_backends.model_spec import InferenceModelSpec
@@ -17,7 +15,7 @@ class ImgGenWorkerFactory:
def make_img_gen_worker(
self,
inference_model: InferenceModelSpec,
- reporting_delegate: Optional[ReportingProtocol] = None,
+ reporting_delegate: ReportingProtocol | None = None,
) -> ImgGenWorkerAbstract:
plugin = Plugin.make_for_inference_model(inference_model=inference_model)
backend = get_models_manager().get_required_inference_backend(inference_model.backend_name)
@@ -28,16 +26,22 @@ class ImgGenWorkerFactory:
try:
fal_api_key = get_secret(secret_id="FAL_API_KEY")
except SecretNotFoundError as exc:
- raise FalCredentialsError("FAL_API_KEY not found") from exc
+ msg = "FAL_API_KEY not found"
+ raise FalCredentialsError(msg) from exc
try:
- from fal_client import AsyncClient as FalAsyncClient
+ from fal_client import AsyncClient as FalAsyncClient # noqa: PLC0415
except ImportError as exc:
+ lib_name = "fal-client"
+ lib_extra_name = "fal"
+ msg = "The fal-client SDK is required to use FAL models (generation of images)."
raise MissingDependencyError(
- "fal-client", "fal", "The fal-client SDK is required to use FAL models (generation of images)."
+ lib_name,
+ lib_extra_name,
+ msg,
) from exc
- from pipelex.plugins.fal.fal_img_gen_worker import FalImgGenWorker
+ from pipelex.plugins.fal.fal_img_gen_worker import FalImgGenWorker # noqa: PLC0415
img_gen_sdk_instance = plugin_sdk_registry.get_sdk_instance(plugin=plugin) or plugin_sdk_registry.set_sdk_instance(
plugin=plugin,
@@ -50,8 +54,8 @@ class ImgGenWorkerFactory:
reporting_delegate=reporting_delegate,
)
case "openai":
- from pipelex.plugins.openai.openai_factory import OpenAIFactory
- from pipelex.plugins.openai.openai_img_gen_worker import OpenAIImgGenWorker
+ from pipelex.plugins.openai.openai_factory import OpenAIFactory # noqa: PLC0415
+ from pipelex.plugins.openai.openai_img_gen_worker import OpenAIImgGenWorker # noqa: PLC0415
img_gen_sdk_instance = plugin_sdk_registry.get_sdk_instance(plugin=plugin) or plugin_sdk_registry.set_sdk_instance(
plugin=plugin,
@@ -67,6 +71,7 @@ class ImgGenWorkerFactory:
reporting_delegate=reporting_delegate,
)
case _:
- raise NotImplementedError(f"Plugin '{plugin}' is not supported for image generation")
+ msg = f"Plugin '{plugin}' is not supported for image generation"
+ raise NotImplementedError(msg)
return img_gen_worker
diff --git a/pipelex/cogt/inference/__init__.py b/pipelex/cogt/inference/__init__.py
index 8b1378917..e69de29bb 100644
--- a/pipelex/cogt/inference/__init__.py
+++ b/pipelex/cogt/inference/__init__.py
@@ -1 +0,0 @@
-
diff --git a/pipelex/cogt/inference/inference_manager.py b/pipelex/cogt/inference/inference_manager.py
index fec461379..b3f96b475 100644
--- a/pipelex/cogt/inference/inference_manager.py
+++ b/pipelex/cogt/inference/inference_manager.py
@@ -1,5 +1,3 @@
-from typing import Dict, Type
-
from typing_extensions import override
from pipelex import log
@@ -21,9 +19,9 @@ class InferenceManager(InferenceManagerProtocol):
def __init__(self):
self.img_gen_worker_factory = ImgGenWorkerFactory()
self.ocr_worker_factory = OcrWorkerFactory()
- self.llm_workers: Dict[str, LLMWorkerAbstract] = {}
- self.img_gen_workers: Dict[str, ImgGenWorkerAbstract] = {}
- self.ocr_workers: Dict[str, OcrWorkerAbstract] = {}
+ self.llm_workers: dict[str, LLMWorkerAbstract] = {}
+ self.img_gen_workers: dict[str, ImgGenWorkerAbstract] = {}
+ self.ocr_workers: dict[str, OcrWorkerAbstract] = {}
@override
def teardown(self):
@@ -75,28 +73,24 @@ class InferenceManager(InferenceManagerProtocol):
if llm_worker := self.llm_workers.get(llm_handle):
return llm_worker
if not get_config().cogt.inference_manager_config.is_auto_setup_preset_llm:
- raise InferenceManagerWorkerSetupError(
- f"No LLM worker for '{llm_handle}', set it up or enable cogt.inference_manager_config.is_auto_setup_preset_llm"
- )
+ msg = f"No LLM worker for '{llm_handle}', set it up or enable cogt.inference_manager_config.is_auto_setup_preset_llm"
+ raise InferenceManagerWorkerSetupError(msg)
inference_model = get_models_manager().get_inference_model(model_handle=llm_handle)
- llm_worker = self._setup_one_internal_llm_worker(
+ return self._setup_one_internal_llm_worker(
inference_model=inference_model,
llm_handle=llm_handle,
)
- return llm_worker
-
@override
def set_llm_worker_from_external_plugin(
self,
llm_handle: str,
- llm_worker_class: Type[LLMWorkerAbstract],
+ llm_worker_class: type[LLMWorkerAbstract],
should_warn_if_already_registered: bool = True,
):
- if llm_handle in self.llm_workers:
- if should_warn_if_already_registered:
- log.warning(f"LLM worker for '{llm_handle}' already registered, skipping")
+ if llm_handle in self.llm_workers and should_warn_if_already_registered:
+ log.warning(f"LLM worker for '{llm_handle}' already registered, skipping")
self.llm_workers[llm_handle] = llm_worker_class(reporting_delegate=get_report_delegate())
####################################################################################################
@@ -118,9 +112,8 @@ class InferenceManager(InferenceManagerProtocol):
img_gen_worker = self.img_gen_workers.get(img_gen_handle)
if img_gen_worker is None:
if not get_config().cogt.inference_manager_config.is_auto_setup_preset_img_gen:
- raise InferenceManagerWorkerSetupError(
- f"Found no Imgg worker for '{img_gen_handle}', set it up or enable cogt.inference_manager_config.is_auto_setup_preset_img_gen"
- )
+ msg = f"Found no ImgGen worker for '{img_gen_handle}', set it up or enable cogt.inference_manager_config.is_auto_setup_preset_img_gen"
+ raise InferenceManagerWorkerSetupError(msg)
img_gen_worker = self._setup_one_img_gen_worker(img_gen_handle=img_gen_handle)
return img_gen_worker
@@ -146,13 +139,11 @@ class InferenceManager(InferenceManagerProtocol):
if ocr_worker := self.ocr_workers.get(model_handle):
return ocr_worker
if not get_config().cogt.inference_manager_config.is_auto_setup_preset_ocr:
- raise InferenceManagerWorkerSetupError(
- f"Found no OCR worker for '{model_handle}', set it up or enable cogt.inference_manager_config.is_auto_setup_preset_ocr"
- )
+ msg = f"Found no OCR worker for '{model_handle}', set it up or enable cogt.inference_manager_config.is_auto_setup_preset_ocr"
+ raise InferenceManagerWorkerSetupError(msg)
inference_model = get_models_manager().get_inference_model(model_handle=model_handle)
- ocr_worker = self._setup_one_ocr_worker(
+ return self._setup_one_ocr_worker(
inference_model=inference_model,
model_handle=model_handle,
)
- return ocr_worker
diff --git a/pipelex/cogt/inference/inference_manager_protocol.py b/pipelex/cogt/inference/inference_manager_protocol.py
index 30d4b82c9..e6ea5d4c8 100644
--- a/pipelex/cogt/inference/inference_manager_protocol.py
+++ b/pipelex/cogt/inference/inference_manager_protocol.py
@@ -1,4 +1,4 @@
-from typing import Protocol, Type
+from typing import Protocol
from pipelex.cogt.img_gen.img_gen_worker_abstract import ImgGenWorkerAbstract
from pipelex.cogt.llm.llm_worker_abstract import LLMWorkerAbstract
@@ -6,8 +6,7 @@ from pipelex.cogt.ocr.ocr_worker_abstract import OcrWorkerAbstract
class InferenceManagerProtocol(Protocol):
- """
- This is the protocol for the inference manager.
+ """This is the protocol for the inference manager.
Its point is only to avoid a circular import.
"""
@@ -22,7 +21,7 @@ class InferenceManagerProtocol(Protocol):
def set_llm_worker_from_external_plugin(
self,
llm_handle: str,
- llm_worker_class: Type[LLMWorkerAbstract],
+ llm_worker_class: type[LLMWorkerAbstract],
should_warn_if_already_registered: bool = True,
): ...
diff --git a/pipelex/cogt/inference/inference_worker_abstract.py b/pipelex/cogt/inference/inference_worker_abstract.py
index 22f4510d4..ff442b53d 100644
--- a/pipelex/cogt/inference/inference_worker_abstract.py
+++ b/pipelex/cogt/inference/inference_worker_abstract.py
@@ -1,5 +1,4 @@
from abc import ABC, abstractmethod
-from typing import Optional
from pipelex.reporting.reporting_protocol import ReportingProtocol
@@ -7,7 +6,7 @@ from pipelex.reporting.reporting_protocol import ReportingProtocol
class InferenceWorkerAbstract(ABC):
def __init__(
self,
- reporting_delegate: Optional[ReportingProtocol] = None,
+ reporting_delegate: ReportingProtocol | None = None,
):
self.reporting_delegate = reporting_delegate
diff --git a/pipelex/cogt/llm/llm_job_components.py b/pipelex/cogt/llm/llm_job_components.py
index 9dfb1b062..d43af1f43 100644
--- a/pipelex/cogt/llm/llm_job_components.py
+++ b/pipelex/cogt/llm/llm_job_components.py
@@ -1,5 +1,3 @@
-from typing import Optional
-
from pydantic import BaseModel, Field
from pipelex.cogt.llm.llm_report import LLMTokensUsage
@@ -11,8 +9,8 @@ from pipelex.cogt.llm.llm_report import LLMTokensUsage
class LLMJobParams(BaseModel):
temperature: float = Field(..., ge=0, le=1)
- max_tokens: Optional[int] = Field(None, gt=0)
- seed: Optional[int] = Field(None, ge=0)
+ max_tokens: int | None = Field(None, gt=0)
+ seed: int | None = Field(None, ge=0)
class LLMJobConfig(BaseModel):
@@ -26,4 +24,4 @@ class LLMJobConfig(BaseModel):
class LLMJobReport(BaseModel):
- llm_tokens_usage: Optional[LLMTokensUsage] = None
+ llm_tokens_usage: LLMTokensUsage | None = None
diff --git a/pipelex/cogt/llm/llm_job_factory.py b/pipelex/cogt/llm/llm_job_factory.py
index 0e0cc7e9f..930a3b3b5 100644
--- a/pipelex/cogt/llm/llm_job_factory.py
+++ b/pipelex/cogt/llm/llm_job_factory.py
@@ -1,4 +1,4 @@
-from typing import Any, Dict, List, Optional
+from typing import Any
from pipelex import log
from pipelex.cogt.image.prompt_image import PromptImage
@@ -18,8 +18,8 @@ class LLMJobFactory:
cls,
llm_prompt: LLMPrompt,
llm_job_params: LLMJobParams,
- llm_job_config: Optional[LLMJobConfig] = None,
- job_metadata: Optional[JobMetadata] = None,
+ llm_job_config: LLMJobConfig | None = None,
+ job_metadata: JobMetadata | None = None,
) -> LLMJob:
config = get_config()
llm_config = config.cogt.llm_config
@@ -27,7 +27,7 @@ class LLMJobFactory:
job_metadata.update(
JobMetadata(
job_category=JobCategory.LLM_JOB,
- )
+ ),
)
else:
job_metadata = JobMetadata(
@@ -49,10 +49,10 @@ class LLMJobFactory:
async def make_llm_job_from_prompt_factory(
cls,
llm_prompt_factory: LLMPromptFactoryAbstract,
- llm_prompt_arguments: Dict[str, Any],
+ llm_prompt_arguments: dict[str, Any],
llm_job_params: LLMJobParams,
- llm_job_config: Optional[LLMJobConfig] = None,
- job_metadata: Optional[JobMetadata] = None,
+ llm_job_config: LLMJobConfig | None = None,
+ job_metadata: JobMetadata | None = None,
) -> LLMJob:
log.debug(llm_prompt_arguments, title="llm_prompt_arguments")
llm_prompt = await llm_prompt_factory.make_llm_prompt_from_args(**llm_prompt_arguments)
@@ -69,10 +69,10 @@ class LLMJobFactory:
async def make_llm_job_from_prompt_template(
cls,
llm_prompt_template: LLMPromptTemplate,
- llm_prompt_arguments: Dict[str, Any],
+ llm_prompt_arguments: dict[str, Any],
llm_job_params: LLMJobParams,
- llm_job_config: Optional[LLMJobConfig] = None,
- job_metadata: Optional[JobMetadata] = None,
+ llm_job_config: LLMJobConfig | None = None,
+ job_metadata: JobMetadata | None = None,
) -> LLMJob:
log.debug(llm_prompt_arguments, title="llm_prompt_arguments")
llm_prompt = await llm_prompt_template.make_llm_prompt_from_args(**llm_prompt_arguments)
@@ -88,11 +88,11 @@ class LLMJobFactory:
def make_llm_job_from_prompt_contents(
cls,
llm_job_params: LLMJobParams,
- user_text: Optional[str] = None,
- system_text: Optional[str] = None,
- user_images: Optional[List[PromptImage]] = None,
- llm_job_config: Optional[LLMJobConfig] = None,
- job_metadata: Optional[JobMetadata] = None,
+ user_text: str | None = None,
+ system_text: str | None = None,
+ user_images: list[PromptImage] | None = None,
+ llm_job_config: LLMJobConfig | None = None,
+ job_metadata: JobMetadata | None = None,
) -> LLMJob:
config = get_config()
llm_config = config.cogt.llm_config
@@ -100,7 +100,7 @@ class LLMJobFactory:
job_metadata.update(
JobMetadata(
job_category=JobCategory.LLM_JOB,
- )
+ ),
)
else:
job_metadata = JobMetadata(
diff --git a/pipelex/cogt/llm/llm_prompt.py b/pipelex/cogt/llm/llm_prompt.py
index 38aa757af..fbcbe1d56 100644
--- a/pipelex/cogt/llm/llm_prompt.py
+++ b/pipelex/cogt/llm/llm_prompt.py
@@ -1,5 +1,3 @@
-from typing import List, Optional
-
from pydantic import BaseModel
from typing_extensions import override
@@ -11,9 +9,9 @@ from pipelex.tools.runtime_manager import ProblemReaction, runtime_manager
class LLMPrompt(BaseModel):
- system_text: Optional[str] = None
- user_text: Optional[str] = None
- user_images: List[PromptImage] = []
+ system_text: str | None = None
+ user_text: str | None = None
+ user_images: list[PromptImage] = []
def validate_before_execution(self):
reaction = runtime_manager.problem_reactions.job
@@ -25,9 +23,11 @@ class LLMPrompt(BaseModel):
if self.system_text == "":
log.warning(f"system_text should be None or contain text. system_text = '{self.system_text}'")
else:
- raise LLMPromptParameterError("system_text should be None or contain text")
+ msg = "system_text should be None or contain text"
+ raise LLMPromptParameterError(msg)
if not is_not_none_and_has_text(text=self.user_text):
- raise LLMPromptParameterError("user_text should contain text")
+ msg = "user_text should contain text"
+ raise LLMPromptParameterError(msg)
case ProblemReaction.LOG:
if not is_none_or_has_text(text=self.system_text):
if self.system_text == "":
@@ -51,7 +51,7 @@ class LLMPrompt(BaseModel):
def __format__(self, format_spec: str) -> str:
return self.desc()
- def desc(self, truncate_text_length: Optional[int] = None) -> str:
+ def desc(self, truncate_text_length: int | None = None) -> str:
description = "LLM Prompt:"
if truncate_text_length:
if self.system_text:
diff --git a/pipelex/cogt/llm/llm_prompt_spec.py b/pipelex/cogt/llm/llm_prompt_spec.py
index 8b4298f7f..59da2e258 100644
--- a/pipelex/cogt/llm/llm_prompt_spec.py
+++ b/pipelex/cogt/llm/llm_prompt_spec.py
@@ -1,11 +1,9 @@
-from typing import Any, Dict, List, Optional, Set
+from typing import TYPE_CHECKING, Any
from pydantic import BaseModel, model_validator
-from typing_extensions import Self
from pipelex import log
from pipelex.cogt.exceptions import LLMPromptSpecError
-from pipelex.cogt.image.prompt_image import PromptImage
from pipelex.cogt.image.prompt_image_factory import PromptImageFactory
from pipelex.cogt.llm.llm_prompt import LLMPrompt
from pipelex.core.stuffs.stuff_content import ImageContent
@@ -15,20 +13,24 @@ from pipelex.tools.templating.jinja2_blueprint import Jinja2Blueprint
from pipelex.tools.templating.jinja2_required_variables import detect_jinja2_required_variables
from pipelex.tools.templating.templating_models import PromptingStyle
from pipelex.tools.typing.validation_utils import has_exactly_one_among_attributes_from_list, has_more_than_one_among_attributes_from_list
+from pipelex.types import Self
+
+if TYPE_CHECKING:
+ from pipelex.cogt.image.prompt_image import PromptImage
class LLMPromptSpec(BaseModel):
- prompting_style: Optional[PromptingStyle] = None
+ prompting_style: PromptingStyle | None = None
- system_prompt_jinja2_blueprint: Optional[Jinja2Blueprint] = None
- system_prompt_verbatim_name: Optional[str] = None
- system_prompt: Optional[str] = None
+ system_prompt_jinja2_blueprint: Jinja2Blueprint | None = None
+ system_prompt_verbatim_name: str | None = None
+ system_prompt: str | None = None
- user_text_jinja2_blueprint: Optional[Jinja2Blueprint] = None
- user_prompt_verbatim_name: Optional[str] = None
- user_text: Optional[str] = None
+ user_text_jinja2_blueprint: Jinja2Blueprint | None = None
+ user_prompt_verbatim_name: str | None = None
+ user_text: str | None = None
- user_images: Optional[List[str]] = None
+ user_images: list[str] | None = None
@model_validator(mode="after")
def validate_user_text(self) -> Self:
@@ -40,9 +42,8 @@ class LLMPromptSpec(BaseModel):
"user_text",
],
):
- raise LLMPromptSpecError(
- f"LLMPromptSpec user text must have exactly one of user_text, user_text_jinja2_blueprint or user_prompt_verbatim_name: {self}"
- )
+ msg = f"LLMPromptSpec user text must have exactly one of user_text, user_text_jinja2_blueprint or user_prompt_verbatim_name: {self}"
+ raise LLMPromptSpecError(msg)
if has_more_than_one_among_attributes_from_list(
obj=self,
attributes_list=[
@@ -51,9 +52,8 @@ class LLMPromptSpec(BaseModel):
"system_prompt",
],
):
- raise LLMPromptSpecError(
- f"LLMPromptSpec system got more than one of system_prompt, system_prompt_jinja2_blueprint, system_prompt_verbatim_name: {self}"
- )
+ msg = f"LLMPromptSpec system got more than one of system_prompt, system_prompt_jinja2_blueprint, system_prompt_verbatim_name: {self}"
+ raise LLMPromptSpecError(msg)
return self
def validate_with_libraries(self):
@@ -69,8 +69,8 @@ class LLMPromptSpec(BaseModel):
the_template = get_template(template_name=self.system_prompt_jinja2_blueprint.jinja2_name)
log.debug(f"Validated jinja2 template '{self.system_prompt_jinja2_blueprint.jinja2_name}':\n{the_template}")
- def required_variables(self) -> Set[str]:
- required_variables: Set[str] = set()
+ def required_variables(self) -> set[str]:
+ required_variables: set[str] = set()
if self.user_images:
user_images_top_object_name = [user_image.split(".", 1)[0] for user_image in self.user_images]
required_variables.update(user_images_top_object_name)
@@ -85,7 +85,7 @@ class LLMPromptSpec(BaseModel):
return {
variable_name
for variable_name in required_variables
- if not variable_name.startswith("_") and variable_name != "preliminary_text" and variable_name != "place_holder"
+ if not variable_name.startswith("_") and variable_name not in ("preliminary_text", "place_holder")
}
# TODO: make this consistent with `LLMPromptFactoryAbstract` or `LLMPromptTemplate`,
@@ -94,22 +94,21 @@ class LLMPromptSpec(BaseModel):
self,
output_concept_string: str,
context_provider: ContextProviderAbstract,
- output_structure_prompt: Optional[str] = None,
- extra_params: Optional[Dict[str, Any]] = None,
+ output_structure_prompt: str | None = None,
+ extra_params: dict[str, Any] | None = None,
) -> LLMPrompt:
############################################################
# User images
############################################################
- prompt_user_images: List[PromptImage] = []
+ prompt_user_images: list[PromptImage] = []
if self.user_images:
for user_image_name in self.user_images:
log.debug(f"Getting user image '{user_image_name}' from context")
try:
prompt_image_content = context_provider.get_typed_object_or_attribute(name=user_image_name, wanted_type=ImageContent)
except ContextProviderException as exc:
- raise LLMPromptSpecError(
- f"Could not find a valid user image named '{user_image_name}' from the provided context_provider: {exc}"
- ) from exc
+ msg = f"Could not find a valid user image named '{user_image_name}' from the provided context_provider: {exc}"
+ raise LLMPromptSpecError(msg) from exc
if prompt_image_content is not None: # An ImageContent can be optional..
if base_64 := prompt_image_content.base_64:
@@ -130,7 +129,8 @@ class LLMPromptSpec(BaseModel):
extra_params=extra_params,
)
if not user_text:
- raise ValueError("For user_text we need either a pipe_jinja2, a text_verbatim_name or a fixed user_text")
+ msg = "For user_text we need either a pipe_jinja2, a text_verbatim_name or a fixed user_text"
+ raise ValueError(msg)
if output_structure_prompt:
user_text += output_structure_prompt
@@ -151,30 +151,28 @@ class LLMPromptSpec(BaseModel):
############################################################
# Full LLMPrompt
############################################################
- llm_prompt = LLMPrompt(
+ return LLMPrompt(
system_text=system_text,
user_text=user_text,
user_images=prompt_user_images,
)
- return llm_prompt
-
async def _unravel_text(
self,
context_provider: ContextProviderAbstract,
- jinja2_blueprint: Optional[Jinja2Blueprint],
- text_verbatim_name: Optional[str],
- fixed_text: Optional[str],
- extra_params: Optional[Dict[str, Any]] = None,
- ) -> Optional[str]:
- the_text: Optional[str]
+ jinja2_blueprint: Jinja2Blueprint | None,
+ text_verbatim_name: str | None,
+ fixed_text: str | None,
+ extra_params: dict[str, Any] | None = None,
+ ) -> str | None:
+ the_text: str | None
if jinja2_blueprint:
log.verbose(f"Working with Jinja2 pipe '{jinja2_blueprint.jinja2_name}'")
if (prompting_style := self.prompting_style) and not jinja2_blueprint.prompting_style:
jinja2_blueprint.prompting_style = prompting_style
log.verbose(f"Setting prompting style to {prompting_style}")
- context: Dict[str, Any] = context_provider.generate_context()
+ context: dict[str, Any] = context_provider.generate_context()
if extra_params:
context.update(**extra_params)
if jinja2_blueprint.extra_context:
@@ -192,7 +190,8 @@ class LLMPromptSpec(BaseModel):
template_name=text_verbatim_name,
)
if not user_text_verbatim:
- raise ValueError(f"Could not find text_verbatim template '{text_verbatim_name}'")
+ msg = f"Could not find text_verbatim template '{text_verbatim_name}'"
+ raise ValueError(msg)
the_text = user_text_verbatim
elif fixed_text:
the_text = fixed_text
diff --git a/pipelex/cogt/llm/llm_prompt_template.py b/pipelex/cogt/llm/llm_prompt_template.py
index d5fe01206..6be640f18 100644
--- a/pipelex/cogt/llm/llm_prompt_template.py
+++ b/pipelex/cogt/llm/llm_prompt_template.py
@@ -1,4 +1,4 @@
-from typing import Any, List, Optional
+from typing import Any
from typing_extensions import override
@@ -19,8 +19,8 @@ class LLMPromptTemplate(LLMPromptFactoryAbstract):
base_template_inputs: LLMPromptTemplateInputs = LLMPromptTemplateInputs()
# fields kept for reference and debugging only (they have no effect)
- source_system_template_name: Optional[str] = None
- source_user_template_name: Optional[str] = None
+ source_system_template_name: str | None = None
+ source_user_template_name: str | None = None
@property
@override
@@ -41,18 +41,18 @@ class LLMPromptTemplate(LLMPromptFactoryAbstract):
arguments_dict = prompt_arguments.copy()
# pop the base fields and then use the templating method
- system_text: Optional[str] = arguments_dict.pop("system_text", None)
- user_text: Optional[str] = arguments_dict.pop("user_text", None)
+ system_text: str | None = arguments_dict.pop("system_text", None)
+ user_text: str | None = arguments_dict.pop("user_text", None)
if not user_text:
user_text = self.proto_prompt.user_text
# user_images is Optional here: None means the template is not altering the user_images field
- user_images: Optional[List[PromptImage]] = None
+ user_images: list[PromptImage] | None = None
if "user_images" in arguments_dict:
user_images = arguments_dict.pop("user_images")
elif "user_image" in arguments_dict:
user_images = [arguments_dict.pop("user_image")]
- is_user_images_append: Optional[bool] = arguments_dict.pop("is_user_images_append", None)
+ is_user_images_append: bool | None = arguments_dict.pop("is_user_images_append", None)
return self.make_llm_prompt(
system_text=system_text,
@@ -64,19 +64,21 @@ class LLMPromptTemplate(LLMPromptFactoryAbstract):
def make_llm_prompt(
self,
- system_text: Optional[str] = None,
- user_text: Optional[str] = None,
- user_images: Optional[List[PromptImage]] = None,
- is_user_images_append: Optional[bool] = None,
- template_inputs: Optional[LLMPromptTemplateInputs] = None,
+ system_text: str | None = None,
+ user_text: str | None = None,
+ user_images: list[PromptImage] | None = None,
+ is_user_images_append: bool | None = None,
+ template_inputs: LLMPromptTemplateInputs | None = None,
) -> LLMPrompt:
if not is_none_or_has_text(system_text):
if system_text == "":
log.warning(f"Prompt template system_text should be None or contain text. system_text = '{system_text}'")
else:
- raise LLMPromptTemplateInputsError(f"Prompt template system_text should be None or contain text. system_text = '{system_text}'")
+ msg = f"Prompt template system_text should be None or contain text. system_text = '{system_text}'"
+ raise LLMPromptTemplateInputsError(msg)
if not is_none_or_has_text(user_text):
- raise LLMPromptTemplateInputsError(f"Prompt template user_text should be None or contain text. system_text = '{user_text}'")
+ msg = f"Prompt template user_text should be None or contain text. system_text = '{user_text}'"
+ raise LLMPromptTemplateInputsError(msg)
all_template_inputs = self.base_template_inputs.complemented_by(additional_template_inputs=template_inputs)
@@ -120,11 +122,11 @@ class LLMPromptTemplate(LLMPromptFactoryAbstract):
def from_template_name(
cls,
template_provider: TemplateProviderAbstract,
- system_text: Optional[str] = None,
- user_text: Optional[str] = None,
- user_images: Optional[List[PromptImage]] = None,
- system_template_name: Optional[str] = None,
- user_template_name: Optional[str] = None,
+ system_text: str | None = None,
+ user_text: str | None = None,
+ user_images: list[PromptImage] | None = None,
+ system_template_name: str | None = None,
+ user_template_name: str | None = None,
) -> "LLMPromptTemplate":
proto_prompt: LLMPrompt = LLMPrompt(
system_text=system_text,
@@ -145,9 +147,9 @@ class LLMPromptTemplate(LLMPromptFactoryAbstract):
@classmethod
def from_template_contents(
cls,
- system_text: Optional[str] = None,
- user_text: Optional[str] = None,
- user_images: Optional[List[PromptImage]] = None,
+ system_text: str | None = None,
+ user_text: str | None = None,
+ user_images: list[PromptImage] | None = None,
) -> "LLMPromptTemplate":
proto_prompt = LLMPrompt(
user_text=user_text,
diff --git a/pipelex/cogt/llm/llm_prompt_template_inputs.py b/pipelex/cogt/llm/llm_prompt_template_inputs.py
index 835a1c951..ff0b735fe 100644
--- a/pipelex/cogt/llm/llm_prompt_template_inputs.py
+++ b/pipelex/cogt/llm/llm_prompt_template_inputs.py
@@ -1,7 +1,9 @@
-from typing import Any, Dict, ItemsView, List, Optional
+from __future__ import annotations
+
+from typing import Any
from pydantic import ConfigDict, Field, RootModel, model_validator
-from typing_extensions import Self, override
+from typing_extensions import TYPE_CHECKING, override
from pipelex import log
from pipelex.cogt.exceptions import LLMPromptTemplateInputsError
@@ -9,7 +11,12 @@ from pipelex.tools.misc.json_utils import json_str
from pipelex.tools.misc.string_utils import can_inject_text
from pipelex.tools.runtime_manager import ProblemReaction, runtime_manager
-LLMPromptTemplateInputsDict = Dict[str, Any]
+if TYPE_CHECKING:
+ from collections.abc import ItemsView
+
+ from pipelex.types import Self
+
+LLMPromptTemplateInputsDict = dict[str, Any]
class LLMPromptTemplateInputs(RootModel[LLMPromptTemplateInputsDict]):
@@ -37,16 +44,16 @@ class LLMPromptTemplateInputs(RootModel[LLMPromptTemplateInputsDict]):
def items(self) -> ItemsView[str, Any]:
return self.root.items()
- def list_keys(self) -> List[str]:
+ def list_keys(self) -> list[str]:
return list(self.root.keys())
- def complemented_by(self, additional_template_inputs: Optional["LLMPromptTemplateInputs"]) -> "LLMPromptTemplateInputs":
+ def complemented_by(self, additional_template_inputs: LLMPromptTemplateInputs | None) -> LLMPromptTemplateInputs:
all_template_inputs = self.root.copy()
if additional_template_inputs:
all_template_inputs.update(additional_template_inputs.root)
return LLMPromptTemplateInputs(root=all_template_inputs)
- def complemented_by_dict(self, additional_inputs_dict: Optional[LLMPromptTemplateInputsDict]) -> "LLMPromptTemplateInputs":
+ def complemented_by_dict(self, additional_inputs_dict: LLMPromptTemplateInputsDict | None) -> LLMPromptTemplateInputs:
all_template_inputs = self.root.copy()
if additional_inputs_dict:
all_template_inputs.update(additional_inputs_dict)
@@ -57,5 +64,5 @@ class LLMPromptTemplateInputs(RootModel[LLMPromptTemplateInputsDict]):
return json_str(self.root, title="llm_prompt_template_inputs", is_spaced=True)
@staticmethod
- def from_args(**template_inputs: Any) -> "LLMPromptTemplateInputs":
+ def from_args(**template_inputs: Any) -> LLMPromptTemplateInputs:
return LLMPromptTemplateInputs(root=template_inputs)
diff --git a/pipelex/cogt/llm/llm_report.py b/pipelex/cogt/llm/llm_report.py
index 5febe49a4..24c29ec77 100644
--- a/pipelex/cogt/llm/llm_report.py
+++ b/pipelex/cogt/llm/llm_report.py
@@ -1,4 +1,4 @@
-from typing import Any, Dict
+from typing import Any
from pydantic import BaseModel
@@ -38,11 +38,11 @@ class LLMTokenCostReport(BaseModel):
nb_tokens_by_category: NbTokensByCategoryDict
costs_by_token_category: CostsByCategoryDict
- def as_flat_dictionary(self) -> Dict[str, Any]:
- the_dict: Dict[str, Any] = {}
+ def as_flat_dictionary(self) -> dict[str, Any]:
+ the_dict: dict[str, Any] = {}
dict_for_job_metadata = self.job_metadata.model_dump(serialize_as_any=True)
the_dict.update(dict_for_job_metadata)
- dict_for_llm: Dict[str, Any] = {
+ dict_for_llm: dict[str, Any] = {
LLMTokenCostReportField.LLM_NAME: self.inference_model_name,
LLMTokenCostReportField.PLATFORM_LLM_ID: self.platform_llm_id,
}
diff --git a/pipelex/cogt/llm/llm_setting.py b/pipelex/cogt/llm/llm_setting.py
index 495499b4b..f01b21a78 100644
--- a/pipelex/cogt/llm/llm_setting.py
+++ b/pipelex/cogt/llm/llm_setting.py
@@ -1,31 +1,26 @@
-from typing import Literal, Optional, Set, Union
+from typing import Literal, Union
from pydantic import Field, field_validator
-from typing_extensions import Self
from pipelex.cogt.llm.llm_job_components import LLMJobParams
from pipelex.cogt.model_backends.prompting_target import PromptingTarget
from pipelex.tools.config.config_model import ConfigModel
-from pipelex.tools.exceptions import ConfigValidationError
+from pipelex.types import Self
class LLMSetting(ConfigModel):
llm_handle: str
temperature: float = Field(..., ge=0, le=1)
- max_tokens: Optional[int] = None
- prompting_target: Optional[PromptingTarget] = Field(default=None, strict=False)
+ max_tokens: int | None = None
+ prompting_target: PromptingTarget | None = Field(default=None, strict=False)
@field_validator("max_tokens", mode="before")
@classmethod
- def validate_max_tokens(cls, value: Union[int, Literal["auto"], None]) -> Optional[int]:
- if value is None:
+ def validate_max_tokens(cls, value: int | Literal["auto"] | None) -> int | None:
+ if value is None or (isinstance(value, str) and value == "auto"):
return None
- elif isinstance(value, str) and value == "auto":
- return None
- elif isinstance(value, int): # pyright: ignore[reportUnnecessaryIsInstance]
+ if isinstance(value, int): # pyright: ignore[reportUnnecessaryIsInstance]
return value
- else:
- raise ConfigValidationError(f'Invalid max_tokens shoubd be an int or "auto" but it is a {type(value)}: {value}')
def make_llm_job_params(self) -> LLMJobParams:
return LLMJobParams(
@@ -50,26 +45,17 @@ class LLMSettingChoicesDefaults(ConfigModel):
class LLMSettingChoices(ConfigModel):
- for_text: Optional[LLMChoice]
- for_object: Optional[LLMChoice]
+ for_text: LLMChoice | None
+ for_object: LLMChoice | None
- def list_choices(self) -> Set[str]:
- return set(
- [
- choice
- for choice in [
- self.for_text,
- self.for_object,
- ]
- if isinstance(choice, str)
- ]
- )
+ def list_choices(self) -> set[str]:
+ return {c for c in (self.for_text, self.for_object) if isinstance(c, str)}
@classmethod
def make_completed_with_defaults(
cls,
- for_text: Optional[LLMChoice] = None,
- for_object: Optional[LLMChoice] = None,
+ for_text: LLMChoice | None = None,
+ for_object: LLMChoice | None = None,
) -> Self:
return cls(
for_text=for_text,
diff --git a/pipelex/cogt/llm/llm_worker_abstract.py b/pipelex/cogt/llm/llm_worker_abstract.py
index d8569aea1..12851364e 100644
--- a/pipelex/cogt/llm/llm_worker_abstract.py
+++ b/pipelex/cogt/llm/llm_worker_abstract.py
@@ -1,26 +1,30 @@
+from __future__ import annotations
+
from abc import ABC, abstractmethod
-from typing import Optional, Type
+from typing import TYPE_CHECKING
from typing_extensions import override
from pipelex import log
from pipelex.cogt.inference.inference_worker_abstract import InferenceWorkerAbstract
-from pipelex.cogt.llm.llm_job import LLMJob
from pipelex.pipeline.job_metadata import UnitJobId
-from pipelex.reporting.reporting_protocol import ReportingProtocol
-from pipelex.tools.typing.pydantic_utils import BaseModelTypeVar
+
+if TYPE_CHECKING:
+ from pipelex.cogt.llm.llm_job import LLMJob
+ from pipelex.reporting.reporting_protocol import ReportingProtocol
+ from pipelex.tools.typing.pydantic_utils import BaseModelTypeVar
class LLMWorkerAbstract(InferenceWorkerAbstract, ABC):
def __init__(
self,
- reporting_delegate: Optional[ReportingProtocol] = None,
+ reporting_delegate: ReportingProtocol | None = None,
):
- """
- Initialize the LLMWorker.
+ """Initialize the LLMWorker.
Args:
- reporting_delegate (Optional[ReportingProtocol]): An optional report delegate for reporting unit jobs.
+ reporting_delegate (ReportingProtocol | None): An optional report delegate for reporting unit jobs.
+
"""
InferenceWorkerAbstract.__init__(self, reporting_delegate=reporting_delegate)
@@ -89,7 +93,7 @@ class LLMWorkerAbstract(InferenceWorkerAbstract, ABC):
async def gen_object(
self,
llm_job: LLMJob,
- schema: Type[BaseModelTypeVar],
+ schema: type[BaseModelTypeVar],
) -> BaseModelTypeVar:
log.debug(f"LLM Worker gen_object using {self.desc}")
log.verbose(llm_job.llm_prompt.desc(), title="llm_prompt")
@@ -114,6 +118,6 @@ class LLMWorkerAbstract(InferenceWorkerAbstract, ABC):
async def _gen_object(
self,
llm_job: LLMJob,
- schema: Type[BaseModelTypeVar],
+ schema: type[BaseModelTypeVar],
) -> BaseModelTypeVar:
pass
diff --git a/pipelex/cogt/llm/llm_worker_factory.py b/pipelex/cogt/llm/llm_worker_factory.py
index bd7ac90ef..c239c69a1 100644
--- a/pipelex/cogt/llm/llm_worker_factory.py
+++ b/pipelex/cogt/llm/llm_worker_factory.py
@@ -1,5 +1,3 @@
-from typing import Optional
-
from pipelex.cogt.exceptions import MissingDependencyError
from pipelex.cogt.llm.llm_worker_internal_abstract import LLMWorkerInternalAbstract
from pipelex.cogt.llm.structured_output import StructureMethod
@@ -14,7 +12,7 @@ class LLMWorkerFactory:
@staticmethod
def make_llm_worker(
inference_model: InferenceModelSpec,
- reporting_delegate: Optional[ReportingProtocol] = None,
+ reporting_delegate: ReportingProtocol | None = None,
) -> LLMWorkerInternalAbstract:
plugin = Plugin.make_for_inference_model(inference_model=inference_model)
backend = get_models_manager().get_required_inference_backend(inference_model.backend_name)
@@ -22,13 +20,13 @@ class LLMWorkerFactory:
llm_worker: LLMWorkerInternalAbstract
match plugin.sdk:
case "openai" | "azure_openai":
- from pipelex.plugins.openai.openai_factory import OpenAIFactory
+ from pipelex.plugins.openai.openai_factory import OpenAIFactory # noqa: PLC0415
- structure_method: Optional[StructureMethod] = None
+ structure_method: StructureMethod | None = None
if get_config().cogt.llm_config.instructor_config.is_openai_structured_output_enabled:
structure_method = StructureMethod.INSTRUCTOR_OPENAI_STRUCTURED
- from pipelex.plugins.openai.openai_llm_worker import OpenAILLMWorker
+ from pipelex.plugins.openai.openai_llm_worker import OpenAILLMWorker # noqa: PLC0415
sdk_instance = plugin_sdk_registry.get_sdk_instance(plugin=plugin) or plugin_sdk_registry.set_sdk_instance(
plugin=plugin,
@@ -46,20 +44,23 @@ class LLMWorkerFactory:
)
case "anthropic" | "bedrock_anthropic":
try:
- import anthropic # noqa: F401
+ import anthropic # noqa: PLC0415,F401
except ImportError as exc:
+ lib_name = "anthropic"
+ lib_extra_name = "anthropic"
+ msg = (
+ "The anthropic SDK is required to use Anthropic models via the anthropic client. "
+ "However, you can use Anthropic models through bedrock directly "
+ "by using the 'bedrock-anthropic-claude' llm family. (eg: bedrock-anthropic-claude)"
+ )
raise MissingDependencyError(
- "anthropic",
- "anthropic",
- (
- "The anthropic SDK is required to use Anthropic models via the anthropic client. "
- "However, you can use Anthropic models through bedrock directly "
- "by using the 'bedrock-anthropic-claude' llm family. (eg: bedrock-anthropic-claude)"
- ),
+ lib_name,
+ lib_extra_name,
+ msg,
) from exc
- from pipelex.plugins.anthropic.anthropic_factory import AnthropicFactory
- from pipelex.plugins.anthropic.anthropic_llm_worker import AnthropicLLMWorker
+ from pipelex.plugins.anthropic.anthropic_factory import AnthropicFactory # noqa: PLC0415
+ from pipelex.plugins.anthropic.anthropic_llm_worker import AnthropicLLMWorker # noqa: PLC0415
sdk_instance = plugin_sdk_registry.get_sdk_instance(plugin=plugin) or plugin_sdk_registry.set_sdk_instance(
plugin=plugin,
@@ -75,20 +76,23 @@ class LLMWorkerFactory:
)
case "mistral":
try:
- import mistralai # noqa: F401
+ import mistralai # noqa: PLC0415,F401
except ImportError as exc:
+ lib_name = "mistralai"
+ lib_extra_name = "mistral"
+ msg = (
+ "The mistralai SDK is required to use Mistral models through the mistralai client. "
+ "However, you can use Mistral models through bedrock directly "
+ "by using the 'bedrock-mistral' llm family. (eg: bedrock-mistral-large)"
+ )
raise MissingDependencyError(
- "mistralai",
- "mistral",
- (
- "The mistralai SDK is required to use Mistral models through the mistralai client. "
- "However, you can use Mistral models through bedrock directly "
- "by using the 'bedrock-mistral' llm family. (eg: bedrock-mistral-large)"
- ),
+ lib_name,
+ lib_extra_name,
+ msg,
) from exc
- from pipelex.plugins.mistral.mistral_factory import MistralFactory
- from pipelex.plugins.mistral.mistral_llm_worker import MistralLLMWorker
+ from pipelex.plugins.mistral.mistral_factory import MistralFactory # noqa: PLC0415
+ from pipelex.plugins.mistral.mistral_llm_worker import MistralLLMWorker # noqa: PLC0415
sdk_instance = plugin_sdk_registry.get_sdk_instance(plugin=plugin) or plugin_sdk_registry.set_sdk_instance(
plugin=plugin,
@@ -103,15 +107,20 @@ class LLMWorkerFactory:
)
case "bedrock_boto3" | "bedrock_aioboto3":
try:
- import aioboto3 # noqa: F401
- import boto3 # noqa: F401
+ import aioboto3 # noqa: PLC0415,F401
+ import boto3 # noqa: PLC0415,F401
except ImportError as exc:
+ lib_name = "boto3,aioboto3"
+ lib_extra_name = "bedrock"
+ msg = "The boto3 and aioboto3 SDKs are required to use Bedrock models."
raise MissingDependencyError(
- "boto3,aioboto3", "bedrock", "The boto3 and aioboto3 SDKs are required to use Bedrock models."
+ lib_name,
+ lib_extra_name,
+ msg,
) from exc
- from pipelex.plugins.bedrock.bedrock_factory import BedrockFactory
- from pipelex.plugins.bedrock.bedrock_llm_worker import BedrockLLMWorker
+ from pipelex.plugins.bedrock.bedrock_factory import BedrockFactory # noqa: PLC0415
+ from pipelex.plugins.bedrock.bedrock_llm_worker import BedrockLLMWorker # noqa: PLC0415
sdk_instance = plugin_sdk_registry.get_sdk_instance(plugin=plugin) or plugin_sdk_registry.set_sdk_instance(
plugin=plugin,
@@ -125,16 +134,19 @@ class LLMWorkerFactory:
)
case "google":
try:
- import google.genai # noqa: F401
+ import google.genai # noqa: PLC0415,F401
except ImportError as exc:
+ lib_name = "google-genai"
+ lib_extra_name = "google"
+ msg = "The google-genai SDK is required to use Google Gemini API directly. You can install it with 'pip install google-genai'."
raise MissingDependencyError(
- "google-genai",
- "google",
- ("The google-genai SDK is required to use Google Gemini API directly. You can install it with 'pip install google-genai'."),
+ lib_name,
+ lib_extra_name,
+ msg,
) from exc
- from pipelex.plugins.google.google_factory import GoogleFactory
- from pipelex.plugins.google.google_llm_worker import GoogleLLMWorker
+ from pipelex.plugins.google.google_factory import GoogleFactory # noqa: PLC0415
+ from pipelex.plugins.google.google_llm_worker import GoogleLLMWorker # noqa: PLC0415
sdk_instance = plugin_sdk_registry.get_sdk_instance(plugin=plugin) or plugin_sdk_registry.set_sdk_instance(
plugin=plugin,
@@ -148,5 +160,6 @@ class LLMWorkerFactory:
reporting_delegate=reporting_delegate,
)
case _:
- raise NotImplementedError(f"Plugin '{plugin}' is not supported")
+ msg = f"Plugin '{plugin}' is not supported"
+ raise NotImplementedError(msg)
return llm_worker
diff --git a/pipelex/cogt/llm/llm_worker_internal_abstract.py b/pipelex/cogt/llm/llm_worker_internal_abstract.py
index 4ac2b935b..dd422171f 100644
--- a/pipelex/cogt/llm/llm_worker_internal_abstract.py
+++ b/pipelex/cogt/llm/llm_worker_internal_abstract.py
@@ -1,5 +1,3 @@
-from typing import Optional
-
from typing_extensions import override
from pipelex import log
@@ -15,16 +13,16 @@ class LLMWorkerInternalAbstract(LLMWorkerAbstract):
def __init__(
self,
inference_model: InferenceModelSpec,
- structure_method: Optional[StructureMethod] = None,
- reporting_delegate: Optional[ReportingProtocol] = None,
+ structure_method: StructureMethod | None = None,
+ reporting_delegate: ReportingProtocol | None = None,
):
- """
- Initialize the LLMWorker.
+ """Initialize the LLMWorker.
Args:
inference_model (InferenceModelSpec): The inference model to be used by the worker.
- structure_method (Optional[StructureMethod]): The structure method to be used by the worker.
- reporting_delegate (Optional[ReportingProtocol]): An optional report delegate for reporting unit jobs.
+ structure_method (StructureMethod | None): The structure method to be used by the worker.
+ reporting_delegate (ReportingProtocol | None): An optional report delegate for reporting unit jobs.
+
"""
LLMWorkerAbstract.__init__(self, reporting_delegate=reporting_delegate)
self.inference_model = inference_model
@@ -61,9 +59,11 @@ class LLMWorkerInternalAbstract(LLMWorkerAbstract):
def _check_vision_support(self, llm_job: LLMJob):
if llm_job.llm_prompt.user_images:
if not self.inference_model.is_vision_supported:
- raise LLMCapabilityError(f"LLM Engine '{self.inference_model.tag}' does not support vision.")
+ msg = f"LLM Engine '{self.inference_model.tag}' does not support vision."
+ raise LLMCapabilityError(msg)
nb_images = len(llm_job.llm_prompt.user_images)
max_prompt_images = self.inference_model.max_prompt_images or 5000
if nb_images > max_prompt_images:
- raise LLMCapabilityError(f"LLM Engine '{self.inference_model.tag}' does not accept that many images: {nb_images}.")
+ msg = f"LLM Engine '{self.inference_model.tag}' does not accept that many images: {nb_images}."
+ raise LLMCapabilityError(msg)
diff --git a/pipelex/cogt/model_backends/backend.py b/pipelex/cogt/model_backends/backend.py
index 38046a638..94ec65cd5 100644
--- a/pipelex/cogt/model_backends/backend.py
+++ b/pipelex/cogt/model_backends/backend.py
@@ -1,4 +1,4 @@
-from typing import Any, Dict, List, Optional
+from typing import Any
from pydantic import Field
@@ -8,20 +8,19 @@ from pipelex.tools.config.config_model import ConfigModel
class InferenceBackend(ConfigModel):
name: str
- endpoint: Optional[str] = None
- api_key: Optional[str] = None
- extra_config: Dict[str, Any] = Field(default_factory=dict)
- model_specs: Dict[str, InferenceModelSpec] = Field(default_factory=dict)
+ endpoint: str | None = None
+ api_key: str | None = None
+ extra_config: dict[str, Any] = Field(default_factory=dict)
+ model_specs: dict[str, InferenceModelSpec] = Field(default_factory=dict)
- def list_model_names(self) -> List[str]:
+ def list_model_names(self) -> list[str]:
"""List the names of all models in the backend."""
return list(self.model_specs.keys())
- def get_model_spec(self, model_name: str) -> Optional[InferenceModelSpec]:
+ def get_model_spec(self, model_name: str) -> InferenceModelSpec | None:
"""Get a model spec by name."""
- model_spec = self.model_specs.get(model_name)
- return model_spec
+ return self.model_specs.get(model_name)
- def get_extra_config(self, key: str) -> Optional[Any]:
+ def get_extra_config(self, key: str) -> Any | None:
"""Get an extra config by key."""
return self.extra_config.get(key)
diff --git a/pipelex/cogt/model_backends/backend_factory.py b/pipelex/cogt/model_backends/backend_factory.py
index 17c943fbf..352fe5b75 100644
--- a/pipelex/cogt/model_backends/backend_factory.py
+++ b/pipelex/cogt/model_backends/backend_factory.py
@@ -1,4 +1,4 @@
-from typing import Any, Dict, Optional
+from typing import Any
from pydantic import Field
@@ -10,9 +10,9 @@ from pipelex.tools.config.config_model import ConfigModel
class InferenceBackendBlueprint(ConfigModel):
enabled: bool = True
- endpoint: Optional[str] = None
- api_key: Optional[str] = None
- extra_config: Dict[str, Any] = Field(default_factory=dict)
+ endpoint: str | None = None
+ api_key: str | None = None
+ extra_config: dict[str, Any] = Field(default_factory=dict)
class InferenceBackendFactory:
@@ -21,8 +21,8 @@ class InferenceBackendFactory:
cls,
name: str,
blueprint: InferenceBackendBlueprint,
- extra_config: Dict[str, Any],
- model_specs: Dict[str, InferenceModelSpec],
+ extra_config: dict[str, Any],
+ model_specs: dict[str, InferenceModelSpec],
) -> InferenceBackend:
endpoint = blueprint.endpoint
api_key = blueprint.api_key
diff --git a/pipelex/cogt/model_backends/backend_library.py b/pipelex/cogt/model_backends/backend_library.py
index 6d52df12f..d9d27249c 100644
--- a/pipelex/cogt/model_backends/backend_library.py
+++ b/pipelex/cogt/model_backends/backend_library.py
@@ -1,7 +1,6 @@
-from typing import Any, Dict, List, Optional, Set, cast
+from typing import TYPE_CHECKING, Any, cast
from pydantic import Field, RootModel, ValidationError
-from typing_extensions import Self
from pipelex import log
from pipelex.cogt.exceptions import (
@@ -12,15 +11,18 @@ from pipelex.cogt.exceptions import (
)
from pipelex.cogt.model_backends.backend import InferenceBackend
from pipelex.cogt.model_backends.backend_factory import InferenceBackendBlueprint, InferenceBackendFactory
-from pipelex.cogt.model_backends.model_spec import InferenceModelSpec
from pipelex.cogt.model_backends.model_spec_factory import InferenceModelSpecBlueprint, InferenceModelSpecFactory
from pipelex.config import get_config
from pipelex.tools.misc.dict_utils import apply_to_strings_recursive
from pipelex.tools.misc.toml_utils import load_toml_from_path
from pipelex.tools.runtime_manager import runtime_manager
from pipelex.tools.secrets.secrets_utils import UnknownVarPrefixError, VarFallbackPatternError, VarNotFoundError, substitute_vars
+from pipelex.types import Self
-InferenceBackendLibraryRoot = Dict[str, InferenceBackend]
+if TYPE_CHECKING:
+ from pipelex.cogt.model_backends.model_spec import InferenceModelSpec
+
+InferenceBackendLibraryRoot = dict[str, InferenceBackend]
class InferenceBackendLibrary(RootModel[InferenceBackendLibraryRoot]):
@@ -38,11 +40,12 @@ class InferenceBackendLibrary(RootModel[InferenceBackendLibraryRoot]):
try:
backends_dict = load_toml_from_path(path=backends_library_path)
except (FileNotFoundError, InferenceBackendLibraryError) as exc:
- raise InferenceBackendLibraryError(f"Failed to load inference backend library from file '{backends_library_path}': {exc}") from exc
+ msg = f"Failed to load inference backend library from file '{backends_library_path}': {exc}"
+ raise InferenceBackendLibraryError(msg) from exc
for backend_name, backend_dict in backends_dict.items():
# We'll split the read settings into standard fields and extra config
standard_fields = InferenceBackendBlueprint.model_fields.keys()
- extra_config: Dict[str, Any] = {}
+ extra_config: dict[str, Any] = {}
inference_backend_blueprint_dict_raw = backend_dict.copy()
if not inference_backend_blueprint_dict_raw.get("enabled", True):
continue
@@ -51,20 +54,23 @@ class InferenceBackendLibrary(RootModel[InferenceBackendLibraryRoot]):
try:
inference_backend_blueprint_dict = apply_to_strings_recursive(inference_backend_blueprint_dict_raw, substitute_vars)
except VarFallbackPatternError as var_fallback_pattern_exc:
+ msg = f"Variable substitution failed due to a pattern error in file '{backends_library_path}':\n{var_fallback_pattern_exc}"
+ key_name = "unknown"
raise InferenceBackendCredentialsError(
error_type=InferenceBackendCredentialsErrorType.VAR_FALLBACK_PATTERN,
backend_name=backend_name,
- message=f"Variable substitution failed due to a pattern error in file '{backends_library_path}':\n{var_fallback_pattern_exc}",
- key_name="unknown",
+ message=msg,
+ key_name=key_name,
) from var_fallback_pattern_exc
except VarNotFoundError as var_not_found_exc:
+ msg = (
+ f"Variable substitution failed due to a variable not found error in file '{backends_library_path}':"
+ "\n{var_not_found_exc}\nRun mode: '{runtime_manager.run_mode}'"
+ )
raise InferenceBackendCredentialsError(
error_type=InferenceBackendCredentialsErrorType.VAR_NOT_FOUND,
backend_name=backend_name,
- message=(
- f"Variable substitution failed due to a variable not found error in file '{backends_library_path}':\n"
- f"{var_not_found_exc}\nRun mode: '{runtime_manager.run_mode}'"
- ),
+ message=msg,
key_name=var_not_found_exc.var_name,
) from var_not_found_exc
except UnknownVarPrefixError as unknown_var_prefix_exc:
@@ -78,7 +84,7 @@ class InferenceBackendLibrary(RootModel[InferenceBackendLibraryRoot]):
key_name=unknown_var_prefix_exc.var_name,
) from unknown_var_prefix_exc
- for key in backend_dict.keys():
+ for key in backend_dict:
if key not in standard_fields:
extra_config[key] = inference_backend_blueprint_dict.pop(key)
backend_blueprint = InferenceBackendBlueprint.model_validate(inference_backend_blueprint_dict)
@@ -91,17 +97,18 @@ class InferenceBackendLibrary(RootModel[InferenceBackendLibraryRoot]):
try:
model_specs_dict = apply_to_strings_recursive(model_specs_dict_raw, substitute_vars)
except (VarNotFoundError, UnknownVarPrefixError) as exc:
- raise InferenceModelSpecError(f"Variable substitution failed in file '{path_to_model_specs_toml}': {exc}") from exc
+ msg = f"Variable substitution failed in file '{path_to_model_specs_toml}': {exc}"
+ raise InferenceModelSpecError(msg) from exc
except (FileNotFoundError, InferenceModelSpecError) as exc:
- raise InferenceBackendLibraryError(f"Failed to load inference model specs from file '{path_to_model_specs_toml}': {exc}") from exc
- defaults_dict: Dict[str, Any] = model_specs_dict.pop("defaults", {})
- backend_model_specs: Dict[str, InferenceModelSpec] = {}
+ msg = f"Failed to load inference model specs from file '{path_to_model_specs_toml}': {exc}"
+ raise InferenceBackendLibraryError(msg) from exc
+ defaults_dict: dict[str, Any] = model_specs_dict.pop("defaults", {})
+ backend_model_specs: dict[str, InferenceModelSpec] = {}
for model_spec_name, value in model_specs_dict.items():
if not isinstance(value, dict):
- raise InferenceModelSpecError(
- f"Model spec '{model_spec_name}' for backend '{backend_name}' at path '{path_to_model_specs_toml}' is not a dictionary"
- )
- model_spec_dict: Dict[str, Any] = cast(Dict[str, Any], value)
+ msg = f"Model spec '{model_spec_name}' for backend '{backend_name}' at path '{path_to_model_specs_toml}' is not a dictionary"
+ raise InferenceModelSpecError(msg)
+ model_spec_dict: dict[str, Any] = cast("dict[str, Any]", value)
try:
# Start from the defaults
model_spec_blueprint_dict = defaults_dict.copy()
@@ -115,10 +122,10 @@ class InferenceBackendLibrary(RootModel[InferenceBackendLibraryRoot]):
)
backend_model_specs[model_spec_name] = model_spec
except (InferenceModelSpecError, ValidationError) as exc:
- raise InferenceBackendLibraryError(
- f"Failed to load inference model spec '{model_spec_name}' for backend '{backend_name}' "
- f"from file '{path_to_model_specs_toml}': {exc}"
+ msg = (
+ f"Failed to load inference model spec '{model_spec_name}' for backend '{backend_name}' from file '{path_to_model_specs_toml}'"
)
+ raise InferenceBackendLibraryError(msg) from exc
backend = InferenceBackendFactory.make_inference_backend(
name=backend_name,
blueprint=backend_blueprint,
@@ -128,19 +135,19 @@ class InferenceBackendLibrary(RootModel[InferenceBackendLibraryRoot]):
self.root[backend_name] = backend
log.debug(f"Loaded inference backend '{backend_name}'")
- def list_backend_names(self) -> List[str]:
+ def list_backend_names(self) -> list[str]:
return list(self.root.keys())
- def list_all_model_names(self) -> List[str]:
+ def list_all_model_names(self) -> list[str]:
"""List the names of all models in all backends."""
- all_model_names: Set[str] = set()
+ all_model_names: set[str] = set()
for backend in self.root.values():
all_model_names.update(backend.list_model_names())
return sorted(all_model_names)
- def get_all_models_and_possible_backends(self) -> Dict[str, List[str]]:
+ def get_all_models_and_possible_backends(self) -> dict[str, list[str]]:
"""Get a dictionary of all models and their possible backends."""
- all_models_and_possible_backends: Dict[str, List[str]] = {}
+ all_models_and_possible_backends: dict[str, list[str]] = {}
for backend in self.root.values():
for model_name in backend.list_model_names():
if model_name not in all_models_and_possible_backends:
@@ -148,7 +155,5 @@ class InferenceBackendLibrary(RootModel[InferenceBackendLibraryRoot]):
all_models_and_possible_backends[model_name].append(backend.name)
return all_models_and_possible_backends
- def get_inference_backend(self, backend_name: str) -> Optional[InferenceBackend]:
- """Get a backend by name."""
- backend = self.root.get(backend_name)
- return backend
+ def get_inference_backend(self, backend_name: str) -> InferenceBackend | None:
+ return self.root.get(backend_name)
diff --git a/pipelex/cogt/model_backends/model_spec.py b/pipelex/cogt/model_backends/model_spec.py
index 6a70a0cad..30568f829 100644
--- a/pipelex/cogt/model_backends/model_spec.py
+++ b/pipelex/cogt/model_backends/model_spec.py
@@ -1,5 +1,3 @@
-from typing import List, Optional
-
from pydantic import Field
from pipelex.cogt.model_backends.model_constraints import ModelConstraints
@@ -7,6 +5,7 @@ from pipelex.cogt.model_backends.model_type import ModelType
from pipelex.cogt.model_backends.prompting_target import PromptingTarget
from pipelex.cogt.usage.cost_category import CostsByCategoryDict
from pipelex.tools.config.config_model import ConfigModel
+from pipelex.tools.typing.pydantic_utils import empty_list_factory_of
class InferenceModelSpec(ConfigModel):
@@ -15,13 +14,13 @@ class InferenceModelSpec(ConfigModel):
sdk: str
model_type: ModelType = Field(strict=False)
model_id: str
- inputs: List[str] = Field(default_factory=list)
- outputs: List[str] = Field(default_factory=list)
+ inputs: list[str] = Field(default_factory=list)
+ outputs: list[str] = Field(default_factory=list)
costs: CostsByCategoryDict = Field(strict=False)
- max_tokens: Optional[int]
- max_prompt_images: Optional[int]
- prompting_target: Optional[PromptingTarget] = Field(default=None, strict=False)
- constraints: List[ModelConstraints] = Field(default_factory=list)
+ max_tokens: int | None
+ max_prompt_images: int | None
+ prompting_target: PromptingTarget | None = Field(default=None, strict=False)
+ constraints: list[ModelConstraints] = Field(default_factory=empty_list_factory_of(ModelConstraints))
@property
def tag(self) -> str:
diff --git a/pipelex/cogt/model_backends/model_spec_factory.py b/pipelex/cogt/model_backends/model_spec_factory.py
index ed74d5ea9..c251282d8 100644
--- a/pipelex/cogt/model_backends/model_spec_factory.py
+++ b/pipelex/cogt/model_backends/model_spec_factory.py
@@ -1,5 +1,3 @@
-from typing import Dict, List, Optional
-
from pydantic import BaseModel, Field, field_validator
from pipelex.cogt.model_backends.model_constraints import ModelConstraints
@@ -8,6 +6,7 @@ from pipelex.cogt.model_backends.model_type import ModelType
from pipelex.cogt.model_backends.prompting_target import PromptingTarget
from pipelex.cogt.usage.cost_category import CostCategory, CostsByCategoryDict
from pipelex.tools.config.config_model import ConfigModel
+from pipelex.tools.typing.pydantic_utils import empty_list_factory_of
class InferenceModelSpecBlueprint(ConfigModel):
@@ -15,23 +14,25 @@ class InferenceModelSpecBlueprint(ConfigModel):
sdk: str
model_type: ModelType = Field(default=ModelType.LLM, strict=False)
model_id: str
- inputs: List[str] = Field(default_factory=list)
- outputs: List[str] = Field(default_factory=list)
+ inputs: list[str] = Field(default_factory=list)
+ outputs: list[str] = Field(default_factory=list)
costs: CostsByCategoryDict = Field(strict=False)
- max_tokens: Optional[int] = None
- max_prompt_images: Optional[int] = None
- prompting_target: Optional[PromptingTarget] = Field(default=None, strict=False)
- constraints: List[ModelConstraints] = Field(default_factory=list)
+ max_tokens: int | None = None
+ max_prompt_images: int | None = None
+ prompting_target: PromptingTarget | None = Field(default=None, strict=False)
+ constraints: list[ModelConstraints] = Field(default_factory=empty_list_factory_of(ModelConstraints))
@field_validator("costs", mode="before")
- def validate_costs(cls, value: Dict[str, float]) -> CostsByCategoryDict:
+ @staticmethod
+ def validate_costs(value: dict[str, float]) -> CostsByCategoryDict:
return ConfigModel.transform_dict_of_floats_str_to_enum(
input_dict=value,
key_enum_cls=CostCategory,
)
@field_validator("constraints", mode="before")
- def validate_constraints(cls, value: List[str]) -> List[ModelConstraints]:
+ @staticmethod
+ def validate_constraints(value: list[str]) -> list[ModelConstraints]:
return ConfigModel.transform_list_of_str_to_enum(
input_list=value,
enum_cls=ModelConstraints,
diff --git a/pipelex/cogt/model_routing/routing_models.py b/pipelex/cogt/model_routing/routing_models.py
index e7826aa28..b289b8509 100644
--- a/pipelex/cogt/model_routing/routing_models.py
+++ b/pipelex/cogt/model_routing/routing_models.py
@@ -1,5 +1,3 @@
-from typing import Optional
-
from pydantic import BaseModel
from pipelex.types import StrEnum
@@ -16,4 +14,4 @@ class BackendMatchForModel(BaseModel):
backend_name: str
routing_profile_name: str
matching_method: BackendMatchingMethod
- matched_pattern: Optional[str]
+ matched_pattern: str | None
diff --git a/pipelex/cogt/model_routing/routing_profile.py b/pipelex/cogt/model_routing/routing_profile.py
index fc3081412..109dfbf7a 100644
--- a/pipelex/cogt/model_routing/routing_profile.py
+++ b/pipelex/cogt/model_routing/routing_profile.py
@@ -1,5 +1,3 @@
-from typing import Dict, Optional
-
from pydantic import Field
from pipelex.cogt.model_routing.routing_models import BackendMatchForModel, BackendMatchingMethod
@@ -11,11 +9,11 @@ class RoutingProfile(ConfigModel):
"""Configuration for model routing to backends."""
name: str
- description: Optional[str] = None
- default: Optional[str] = None
- routes: Dict[str, str] = Field(default_factory=dict) # Pattern -> Backend mapping
+ description: str | None = None
+ default: str | None = None
+ routes: dict[str, str] = Field(default_factory=dict) # Pattern -> Backend mapping
- def get_backend_match_for_model(self, model_name: str) -> Optional[BackendMatchForModel]:
+ def get_backend_match_for_model(self, model_name: str) -> BackendMatchForModel | None:
"""Get the backend name for a given model name.
Args:
@@ -23,6 +21,7 @@ class RoutingProfile(ConfigModel):
Returns:
Backend name to use for this model
+
"""
# Check exact matches first
if model_name in self.routes:
@@ -54,5 +53,4 @@ class RoutingProfile(ConfigModel):
matching_method=BackendMatchingMethod.DEFAULT,
matched_pattern=None,
)
- else:
- return None
+ return None
diff --git a/pipelex/cogt/model_routing/routing_profile_factory.py b/pipelex/cogt/model_routing/routing_profile_factory.py
index a08872690..42f225678 100644
--- a/pipelex/cogt/model_routing/routing_profile_factory.py
+++ b/pipelex/cogt/model_routing/routing_profile_factory.py
@@ -1,5 +1,3 @@
-from typing import Dict, Optional
-
from pydantic import Field
from pipelex.cogt.model_routing.routing_profile import RoutingProfile
@@ -10,15 +8,15 @@ class RoutingProfileBlueprint(ConfigModel):
"""Blueprint for creating RoutingProfile instances."""
description: str
- default: Optional[str] = None
- routes: Dict[str, str] = Field(default_factory=dict)
+ default: str | None = None
+ routes: dict[str, str] = Field(default_factory=dict)
class RoutingProfileLibraryBlueprint(ConfigModel):
"""Blueprint for the entire routing profile library."""
active: str
- profiles: Dict[str, RoutingProfileBlueprint] = Field(default_factory=dict)
+ profiles: dict[str, RoutingProfileBlueprint] = Field(default_factory=dict)
class RoutingProfileFactory:
@@ -38,6 +36,7 @@ class RoutingProfileFactory:
Returns:
RoutingProfile instance
+
"""
return RoutingProfile(
name=name,
diff --git a/pipelex/cogt/model_routing/routing_profile_library.py b/pipelex/cogt/model_routing/routing_profile_library.py
index 8e7ec0473..949e8a60c 100644
--- a/pipelex/cogt/model_routing/routing_profile_library.py
+++ b/pipelex/cogt/model_routing/routing_profile_library.py
@@ -1,7 +1,4 @@
-from typing import Dict, Optional
-
from pydantic import Field, RootModel, ValidationError
-from typing_extensions import Self
from pipelex import log
from pipelex.cogt.exceptions import RoutingProfileLibraryError, RoutingProfileLibraryNotFoundError
@@ -13,22 +10,25 @@ from pipelex.cogt.model_routing.routing_profile_factory import (
)
from pipelex.config import get_config
from pipelex.tools.misc.toml_utils import TOMLValidationError, load_toml_from_path
+from pipelex.types import Self
-RoutingProfileLibraryRoot = Dict[str, RoutingProfile]
+RoutingProfileLibraryRoot = dict[str, RoutingProfile]
class RoutingProfileLibrary(RootModel[RoutingProfileLibraryRoot]):
"""Library for managing routing profile configurations."""
root: RoutingProfileLibraryRoot = Field(default_factory=dict)
- _active_config: Optional[str] = None
+ _active_config: str | None = None
@property
def active_profile(self) -> RoutingProfile:
if not self._active_config:
- raise RoutingProfileLibraryError("No active routing profile loaded")
+ msg = "No active routing profile loaded"
+ raise RoutingProfileLibraryError(msg)
if self._active_config not in self.root:
- raise RoutingProfileLibraryError(f"Active routing profile '{self._active_config}' not found in loaded routing profile library")
+ msg = f"Active routing profile '{self._active_config}' not found in loaded routing profile library"
+ raise RoutingProfileLibraryError(msg)
return self.root[self._active_config]
@classmethod
@@ -45,24 +45,22 @@ class RoutingProfileLibrary(RootModel[RoutingProfileLibraryRoot]):
try:
catalog_dict = load_toml_from_path(path=routing_profile_library_path)
except FileNotFoundError as not_found_exc:
- raise RoutingProfileLibraryNotFoundError(
- f"Failed to load routing profile library from file '{routing_profile_library_path}': {not_found_exc}"
- ) from not_found_exc
+ msg = f"Failed to load routing profile library from file '{routing_profile_library_path}': {not_found_exc}"
+ raise RoutingProfileLibraryNotFoundError(msg) from not_found_exc
except TOMLValidationError as toml_validation_exc:
- raise RoutingProfileLibraryError(
- f"Failed to load routing profile library from file '{routing_profile_library_path}': {toml_validation_exc}"
- ) from toml_validation_exc
+ msg = f"Failed to load routing profile library from file '{routing_profile_library_path}': {toml_validation_exc}"
+ raise RoutingProfileLibraryError(msg) from toml_validation_exc
try:
catalog_blueprint = RoutingProfileLibraryBlueprint.model_validate(catalog_dict)
except ValidationError as exc:
- raise RoutingProfileLibraryError(f"Invalid routing profile library configuration in '{routing_profile_library_path}': {exc}") from exc
+ msg = f"Invalid routing profile library configuration in '{routing_profile_library_path}': {exc}"
+ raise RoutingProfileLibraryError(msg) from exc
# Validate that the active config exists
if catalog_blueprint.active not in catalog_blueprint.profiles:
- raise RoutingProfileLibraryError(
- f"Active profile '{catalog_blueprint.active}' not found in library. Available profiles: {list(catalog_blueprint.profiles.keys())}"
- )
+ msg = f"Active profile '{catalog_blueprint.active}' not found in library. Available profiles: {list(catalog_blueprint.profiles.keys())}"
+ raise RoutingProfileLibraryError(msg)
# Load all profiles
self.root = {}
@@ -76,7 +74,7 @@ class RoutingProfileLibrary(RootModel[RoutingProfileLibraryRoot]):
log.debug(f"Loaded routing profile library with active profile: '{self._active_config}'")
log.debug(f"Available profiles: {list(self.root.keys())}")
- def get_backend_match_for_model_from_active_routing_profile(self, model_name: str) -> Optional[BackendMatchForModel]:
+ def get_backend_match_for_model_from_active_routing_profile(self, model_name: str) -> BackendMatchForModel | None:
"""Get the backend name for a given model.
Args:
@@ -87,10 +85,10 @@ class RoutingProfileLibrary(RootModel[RoutingProfileLibraryRoot]):
Raises:
RoutingProfileLibraryError: If no active profile is set or config not found
+
"""
profile = self.active_profile
- backend_match_for_model = profile.get_backend_match_for_model(model_name)
- return backend_match_for_model
+ return profile.get_backend_match_for_model(model_name)
def list_routing_profile_names(self) -> list[str]:
"""Get a list of all available routing profile names."""
@@ -99,5 +97,6 @@ class RoutingProfileLibrary(RootModel[RoutingProfileLibraryRoot]):
def get_required_routing_profile(self, routing_profile_name: str) -> RoutingProfile:
routing_profile = self.root.get(routing_profile_name)
if not routing_profile:
- raise RoutingProfileLibraryError(f"Routing profile '{routing_profile_name}' not found in loaded routing profile library")
+ msg = f"Routing profile '{routing_profile_name}' not found in loaded routing profile library"
+ raise RoutingProfileLibraryError(msg)
return routing_profile
diff --git a/pipelex/cogt/models/model_deck.py b/pipelex/cogt/models/model_deck.py
index 45ac09e87..93e780771 100644
--- a/pipelex/cogt/models/model_deck.py
+++ b/pipelex/cogt/models/model_deck.py
@@ -1,7 +1,6 @@
-from typing import Dict, List, Optional, Union
+from typing import Union
from pydantic import Field, field_validator, model_validator
-from typing_extensions import Self
from pipelex import log
from pipelex.cogt.exceptions import (
@@ -19,14 +18,15 @@ from pipelex.cogt.model_backends.model_spec import InferenceModelSpec
from pipelex.cogt.ocr.ocr_setting import OcrChoice, OcrSetting
from pipelex.tools.config.config_model import ConfigModel
from pipelex.tools.exceptions import ConfigValidationError
+from pipelex.types import Self
LLM_PRESET_DISABLED = "disabled"
-Waterfall = Union[str, List[str]]
+Waterfall = Union[str, list[str]]
class LLMDeckBlueprint(ConfigModel):
- presets: Dict[str, LLMSetting] = Field(default_factory=dict)
+ presets: dict[str, LLMSetting] = Field(default_factory=dict)
choice_defaults: LLMSettingChoicesDefaults
choice_overrides: LLMSettingChoices = LLMSettingChoices(
for_text=None,
@@ -35,17 +35,17 @@ class LLMDeckBlueprint(ConfigModel):
class OCRDeckBlueprint(ConfigModel):
- presets: Dict[str, OcrSetting] = Field(default_factory=dict)
+ presets: dict[str, OcrSetting] = Field(default_factory=dict)
choice_default: OcrChoice
class ImgGenDeckBlueprint(ConfigModel):
- presets: Dict[str, ImgGenSetting] = Field(default_factory=dict)
+ presets: dict[str, ImgGenSetting] = Field(default_factory=dict)
choice_default: ImgGenChoice
class ModelDeckBlueprint(ConfigModel):
- aliases: Dict[str, Waterfall] = Field(default_factory=dict)
+ aliases: dict[str, Waterfall] = Field(default_factory=dict)
llm: LLMDeckBlueprint
ocr: OCRDeckBlueprint
@@ -53,20 +53,20 @@ class ModelDeckBlueprint(ConfigModel):
class ModelDeck(ConfigModel):
- inference_models: Dict[str, InferenceModelSpec] = Field(default_factory=dict)
- aliases: Dict[str, Waterfall] = Field(default_factory=dict)
+ inference_models: dict[str, InferenceModelSpec] = Field(default_factory=dict)
+ aliases: dict[str, Waterfall] = Field(default_factory=dict)
- llm_presets: Dict[str, LLMSetting] = Field(default_factory=dict)
+ llm_presets: dict[str, LLMSetting] = Field(default_factory=dict)
llm_choice_defaults: LLMSettingChoicesDefaults
llm_choice_overrides: LLMSettingChoices = LLMSettingChoices(
for_text=None,
for_object=None,
)
- ocr_presets: Dict[str, OcrSetting] = Field(default_factory=dict)
+ ocr_presets: dict[str, OcrSetting] = Field(default_factory=dict)
ocr_choice_default: OcrChoice
- img_gen_presets: Dict[str, ImgGenSetting] = Field(default_factory=dict)
+ img_gen_presets: dict[str, ImgGenSetting] = Field(default_factory=dict)
img_gen_choice_default: ImgGenChoice
def check_llm_setting(self, llm_setting_or_preset_id: LLMChoice, is_disabled_allowed: bool = False):
@@ -77,43 +77,41 @@ class ModelDeck(ConfigModel):
return
if preset_id == LLM_PRESET_DISABLED and is_disabled_allowed:
return
- raise LLMChoiceNotFoundError(f"llm preset id '{preset_id}' not found in deck")
+ msg = f"llm preset id '{preset_id}' not found in deck"
+ raise LLMChoiceNotFoundError(msg)
def get_llm_setting(self, llm_choice: LLMChoice) -> LLMSetting:
if isinstance(llm_choice, LLMSetting):
return llm_choice
- else:
- # it's a string, so either an llm preset id or an llm handle
- if llm_preset := self.llm_presets.get(llm_choice):
- return llm_preset
- elif self.is_handle_defined(model_handle=llm_choice):
- return LLMSetting(llm_handle=llm_choice, temperature=0.7, max_tokens=None)
- else:
- raise LLMChoiceNotFoundError(f"LLM choice '{llm_choice}' not found in deck")
+ # it's a string, so either an llm preset id or an llm handle
+ if llm_preset := self.llm_presets.get(llm_choice):
+ return llm_preset
+ if self.is_handle_defined(model_handle=llm_choice):
+ return LLMSetting(llm_handle=llm_choice, temperature=0.7, max_tokens=None)
+ msg = f"LLM choice '{llm_choice}' not found in deck"
+ raise LLMChoiceNotFoundError(msg)
def get_ocr_setting(self, ocr_choice: OcrChoice) -> OcrSetting:
if isinstance(ocr_choice, OcrSetting):
return ocr_choice
- else:
- # it's a string, so either an ocr preset id or an ocr handle
- if ocr_preset := self.ocr_presets.get(ocr_choice):
- return ocr_preset
- elif self.is_handle_defined(model_handle=ocr_choice):
- return OcrSetting(ocr_handle=ocr_choice)
- else:
- raise OcrChoiceNotFoundError(f"OCR choice '{ocr_choice}' not found in deck")
+ # it's a string, so either an ocr preset id or an ocr handle
+ if ocr_preset := self.ocr_presets.get(ocr_choice):
+ return ocr_preset
+ if self.is_handle_defined(model_handle=ocr_choice):
+ return OcrSetting(ocr_handle=ocr_choice)
+ msg = f"OCR choice '{ocr_choice}' not found in deck"
+ raise OcrChoiceNotFoundError(msg)
def get_img_gen_setting(self, img_gen_choice: ImgGenChoice) -> ImgGenSetting:
if isinstance(img_gen_choice, ImgGenSetting):
return img_gen_choice
- else:
- # it's a string, so either an img gen preset id or an img gen handle
- if img_gen_preset := self.img_gen_presets.get(img_gen_choice):
- return img_gen_preset
- elif self.is_handle_defined(model_handle=img_gen_choice):
- return ImgGenSetting(img_gen_handle=img_gen_choice)
- else:
- raise ImgGenChoiceNotFoundError(f"Image generation choice '{img_gen_choice}' not found in deck")
+ # it's a string, so either an img gen preset id or an img gen handle
+ if img_gen_preset := self.img_gen_presets.get(img_gen_choice):
+ return img_gen_preset
+ if self.is_handle_defined(model_handle=img_gen_choice):
+ return ImgGenSetting(img_gen_handle=img_gen_choice)
+ msg = f"Image generation choice '{img_gen_choice}' not found in deck"
+ raise ImgGenChoiceNotFoundError(msg)
@classmethod
def final_validate(cls, deck: Self): # pyright: ignore[reportIncompatibleMethodOverride]
@@ -122,7 +120,8 @@ class ModelDeck(ConfigModel):
try:
cls._validate_llm_setting(llm_setting=llm_setting, inference_model=inference_model)
except ConfigValidationError as exc:
- raise ModelDeckValidatonError(f"LLM preset '{llm_preset_id}' is invalid: {exc}")
+ msg = f"LLM preset '{llm_preset_id}' is invalid: {exc}"
+ raise ModelDeckValidatonError(msg) from exc
############################################################
#### ModelDeck validations
@@ -132,25 +131,27 @@ class ModelDeck(ConfigModel):
def _validate_llm_setting(cls, llm_setting: LLMSetting, inference_model: InferenceModelSpec):
if inference_model.max_tokens is not None and (llm_setting_max_tokens := llm_setting.max_tokens):
if llm_setting_max_tokens > inference_model.max_tokens:
- raise LLMSettingsValidationError(
- (
- f"LLM setting '{llm_setting.llm_handle}' has a max_tokens of {llm_setting_max_tokens}, "
- f"which is greater than the model's max_tokens of {inference_model.max_tokens}"
- )
+ msg = (
+ f"LLM setting '{llm_setting.llm_handle}' has a max_tokens of {llm_setting_max_tokens}, "
+ f"which is greater than the model's max_tokens of {inference_model.max_tokens}"
)
+ raise LLMSettingsValidationError(msg)
if ModelConstraints.TEMPERATURE_MUST_BE_1 in inference_model.constraints and llm_setting.temperature != 1:
- raise LLMSettingsValidationError(
+ msg = (
f"LLM setting '{llm_setting.llm_handle}' has a temperature of {llm_setting.temperature}, "
f"which is not allowed by the model's constraints: it must be 1"
)
+ raise LLMSettingsValidationError(msg)
@field_validator("llm_choice_defaults", mode="after")
@classmethod
def validate_llm_choice_defaults(cls, llm_choice_defaults: LLMSettingChoices) -> LLMSettingChoices:
if llm_choice_defaults.for_text is None:
- raise ConfigValidationError("llm_choice_defaults.for_text cannot be None")
+ msg = "llm_choice_defaults.for_text cannot be None"
+ raise ConfigValidationError(msg)
if llm_choice_defaults.for_object is None:
- raise ConfigValidationError("llm_choice_defaults.for_object cannot be None")
+ msg = "llm_choice_defaults.for_object cannot be None"
+ raise ConfigValidationError(msg)
return llm_choice_defaults
@field_validator("llm_choice_overrides", mode="after")
@@ -165,7 +166,8 @@ class ModelDeck(ConfigModel):
def validate_llm_presets(self) -> Self:
for llm_preset_id, llm_setting in self.llm_presets.items():
if llm_setting.llm_handle not in self.inference_models:
- raise LLMHandleNotFoundError(f"llm_handle '{llm_setting.llm_handle}' for llm_preset '{llm_preset_id}' not found in deck")
+ msg = f"llm_handle '{llm_setting.llm_handle}' for llm_preset '{llm_preset_id}' not found in deck"
+ raise LLMHandleNotFoundError(msg)
return self
@model_validator(mode="after")
@@ -176,12 +178,11 @@ class ModelDeck(ConfigModel):
def _validate_llm_choices(self, llm_choices: LLMSettingChoices):
for llm_setting in llm_choices.list_choices():
self.check_llm_setting(llm_setting_or_preset_id=llm_setting)
- return
- def get_optional_inference_model(self, model_handle: str) -> Optional[InferenceModelSpec]:
+ def get_optional_inference_model(self, model_handle: str) -> InferenceModelSpec | None:
if inference_model := self.inference_models.get(model_handle):
return inference_model
- elif redirection := self.aliases.get(model_handle):
+ if redirection := self.aliases.get(model_handle):
log.debug(f"Redirection for '{model_handle}': {redirection}")
if isinstance(redirection, str):
alias_list = [redirection]
@@ -199,7 +200,8 @@ class ModelDeck(ConfigModel):
def get_required_inference_model(self, model_handle: str) -> InferenceModelSpec:
inference_model = self.get_optional_inference_model(model_handle=model_handle)
if inference_model is None:
- raise LLMHandleNotFoundError(f"Model handle '{model_handle}' not found in deck")
+ msg = f"Model handle '{model_handle}' not found in deck"
+ raise LLMHandleNotFoundError(msg)
if model_handle not in self.inference_models:
log.dev(f"Model handle '{model_handle}' is an alias which resolves to '{inference_model.name}'")
return inference_model
diff --git a/pipelex/cogt/models/model_deck_check.py b/pipelex/cogt/models/model_deck_check.py
index 74891c158..4faf808aa 100644
--- a/pipelex/cogt/models/model_deck_check.py
+++ b/pipelex/cogt/models/model_deck_check.py
@@ -8,31 +8,30 @@ from pipelex.hub import get_model_deck
def check_llm_choice_with_deck(llm_choice: LLMChoice):
if isinstance(llm_choice, LLMSetting):
return
+
llm_deck = get_model_deck()
- if llm_choice in llm_deck.llm_presets:
- return
- elif llm_deck.is_handle_defined(model_handle=llm_choice):
+
+ if llm_choice in llm_deck.llm_presets or llm_deck.is_handle_defined(model_handle=llm_choice):
return
- raise LLMChoiceNotFoundError(f"LLM choice '{llm_choice}' not found in deck")
+ msg = f"LLM choice '{llm_choice}' not found in deck"
+ raise LLMChoiceNotFoundError(msg)
def check_ocr_choice_with_deck(ocr_choice: OcrChoice):
if isinstance(ocr_choice, OcrSetting):
return
ocr_deck = get_model_deck()
- if ocr_choice in ocr_deck.ocr_presets:
+ if ocr_choice in ocr_deck.ocr_presets or ocr_deck.is_handle_defined(model_handle=ocr_choice):
return
- elif ocr_deck.is_handle_defined(model_handle=ocr_choice):
- return
- raise OcrChoiceNotFoundError(f"OCR choice '{ocr_choice}' not found in deck")
+ msg = f"OCR choice '{ocr_choice}' not found in deck"
+ raise OcrChoiceNotFoundError(msg)
def check_img_gen_choice_with_deck(img_gen_choice: ImgGenChoice):
if isinstance(img_gen_choice, ImgGenSetting):
return
img_gen_deck = get_model_deck()
- if img_gen_choice in img_gen_deck.img_gen_presets:
- return
- elif img_gen_deck.is_handle_defined(model_handle=img_gen_choice):
+ if img_gen_choice in img_gen_deck.img_gen_presets or img_gen_deck.is_handle_defined(model_handle=img_gen_choice):
return
- raise ImgGenChoiceNotFoundError(f"Image generation choice '{img_gen_choice}' not found in deck")
+ msg = f"Image generation choice '{img_gen_choice}' not found in deck"
+ raise ImgGenChoiceNotFoundError(msg)
diff --git a/pipelex/cogt/models/model_manager.py b/pipelex/cogt/models/model_manager.py
index ba7966f29..a314ac2ff 100644
--- a/pipelex/cogt/models/model_manager.py
+++ b/pipelex/cogt/models/model_manager.py
@@ -1,5 +1,5 @@
import os
-from typing import Any, Dict, Optional
+from typing import Any
from typing_extensions import override
@@ -22,12 +22,13 @@ class ModelManager(ModelManagerAbstract):
def __init__(self) -> None:
self.routing_profile_library = RoutingProfileLibrary.make_empty()
self.inference_backend_library = InferenceBackendLibrary.make_empty()
- self.model_deck: Optional[ModelDeck] = None
+ self.model_deck: ModelDeck | None = None
@override
def get_model_deck(self) -> ModelDeck:
if self.model_deck is None:
- raise RuntimeError("Model deck is not initialized")
+ msg = "Model deck is not initialized"
+ raise RuntimeError(msg)
return self.model_deck
@override
@@ -53,33 +54,36 @@ class ModelManager(ModelManagerAbstract):
try:
validate_toml_file(deck_path)
except TOMLValidationError as exc:
- log.error(f"TOML formatting issues in LLM deck file '{deck_path}': {exc}")
- raise LibraryError(f"TOML validation failed for LLM deck file '{deck_path}': {exc}") from exc
+ msg = f"TOML validation failed for LLM deck file '{deck_path}': {exc}"
+ log.error(msg)
+ raise LibraryError(msg) from exc
@classmethod
def load_deck_blueprint(cls) -> ModelDeckBlueprint:
deck_paths = get_config().cogt.inference_config.get_model_deck_paths()
- full_deck_dict: Dict[str, Any] = {}
+ full_deck_dict: dict[str, Any] = {}
if not deck_paths:
- raise ModelDeckNotFoundError("No LLM deck paths found. Please run `pipelex init-libraries` to create it.")
+ msg = "No LLM deck paths found. Please run `pipelex init-libraries` to create it."
+ raise ModelDeckNotFoundError(msg)
for deck_path in deck_paths:
if not os.path.exists(deck_path):
- raise ModelDeckNotFoundError(f"LLM deck path `{deck_path}` not found. Please run `pipelex init-libraries` to create it.")
+ msg = f"LLM deck path `{deck_path}` not found. Please run `pipelex init-libraries` to create it."
+ raise ModelDeckNotFoundError(msg)
try:
deck_dict = load_toml_from_path(path=deck_path)
log.debug(f"Loaded LLM deck from {deck_path}")
deep_update(full_deck_dict, deck_dict)
except Exception as exc:
- log.error(f"Failed to load LLM deck file '{deck_path}': {exc}")
+ msg = f"Failed to load LLM deck file '{deck_path}': {exc}"
+ log.error(msg)
raise
- model_deck_blueprint = ModelDeckBlueprint.model_validate(full_deck_dict)
- return model_deck_blueprint
+ return ModelDeckBlueprint.model_validate(full_deck_dict)
def build_deck(self, model_deck_blueprint: ModelDeckBlueprint) -> ModelDeck:
all_models_and_possible_backends = self.inference_backend_library.get_all_models_and_possible_backends()
- inference_models: Dict[str, InferenceModelSpec] = {}
+ inference_models: dict[str, InferenceModelSpec] = {}
for model_name, available_backends in all_models_and_possible_backends.items():
backend_match_for_model = self.routing_profile_library.get_backend_match_for_model_from_active_routing_profile(
@@ -91,20 +95,22 @@ class ModelManager(ModelManagerAbstract):
matched_backend_name = backend_match_for_model.backend_name
backend = self.inference_backend_library.get_inference_backend(backend_name=matched_backend_name)
if backend is None:
- raise ModelsManagerError(f"Backend '{matched_backend_name}', requested for model '{model_name}', could not be found")
+ msg = f"Backend '{matched_backend_name}', requested for model '{model_name}', could not be found"
+ raise ModelsManagerError(msg)
model_spec = backend.get_model_spec(model_name)
if model_spec is None:
# Not finding the model spec can be an error or not according to the matching method
match backend_match_for_model.matching_method:
case BackendMatchingMethod.EXACT_MATCH:
- raise ModelsManagerError(
+ msg = (
f"Model spec '{model_name}' not found in backend '{matched_backend_name}' "
f"which was matched exactly in routing profile '{backend_match_for_model.routing_profile_name}'"
)
+ raise ModelsManagerError(msg)
case BackendMatchingMethod.PATTERN_MATCH:
log.verbose(
f"Model spec '{model_name}' not found in backend '{matched_backend_name}' but it's OK because "
- f"it was only matched by pattern in routing profile '{backend_match_for_model.routing_profile_name}'"
+ f"it was only matched by pattern in routing profile '{backend_match_for_model.routing_profile_name}'",
)
# We can skip it because it was only a pattern match
continue
@@ -118,18 +124,20 @@ class ModelManager(ModelManagerAbstract):
continue
backend = self.inference_backend_library.get_inference_backend(backend_name=available_backend)
if backend is None:
- raise ModelsManagerError(f"Backend '{available_backend}' not found for model '{model_name}'")
+ msg = f"Backend '{available_backend}' not found for model '{model_name}'"
+ raise ModelsManagerError(msg)
model_spec = backend.get_model_spec(model_name)
if model_spec is not None:
break
if model_spec is None:
- raise ModelsManagerError(
+ msg = (
f"Model spec '{model_name}' not found in any of the available backends '{available_backends}' "
f"which was set as default in routing profile '{backend_match_for_model.routing_profile_name}'"
)
+ raise ModelsManagerError(msg)
inference_models[model_name] = model_spec
- model_deck = ModelDeck(
+ return ModelDeck(
inference_models=inference_models,
aliases=model_deck_blueprint.aliases,
llm_presets=model_deck_blueprint.llm.presets,
@@ -140,17 +148,18 @@ class ModelManager(ModelManagerAbstract):
img_gen_presets=model_deck_blueprint.img_gen.presets,
img_gen_choice_default=model_deck_blueprint.img_gen.choice_default,
)
- return model_deck
@override
def get_inference_model(self, model_handle: str) -> InferenceModelSpec:
if self.model_deck is None:
- raise RuntimeError("Model deck is not initialized")
+ msg = "Model deck is not initialized"
+ raise RuntimeError(msg)
return self.model_deck.get_required_inference_model(model_handle=model_handle)
@override
def get_required_inference_backend(self, backend_name: str) -> InferenceBackend:
backend = self.inference_backend_library.get_inference_backend(backend_name)
if backend is None:
- raise ModelsManagerError(f"Inference backend '{backend_name}' not found")
+ msg = f"Inference backend '{backend_name}' not found"
+ raise ModelsManagerError(msg)
return backend
diff --git a/pipelex/cogt/ocr/ocr_input.py b/pipelex/cogt/ocr/ocr_input.py
index 8c6dbb420..a364aa014 100644
--- a/pipelex/cogt/ocr/ocr_input.py
+++ b/pipelex/cogt/ocr/ocr_input.py
@@ -1,10 +1,8 @@
-from typing import Optional
-
from pydantic import BaseModel, model_validator
-from typing_extensions import Self
from pipelex.cogt.exceptions import CogtError
from pipelex.tools.typing.validation_utils import has_exactly_one_among_attributes_from_list
+from pipelex.types import Self
class OcrInputError(CogtError):
@@ -12,11 +10,12 @@ class OcrInputError(CogtError):
class OcrInput(BaseModel):
- image_uri: Optional[str] = None
- pdf_uri: Optional[str] = None
+ image_uri: str | None = None
+ pdf_uri: str | None = None
@model_validator(mode="after")
def validate_at_exactly_one_input(self) -> Self:
if not has_exactly_one_among_attributes_from_list(self, attributes_list=["image_uri", "pdf_uri"]):
- raise OcrInputError("Exactly one of 'image_uri' or 'pdf_uri' must be provided")
+ msg = "Exactly one of 'image_uri' or 'pdf_uri' must be provided"
+ raise OcrInputError(msg)
return self
diff --git a/pipelex/cogt/ocr/ocr_job.py b/pipelex/cogt/ocr/ocr_job.py
index 8a2edd17a..3047498bf 100644
--- a/pipelex/cogt/ocr/ocr_job.py
+++ b/pipelex/cogt/ocr/ocr_job.py
@@ -3,7 +3,6 @@ from datetime import datetime
from typing_extensions import override
from pipelex.cogt.inference.inference_job_abstract import InferenceJobAbstract
-from pipelex.cogt.model_backends.model_spec import InferenceModelSpec
from pipelex.cogt.ocr.ocr_input import OcrInput
from pipelex.cogt.ocr.ocr_job_components import OcrJobConfig, OcrJobParams, OcrJobReport
@@ -18,7 +17,7 @@ class OcrJob(InferenceJobAbstract):
def validate_before_execution(self):
pass
- def ocr_job_before_start(self, inference_model: InferenceModelSpec):
+ def ocr_job_before_start(self):
# Reset metadata
self.job_metadata.started_at = datetime.now()
diff --git a/pipelex/cogt/ocr/ocr_job_components.py b/pipelex/cogt/ocr/ocr_job_components.py
index 788258065..e07317231 100644
--- a/pipelex/cogt/ocr/ocr_job_components.py
+++ b/pipelex/cogt/ocr/ocr_job_components.py
@@ -1,5 +1,3 @@
-from typing import Optional
-
from pydantic import BaseModel
from pipelex.tools.config.config_model import ConfigModel
@@ -7,11 +5,11 @@ from pipelex.tools.config.config_model import ConfigModel
class OcrJobParams(BaseModel):
should_include_images: bool
- max_nb_images: Optional[int]
- image_min_size: Optional[int]
+ max_nb_images: int | None
+ image_min_size: int | None
should_caption_images: bool
should_include_page_views: bool
- page_views_dpi: Optional[int]
+ page_views_dpi: int | None
@classmethod
def make_default_ocr_job_params(cls) -> "OcrJobParams":
diff --git a/pipelex/cogt/ocr/ocr_job_factory.py b/pipelex/cogt/ocr/ocr_job_factory.py
index 2633b0d0a..c89601f4c 100644
--- a/pipelex/cogt/ocr/ocr_job_factory.py
+++ b/pipelex/cogt/ocr/ocr_job_factory.py
@@ -1,5 +1,3 @@
-from typing import Optional
-
from pipelex.cogt.ocr.ocr_input import OcrInput
from pipelex.cogt.ocr.ocr_job import OcrJob
from pipelex.cogt.ocr.ocr_job_components import OcrJobConfig, OcrJobParams, OcrJobReport
@@ -11,9 +9,9 @@ class OcrJobFactory:
def make_ocr_job(
cls,
ocr_input: OcrInput,
- ocr_job_params: Optional[OcrJobParams] = None,
- ocr_job_config: Optional[OcrJobConfig] = None,
- job_metadata: Optional[JobMetadata] = None,
+ ocr_job_params: OcrJobParams | None = None,
+ ocr_job_config: OcrJobConfig | None = None,
+ job_metadata: JobMetadata | None = None,
) -> OcrJob:
# TODO: manage the param default through the config
# ocr_config = get_config().cogt.ocr_config
diff --git a/pipelex/cogt/ocr/ocr_output.py b/pipelex/cogt/ocr/ocr_output.py
index d3d038551..186e5600c 100644
--- a/pipelex/cogt/ocr/ocr_output.py
+++ b/pipelex/cogt/ocr/ocr_output.py
@@ -1,17 +1,15 @@
-from typing import Dict, List, Optional
-
from pydantic import Field
from pipelex import log
from pipelex.tools.misc.base_64_utils import save_base64_to_binary_file
from pipelex.tools.misc.file_utils import ensure_directory_exists, save_text_to_path
-from pipelex.tools.typing.pydantic_utils import CustomBaseModel
+from pipelex.tools.typing.pydantic_utils import CustomBaseModel, empty_list_factory_of
class ExtractedImage(CustomBaseModel):
image_id: str
- base_64: Optional[str] = None
- caption: Optional[str] = None
+ base_64: str | None = None
+ caption: str | None = None
def save_to_directory(self, directory: str):
ensure_directory_exists(directory)
@@ -23,16 +21,16 @@ class ExtractedImage(CustomBaseModel):
class ExtractedImageFromPage(ExtractedImage):
- top_left_x: Optional[int] = None
- top_left_y: Optional[int] = None
- bottom_right_x: Optional[int] = None
- bottom_right_y: Optional[int] = None
+ top_left_x: int | None = None
+ top_left_y: int | None = None
+ bottom_right_x: int | None = None
+ bottom_right_y: int | None = None
class Page(CustomBaseModel):
- text: Optional[str] = None
- extracted_images: List[ExtractedImageFromPage] = Field(default_factory=list)
- page_view: Optional[ExtractedImageFromPage] = None
+ text: str | None = None
+ extracted_images: list[ExtractedImageFromPage] = Field(default_factory=empty_list_factory_of(ExtractedImageFromPage))
+ page_view: ExtractedImageFromPage | None = None
def save_to_directory(self, directory: str, page_text_file_name: str):
ensure_directory_exists(directory)
@@ -47,7 +45,7 @@ class Page(CustomBaseModel):
class OcrOutput(CustomBaseModel):
- pages: Dict[int, Page]
+ pages: dict[int, Page]
@property
def concatenated_text(self) -> str:
diff --git a/pipelex/cogt/ocr/ocr_setting.py b/pipelex/cogt/ocr/ocr_setting.py
index 942320362..9ec34a603 100644
--- a/pipelex/cogt/ocr/ocr_setting.py
+++ b/pipelex/cogt/ocr/ocr_setting.py
@@ -1,4 +1,4 @@
-from typing import Optional, Union
+from typing import Union
from pydantic import Field
@@ -7,8 +7,8 @@ from pipelex.tools.config.config_model import ConfigModel
class OcrSetting(ConfigModel):
ocr_handle: str
- max_nb_images: Optional[int] = Field(default=None, ge=0)
- image_min_size: Optional[int] = Field(default=None, ge=0)
+ max_nb_images: int | None = Field(default=None, ge=0)
+ image_min_size: int | None = Field(default=None, ge=0)
def desc(self) -> str:
return f"OcrSetting(ocr_handle={self.ocr_handle}, max_nb_images={self.max_nb_images}, image_min_size={self.image_min_size})"
diff --git a/pipelex/cogt/ocr/ocr_worker_abstract.py b/pipelex/cogt/ocr/ocr_worker_abstract.py
index 42570da61..e80464a5d 100644
--- a/pipelex/cogt/ocr/ocr_worker_abstract.py
+++ b/pipelex/cogt/ocr/ocr_worker_abstract.py
@@ -1,5 +1,5 @@
from abc import abstractmethod
-from typing import Any, Dict, Optional
+from typing import Any
from typing_extensions import override
@@ -15,9 +15,9 @@ from pipelex.reporting.reporting_protocol import ReportingProtocol
class OcrWorkerAbstract(InferenceWorkerAbstract):
def __init__(
self,
- extra_config: Dict[str, Any],
+ extra_config: dict[str, Any],
inference_model: InferenceModelSpec,
- reporting_delegate: Optional[ReportingProtocol] = None,
+ reporting_delegate: ReportingProtocol | None = None,
):
InferenceWorkerAbstract.__init__(self, reporting_delegate=reporting_delegate)
self.extra_config = extra_config
@@ -53,7 +53,7 @@ class OcrWorkerAbstract(InferenceWorkerAbstract):
ocr_job.job_metadata.unit_job_id = UnitJobId.OCR_EXTRACT_PAGES
# Prepare job
- ocr_job.ocr_job_before_start(inference_model=self.inference_model)
+ ocr_job.ocr_job_before_start()
# Execute job
result = await self._ocr_extract_pages(ocr_job=ocr_job)
diff --git a/pipelex/cogt/ocr/ocr_worker_factory.py b/pipelex/cogt/ocr/ocr_worker_factory.py
index 718d50417..d66c563b1 100644
--- a/pipelex/cogt/ocr/ocr_worker_factory.py
+++ b/pipelex/cogt/ocr/ocr_worker_factory.py
@@ -1,5 +1,3 @@
-from typing import Optional
-
from pipelex.cogt.exceptions import MissingDependencyError
from pipelex.cogt.model_backends.model_spec import InferenceModelSpec
from pipelex.cogt.ocr.ocr_worker_abstract import OcrWorkerAbstract
@@ -12,7 +10,7 @@ class OcrWorkerFactory:
def make_ocr_worker(
self,
inference_model: InferenceModelSpec,
- reporting_delegate: Optional[ReportingProtocol] = None,
+ reporting_delegate: ReportingProtocol | None = None,
) -> OcrWorkerAbstract:
plugin = Plugin.make_for_inference_model(inference_model=inference_model)
backend = get_models_manager().get_required_inference_backend(inference_model.backend_name)
@@ -21,16 +19,19 @@ class OcrWorkerFactory:
match plugin.sdk:
case "mistral":
try:
- import mistralai # noqa: F401
+ import mistralai # noqa: PLC0415,F401
except ImportError as exc:
+ lib_name = "mistralai"
+ lib_extra_name = "mistral"
+ msg = "The mistralai SDK is required to use Mistral OCR models through the mistralai client."
raise MissingDependencyError(
- "mistralai",
- "mistral",
- "The mistralai SDK is required to use Mistral OCR models through the mistralai client.",
+ lib_name,
+ lib_extra_name,
+ msg,
) from exc
- from pipelex.plugins.mistral.mistral_factory import MistralFactory
- from pipelex.plugins.mistral.mistral_ocr_worker import MistralOcrWorker
+ from pipelex.plugins.mistral.mistral_factory import MistralFactory # noqa: PLC0415
+ from pipelex.plugins.mistral.mistral_ocr_worker import MistralOcrWorker # noqa: PLC0415
ocr_sdk_instance = plugin_sdk_registry.get_sdk_instance(plugin=plugin) or plugin_sdk_registry.set_sdk_instance(
plugin=plugin,
@@ -44,15 +45,15 @@ class OcrWorkerFactory:
reporting_delegate=reporting_delegate,
)
case "pypdfium2":
- from pipelex.plugins.pypdfium2.pypdfium2_worker import Pypdfium2Worker
+ from pipelex.plugins.pypdfium2.pypdfium2_worker import Pypdfium2Worker # noqa: PLC0415
ocr_worker = Pypdfium2Worker(
- sdk_instance=None,
extra_config=backend.extra_config,
inference_model=inference_model,
reporting_delegate=reporting_delegate,
)
case _:
- raise NotImplementedError(f"Plugin '{plugin}' is not supported")
+ msg = f"Plugin '{plugin}' is not supported"
+ raise NotImplementedError(msg)
return ocr_worker
diff --git a/pipelex/cogt/usage/__init__.py b/pipelex/cogt/usage/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/pipelex/cogt/usage/cost_category.py b/pipelex/cogt/usage/cost_category.py
index 3330090aa..bdbf1be69 100644
--- a/pipelex/cogt/usage/cost_category.py
+++ b/pipelex/cogt/usage/cost_category.py
@@ -1,5 +1,3 @@
-from typing import Dict
-
from pipelex.types import StrEnum
@@ -16,4 +14,4 @@ class CostCategory(StrEnum):
OUTPUT_REJECTED_PREDICTION = "output_rejected_prediction"
-CostsByCategoryDict = Dict[CostCategory, float]
+CostsByCategoryDict = dict[CostCategory, float]
diff --git a/pipelex/cogt/usage/cost_registry.py b/pipelex/cogt/usage/cost_registry.py
index 18b807c21..36538c696 100644
--- a/pipelex/cogt/usage/cost_registry.py
+++ b/pipelex/cogt/usage/cost_registry.py
@@ -1,4 +1,4 @@
-from typing import Any, Dict, List, Optional
+from typing import Any
import pandas as pd
from pydantic import Field, RootModel
@@ -12,28 +12,28 @@ from pipelex.cogt.llm.llm_report import LLMTokenCostReport, LLMTokenCostReportFi
from pipelex.cogt.usage.cost_category import CostCategory, CostsByCategoryDict
from pipelex.cogt.usage.costs_per_token import model_cost_per_token
from pipelex.cogt.usage.token_category import TokenCategory
+from pipelex.tools.typing.pydantic_utils import empty_list_factory_of
-CostRegistryRoot = List[LLMTokenCostReport]
+CostRegistryRoot = list[LLMTokenCostReport]
class CostRegistry(RootModel[CostRegistryRoot]):
- root: CostRegistryRoot = Field(default_factory=list)
+ root: CostRegistryRoot = Field(default_factory=empty_list_factory_of(LLMTokenCostReport))
def to_dataframe(self) -> pd.DataFrame:
- records: List[Dict[str, Any]] = []
+ records: list[dict[str, Any]] = []
for token_cost_report in self.root:
record_dict = token_cost_report.as_flat_dictionary()
records.append(record_dict)
- df = pd.DataFrame(records)
- return df
+ return pd.DataFrame(records)
@classmethod
def generate_report(
cls,
pipeline_run_id: str,
- llm_tokens_usages: List[LLMTokensUsage],
+ llm_tokens_usages: list[LLMTokensUsage],
unit_scale: float,
- cost_report_file_path: Optional[str] = None,
+ cost_report_file_path: str | None = None,
):
if not llm_tokens_usages:
if pipeline_run_id != "untitled":
@@ -75,10 +75,11 @@ class CostRegistry(RootModel[CostRegistryRoot]):
LLMTokenCostReportField.COST_INPUT_NON_CACHED: "sum",
LLMTokenCostReportField.COST_INPUT_JOINED: "sum",
LLMTokenCostReportField.COST_OUTPUT: "sum",
- }
+ },
).reset_index()
if agg_by_llm_name is None or agg_by_llm_name.empty: # pyright: ignore[reportUnnecessaryComparison]
- raise CostRegistryError("Empty report aggregation by LLM name")
+ msg = "Empty report aggregation by LLM name"
+ raise CostRegistryError(msg)
console = Console()
title = "Costs by LLM model"
@@ -160,21 +161,21 @@ class CostRegistry(RootModel[CostRegistryRoot]):
cost_category=token_type.to_cost_category,
)
costs_by_token_category[token_type.to_cost_category] = cost_per_token * nb_tokens
- token_cost_report = LLMTokenCostReport(
+ return LLMTokenCostReport(
job_metadata=llm_tokens_usage.job_metadata,
inference_model_name=llm_tokens_usage.inference_model_name,
platform_llm_id=llm_tokens_usage.inference_model_id,
nb_tokens_by_category=llm_tokens_usage.nb_tokens_by_category,
costs_by_token_category=costs_by_token_category,
)
- return token_cost_report
@classmethod
def complete_cost_report(cls, llm_tokens_usage: LLMTokensUsage) -> LLMTokenCostReport:
cost_report = cls.compute_cost_report(llm_tokens_usage=llm_tokens_usage)
# compute the input_non_cached tokens
if cost_report.nb_tokens_by_category.get(TokenCategory.INPUT_NON_CACHED) is not None:
- raise CostRegistryError("CostCategory.INPUT_NON_CACHED already exists in the cost report")
+ msg = "CostCategory.INPUT_NON_CACHED already exists in the cost report"
+ raise CostRegistryError(msg)
# we use pop to remove input tokens which will be replaced by "input joined"
nb_tokens_input_joined = cost_report.nb_tokens_by_category.pop(TokenCategory.INPUT, 0)
cost_report.costs_by_token_category.pop(CostCategory.INPUT, None)
@@ -186,7 +187,8 @@ class CostRegistry(RootModel[CostRegistryRoot]):
cost_report.nb_tokens_by_category[TokenCategory.INPUT_CACHED] = nb_tokens_input_cached
cost_report.costs_by_token_category[CostCategory.INPUT_NON_CACHED] = nb_tokens_input_non_cached * model_cost_per_token(
- costs=llm_tokens_usage.unit_costs, cost_category=CostCategory.INPUT_NON_CACHED
+ costs=llm_tokens_usage.unit_costs,
+ cost_category=CostCategory.INPUT_NON_CACHED,
)
costs_input_cached = cost_report.costs_by_token_category.get(CostCategory.INPUT_CACHED, 0)
cost_report.costs_by_token_category[CostCategory.INPUT_CACHED] = costs_input_cached
diff --git a/pipelex/cogt/usage/costs_per_token.py b/pipelex/cogt/usage/costs_per_token.py
index 6486e5a51..05027958d 100644
--- a/pipelex/cogt/usage/costs_per_token.py
+++ b/pipelex/cogt/usage/costs_per_token.py
@@ -4,17 +4,28 @@ from pipelex.cogt.usage.cost_category import CostCategory, CostsByCategoryDict
def model_cost_per_token(costs: CostsByCategoryDict, cost_category: CostCategory) -> float:
# cost_per_million_tokens_usd should be missing only for models that we run on our own GPUs
# all token types are not used for all models
- if cost_category == CostCategory.INPUT_CACHED:
- if cost_per_million_tokens := costs.get(CostCategory.INPUT_CACHED):
- return cost_per_million_tokens / 1000000
- elif cost_per_million_tokens := costs.get(CostCategory.INPUT):
- # according to openai docs, cached input tokens are discounted 50%
- return 0.5 * cost_per_million_tokens / 1000000
- else:
- return 0.0
- elif cost_category == CostCategory.INPUT_NON_CACHED:
- return model_cost_per_token(costs=costs, cost_category=CostCategory.INPUT)
- elif cost_per_million_tokens := costs.get(cost_category):
- return cost_per_million_tokens / 1000000
- else:
- return 0.0
+ match cost_category:
+ case CostCategory.INPUT_CACHED:
+ if cost_per_million_tokens := costs.get(CostCategory.INPUT_CACHED):
+ return cost_per_million_tokens / 1000000
+ elif cost_per_million_tokens := costs.get(CostCategory.INPUT):
+ # according to openai docs, cached input tokens are discounted 50%
+ return 0.5 * cost_per_million_tokens / 1000000
+ else:
+ return 0.0
+ case CostCategory.INPUT_NON_CACHED:
+ return model_cost_per_token(costs=costs, cost_category=CostCategory.INPUT)
+ case (
+ CostCategory.INPUT
+ | CostCategory.INPUT_JOINED
+ | CostCategory.INPUT_AUDIO
+ | CostCategory.OUTPUT
+ | CostCategory.OUTPUT_AUDIO
+ | CostCategory.OUTPUT_REASONING
+ | CostCategory.OUTPUT_ACCEPTED_PREDICTION
+ | CostCategory.OUTPUT_REJECTED_PREDICTION
+ ):
+ if cost_per_million_tokens := costs.get(cost_category):
+ return cost_per_million_tokens / 1000000
+ else:
+ return 0.0
diff --git a/pipelex/cogt/usage/token_category.py b/pipelex/cogt/usage/token_category.py
index 7e93b9082..f31e8d0ad 100644
--- a/pipelex/cogt/usage/token_category.py
+++ b/pipelex/cogt/usage/token_category.py
@@ -1,5 +1,3 @@
-from typing import Dict
-
from pipelex.cogt.usage.cost_category import CostCategory
from pipelex.types import StrEnum
@@ -21,4 +19,4 @@ class TokenCategory(StrEnum):
return CostCategory(self)
-NbTokensByCategoryDict = Dict[TokenCategory, int]
+NbTokensByCategoryDict = dict[TokenCategory, int]
diff --git a/pipelex/config.py b/pipelex/config.py
index 7bfc7cbfc..a41a7def0 100644
--- a/pipelex/config.py
+++ b/pipelex/config.py
@@ -1,4 +1,4 @@
-from typing import Dict, List, Optional, cast
+from typing import cast
import shortuuid
from pydantic import Field, field_validator
@@ -25,19 +25,19 @@ class StaticValidationReaction(StrEnum):
class StaticValidationConfig(ConfigModel):
default_reaction: StaticValidationReaction = Field(strict=False)
- reactions: Dict[StaticValidationErrorType, StaticValidationReaction]
+ reactions: dict[StaticValidationErrorType, StaticValidationReaction]
@field_validator("reactions", mode="before")
- def validate_reactions(cls, value: Dict[str, str]) -> Dict[StaticValidationErrorType, StaticValidationReaction]:
- the_dict = cast(
- Dict[StaticValidationErrorType, StaticValidationReaction],
+ @staticmethod
+ def validate_reactions(value: dict[str, str]) -> dict[StaticValidationErrorType, StaticValidationReaction]:
+ return cast(
+ "dict[StaticValidationErrorType, StaticValidationReaction]",
ConfigModel.transform_dict_str_to_enum(
input_dict=value,
key_enum_cls=StaticValidationErrorType,
value_enum_cls=StaticValidationReaction,
),
)
- return the_dict
class PipeRunConfig(ConfigModel):
@@ -49,13 +49,15 @@ class DryRunConfig(ConfigModel):
text_gen_truncate_length: int
nb_list_items: int
nb_ocr_pages: int
- image_urls: List[str]
- allowed_to_fail_pipes: List[str] = Field(default_factory=list)
+ image_urls: list[str]
+ allowed_to_fail_pipes: list[str] = Field(default_factory=list)
@field_validator("image_urls", mode="before")
- def validate_image_urls(cls, value: List[str]) -> List[str]:
+ @staticmethod
+ def validate_image_urls(value: list[str]) -> list[str]:
if not value:
- raise PipelexConfigError("dry_run_config.image_urls must be a non-empty list")
+ msg = "dry_run_config.image_urls must be a non-empty list"
+ raise PipelexConfigError(msg)
return value
@@ -70,13 +72,12 @@ class StructureConfig(ConfigModel):
class PromptingConfig(ConfigModel):
default_prompting_style: PromptingStyle
- prompting_styles: Dict[str, PromptingStyle]
+ prompting_styles: dict[str, PromptingStyle]
- def get_prompting_style(self, prompting_target: Optional[PromptingTarget] = None) -> Optional[PromptingStyle]:
+ def get_prompting_style(self, prompting_target: PromptingTarget | None = None) -> PromptingStyle | None:
if prompting_target:
return self.prompting_styles.get(prompting_target, self.default_prompting_style)
- else:
- return None
+ return None
class FeatureConfig(ConfigModel):
@@ -125,5 +126,6 @@ class PipelexConfig(ConfigRoot):
def get_config() -> PipelexConfig:
singleton_config = get_required_config()
if not isinstance(singleton_config, PipelexConfig):
- raise RuntimeError(f"Expected {PipelexConfig}, but got {type(singleton_config)}")
+ msg = f"Expected {PipelexConfig}, but got {type(singleton_config)}"
+ raise TypeError(msg)
return singleton_config
diff --git a/pipelex/core/bundles/__init__.py b/pipelex/core/bundles/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/pipelex/core/bundles/pipelex_bundle_blueprint.py b/pipelex/core/bundles/pipelex_bundle_blueprint.py
index d649b3826..66ce37b84 100644
--- a/pipelex/core/bundles/pipelex_bundle_blueprint.py
+++ b/pipelex/core/bundles/pipelex_bundle_blueprint.py
@@ -1,4 +1,4 @@
-from typing import Annotated, Dict, Optional, Union
+from typing import Annotated
from pydantic import BaseModel, ConfigDict, Field, field_validator
@@ -15,19 +15,15 @@ from pipelex.pipe_operators.llm.pipe_llm_blueprint import PipeLLMBlueprint
from pipelex.pipe_operators.ocr.pipe_ocr_blueprint import PipeOcrBlueprint
PipeBlueprintUnion = Annotated[
- Union[
- # Pipe operators
- PipeFuncBlueprint,
- PipeImgGenBlueprint,
- PipeJinja2Blueprint,
- PipeLLMBlueprint,
- PipeOcrBlueprint,
- # Pipe controllers
- PipeBatchBlueprint,
- PipeConditionBlueprint,
- PipeParallelBlueprint,
- PipeSequenceBlueprint,
- ],
+ PipeFuncBlueprint
+ | PipeImgGenBlueprint
+ | PipeJinja2Blueprint
+ | PipeLLMBlueprint
+ | PipeOcrBlueprint
+ | PipeBatchBlueprint
+ | PipeConditionBlueprint
+ | PipeParallelBlueprint
+ | PipeSequenceBlueprint,
Field(discriminator="type"),
]
@@ -36,14 +32,14 @@ class PipelexBundleBlueprint(BaseModel):
model_config = ConfigDict(extra="forbid")
domain: str
- definition: Optional[str] = None
- system_prompt: Optional[str] = None
- system_prompt_to_structure: Optional[str] = None
- prompt_template_to_structure: Optional[str] = None
+ definition: str | None = None
+ system_prompt: str | None = None
+ system_prompt_to_structure: str | None = None
+ prompt_template_to_structure: str | None = None
- concept: Optional[Dict[str, Union[ConceptBlueprint, str]]] = Field(default_factory=dict)
+ concept: dict[str, ConceptBlueprint | str] | None = Field(default_factory=dict)
- pipe: Optional[Dict[str, PipeBlueprintUnion]] = Field(default_factory=dict)
+ pipe: dict[str, PipeBlueprintUnion] | None = Field(default_factory=dict)
@field_validator("domain", mode="before")
@classmethod
diff --git a/pipelex/core/concepts/concept.py b/pipelex/core/concepts/concept.py
index e88f925be..e8630ed25 100644
--- a/pipelex/core/concepts/concept.py
+++ b/pipelex/core/concepts/concept.py
@@ -1,5 +1,3 @@
-from typing import Optional
-
from kajson.kajson_manager import KajsonManager
from pydantic import BaseModel, ConfigDict, field_validator
@@ -20,7 +18,7 @@ class Concept(BaseModel):
domain: str
definition: str
structure_class_name: str
- refines: Optional[str] = None
+ refines: str | None = None
@property
def concept_string(self) -> str:
@@ -32,17 +30,20 @@ class Concept(BaseModel):
return concept_string.startswith(SpecialDomain.IMPLICIT)
@field_validator("code")
- def validate_code(cls, code: str) -> str:
+ @staticmethod
+ def validate_code(code: str) -> str:
ConceptBlueprint.validate_concept_code(concept_code=code)
return code
@field_validator("domain")
- def validate_domain(cls, domain: str) -> str:
+ @staticmethod
+ def validate_domain(domain: str) -> str:
DomainBlueprint.validate_domain_code(code=domain)
return domain
@field_validator("refines", mode="before")
- def validate_refines(cls, refines: Optional[str]) -> Optional[str]:
+ @staticmethod
+ def validate_refines(refines: str | None) -> str | None:
if refines is None:
return None
ConceptBlueprint.validate_concept_string(concept_string=refines)
@@ -72,16 +73,15 @@ class Concept(BaseModel):
if strict:
# Check if classes are equivalent (same fields, types, descriptions)
return ClassRegistryUtils.are_classes_equivalent(concept_1_class, concept_2_class)
- else:
- # Check if concept_1 is a subclass of concept_2
- try:
- if issubclass(concept_1_class, concept_2_class):
- return True
- except TypeError:
- pass
-
- # Check if concept_1 has a field that is of type concept_2_class
- return ClassRegistryUtils.has_compatible_field(concept_1_class, concept_2_class)
+ # Check if concept_1 is a subclass of concept_2
+ try:
+ if issubclass(concept_1_class, concept_2_class):
+ return True
+ except TypeError:
+ pass
+
+ # Check if concept_1 has a field that is of type concept_2_class
+ return ClassRegistryUtils.has_compatible_field(concept_1_class, concept_2_class)
return False
@classmethod
@@ -89,8 +89,7 @@ class Concept(BaseModel):
# We get_class_registry directly from KajsonManager instead of pipelex hub to avoid circular import
if KajsonManager.get_class_registry().has_subclass(name=structure_class_name, base_class=StuffContent):
return True
- else:
- # We get_class_registry directly from KajsonManager instead of pipelex hub to avoid circular import
- if KajsonManager.get_class_registry().has_class(name=structure_class_name):
- log.warning(f"Concept class '{structure_class_name}' is registered but it's not a subclass of StuffContent")
- return False
+ # We get_class_registry directly from KajsonManager instead of pipelex hub to avoid circular import
+ if KajsonManager.get_class_registry().has_class(name=structure_class_name):
+ log.warning(f"Concept class '{structure_class_name}' is registered but it's not a subclass of StuffContent")
+ return False
diff --git a/pipelex/core/concepts/concept_blueprint.py b/pipelex/core/concepts/concept_blueprint.py
index 5ca8655ae..493d4c7b0 100644
--- a/pipelex/core/concepts/concept_blueprint.py
+++ b/pipelex/core/concepts/concept_blueprint.py
@@ -1,15 +1,14 @@
from datetime import datetime
-from typing import Any, Dict, List, Optional, Union
+from typing import Any
from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
-from typing_extensions import Self
from pipelex.core.concepts.concept_native import NativeConceptEnum, NativeConceptManager
from pipelex.core.concepts.exceptions import ConceptCodeError, ConceptStringError, ConceptStringOrConceptCodeError
from pipelex.core.domains.domain import SpecialDomain
from pipelex.core.domains.domain_blueprint import DomainBlueprint
from pipelex.tools.misc.string_utils import is_pascal_case
-from pipelex.types import StrEnum
+from pipelex.types import Self, StrEnum
class ConceptBlueprintError(Exception):
@@ -32,13 +31,13 @@ class ConceptStructureBlueprintFieldType(StrEnum):
class ConceptStructureBlueprint(BaseModel):
definition: str
- type: Optional[ConceptStructureBlueprintFieldType] = None
- item_type: Optional[str] = None
- key_type: Optional[str] = None
- value_type: Optional[str] = None
- choices: Optional[List[str]] = Field(default_factory=list)
- required: Optional[bool] = Field(default=True)
- default_value: Optional[Any] = None
+ type: ConceptStructureBlueprintFieldType | None = None
+ item_type: str | None = None
+ key_type: str | None = None
+ value_type: str | None = None
+ choices: list[str] | None = Field(default_factory=list)
+ required: bool | None = Field(default=True)
+ default_value: Any | None = None
# TODO: date translator for default_value
@@ -47,27 +46,25 @@ class ConceptStructureBlueprint(BaseModel):
"""Validate the structure blueprint according to type rules."""
# If type is None (array), choices must not be None
if self.type is None and not self.choices:
- raise ConceptStructureBlueprintError(
- f"When type is None (array), choices must not be empty. Actual type: {self.type}, choices: {self.choices}"
- )
+ msg = f"When type is None (array), choices must not be empty. Actual type: {self.type}, choices: {self.choices}"
+ raise ConceptStructureBlueprintError(msg)
# If type is "dict", key_type and value_type must not be empty
if self.type == ConceptStructureBlueprintFieldType.DICT:
if not self.key_type:
- raise ConceptStructureBlueprintError(
- f"When type is '{ConceptStructureBlueprintFieldType.DICT}', key_type must not be empty. Actual key_type: {self.key_type}"
- )
+ msg = f"When type is '{ConceptStructureBlueprintFieldType.DICT}', key_type must not be empty. Actual key_type: {self.key_type}"
+ raise ConceptStructureBlueprintError(msg)
if not self.value_type:
- raise ConceptStructureBlueprintError(
- f"When type is '{ConceptStructureBlueprintFieldType.DICT}', value_type must not be empty. Actual value_type: {self.value_type}"
- )
+ msg = f"When type is '{ConceptStructureBlueprintFieldType.DICT}', value_type must not be empty. Actual value_type: {self.value_type}"
+ raise ConceptStructureBlueprintError(msg)
# Check when default_value is not None, type is not None (except for choice fields)
if self.default_value is not None and self.type is None and not self.choices:
- raise ConceptStructureBlueprintError(
- f"When default_value is not None, type must be specified (unless choices are provided). "
- f"Actual type: {self.type}, default_value: {self.default_value}, choices: {self.choices}"
+ msg = (
+ f"When default_value is not None, type must be specified (unless choices are provided). Actual type: {self.type},"
+ f"default_value: {self.default_value}, choices: {self.choices}"
)
+ raise ConceptStructureBlueprintError(msg)
# Check default_value type is the same as type
if self.default_value is not None and self.type is not None:
@@ -76,9 +73,8 @@ class ConceptStructureBlueprint(BaseModel):
# Check default_value is valid for choice fields
if self.default_value is not None and self.type is None and self.choices:
if self.default_value not in self.choices:
- raise ConceptStructureBlueprintError(
- f"default_value must be one of the valid choices. Got '{self.default_value}', valid choices: {self.choices}"
- )
+ msg = f"default_value must be one of the valid choices. Got '{self.default_value}', valid choices: {self.choices}"
+ raise ConceptStructureBlueprintError(msg)
return self
@@ -111,10 +107,8 @@ class ConceptStructureBlueprint(BaseModel):
self._raise_type_mismatch_error("date", type(self.default_value).__name__)
def _raise_type_mismatch_error(self, expected_type_name: str, actual_type_name: str) -> None:
- """Raise a type mismatch error with consistent formatting."""
- raise ConceptStructureBlueprintError(
- f"default_value type mismatch: expected {expected_type_name} for type '{self.type}', but got {actual_type_name}"
- )
+ msg = f"default_value type mismatch: expected {expected_type_name} for type '{self.type}', but got {actual_type_name}"
+ raise ConceptStructureBlueprintError(msg)
class ConceptBlueprint(BaseModel):
@@ -122,32 +116,32 @@ class ConceptBlueprint(BaseModel):
definition: str
# TODO (non-blockiing): define a type for Union[str, ConceptStructureBlueprint] (ConceptChoice to be consistent with LLMChoice)
- structure: Optional[Union[str, Dict[str, Union[str, ConceptStructureBlueprint]]]] = None
+ structure: str | dict[str, str | ConceptStructureBlueprint] | None = None
# TODO: restore possibility of multiple refiles
- refines: Optional[str] = None
+ refines: str | None = None
@classmethod
def is_native_concept_code(cls, concept_code: str) -> bool:
ConceptBlueprint.validate_concept_code(concept_code=concept_code)
- return concept_code in [native_concept for native_concept in NativeConceptEnum]
+ return concept_code in NativeConceptEnum.values_list()
@classmethod
def validate_concept_code(cls, concept_code: str) -> None:
if not is_pascal_case(concept_code):
- raise ConceptCodeError(
- f"Concept code '{concept_code}' must be PascalCase (letters and numbers only, starting with uppercase, without `.`)"
- )
+ msg = f"Concept code '{concept_code}' must be PascalCase (letters and numbers only, starting with uppercase, without `.`)"
+ raise ConceptCodeError(msg)
@classmethod
def validate_concept_string_or_code(cls, concept_string_or_code: str) -> None:
if concept_string_or_code.count(".") > 1:
- raise ConceptStringOrConceptCodeError(
+ msg = (
f"concept_string_or_code '{concept_string_or_code}' is invalid. "
"It should either contain a domain in snake_case and a concept code in PascalCase separated by one dot, "
- "or be a concept code in PascalCase."
+ "or be a concept code in PascalCase.",
)
+ raise ConceptStringOrConceptCodeError(msg)
- elif concept_string_or_code.count(".") == 1:
+ if concept_string_or_code.count(".") == 1:
domain, concept_code = concept_string_or_code.split(".")
DomainBlueprint.validate_domain_code(code=domain)
cls.validate_concept_code(concept_code=concept_code)
@@ -157,59 +151,57 @@ class ConceptBlueprint(BaseModel):
@staticmethod
def validate_concept_string(concept_string: str) -> None:
"""Validate that a concept code follows PascalCase convention."""
- if "." not in concept_string:
- raise ConceptStringError(
- f"Concept string '{concept_string}' is invalid. It should contain a domain in snake_case "
- "and a concept code in PascalCase separated by one dot."
- )
- elif concept_string.count(".") > 1:
- raise ConceptStringError(
+ if "." not in concept_string or concept_string.count(".") > 1:
+ msg = (
f"Concept string '{concept_string}' is invalid. It should contain a domain in snake_case "
"and a concept code in PascalCase separated by one dot."
)
- else:
- domain, concept_code = concept_string.split(".", 1)
+ raise ConceptStringError(msg)
+ domain, concept_code = concept_string.split(".", 1)
# Validate domain
DomainBlueprint.validate_domain_code(domain)
# Validate concept code
if not is_pascal_case(concept_code):
- raise ConceptCodeError(
- f"Concept code '{concept_code}' must be PascalCase (letters and numbers only, starting with uppercase, without `.`)"
- )
+ msg = f"Concept code '{concept_code}' must be PascalCase (letters and numbers only, starting with uppercase, without `.`)"
+ raise ConceptCodeError(msg)
# Validate that if the concept code is among the native concepts, the domain MUST be native.
- if concept_code in [concept.value for concept in [native_concept for native_concept in NativeConceptEnum]]:
+ if concept_code in NativeConceptEnum.values_list():
if not SpecialDomain.is_native(domain=domain):
- raise ConceptStringError(
+ msg = (
f"Concept string '{concept_string}' is invalid. "
f"Concept code '{concept_code}' is a native concept, so the domain must be '{SpecialDomain.NATIVE}', "
f"or nothing, but not '{domain}'"
)
-
+ raise ConceptStringError(msg)
# Validate that if the domain is native, the concept code is a native concept
if SpecialDomain.is_native(domain=domain):
- if concept_code not in [native_concept for native_concept in NativeConceptEnum]:
- raise ConceptStringError(
+ if concept_code not in NativeConceptEnum.values_list():
+ msg = (
f"Concept string '{concept_string}' is invalid. "
f"Concept code '{concept_code}' is not a native concept, so the domain must not be '{SpecialDomain.NATIVE}'."
)
+ raise ConceptStringError(msg)
@field_validator("refines", mode="before")
@classmethod
- def validate_refines(cls, refines: Optional[str] = None) -> Optional[str]:
+ def validate_refines(cls, refines: str | None = None) -> str | None:
if refines is not None:
if not NativeConceptManager.is_native_concept(refines):
- raise ConceptBlueprintError(f"Forbidden to refine a non-native concept: '{refines}'. Refining non-native concepts will come soon.")
+ msg = f"Forbidden to refine a non-native concept: '{refines}'. Refining non-native concepts will come soon."
+ raise ConceptBlueprintError(msg)
cls.validate_concept_string_or_code(concept_string_or_code=refines)
return refines
@model_validator(mode="before")
- def model_validate_blueprint(cls, values: Union[Dict[str, Any], str]) -> Union[Dict[str, Any], str]:
+ @classmethod
+ def model_validate_blueprint(cls, values: dict[str, Any] | str) -> dict[str, Any] | str:
if isinstance(values, dict) and values.get("refines") and values.get("structure"):
- raise ConceptBlueprintError(
+ msg = (
f"Forbidden to have refines and structure at the same time: `{values.get('refines')}` "
f"and `{values.get('structure')}` for concept that has the definition `{values.get('definition')}`"
)
+ raise ConceptBlueprintError(msg)
return values
diff --git a/pipelex/core/concepts/concept_factory.py b/pipelex/core/concepts/concept_factory.py
index 90f8643bc..f501974be 100644
--- a/pipelex/core/concepts/concept_factory.py
+++ b/pipelex/core/concepts/concept_factory.py
@@ -1,4 +1,4 @@
-from typing import Any, Dict, List, Optional, Union
+from typing import Any
from kajson.kajson_manager import KajsonManager
from pydantic import BaseModel
@@ -25,7 +25,7 @@ class DomainAndConceptCode(BaseModel):
class ConceptFactory:
@classmethod
- def normalize_structure_blueprint(cls, structure_dict: Dict[str, Union[str, ConceptStructureBlueprint]]) -> Dict[str, ConceptStructureBlueprint]:
+ def normalize_structure_blueprint(cls, structure_dict: dict[str, str | ConceptStructureBlueprint]) -> dict[str, ConceptStructureBlueprint]:
"""Convert a mixed structure dictionary to a proper ConceptStructureBlueprint dictionary.
Args:
@@ -33,8 +33,9 @@ class ConceptFactory:
Returns:
Dictionary with all values as ConceptStructureBlueprint objects
+
"""
- normalized: Dict[str, ConceptStructureBlueprint] = {}
+ normalized: dict[str, ConceptStructureBlueprint] = {}
for field_name, field_value in structure_dict.items():
if isinstance(field_value, str):
@@ -64,7 +65,7 @@ class ConceptFactory:
return f"{domain}.{concept_code}"
@classmethod
- def make(cls, concept_code: str, domain: str, definition: str, structure_class_name: str, refines: Optional[str] = None) -> Concept:
+ def make(cls, concept_code: str, domain: str, definition: str, structure_class_name: str, refines: str | None = None) -> Concept:
return Concept(
code=concept_code,
domain=domain,
@@ -84,32 +85,34 @@ class ConceptFactory:
@classmethod
def make_domain_and_concept_code_from_concept_string_or_code(
- cls, domain: str, concept_string_or_code: str, concept_codes_from_the_same_domain: Optional[List[str]] = None
+ cls,
+ domain: str,
+ concept_string_or_code: str,
+ concept_codes_from_the_same_domain: list[str] | None = None,
) -> DomainAndConceptCode:
# At this point, the concept_string_or_code is already validated
if "." in concept_string_or_code:
# Is a concept string.
parts = concept_string_or_code.rsplit(".")
return DomainAndConceptCode(domain=parts[0], concept_code=parts[1])
- else:
- if NativeConceptManager.is_native_concept(concept_string_or_code=concept_string_or_code):
- return DomainAndConceptCode(domain=SpecialDomain.NATIVE, concept_code=concept_string_or_code)
+ if NativeConceptManager.is_native_concept(concept_string_or_code=concept_string_or_code):
+ return DomainAndConceptCode(domain=SpecialDomain.NATIVE, concept_code=concept_string_or_code)
- elif (
- concept_codes_from_the_same_domain and concept_string_or_code in concept_codes_from_the_same_domain
- ): # Is a concept code from the same domain
- return DomainAndConceptCode(domain=domain, concept_code=concept_string_or_code)
- else:
- return DomainAndConceptCode(domain=SpecialDomain.IMPLICIT, concept_code=concept_string_or_code)
+ if (
+ concept_codes_from_the_same_domain and concept_string_or_code in concept_codes_from_the_same_domain
+ ): # Is a concept code from the same domain
+ return DomainAndConceptCode(domain=domain, concept_code=concept_string_or_code)
+ return DomainAndConceptCode(domain=SpecialDomain.IMPLICIT, concept_code=concept_string_or_code)
@classmethod
def make_refine(cls, refine: str) -> str:
if NativeConceptManager.is_native_concept(concept_string_or_code=refine):
return NativeConceptManager.get_native_concept_string(concept_string_or_code=refine)
- raise ConceptFactoryError(f"Refine '{refine}' is not a native concept")
+ msg = f"Refine '{refine}' is not a native concept"
+ raise ConceptFactoryError(msg)
@classmethod
- def make_refines(cls, blueprint: ConceptBlueprint) -> Optional[str]:
+ def make_refines(cls, blueprint: ConceptBlueprint) -> str | None:
if blueprint.refines:
return cls.make_refine(refine=blueprint.refines)
return None
@@ -120,21 +123,22 @@ class ConceptFactory:
domain: str,
concept_code: str,
blueprint: ConceptBlueprint,
- concept_codes_from_the_same_domain: Optional[List[str]] = None,
+ concept_codes_from_the_same_domain: list[str] | None = None,
) -> Concept:
ConceptBlueprint.validate_concept_code(concept_code=concept_code)
structure_class_name: str
- current_refine: Optional[str] = None
+ current_refine: str | None = None
# Handle structure definition
if blueprint.structure:
if isinstance(blueprint.structure, str):
# Structure is defined as a string - check if the class is in the registry and is valid
if not Concept.is_valid_structure_class(structure_class_name=blueprint.structure):
- raise StructureClassError(
+ msg = (
f"Structure class '{blueprint.structure}' set for concept '{concept_code}' in domain '{domain}' "
"is not a registered subclass of StuffContent"
)
+ raise StructureClassError(msg)
structure_class_name = blueprint.structure
else:
# Structure is defined as a ConceptStructureBlueprint - run the structure generator and put it in the class registry
@@ -148,7 +152,7 @@ class ConceptFactory:
)
# Execute the generated Python code to register the class
- exec_globals: Dict[str, Any] = {}
+ exec_globals: dict[str, Any] = {}
exec(python_code, exec_globals)
# Register the generated class
@@ -158,26 +162,27 @@ class ConceptFactory:
structure_class_name = concept_code
except Exception as exc:
- raise ConceptFactoryError(f"Error generating structure class for concept '{concept_code}' in domain '{domain}': {exc}") from exc
+ msg = f"Error generating structure class for concept '{concept_code}' in domain '{domain}': {exc}"
+ raise ConceptFactoryError(msg) from exc
# Handle refines definition
elif blueprint.refines:
# If we have refines, validate that there is no structure related to the concept code in the class registry
if Concept.is_valid_structure_class(structure_class_name=concept_code):
- raise ConceptFactoryError(
+ msg = (
f"Concept '{concept_code}' in domain '{domain}' has refines but also has a structure class registered. "
"A concept cannot have both structure and refines."
)
+ raise ConceptFactoryError(msg)
current_refine = cls.make_refines(blueprint=blueprint)
structure_class_name = current_refine.split(".")[1] + "Content" if current_refine else TextContent.__name__
# Handle neither structure nor refines - check the class registry
+ # If there is a class, use it. structure_class_name is then the concept_code
+ elif Concept.is_valid_structure_class(structure_class_name=concept_code):
+ structure_class_name = concept_code
else:
- # If there is a class, use it. structure_class_name is then the concept_code
- if Concept.is_valid_structure_class(structure_class_name=concept_code):
- structure_class_name = concept_code
- else:
- # If there is NO class, the fallback class is TextContent.__name__
- structure_class_name = TextContent.__name__
+ # If there is NO class, the fallback class is TextContent.__name__
+ structure_class_name = TextContent.__name__
domain_and_concept_code = cls.make_domain_and_concept_code_from_concept_string_or_code(
domain=domain,
diff --git a/pipelex/core/concepts/concept_library.py b/pipelex/core/concepts/concept_library.py
index 716c44adb..2904820f7 100644
--- a/pipelex/core/concepts/concept_library.py
+++ b/pipelex/core/concepts/concept_library.py
@@ -1,7 +1,7 @@
-from typing import Any, Dict, List, Optional, Type
+from typing import Any
from pydantic import Field, RootModel
-from typing_extensions import Self, override
+from typing_extensions import override
from pipelex.core.concepts.concept import Concept
from pipelex.core.concepts.concept_blueprint import ConceptBlueprint
@@ -12,8 +12,9 @@ from pipelex.core.domains.domain import SpecialDomain
from pipelex.core.stuffs.stuff_content import ImageContent
from pipelex.exceptions import ConceptLibraryConceptNotFoundError, ConceptLibraryError
from pipelex.hub import get_class_registry
+from pipelex.types import Self
-ConceptLibraryRoot = Dict[str, Concept]
+ConceptLibraryRoot = dict[str, Concept]
class ConceptLibrary(RootModel[ConceptLibraryRoot], ConceptProviderAbstract):
@@ -22,16 +23,15 @@ class ConceptLibrary(RootModel[ConceptLibraryRoot], ConceptProviderAbstract):
def validate_with_libraries(self):
"""Validates that the each refine concept code in the refines array of each concept in the library exists in the library"""
for concept in self.root.values():
- if concept.refines and concept.refines not in self.root.keys():
- raise ConceptLibraryError(
- f"Concept '{concept.code}' refines '{concept.refines}' but no concept with the code '{concept.refines}' exists"
- )
+ if concept.refines and concept.refines not in self.root:
+ msg = f"Concept '{concept.code}' refines '{concept.refines}' but no concept with the code '{concept.refines}' exists"
+ raise ConceptLibraryError(msg)
@override
def setup(self):
native_concepts = [
ConceptFactory.make_native_concept(native_concept_data=NATIVE_CONCEPTS_DATA[native_concept])
- for native_concept in [native_concept for native_concept in NativeConceptEnum]
+ for native_concept in NativeConceptEnum.values_list()
]
self.add_concepts(native_concepts)
@@ -52,79 +52,78 @@ class ConceptLibrary(RootModel[ConceptLibraryRoot], ConceptProviderAbstract):
def get_native_concept(self, native_concept: NativeConceptEnum) -> Concept:
try:
return self.root[f"{SpecialDomain.NATIVE}.{native_concept}"]
- except KeyError:
- raise ConceptLibraryConceptNotFoundError(f"Native concept '{native_concept}' not found in the library")
+ except KeyError as key_error:
+ msg = f"Native concept '{native_concept}' not found in the library"
+ raise ConceptLibraryConceptNotFoundError(msg) from key_error
- def get_native_concepts(self) -> List[Concept]:
+ def get_native_concepts(self) -> list[Concept]:
"""Create all native concepts from the hardcoded data"""
- return [self.get_native_concept(native_concept=native_concept) for native_concept in [native_concept for native_concept in NativeConceptEnum]]
+ return [self.get_native_concept(native_concept=native_concept) for native_concept in NativeConceptEnum.values_list()]
@override
- def list_concepts(self) -> List[Concept]:
+ def list_concepts(self) -> list[Concept]:
return list(self.root.values())
@override
- def list_concepts_by_domain(self, domain: str) -> List[Concept]:
+ def list_concepts_by_domain(self, domain: str) -> list[Concept]:
return [concept for key, concept in self.root.items() if key.startswith(f"{domain}.")]
@override
def add_new_concept(self, concept: Concept):
if concept.concept_string in self.root:
- raise ConceptLibraryError(f"Concept '{concept.concept_string}' already exists in the library")
+ msg = f"Concept '{concept.concept_string}' already exists in the library"
+ raise ConceptLibraryError(msg)
self.root[concept.concept_string] = concept
@override
- def add_concepts(self, concepts: List[Concept]):
+ def add_concepts(self, concepts: list[Concept]):
for concept in concepts:
self.add_new_concept(concept=concept)
- def remove_concepts_by_codes(self, concept_codes: List[str]) -> None:
+ def remove_concepts_by_codes(self, concept_codes: list[str]) -> None:
for concept_code in concept_codes:
if concept_code in self.root:
del self.root[concept_code]
@override
def is_compatible(self, tested_concept: Concept, wanted_concept: Concept, strict: bool = False) -> bool:
- if Concept.are_concept_compatible(concept_1=tested_concept, concept_2=wanted_concept, strict=strict):
- return True
- return False
+ return Concept.are_concept_compatible(concept_1=tested_concept, concept_2=wanted_concept, strict=strict)
@override
def get_required_concept(self, concept_string: str) -> Concept:
- """
- `concept_string` can have the domain or not. If it doesn't have the domain, it is assumed to be native.
+ """`concept_string` can have the domain or not. If it doesn't have the domain, it is assumed to be native.
If it is not native and doesnt have a domain, it should raise an error
"""
if Concept.is_implicit_concept(concept_string=concept_string):
return ConceptFactory.make_implicit_concept(concept_string=concept_string)
ConceptBlueprint.validate_concept_string(concept_string=concept_string)
- concept = self.root[concept_string]
- return concept
+ return self.root[concept_string]
@override
- def get_class(self, concept_code: str) -> Optional[Type[Any]]:
+ def get_class(self, concept_code: str) -> type[Any] | None:
return get_class_registry().get_class(concept_code)
@override
def is_image_concept(self, concept: Concept) -> bool:
- """
- Check if the concept is an image concept.
+ """Check if the concept is an image concept.
It is an image concept if its structure class is a subclass of ImageContent
or if it refines the native Image concept.
"""
pydantic_model = self.get_class(concept_code=concept.structure_class_name)
is_image_class = bool(pydantic_model and issubclass(pydantic_model, ImageContent))
refines_image = self.is_compatible(
- tested_concept=concept, wanted_concept=self.get_native_concept(native_concept=NativeConceptEnum.IMAGE), strict=True
+ tested_concept=concept,
+ wanted_concept=self.get_native_concept(native_concept=NativeConceptEnum.IMAGE),
+ strict=True,
)
return is_image_class or refines_image
@override
- def search_for_concept_in_domains(self, concept_code: str, search_domains: List[str]) -> Optional[Concept]:
+ def search_for_concept_in_domains(self, concept_code: str, search_domains: list[str]) -> Concept | None:
ConceptBlueprint.validate_concept_code(concept_code=concept_code)
for domain in search_domains:
if found_concept := self.get_required_concept(
- concept_string=ConceptFactory.construct_concept_string_with_domain(domain=domain, concept_code=concept_code)
+ concept_string=ConceptFactory.construct_concept_string_with_domain(domain=domain, concept_code=concept_code),
):
return found_concept
diff --git a/pipelex/core/concepts/concept_native.py b/pipelex/core/concepts/concept_native.py
index 7732bac45..4e38c24ec 100644
--- a/pipelex/core/concepts/concept_native.py
+++ b/pipelex/core/concepts/concept_native.py
@@ -1,4 +1,4 @@
-from typing import Dict, NamedTuple
+from typing import NamedTuple
from pipelex.core.domains.domain import SpecialDomain
from pipelex.types import StrEnum
@@ -25,6 +25,10 @@ class NativeConceptEnum(StrEnum):
PAGE = "Page"
ANYTHING = "Anything"
+ @classmethod
+ def values_list(cls) -> list["NativeConceptEnum"]:
+ return list(cls)
+
@classmethod
def is_text(cls, concept_code: str) -> bool:
try:
@@ -48,27 +52,41 @@ class NativeConceptEnum(StrEnum):
return False
-NATIVE_CONCEPTS_DATA: Dict[NativeConceptEnum, NativeConceptEnumData] = {
+NATIVE_CONCEPTS_DATA: dict[NativeConceptEnum, NativeConceptEnumData] = {
NativeConceptEnum.DYNAMIC: NativeConceptEnumData(
- code=NativeConceptEnum.DYNAMIC, content_class_name=f"{NativeConceptEnum.DYNAMIC}Content", definition="A dynamic concept"
+ code=NativeConceptEnum.DYNAMIC,
+ content_class_name=f"{NativeConceptEnum.DYNAMIC}Content",
+ definition="A dynamic concept",
),
NativeConceptEnum.TEXT: NativeConceptEnumData(
- code=NativeConceptEnum.TEXT, content_class_name=f"{NativeConceptEnum.TEXT}Content", definition="A text"
+ code=NativeConceptEnum.TEXT,
+ content_class_name=f"{NativeConceptEnum.TEXT}Content",
+ definition="A text",
),
NativeConceptEnum.IMAGE: NativeConceptEnumData(
- code=NativeConceptEnum.IMAGE, content_class_name=f"{NativeConceptEnum.IMAGE}Content", definition="An image"
+ code=NativeConceptEnum.IMAGE,
+ content_class_name=f"{NativeConceptEnum.IMAGE}Content",
+ definition="An image",
),
NativeConceptEnum.PDF: NativeConceptEnumData(
- code=NativeConceptEnum.PDF, content_class_name=f"{NativeConceptEnum.PDF}Content", definition="A PDF"
+ code=NativeConceptEnum.PDF,
+ content_class_name=f"{NativeConceptEnum.PDF}Content",
+ definition="A PDF",
),
NativeConceptEnum.TEXT_AND_IMAGES: NativeConceptEnumData(
- code=NativeConceptEnum.TEXT_AND_IMAGES, content_class_name=f"{NativeConceptEnum.TEXT_AND_IMAGES}Content", definition="A text and an image"
+ code=NativeConceptEnum.TEXT_AND_IMAGES,
+ content_class_name=f"{NativeConceptEnum.TEXT_AND_IMAGES}Content",
+ definition="A text and an image",
),
NativeConceptEnum.NUMBER: NativeConceptEnumData(
- code=NativeConceptEnum.NUMBER, content_class_name=f"{NativeConceptEnum.NUMBER}Content", definition="A number"
+ code=NativeConceptEnum.NUMBER,
+ content_class_name=f"{NativeConceptEnum.NUMBER}Content",
+ definition="A number",
),
NativeConceptEnum.LLM_PROMPT: NativeConceptEnumData(
- code=NativeConceptEnum.LLM_PROMPT, content_class_name=f"{NativeConceptEnum.LLM_PROMPT}Content", definition="A prompt for an LLM"
+ code=NativeConceptEnum.LLM_PROMPT,
+ content_class_name=f"{NativeConceptEnum.LLM_PROMPT}Content",
+ definition="A prompt for an LLM",
),
NativeConceptEnum.PAGE: NativeConceptEnumData(
code=NativeConceptEnum.PAGE,
@@ -76,7 +94,9 @@ NATIVE_CONCEPTS_DATA: Dict[NativeConceptEnum, NativeConceptEnumData] = {
definition="The content of a page of a document, comprising text and linked images and an optional page view image",
),
NativeConceptEnum.ANYTHING: NativeConceptEnumData(
- code=NativeConceptEnum.ANYTHING, content_class_name=f"{NativeConceptEnum.ANYTHING}Content", definition="Anything"
+ code=NativeConceptEnum.ANYTHING,
+ content_class_name=f"{NativeConceptEnum.ANYTHING}Content",
+ definition="Anything",
),
}
@@ -84,17 +104,14 @@ NATIVE_CONCEPTS_DATA: Dict[NativeConceptEnum, NativeConceptEnumData] = {
class NativeConceptManager:
@classmethod
def is_native_concept(cls, concept_string_or_code: str) -> bool:
- native_concept_values = [concept.value for concept in NativeConceptEnum]
+ native_concept_values = NativeConceptEnum.values_list()
if "." in concept_string_or_code:
domain, concept_code = concept_string_or_code.split(".", 1)
if SpecialDomain.is_native(domain=domain) and concept_code in native_concept_values:
return True
- if concept_string_or_code in native_concept_values:
- return True
-
- return False
+ return concept_string_or_code in native_concept_values
@classmethod
def get_native_concept_string(cls, concept_string_or_code: str) -> str:
diff --git a/pipelex/core/concepts/concept_provider_abstract.py b/pipelex/core/concepts/concept_provider_abstract.py
index 00ab47841..cbfbb404c 100644
--- a/pipelex/core/concepts/concept_provider_abstract.py
+++ b/pipelex/core/concepts/concept_provider_abstract.py
@@ -1,10 +1,10 @@
from abc import ABC, abstractmethod
-from typing import Any, Dict, List, Optional, Type
+from typing import Any
from pipelex.core.concepts.concept import Concept
from pipelex.core.concepts.concept_native import NativeConceptEnum
-ConceptLibraryRoot = Dict[str, Concept]
+ConceptLibraryRoot = dict[str, Concept]
class ConceptProviderAbstract(ABC):
@@ -13,15 +13,15 @@ class ConceptProviderAbstract(ABC):
pass
@abstractmethod
- def add_concepts(self, concepts: List[Concept]) -> None:
+ def add_concepts(self, concepts: list[Concept]) -> None:
pass
@abstractmethod
- def list_concepts_by_domain(self, domain: str) -> List[Concept]:
+ def list_concepts_by_domain(self, domain: str) -> list[Concept]:
pass
@abstractmethod
- def list_concepts(self) -> List[Concept]:
+ def list_concepts(self) -> list[Concept]:
pass
@abstractmethod
@@ -49,11 +49,11 @@ class ConceptProviderAbstract(ABC):
pass
@abstractmethod
- def search_for_concept_in_domains(self, concept_code: str, search_domains: List[str]) -> Optional[Concept]:
+ def search_for_concept_in_domains(self, concept_code: str, search_domains: list[str]) -> Concept | None:
pass
@abstractmethod
- def get_class(self, concept_code: str) -> Optional[Type[Any]]:
+ def get_class(self, concept_code: str) -> type[Any] | None:
pass
@abstractmethod
diff --git a/pipelex/core/concepts/structure_generator.py b/pipelex/core/concepts/structure_generator.py
index 2c6479783..b740d0f2e 100644
--- a/pipelex/core/concepts/structure_generator.py
+++ b/pipelex/core/concepts/structure_generator.py
@@ -1,12 +1,13 @@
-"""Generate Pydantic BaseModel classes from concept structure blueprints for structured outputs."""
-
-from __future__ import annotations
-
import ast
-from typing import Any, Dict, List, Type, Union, cast
+from datetime import datetime
+from enum import Enum
+from typing import Any, Literal, Optional, cast
+
+from pydantic import Field
from pipelex import log
from pipelex.core.concepts.concept_blueprint import ConceptStructureBlueprint, ConceptStructureBlueprintFieldType
+from pipelex.core.stuffs.stuff_content import StructuredContent
class StructureGenerator:
@@ -21,16 +22,15 @@ class StructureGenerator:
"from pipelex.core.stuffs.stuff_content import StructuredContent",
"from pydantic import Field",
}
- self.enum_definitions: Dict[str, Dict[str, Any]] = {} # Store enum definitions
+ self.enum_definitions: dict[str, dict[str, Any]] = {} # Store enum definitions
def _format_default_value(self, value: Any) -> str:
"""Format default value for Python code, ensuring strings use double quotes."""
if isinstance(value, str):
return f'"{value}"'
- else:
- return repr(value)
+ return repr(value)
- def generate_from_structure_blueprint(self, class_name: str, structure_blueprint: Dict[str, ConceptStructureBlueprint]) -> str:
+ def generate_from_structure_blueprint(self, class_name: str, structure_blueprint: dict[str, ConceptStructureBlueprint]) -> str:
"""Generate Python module content from structure blueprint.
Args:
@@ -39,6 +39,7 @@ class StructureGenerator:
Returns:
Generated Python module content
+
"""
# Generate the class
class_code = self._generate_class_from_blueprint(class_name, structure_blueprint)
@@ -50,11 +51,12 @@ class StructureGenerator:
# Validate the generated code
if not self.validate_generated_code(generated_code, class_name):
- raise ValueError(f"Generated code for class '{class_name}' failed validation")
+ msg = f"Generated code for class '{class_name}' failed validation"
+ raise ValueError(msg)
return generated_code
- def _generate_class_from_blueprint(self, class_name: str, structure_blueprint: Dict[str, ConceptStructureBlueprint]) -> str:
+ def _generate_class_from_blueprint(self, class_name: str, structure_blueprint: dict[str, ConceptStructureBlueprint]) -> str:
"""Generate a class definition from ConceptStructureBlueprint.
Args:
@@ -63,12 +65,13 @@ class StructureGenerator:
Returns:
Generated class code
+
"""
# Generate class header
class_header = f'class {class_name}(StructuredContent):\n """Generated {class_name} class"""\n'
# Generate fields
- field_definitions: List[str] = []
+ field_definitions: list[str] = []
for field_name, field_blueprint in structure_blueprint.items():
field_code = self._generate_field_from_blueprint(field_name, field_blueprint)
field_definitions.append(field_code)
@@ -89,6 +92,7 @@ class StructureGenerator:
Returns:
Generated field code
+
"""
# Determine Python type
if field_blueprint.choices:
@@ -110,11 +114,10 @@ class StructureGenerator:
field_params.insert(0, f"default={self._format_default_value(field_blueprint.default_value)}")
else:
field_params.insert(0, "...")
+ elif field_blueprint.default_value is not None:
+ field_params.insert(0, f"default={self._format_default_value(field_blueprint.default_value)}")
else:
- if field_blueprint.default_value is not None:
- field_params.insert(0, f"default={self._format_default_value(field_blueprint.default_value)}")
- else:
- field_params.insert(0, "default=None")
+ field_params.insert(0, "default=None")
field_call = f"Field({', '.join(field_params)})"
@@ -128,6 +131,7 @@ class StructureGenerator:
Returns:
Python type annotation string
+
"""
if field_blueprint.type is None:
# This should not happen based on validation, but handle gracefully
@@ -176,7 +180,7 @@ class StructureGenerator:
pass
return f"Dict[{key_type}, {value_type}]"
- def _generate_field(self, field_name: str, field_def: Union[Dict[str, Any], str]) -> str:
+ def _generate_field(self, field_name: str, field_def: dict[str, Any] | str) -> str:
"""Generate a single field definition.
Args:
@@ -185,6 +189,7 @@ class StructureGenerator:
Returns:
Generated field code
+
"""
# Handle simple string definitions (just the definition text)
if isinstance(field_def, str):
@@ -216,17 +221,16 @@ class StructureGenerator:
field_params.insert(0, f"default={self._format_default_value(default_value)}")
else:
field_params.insert(0, "...")
+ elif default_value is not None:
+ field_params.insert(0, f"default={self._format_default_value(default_value)}")
else:
- if default_value is not None:
- field_params.insert(0, f"default={self._format_default_value(default_value)}")
- else:
- field_params.insert(0, "default=None")
+ field_params.insert(0, "default=None")
field_call = f"Field({', '.join(field_params)})"
return f" {field_name}: {python_type} = {field_call}"
- def _get_python_type(self, field_type: Any, field_def: Dict[str, Any]) -> str:
+ def _get_python_type(self, field_type: Any, field_def: dict[str, Any]) -> str:
"""Convert high-level type to Python type annotation.
Args:
@@ -235,6 +239,7 @@ class StructureGenerator:
Returns:
Python type annotation string
+
"""
# Check if it's a reference to a defined enum
if isinstance(field_type, str) and field_type in self.enum_definitions:
@@ -306,6 +311,7 @@ class StructureGenerator:
Returns:
True if the code is valid, False otherwise
+
"""
# Step 1: Syntax validation
if not self._validate_syntax(python_code):
@@ -320,10 +326,7 @@ class StructureGenerator:
return False
# Step 4: Class instantiation validation
- if not self._validate_instantiation(python_code, expected_class_name):
- return False
-
- return True
+ return self._validate_instantiation(python_code, expected_class_name)
def _validate_syntax(self, python_code: str) -> bool:
"""Validate that the code has valid Python syntax."""
@@ -347,13 +350,13 @@ class StructureGenerator:
"""Validate that the code executes and creates the expected class."""
try:
# Import necessary modules for the execution context
- from datetime import datetime
- from enum import Enum
- from typing import Any, Dict, List, Literal, Optional
+ from datetime import datetime # noqa: PLC0415
+ from enum import Enum # noqa: PLC0415
+ from typing import Any, Dict, List, Literal, Optional # noqa: PLC0415,F401
- from pydantic import Field
+ from pydantic import Field # noqa: PLC0415
- from pipelex.core.stuffs.stuff_content import StructuredContent
+ from pipelex.core.stuffs.stuff_content import StructuredContent # noqa: PLC0415
# Provide necessary imports in the execution context
exec_globals = {
@@ -361,14 +364,14 @@ class StructureGenerator:
"datetime": datetime,
"Enum": Enum,
"Optional": Optional,
- "List": List,
- "Dict": Dict,
+ "List": list,
+ "Dict": dict,
"Any": Any,
"Literal": Literal,
"Field": Field,
"StructuredContent": StructuredContent,
}
- exec_locals: Dict[str, Any] = {}
+ exec_locals: dict[str, Any] = {}
exec(python_code, exec_globals, exec_locals)
# Verify the expected class was created
@@ -391,34 +394,23 @@ class StructureGenerator:
return False
def _validate_instantiation(self, python_code: str, expected_class_name: str) -> bool:
- """Validate that the generated class can be instantiated."""
try:
- # Import necessary modules for the execution context
- from datetime import datetime
- from enum import Enum
- from typing import Any, Dict, List, Literal, Optional
-
- from pydantic import Field
-
- from pipelex.core.stuffs.stuff_content import StructuredContent
-
- # Provide necessary imports in the execution context
exec_globals = {
"__builtins__": __builtins__,
"datetime": datetime,
"Enum": Enum,
"Optional": Optional,
- "List": List,
- "Dict": Dict,
+ "List": list,
+ "Dict": dict,
"Any": Any,
"Literal": Literal,
"Field": Field,
"StructuredContent": StructuredContent,
}
- exec_locals: Dict[str, Any] = {}
+ exec_locals: dict[str, Any] = {}
exec(python_code, exec_globals, exec_locals)
- generated_class = cast(Type[Any], exec_locals[expected_class_name])
+ generated_class = cast("type[Any]", exec_locals[expected_class_name])
# Try to create an instance (this will catch Pydantic validation issues)
# For validation purposes, we'll try to create an instance with minimal valid data
@@ -429,7 +421,7 @@ class StructureGenerator:
except Exception:
# If that fails, try with empty dict (some models accept this)
try:
- instance = generated_class(**{})
+ instance = generated_class()
except Exception:
# If that fails too, the class structure is probably fine but requires specific data
# For validation purposes, we'll just check that it's a valid Pydantic model class
diff --git a/pipelex/core/domains/__init__.py b/pipelex/core/domains/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/pipelex/core/domains/domain.py b/pipelex/core/domains/domain.py
index 95f15f13d..59d1d08a4 100644
--- a/pipelex/core/domains/domain.py
+++ b/pipelex/core/domains/domain.py
@@ -1,9 +1,6 @@
-from typing import Optional
-
from pydantic import BaseModel
-from typing_extensions import Self
-from pipelex.types import StrEnum
+from pipelex.types import Self, StrEnum
class SpecialDomain(StrEnum):
@@ -26,10 +23,10 @@ class SpecialDomain(StrEnum):
class Domain(BaseModel):
code: str
- definition: Optional[str] = None
- system_prompt: Optional[str] = None
- system_prompt_to_structure: Optional[str] = None
- prompt_template_to_structure: Optional[str] = None
+ definition: str | None = None
+ system_prompt: str | None = None
+ system_prompt_to_structure: str | None = None
+ prompt_template_to_structure: str | None = None
@classmethod
def make_default(cls) -> Self:
diff --git a/pipelex/core/domains/domain_blueprint.py b/pipelex/core/domains/domain_blueprint.py
index e73e3e6ef..844f3b380 100644
--- a/pipelex/core/domains/domain_blueprint.py
+++ b/pipelex/core/domains/domain_blueprint.py
@@ -1,5 +1,3 @@
-from typing import Optional
-
from pydantic import BaseModel
from pipelex.core.domains.exceptions import DomainError
@@ -8,13 +6,14 @@ from pipelex.tools.misc.string_utils import is_snake_case
class DomainBlueprint(BaseModel):
code: str
- definition: Optional[str] = None
- system_prompt: Optional[str] = None
- system_prompt_to_structure: Optional[str] = None
- prompt_template_to_structure: Optional[str] = None
+ definition: str | None = None
+ system_prompt: str | None = None
+ system_prompt_to_structure: str | None = None
+ prompt_template_to_structure: str | None = None
@staticmethod
def validate_domain_code(code: str) -> None:
"""Validate that a domain code follows snake_case convention."""
if not is_snake_case(code):
- raise DomainError(f"Domain code '{code}' must be snake_case (lowercase letters, numbers, and underscores only)")
+ msg = f"Domain code '{code}' must be snake_case (lowercase letters, numbers, and underscores only)"
+ raise DomainError(msg)
diff --git a/pipelex/core/domains/domain_library.py b/pipelex/core/domains/domain_library.py
index 4e863de48..f9a0abe14 100644
--- a/pipelex/core/domains/domain_library.py
+++ b/pipelex/core/domains/domain_library.py
@@ -1,13 +1,12 @@
-from typing import Dict, Optional
-
from pydantic import RootModel
-from typing_extensions import Self, override
+from typing_extensions import override
from pipelex.core.domains.domain import Domain
from pipelex.core.domains.domain_provider_abstract import DomainProviderAbstract
from pipelex.exceptions import DomainLibraryError
+from pipelex.types import Self
-DomainLibraryRoot = Dict[str, Domain]
+DomainLibraryRoot = dict[str, Domain]
class DomainLibrary(RootModel[DomainLibraryRoot], DomainProviderAbstract):
@@ -34,14 +33,15 @@ class DomainLibrary(RootModel[DomainLibraryRoot], DomainProviderAbstract):
del self.root[domain_code]
@override
- def get_domain(self, domain: str) -> Optional[Domain]:
+ def get_domain(self, domain: str) -> Domain | None:
return self.root.get(domain)
@override
def get_required_domain(self, domain: str) -> Domain:
the_domain = self.get_domain(domain=domain)
if not the_domain:
- raise DomainLibraryError(f"Domain '{domain}' not found. Check for typos and make sure it is declared in a pipeline library.")
+ msg = f"Domain '{domain}' not found. Check for typos and make sure it is declared in a pipeline library."
+ raise DomainLibraryError(msg)
return the_domain
@override
diff --git a/pipelex/core/domains/domain_provider_abstract.py b/pipelex/core/domains/domain_provider_abstract.py
index bb2b7bfc8..0a9cc769c 100644
--- a/pipelex/core/domains/domain_provider_abstract.py
+++ b/pipelex/core/domains/domain_provider_abstract.py
@@ -1,12 +1,11 @@
from abc import ABC, abstractmethod
-from typing import Optional
from pipelex.core.domains.domain import Domain
class DomainProviderAbstract(ABC):
@abstractmethod
- def get_domain(self, domain: str) -> Optional[Domain]:
+ def get_domain(self, domain: str) -> Domain | None:
pass
@abstractmethod
diff --git a/pipelex/core/exceptions.py b/pipelex/core/exceptions.py
index 62f41fe28..2d2f5ccae 100644
--- a/pipelex/core/exceptions.py
+++ b/pipelex/core/exceptions.py
@@ -4,16 +4,10 @@
class PipelexInterpreterError(Exception):
"""Base exception class for PipelexInterpreter errors."""
- pass
-
class PipelexConfigurationError(PipelexInterpreterError):
"""Raised when there are configuration issues with the PipelexInterpreter."""
- pass
-
class PipelexUnknownPipeError(PipelexInterpreterError):
"""Raised when encountering an unknown pipe blueprint type."""
-
- pass
diff --git a/pipelex/core/interpreter.py b/pipelex/core/interpreter.py
index 9413cd7a3..0d6d9509b 100644
--- a/pipelex/core/interpreter.py
+++ b/pipelex/core/interpreter.py
@@ -1,12 +1,12 @@
+from collections.abc import Mapping
from pathlib import Path
-from typing import Any, Dict, Mapping, Optional, Union, cast
+from typing import Any, cast
import toml
from pydantic import BaseModel, model_validator
-from typing_extensions import Self
from pipelex.core.bundles.pipelex_bundle_blueprint import PipelexBundleBlueprint
-from pipelex.core.concepts.concept_blueprint import ConceptBlueprint
+from pipelex.core.concepts.concept_blueprint import ConceptBlueprint, ConceptStructureBlueprint
from pipelex.core.exceptions import (
PipelexConfigurationError,
PipelexUnknownPipeError,
@@ -32,22 +32,21 @@ from pipelex.tools.misc.toml_utils import (
validate_toml_content,
validate_toml_file,
)
+from pipelex.types import Self
class PLXDecodeError(toml.TomlDecodeError):
"""Raised when PLX decoding fails."""
- pass
-
class PipelexInterpreter(BaseModel):
"""plx -> PipelexBundleBlueprint"""
- file_path: Optional[Path] = None
- file_content: Optional[str] = None
+ file_path: Path | None = None
+ file_content: str | None = None
@staticmethod
- def escape_plx_string(value: Optional[str]) -> str:
+ def escape_plx_string(value: str | None) -> str:
"""Escape a string for plx serialization."""
if value is None:
return ""
@@ -58,14 +57,14 @@ class PipelexInterpreter(BaseModel):
# Replace actual newlines with escaped newlines
value = value.replace("\n", "\\n")
value = value.replace("\r", "\\r")
- value = value.replace("\t", "\\t")
- return value
+ return value.replace("\t", "\\t")
@model_validator(mode="after")
def check_file_path_or_file_content(self) -> Self:
"""Need to check if there is at least one of file_path or file_content"""
if self.file_path is None and self.file_content is None:
- raise PipelexConfigurationError("Either file_path or file_content must be provided")
+ msg = "Either file_path or file_content must be provided"
+ raise PipelexConfigurationError(msg)
return self
@model_validator(mode="after")
@@ -79,7 +78,7 @@ class PipelexInterpreter(BaseModel):
def _read_and_reformat(self) -> str:
"""Load PLX content from file_path or use file_content directly."""
if self.file_path:
- with open(self.file_path, "r", encoding="utf-8") as file:
+ with open(self.file_path, encoding="utf-8") as file:
file_content = file.read()
# Clean trailing whitespace and write back if needed
@@ -89,8 +88,9 @@ class PipelexInterpreter(BaseModel):
file.write(cleaned_content)
return cleaned_content
return file_content
- elif self.file_content is None:
- raise PipelexConfigurationError("file_content must be provided if file_path is not provided")
+ if self.file_content is None:
+ msg = "file_content must be provided if file_path is not provided"
+ raise PipelexConfigurationError(msg)
return self.file_content
@staticmethod
@@ -106,6 +106,7 @@ class PipelexInterpreter(BaseModel):
Criteria:
- Has .plx extension
- Starts with "domain =" (ignoring leading whitespace)
+
"""
# Check if it has .toml extension
if file_path.suffix != ".plx":
@@ -117,7 +118,7 @@ class PipelexInterpreter(BaseModel):
try:
# Read the first few lines to check for "domain ="
- with open(file_path, "r", encoding="utf-8") as f:
+ with open(file_path, encoding="utf-8") as f:
# Read first 100 characters (should be enough to find domain)
content = f.read(100)
# Remove leading whitespace and check if it starts with "domain ="
@@ -127,13 +128,14 @@ class PipelexInterpreter(BaseModel):
# If we can't read the file, it's not a valid Pipelex file
return False
- def _parse_plx_content(self, content: str) -> Dict[str, Any]:
+ def _parse_plx_content(self, content: str) -> dict[str, Any]:
"""Parse PLX content and return the dictionary."""
try:
return toml.loads(content)
except toml.TomlDecodeError as exc:
file_path_str = str(self.file_path) if self.file_path else "content"
- raise PLXDecodeError(f"PLX parsing error in '{file_path_str}': {exc}", exc.doc, exc.pos) from exc
+ msg = f"PLX parsing error in '{file_path_str}': {exc}"
+ raise PLXDecodeError(msg, exc.doc, exc.pos) from exc
def make_pipelex_bundle_blueprint(self) -> PipelexBundleBlueprint:
"""Make a PipelexBundleBlueprint from the file_path or file_content"""
@@ -162,13 +164,13 @@ class PipelexInterpreter(BaseModel):
# Concepts section
if blueprint.concept:
- concept_plx = PipelexInterpreter.concepts_to_plx_string(blueprint.concept, blueprint.domain)
+ concept_plx = PipelexInterpreter.concepts_to_plx_string(blueprint.concept)
if concept_plx: # Only add if not empty
plx_parts.append(concept_plx)
# Pipes section
if blueprint.pipe:
- pipes_plx = PipelexInterpreter.pipes_to_plx_string(blueprint.pipe, blueprint.domain)
+ pipes_plx = PipelexInterpreter.pipes_to_plx_string(blueprint.pipe)
if pipes_plx: # Only add if not empty
plx_parts.append(pipes_plx)
@@ -178,7 +180,7 @@ class PipelexInterpreter(BaseModel):
return result
@staticmethod
- def concepts_to_plx_string(concepts: Dict[str, ConceptBlueprint | str], domain: str) -> str:
+ def concepts_to_plx_string(concepts: dict[str, ConceptBlueprint | str]) -> str:
"""Convert concepts dict to PLX string."""
if not concepts:
return ""
@@ -190,15 +192,14 @@ class PipelexInterpreter(BaseModel):
for concept_name, concept_blueprint in concepts.items():
if isinstance(concept_blueprint, str):
simple_concepts.append(f'{concept_name} = "{PipelexInterpreter.escape_plx_string(concept_blueprint)}"')
+ # Handle ConceptBlueprint objects
+ elif concept_blueprint.structure is None and concept_blueprint.refines is None:
+ # Simple concept with just definition
+ simple_concepts.append(f'{concept_name} = "{PipelexInterpreter.escape_plx_string(concept_blueprint.definition)}"')
else:
- # Handle ConceptBlueprint objects
- if concept_blueprint.structure is None and concept_blueprint.refines is None:
- # Simple concept with just definition
- simple_concepts.append(f'{concept_name} = "{PipelexInterpreter.escape_plx_string(concept_blueprint.definition)}"')
- else:
- # Complex concept needs its own section
- complex_concept_plx = PipelexInterpreter.complex_concept_to_plx_string(concept_name, concept_blueprint)
- complex_concepts.append(complex_concept_plx)
+ # Complex concept needs its own section
+ complex_concept_plx = PipelexInterpreter.complex_concept_to_plx_string(concept_name, concept_blueprint)
+ complex_concepts.append(complex_concept_plx)
# Add simple concepts section if we have any
if simple_concepts:
@@ -247,59 +248,58 @@ class PipelexInterpreter(BaseModel):
"""Convert a structure field to PLX string."""
if isinstance(field_def, str):
return f'{field_name} = "{PipelexInterpreter.escape_plx_string(field_def)}"'
- else:
- # Complex field with type, definition, required, etc.
- field_parts: list[str] = []
+ # Complex field with type, definition, required, etc.
+ field_parts: list[str] = []
- if hasattr(field_def, "type") and field_def.type:
- type_value = field_def.type.value if hasattr(field_def.type, "value") else field_def.type
- field_parts.append(f'type = "{PipelexInterpreter.escape_plx_string(str(type_value))}"')
+ if hasattr(field_def, "type") and field_def.type:
+ type_value = field_def.type.value if hasattr(field_def.type, "value") else field_def.type
+ field_parts.append(f'type = "{PipelexInterpreter.escape_plx_string(str(type_value))}"')
- if hasattr(field_def, "definition") and field_def.definition:
- field_parts.append(f'definition = "{PipelexInterpreter.escape_plx_string(field_def.definition)}"')
+ if hasattr(field_def, "definition") and field_def.definition:
+ field_parts.append(f'definition = "{PipelexInterpreter.escape_plx_string(field_def.definition)}"')
- if hasattr(field_def, "required") and field_def.required is not None:
- field_parts.append(f"required = {str(field_def.required).lower()}")
+ if hasattr(field_def, "required") and field_def.required is not None:
+ field_parts.append(f"required = {str(field_def.required).lower()}")
- return f"{field_name} = {{ {', '.join(field_parts)} }}"
+ return f"{field_name} = {{ {', '.join(field_parts)} }}"
@staticmethod
- def pipes_to_plx_string(pipes: Dict[str, Any], domain: str) -> str:
+ def pipes_to_plx_string(pipes: dict[str, Any]) -> str:
"""Convert pipes dict to PLX string."""
plx_parts: list[str] = []
for pipe_name, blueprint in pipes.items():
- pipe_plx = PipelexInterpreter.pipe_to_plx_string(pipe_name, blueprint, domain)
+ pipe_plx = PipelexInterpreter.pipe_to_plx_string(pipe_name, blueprint)
plx_parts.append(pipe_plx)
return "\n\n".join(plx_parts)
@staticmethod
- def pipe_to_plx_string(pipe_name: str, blueprint: Any, domain: str) -> str:
+ def pipe_to_plx_string(pipe_name: str, blueprint: Any) -> str:
"""Convert a single pipe blueprint to PLX string."""
if isinstance(blueprint, PipeLLMBlueprint):
- return PipelexInterpreter.llm_pipe_to_plx_string(pipe_name, blueprint, domain)
+ return PipelexInterpreter.llm_pipe_to_plx_string(pipe_name, blueprint)
elif isinstance(blueprint, PipeSequenceBlueprint):
- return PipelexInterpreter.sequence_pipe_to_plx_string(pipe_name, blueprint, domain)
+ return PipelexInterpreter.sequence_pipe_to_plx_string(pipe_name, blueprint)
elif isinstance(blueprint, PipeOcrBlueprint):
- return PipelexInterpreter.ocr_pipe_to_plx_string(pipe_name, blueprint, domain)
+ return PipelexInterpreter.ocr_pipe_to_plx_string(pipe_name, blueprint)
elif isinstance(blueprint, PipeFuncBlueprint):
- return PipelexInterpreter.func_pipe_to_plx_string(pipe_name, blueprint, domain)
+ return PipelexInterpreter.func_pipe_to_plx_string(pipe_name, blueprint)
elif isinstance(blueprint, PipeImgGenBlueprint):
- return PipelexInterpreter.img_gen_pipe_to_plx_string(pipe_name, blueprint, domain)
+ return PipelexInterpreter.img_gen_pipe_to_plx_string(pipe_name, blueprint)
elif isinstance(blueprint, PipeJinja2Blueprint):
- return PipelexInterpreter.jinja2_pipe_to_plx_string(pipe_name, blueprint, domain)
+ return PipelexInterpreter.jinja2_pipe_to_plx_string(pipe_name, blueprint)
elif isinstance(blueprint, PipeConditionBlueprint):
- return PipelexInterpreter.condition_pipe_to_plx_string(pipe_name, blueprint, domain)
+ return PipelexInterpreter.condition_pipe_to_plx_string(pipe_name, blueprint)
elif isinstance(blueprint, PipeParallelBlueprint):
- return PipelexInterpreter.parallel_pipe_to_plx_string(pipe_name, blueprint, domain)
+ return PipelexInterpreter.parallel_pipe_to_plx_string(pipe_name, blueprint)
elif isinstance(blueprint, PipeBatchBlueprint):
- return PipelexInterpreter.batch_pipe_to_plx_string(pipe_name, blueprint, domain)
+ return PipelexInterpreter.batch_pipe_to_plx_string(pipe_name, blueprint)
else:
# Fallback to old dict approach for unknown pipe types
- pipe_dict = PipelexInterpreter.serialize_pipe(blueprint, domain)
+ pipe_dict = PipelexInterpreter.serialize_pipe(blueprint)
return f"[pipe.{pipe_name}]\n" + "\n".join([f'{k} = "{v}"' if isinstance(v, str) else f"{k} = {v}" for k, v in pipe_dict.items()])
@staticmethod
- def llm_pipe_to_plx_string(pipe_name: str, pipe: PipeLLMBlueprint, domain: str) -> str:
+ def llm_pipe_to_plx_string(pipe_name: str, pipe: PipeLLMBlueprint) -> str:
"""Convert a PipeLLM blueprint directly to PLX section string."""
lines: list[str] = [
f"[pipe.{pipe_name}]",
@@ -332,7 +332,7 @@ class PipelexInterpreter(BaseModel):
return "\n".join(lines)
@staticmethod
- def ocr_pipe_to_plx_string(pipe_name: str, pipe: PipeOcrBlueprint, domain: str) -> str:
+ def ocr_pipe_to_plx_string(pipe_name: str, pipe: PipeOcrBlueprint) -> str:
"""Convert a PipeOcr blueprint directly to PLX section string."""
lines: list[str] = [
f"[pipe.{pipe_name}]",
@@ -349,7 +349,7 @@ class PipelexInterpreter(BaseModel):
return "\n".join(lines)
@staticmethod
- def func_pipe_to_plx_string(pipe_name: str, pipe: PipeFuncBlueprint, domain: str) -> str:
+ def func_pipe_to_plx_string(pipe_name: str, pipe: PipeFuncBlueprint) -> str:
"""Convert a PipeFunc blueprint directly to PLX section string."""
lines: list[str] = [
f"[pipe.{pipe_name}]",
@@ -366,7 +366,7 @@ class PipelexInterpreter(BaseModel):
return "\n".join(lines)
@staticmethod
- def img_gen_pipe_to_plx_string(pipe_name: str, pipe: PipeImgGenBlueprint, domain: str) -> str:
+ def img_gen_pipe_to_plx_string(pipe_name: str, pipe: PipeImgGenBlueprint) -> str:
"""Convert a PipeImgGen blueprint directly to PLX section string."""
lines: list[str] = [
f"[pipe.{pipe_name}]",
@@ -394,7 +394,7 @@ class PipelexInterpreter(BaseModel):
return "\n".join(lines)
@staticmethod
- def jinja2_pipe_to_plx_string(pipe_name: str, pipe: PipeJinja2Blueprint, domain: str) -> str:
+ def jinja2_pipe_to_plx_string(pipe_name: str, pipe: PipeJinja2Blueprint) -> str:
"""Convert a PipeJinja2 blueprint directly to PLX section string."""
lines: list[str] = [
f"[pipe.{pipe_name}]",
@@ -416,7 +416,7 @@ class PipelexInterpreter(BaseModel):
return "\n".join(lines)
@staticmethod
- def condition_pipe_to_plx_string(pipe_name: str, pipe: PipeConditionBlueprint, domain: str) -> str:
+ def condition_pipe_to_plx_string(pipe_name: str, pipe: PipeConditionBlueprint) -> str:
"""Convert a PipeCondition blueprint directly to PLX section string."""
lines: list[str] = [
f"[pipe.{pipe_name}]",
@@ -450,7 +450,7 @@ class PipelexInterpreter(BaseModel):
return "\n".join(lines)
@staticmethod
- def parallel_pipe_to_plx_string(pipe_name: str, pipe: PipeParallelBlueprint, domain: str) -> str:
+ def parallel_pipe_to_plx_string(pipe_name: str, pipe: PipeParallelBlueprint) -> str:
"""Convert a PipeParallel blueprint directly to PLX section string."""
lines: list[str] = [
f"[pipe.{pipe_name}]",
@@ -480,7 +480,7 @@ class PipelexInterpreter(BaseModel):
return "\n".join(lines)
@staticmethod
- def batch_pipe_to_plx_string(pipe_name: str, pipe: PipeBatchBlueprint, domain: str) -> str:
+ def batch_pipe_to_plx_string(pipe_name: str, pipe: PipeBatchBlueprint) -> str:
"""Convert a PipeBatch blueprint directly to PLX section string."""
lines: list[str] = [
f"[pipe.{pipe_name}]",
@@ -505,12 +505,10 @@ class PipelexInterpreter(BaseModel):
@staticmethod
def serialize_concept_structure_field(field_value: Any) -> Any:
"""Serialize a single concept structure field."""
- from pipelex.core.concepts.concept_blueprint import ConceptStructureBlueprint
-
if isinstance(field_value, str):
return field_value
- elif isinstance(field_value, ConceptStructureBlueprint):
- field_data: Dict[str, Any] = {
+ if isinstance(field_value, ConceptStructureBlueprint):
+ field_data: dict[str, Any] = {
"definition": field_value.definition,
"required": field_value.required,
}
@@ -527,26 +525,24 @@ class PipelexInterpreter(BaseModel):
if field_value.default_value is not None:
field_data["default_value"] = field_value.default_value
return field_data
- else:
- return field_value
+ return field_value
@staticmethod
def serialize_concept_structure(
- structure: Union[str, Dict[str, Any]],
- ) -> Union[str, Dict[str, Any]]:
+ structure: str | dict[str, Any],
+ ) -> str | dict[str, Any]:
"""Serialize concept structure field."""
if isinstance(structure, str):
return structure
- else:
- result: Dict[str, Any] = {}
- for field_name, field_value in structure.items():
- result[field_name] = PipelexInterpreter.serialize_concept_structure_field(field_value)
- return result
+ result: dict[str, Any] = {}
+ for field_name, field_value in structure.items():
+ result[field_name] = PipelexInterpreter.serialize_concept_structure_field(field_value)
+ return result
@staticmethod
def serialize_single_concept(
- concept_blueprint: Union[ConceptBlueprint, str],
- ) -> Union[str, Dict[str, Any]]:
+ concept_blueprint: ConceptBlueprint | str,
+ ) -> str | dict[str, Any]:
"""Serialize a single concept blueprint."""
if isinstance(concept_blueprint, str):
return concept_blueprint
@@ -554,70 +550,66 @@ class PipelexInterpreter(BaseModel):
# Handle ConceptBlueprint object
if concept_blueprint.structure is not None:
# Structured concept
- concept_data = {
+ return {
"definition": concept_blueprint.definition,
"structure": PipelexInterpreter.serialize_concept_structure(concept_blueprint.structure),
}
- return concept_data
- elif concept_blueprint.refines is not None:
+ if concept_blueprint.refines is not None:
# Concept with refines
- concept_data = {
+ return {
"definition": concept_blueprint.definition,
"refines": concept_blueprint.refines,
}
- return concept_data
- else:
- # Simple concept with just definition
- return concept_blueprint.definition
+ # Simple concept with just definition
+ return concept_blueprint.definition
@staticmethod
- def serialize_concepts(concepts: Optional[Dict[str, ConceptBlueprint | str]], domain: str) -> Dict[str, Any]:
+ def serialize_concepts(concepts: dict[str, ConceptBlueprint | str] | None) -> dict[str, Any]:
"""Serialize concepts section with domain context."""
if concepts is None:
return {}
- result: Dict[str, Any] = {}
+ result: dict[str, Any] = {}
for concept_name, concept_blueprint in concepts.items():
result[concept_name] = PipelexInterpreter.serialize_single_concept(concept_blueprint)
return result
@staticmethod
- def serialize_pipes(pipes: Dict[str, Any], domain: str) -> Dict[str, Any]:
+ def serialize_pipes(pipes: dict[str, Any]) -> dict[str, Any]:
"""Serialize pipes section with domain context."""
- result: Dict[str, Any] = {}
+ result: dict[str, Any] = {}
for pipe_name, blueprint in pipes.items():
- result[pipe_name] = PipelexInterpreter.serialize_pipe(blueprint, domain)
+ result[pipe_name] = PipelexInterpreter.serialize_pipe(blueprint)
return result
@staticmethod
- def serialize_pipe(blueprint: Any, domain: str) -> Dict[str, Any]:
- """Serialize a single pipe blueprint with domain context."""
-
+ def serialize_pipe(blueprint: Any) -> dict[str, Any]:
if isinstance(blueprint, PipeLLMBlueprint):
- return PipelexInterpreter.serialize_llm_pipe(blueprint, domain)
+ return PipelexInterpreter.serialize_llm_pipe(blueprint)
elif isinstance(blueprint, PipeOcrBlueprint):
- return PipelexInterpreter.serialize_ocr_pipe(blueprint, domain)
+ return PipelexInterpreter.serialize_ocr_pipe(blueprint)
elif isinstance(blueprint, PipeFuncBlueprint):
- return PipelexInterpreter._serialize_func_pipe(blueprint, domain)
+ return PipelexInterpreter._serialize_func_pipe(blueprint)
elif isinstance(blueprint, PipeImgGenBlueprint):
- return PipelexInterpreter._serialize_img_gen_pipe(blueprint, domain)
+ return PipelexInterpreter._serialize_img_gen_pipe(blueprint)
elif isinstance(blueprint, PipeJinja2Blueprint):
- return PipelexInterpreter.serialize_jinja2_pipe(blueprint, domain)
+ return PipelexInterpreter.serialize_jinja2_pipe(blueprint)
elif isinstance(blueprint, PipeSequenceBlueprint):
- return PipelexInterpreter.serialize_sequence_pipe(blueprint, domain)
+ return PipelexInterpreter.serialize_sequence_pipe(blueprint)
elif isinstance(blueprint, PipeConditionBlueprint):
- return PipelexInterpreter._serialize_condition_pipe(blueprint, domain)
+ return PipelexInterpreter._serialize_condition_pipe(blueprint)
elif isinstance(blueprint, PipeParallelBlueprint):
- return PipelexInterpreter._serialize_parallel_pipe(blueprint, domain)
+ return PipelexInterpreter._serialize_parallel_pipe(blueprint)
elif isinstance(blueprint, PipeBatchBlueprint):
- return PipelexInterpreter._serialize_batch_pipe(blueprint, domain)
+ return PipelexInterpreter._serialize_batch_pipe(blueprint)
else:
- raise PipelexUnknownPipeError(f"Unknown pipe blueprint type: {type(blueprint)}")
+ msg = f"Unknown pipe blueprint type: {type(blueprint)}"
+ raise PipelexUnknownPipeError(msg)
@staticmethod
- def serialize_llm_pipe(pipe: PipeLLMBlueprint, domain: str) -> Dict[str, Any]:
+ def serialize_llm_pipe(pipe: PipeLLMBlueprint) -> dict[str, Any]:
"""Serialize PipeLLM blueprint."""
- result: Dict[str, Any] = {
+ result: dict[str, Any] = {
"type": pipe.type,
"definition": pipe.definition,
}
@@ -668,15 +660,15 @@ class PipelexInterpreter(BaseModel):
return result
@staticmethod
- def add_common_pipe_fields(result: Dict[str, Any], pipe: Any) -> None:
+ def add_common_pipe_fields(result: dict[str, Any], pipe: Any) -> None:
"""Add common pipe fields (inputs) to result dict if they exist."""
if hasattr(pipe, "inputs") and pipe.inputs:
result["inputs"] = PipelexInterpreter.serialize_inputs(pipe.inputs)
@staticmethod
- def _serialize_ocr_pipe(pipe: PipeOcrBlueprint, domain: str) -> Dict[str, Any]:
+ def _serialize_ocr_pipe(pipe: PipeOcrBlueprint) -> dict[str, Any]:
"""Serialize PipeOcr blueprint."""
- result: Dict[str, Any] = {
+ result: dict[str, Any] = {
"type": pipe.type,
"definition": pipe.definition,
"output": pipe.output,
@@ -686,9 +678,9 @@ class PipelexInterpreter(BaseModel):
return result
@staticmethod
- def _serialize_func_pipe(pipe: PipeFuncBlueprint, domain: str) -> Dict[str, Any]:
+ def _serialize_func_pipe(pipe: PipeFuncBlueprint) -> dict[str, Any]:
"""Serialize PipeFunc blueprint."""
- result: Dict[str, Any] = {
+ result: dict[str, Any] = {
"type": pipe.type,
"definition": pipe.definition,
"output": pipe.output,
@@ -699,9 +691,9 @@ class PipelexInterpreter(BaseModel):
return result
@staticmethod
- def _serialize_img_gen_pipe(pipe: PipeImgGenBlueprint, domain: str) -> Dict[str, Any]:
+ def _serialize_img_gen_pipe(pipe: PipeImgGenBlueprint) -> dict[str, Any]:
"""Serialize PipeImgGen blueprint."""
- result: Dict[str, Any] = {
+ result: dict[str, Any] = {
"type": pipe.type,
"definition": pipe.definition,
"output": pipe.output,
@@ -724,9 +716,9 @@ class PipelexInterpreter(BaseModel):
return result
@staticmethod
- def serialize_ocr_pipe(pipe: PipeOcrBlueprint, domain: str) -> Dict[str, Any]:
+ def serialize_ocr_pipe(pipe: PipeOcrBlueprint) -> dict[str, Any]:
"""Serialize a PipeOcr blueprint."""
- result: Dict[str, Any] = {
+ result: dict[str, Any] = {
"type": pipe.type,
"definition": pipe.definition,
}
@@ -753,9 +745,9 @@ class PipelexInterpreter(BaseModel):
return result
@staticmethod
- def serialize_sequence_pipe(pipe: PipeSequenceBlueprint, domain: str) -> Dict[str, Any]:
+ def serialize_sequence_pipe(pipe: PipeSequenceBlueprint) -> dict[str, Any]:
"""Serialize a PipeSequence blueprint."""
- result: Dict[str, Any] = {
+ result: dict[str, Any] = {
"type": pipe.type,
"definition": pipe.definition,
}
@@ -773,9 +765,9 @@ class PipelexInterpreter(BaseModel):
return result
@staticmethod
- def serialize_jinja2_pipe(pipe: PipeJinja2Blueprint, domain: str) -> Dict[str, Any]:
+ def serialize_jinja2_pipe(pipe: PipeJinja2Blueprint) -> dict[str, Any]:
"""Serialize PipeJinja2 blueprint."""
- result: Dict[str, Any] = {
+ result: dict[str, Any] = {
"type": pipe.type,
"definition": pipe.definition,
"output": pipe.output,
@@ -795,9 +787,9 @@ class PipelexInterpreter(BaseModel):
return result
@staticmethod
- def serialize_sub_pipe(sub_pipe: Any) -> Dict[str, Any]:
+ def serialize_sub_pipe(sub_pipe: Any) -> dict[str, Any]:
"""Serialize a sub pipe (step or parallel) blueprint."""
- step_data: Dict[str, Any] = {
+ step_data: dict[str, Any] = {
"pipe": sub_pipe.pipe,
"result": sub_pipe.result,
}
@@ -833,9 +825,9 @@ class PipelexInterpreter(BaseModel):
return "{ " + ", ".join(parts) + " }"
@staticmethod
- def _serialize_sequence_pipe(pipe: PipeSequenceBlueprint, domain: str) -> Dict[str, Any]:
+ def _serialize_sequence_pipe(pipe: PipeSequenceBlueprint) -> dict[str, Any]:
"""Serialize PipeSequence blueprint."""
- result: Dict[str, Any] = {
+ result: dict[str, Any] = {
"type": pipe.type,
"definition": pipe.definition,
"output": pipe.output,
@@ -849,7 +841,7 @@ class PipelexInterpreter(BaseModel):
return result
@staticmethod
- def sequence_pipe_to_plx_string(pipe_name: str, pipe: PipeSequenceBlueprint, domain: str) -> str:
+ def sequence_pipe_to_plx_string(pipe_name: str, pipe: PipeSequenceBlueprint) -> str:
"""Convert a PipeSequence blueprint directly to PLX section string."""
lines: list[str] = [
f"[pipe.{pipe_name}]",
@@ -875,9 +867,9 @@ class PipelexInterpreter(BaseModel):
return "\n".join(lines)
@staticmethod
- def _serialize_condition_pipe(pipe: PipeConditionBlueprint, domain: str) -> Dict[str, Any]:
+ def _serialize_condition_pipe(pipe: PipeConditionBlueprint) -> dict[str, Any]:
"""Serialize PipeCondition blueprint."""
- result: Dict[str, Any] = {
+ result: dict[str, Any] = {
"type": pipe.type,
"definition": pipe.definition,
"output": pipe.output,
@@ -899,9 +891,9 @@ class PipelexInterpreter(BaseModel):
return result
@staticmethod
- def _serialize_parallel_pipe(pipe: PipeParallelBlueprint, domain: str) -> Dict[str, Any]:
+ def _serialize_parallel_pipe(pipe: PipeParallelBlueprint) -> dict[str, Any]:
"""Serialize PipeParallel blueprint."""
- result: Dict[str, Any] = {
+ result: dict[str, Any] = {
"type": pipe.type,
"definition": pipe.definition,
"output": pipe.output,
@@ -919,9 +911,9 @@ class PipelexInterpreter(BaseModel):
return result
@staticmethod
- def _serialize_batch_pipe(pipe: PipeBatchBlueprint, domain: str) -> Dict[str, Any]:
+ def _serialize_batch_pipe(pipe: PipeBatchBlueprint) -> dict[str, Any]:
"""Serialize PipeBatch blueprint."""
- result: Dict[str, Any] = {
+ result: dict[str, Any] = {
"type": pipe.type,
"definition": pipe.definition,
"output": pipe.output,
@@ -940,19 +932,19 @@ class PipelexInterpreter(BaseModel):
@staticmethod
def serialize_input_requirement(
input_req: InputRequirementBlueprint,
- ) -> Dict[str, Any]:
+ ) -> dict[str, Any]:
"""Serialize a single InputRequirementBlueprint to PLX format."""
- result: Dict[str, Any] = {"concept": input_req.concept}
+ result: dict[str, Any] = {"concept": input_req.concept}
if input_req.multiplicity is not None:
result["multiplicity"] = input_req.multiplicity
return result
@staticmethod
def serialize_inputs(
- inputs: Mapping[str, Union[str, InputRequirementBlueprint]],
- ) -> Dict[str, Any]:
+ inputs: Mapping[str, str | InputRequirementBlueprint],
+ ) -> dict[str, Any]:
"""Convert InputRequirementBlueprint objects to proper PLX format for serialization."""
- result: Dict[str, Any] = {}
+ result: dict[str, Any] = {}
for key, value in inputs.items():
if isinstance(value, InputRequirementBlueprint):
result[key] = PipelexInterpreter.serialize_input_requirement(value)
@@ -963,7 +955,7 @@ class PipelexInterpreter(BaseModel):
@staticmethod
def inputs_to_plx_string(
- inputs: Mapping[str, Union[str, InputRequirementBlueprint]],
+ inputs: Mapping[str, str | InputRequirementBlueprint],
) -> str:
"""Convert inputs dictionary to PLX string format."""
inputs_dict = PipelexInterpreter.serialize_inputs(inputs)
@@ -973,27 +965,26 @@ class PipelexInterpreter(BaseModel):
if isinstance(value, dict):
# Nested dict (like InputRequirementBlueprint)
nested_parts: list[str] = []
- nested_dict: Dict[str, Any] = cast(Dict[str, Any], value)
+ nested_dict: dict[str, Any] = cast("dict[str, Any]", value)
for nested_key, nested_value in nested_dict.items():
if isinstance(nested_value, str):
nested_parts.append(f'{nested_key} = "{PipelexInterpreter.escape_plx_string(nested_value)}"')
else:
nested_parts.append(f"{nested_key} = {str(nested_value).lower()}")
inputs_parts.append(f"{key} = {{ {', '.join(nested_parts)} }}")
+ # Simple string value
+ elif isinstance(value, str):
+ inputs_parts.append(f'{key} = "{PipelexInterpreter.escape_plx_string(value)}"')
else:
- # Simple string value
- if isinstance(value, str):
- inputs_parts.append(f'{key} = "{PipelexInterpreter.escape_plx_string(value)}"')
- else:
- # Fallback for any other type
- inputs_parts.append(f'{key} = "{PipelexInterpreter.escape_plx_string(str(value))}"')
+ # Fallback for any other type
+ inputs_parts.append(f'{key} = "{PipelexInterpreter.escape_plx_string(str(value))}"')
return f"{{ {', '.join(inputs_parts)} }}"
@staticmethod
def add_inputs_to_lines_if_exist(
lines: list[str],
- pipe_inputs: Optional[Mapping[str, Union[str, InputRequirementBlueprint]]],
+ pipe_inputs: Mapping[str, str | InputRequirementBlueprint] | None,
) -> None:
"""Add inputs line to PLX lines if inputs exist."""
if pipe_inputs:
@@ -1002,16 +993,15 @@ class PipelexInterpreter(BaseModel):
for key, value in inputs_dict.items():
if isinstance(value, dict):
nested_parts: list[str] = []
- nested_dict: Dict[str, Any] = cast(Dict[str, Any], value)
+ nested_dict: dict[str, Any] = cast("dict[str, Any]", value)
for nested_key, nested_value in nested_dict.items():
if isinstance(nested_value, str):
nested_parts.append(f'{nested_key} = "{PipelexInterpreter.escape_plx_string(nested_value)}"')
else:
nested_parts.append(f"{nested_key} = {str(nested_value).lower()}")
inputs_parts.append(f"{key} = {{ {', '.join(nested_parts)} }}")
+ elif isinstance(value, str):
+ inputs_parts.append(f'{key} = "{PipelexInterpreter.escape_plx_string(value)}"')
else:
- if isinstance(value, str):
- inputs_parts.append(f'{key} = "{PipelexInterpreter.escape_plx_string(value)}"')
- else:
- inputs_parts.append(f'{key} = "{PipelexInterpreter.escape_plx_string(str(value))}"')
+ inputs_parts.append(f'{key} = "{PipelexInterpreter.escape_plx_string(str(value))}"')
lines.append(f"inputs = {{ {', '.join(inputs_parts)} }}")
diff --git a/pipelex/core/memory/__init__.py b/pipelex/core/memory/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/pipelex/core/memory/working_memory.py b/pipelex/core/memory/working_memory.py
index a430f6855..349a125cc 100644
--- a/pipelex/core/memory/working_memory.py
+++ b/pipelex/core/memory/working_memory.py
@@ -1,8 +1,8 @@
from operator import attrgetter
-from typing import Any, Dict, List, Optional, Set, Type
+from typing import Any
from pydantic import BaseModel, Field, model_validator
-from typing_extensions import Self, override
+from typing_extensions import override
from pipelex import log, pretty_print
from pipelex.core.stuffs.stuff import Stuff
@@ -25,28 +25,29 @@ from pipelex.exceptions import (
WorkingMemoryTypeError,
)
from pipelex.tools.misc.context_provider_abstract import ContextProviderAbstract
+from pipelex.types import Self
MAIN_STUFF_NAME = "main_stuff"
BATCH_ITEM_STUFF_NAME = "BATCH_ITEM"
PRETTY_PRINT_MAX_LENGTH = 1000
-StuffDict = Dict[str, Stuff]
-StuffArtefactDict = Dict[str, StuffArtefact]
+StuffDict = dict[str, Stuff]
+StuffArtefactDict = dict[str, StuffArtefact]
class WorkingMemory(BaseModel, ContextProviderAbstract):
root: StuffDict = Field(default_factory=dict)
- aliases: Dict[str, str] = Field(default_factory=dict)
+ aliases: dict[str, str] = Field(default_factory=dict)
@model_validator(mode="after")
def validate_stuff_names(self) -> Self:
for key, stuff in self.root.items():
- if key.startswith("_") and not key == BATCH_ITEM_STUFF_NAME:
+ if key.startswith("_") and key != BATCH_ITEM_STUFF_NAME:
log.warning(f"Stuff key '{key}' starts with '_', which is reserved for params")
if not stuff.stuff_name:
self.root[key].stuff_name = key
- elif key != MAIN_STUFF_NAME and stuff.stuff_name != key:
+ elif key not in (MAIN_STUFF_NAME, stuff.stuff_name):
log.warning(f"Stuff name '{stuff.stuff_name}' does not match the key '{key}'")
elif stuff.stuff_name.startswith("_") and stuff.stuff_name != BATCH_ITEM_STUFF_NAME:
log.warning(f"Stuff name '{stuff.stuff_name}' starts with '_', which is reserved for params")
@@ -63,14 +64,14 @@ class WorkingMemory(BaseModel, ContextProviderAbstract):
def make_deep_copy(self) -> Self:
return self.model_copy(deep=True)
- def get_optional_stuff(self, name: str) -> Optional[Stuff]:
+ def get_optional_stuff(self, name: str) -> Stuff | None:
if named_stuff := self.root.get(name):
return named_stuff
if alias := self.aliases.get(name):
return self.root.get(alias)
return None
- def get_optional_main_stuff(self) -> Optional[Stuff]:
+ def get_optional_main_stuff(self) -> Stuff | None:
return self.get_optional_stuff(name=MAIN_STUFF_NAME)
# TODO: all calls to get_stuff should catch WorkingMemoryStuffNotFoundError in order to indicate what pipe is missing a required stuff
@@ -90,24 +91,21 @@ class WorkingMemory(BaseModel, ContextProviderAbstract):
message=f"Stuff '{name}' not found in working memory, valid keys are: {self.list_keys()}",
)
- def get_stuffs(self, names: Set[str]) -> List[Stuff]:
- the_stuffs: List[Stuff] = []
+ def get_stuffs(self, names: set[str]) -> list[Stuff]:
+ the_stuffs: list[Stuff] = []
for name in names:
the_stuffs.append(self.get_stuff(name=name))
return the_stuffs
- def get_existing_stuffs(self, names: Set[str]) -> List[Stuff]:
- the_stuffs: List[Stuff] = []
+ def get_existing_stuffs(self, names: set[str]) -> list[Stuff]:
+ the_stuffs: list[Stuff] = []
for name in names:
if stuff := self.get_optional_stuff(name=name):
the_stuffs.append(stuff)
return the_stuffs
def is_stuff_code_used(self, stuff_code: str) -> bool:
- for stuff in self.root.values():
- if stuff.concept.code == stuff_code:
- return True
- return False
+ return any(stuff.concept.code == stuff_code for stuff in self.root.values())
def get_main_stuff(self) -> Stuff:
return self.get_stuff(name=MAIN_STUFF_NAME)
@@ -122,20 +120,20 @@ class WorkingMemory(BaseModel, ContextProviderAbstract):
def set_stuff(self, name: str, stuff: Stuff):
self.root[name] = stuff
- def add_new_stuff(self, name: str, stuff: Stuff, aliases: Optional[List[str]] = None):
+ def add_new_stuff(self, name: str, stuff: Stuff, aliases: list[str] | None = None):
# TODO: Add unit tests for this method
log.debug(f"Adding new stuff '{name}' to WorkingMemory with aliases: {aliases}")
if self.is_stuff_code_used(stuff_code=stuff.stuff_code):
- raise WorkingMemoryConsistencyError(f"Stuff code '{stuff.stuff_code}' is already used by another stuff")
+ msg = f"Stuff code '{stuff.stuff_code}' is already used by another stuff"
+ raise WorkingMemoryConsistencyError(msg)
if name in self.root or name in self.aliases:
existing_stuff = self.get_stuff(name=name)
if existing_stuff == stuff:
log.warning(f"Key '{name}' already exists in WorkingMemory with the same stuff")
return
- else:
- log.warning(f"Key '{name}' already exists in WorkingMemory and will be replaced by something different")
- log.verbose(f"Existing stuff: {existing_stuff}")
- log.verbose(f"New stuff: {stuff}")
+ log.warning(f"Key '{name}' already exists in WorkingMemory and will be replaced by something different")
+ log.verbose(f"Existing stuff: {existing_stuff}")
+ log.verbose(f"New stuff: {stuff}")
# it's a new stuff
self.set_stuff(name=name, stuff=stuff)
@@ -143,7 +141,7 @@ class WorkingMemory(BaseModel, ContextProviderAbstract):
for alias in aliases:
self.set_alias(alias, name)
- def set_new_main_stuff(self, stuff: Stuff, name: Optional[str] = None):
+ def set_new_main_stuff(self, stuff: Stuff, name: str | None = None):
# TODO: Add unit tests for this method
if name:
self.remove_main_stuff()
@@ -157,16 +155,19 @@ class WorkingMemory(BaseModel, ContextProviderAbstract):
def set_alias(self, alias: str, target: str) -> None:
"""Add an alias pointing to a target name."""
if alias == target:
- raise WorkingMemoryConsistencyError(f"Cannot create alias '{alias}' pointing to itself")
+ msg = f"Cannot create alias '{alias}' pointing to itself"
+ raise WorkingMemoryConsistencyError(msg)
if target not in self.root:
- raise WorkingMemoryConsistencyError(f"Cannot create alias to non-existent target '{target}'")
+ msg = f"Cannot create alias to non-existent target '{target}'"
+ raise WorkingMemoryConsistencyError(msg)
log.debug(f"Setting alias '{alias}' pointing to target '{target}'")
self.aliases[alias] = target
def add_alias(self, alias: str, target: str) -> None:
"""Add an alias pointing to a target name."""
if alias in self.root:
- raise WorkingMemoryConsistencyError(f"Cannot add alias '{alias}' as it already exists")
+ msg = f"Cannot add alias '{alias}' as it already exists"
+ raise WorkingMemoryConsistencyError(msg)
self.set_alias(alias=alias, target=target)
log.debug(f"Added alias '{alias}' pointing to target '{target}'")
@@ -179,11 +180,11 @@ class WorkingMemory(BaseModel, ContextProviderAbstract):
"""Remove the alias pointing to the main stuff if it exists."""
self.remove_alias(alias=MAIN_STUFF_NAME)
- def get_aliases_for(self, target: str) -> List[str]:
+ def get_aliases_for(self, target: str) -> list[str]:
"""Get all aliases pointing to a target name."""
return [alias for alias, t in self.aliases.items() if t == target]
- def list_keys(self) -> List[str]:
+ def list_keys(self) -> list[str]:
return list(self.root.keys()) + list(self.aliases.keys())
def pretty_print(self):
@@ -195,7 +196,7 @@ class WorkingMemory(BaseModel, ContextProviderAbstract):
################################################################################################
@override
- def generate_context(self) -> Dict[str, Any]:
+ def generate_context(self) -> dict[str, Any]:
# TODO: Add unit tests for this method
artefact_dict: StuffArtefactDict = {}
for name, stuff in self.root.items():
@@ -206,7 +207,7 @@ class WorkingMemory(BaseModel, ContextProviderAbstract):
return artefact_dict
@override
- def get_typed_object_or_attribute(self, name: str, wanted_type: Optional[Type[Any]] = None) -> Any:
+ def get_typed_object_or_attribute(self, name: str, wanted_type: type[Any] | None = None) -> Any:
# TODO: Add unit tests for this method
# TODO: Refactor this method. In the python paradigm, we should not have those ".", but arrays with field names.
if "." in name:
@@ -232,33 +233,31 @@ class WorkingMemory(BaseModel, ContextProviderAbstract):
)
return stuff_content
- else:
- content = self.get_stuff(name).content
+ content = self.get_stuff(name).content
- if wanted_type is not None and not isinstance(content, wanted_type):
- raise WorkingMemoryTypeError(
- variable_name=name,
- message=f"Content of '{name}' is of type '{type(content).__name__}', it should be '{wanted_type.__name__}'",
- )
+ if wanted_type is not None and not isinstance(content, wanted_type):
+ raise WorkingMemoryTypeError(
+ variable_name=name,
+ message=f"Content of '{name}' is of type '{type(content).__name__}', it should be '{wanted_type.__name__}'",
+ )
- return content
+ return content
################################################################################################
# Stuff accessors
################################################################################################
- def get_stuff_as(self, name: str, content_type: Type[StuffContentType]) -> StuffContentType:
+ def get_stuff_as(self, name: str, content_type: type[StuffContentType]) -> StuffContentType:
"""Get stuff content as StuffContentType."""
return self.get_stuff(name=name).content_as(content_type=content_type)
- def get_stuff_as_list(self, name: str, item_type: Type[StuffContentType]) -> ListContent[StuffContentType]:
- """
- Get stuff content as ListContent with items of type StuffContentType.
+ def get_stuff_as_list(self, name: str, item_type: type[StuffContentType]) -> ListContent[StuffContentType]:
+ """Get stuff content as ListContent with items of type StuffContentType.
If the items are of possibly various types, use item_type=StuffContent.
"""
return self.get_stuff(name=name).as_list_of_fixed_content_type(item_type=item_type)
- def get_list_stuff_first_item_as(self, name: str, item_type: Type[StuffContentType]) -> StuffContentType:
+ def get_list_stuff_first_item_as(self, name: str, item_type: type[StuffContentType]) -> StuffContentType:
"""Get stuff content as ListContent with items of type StuffContentType then return the first item."""
return self.get_stuff_as_list(name=name, item_type=item_type).items[0]
@@ -298,18 +297,17 @@ class WorkingMemory(BaseModel, ContextProviderAbstract):
# Main stuff accessors
################################################################################################
- def main_stuff_as(self, content_type: Type[StuffContentType]) -> StuffContentType:
+ def main_stuff_as(self, content_type: type[StuffContentType]) -> StuffContentType:
"""Get main stuff content as StuffContentType."""
return self.get_stuff_as(name=MAIN_STUFF_NAME, content_type=content_type)
- def main_stuff_as_list(self, item_type: Type[StuffContentType]) -> ListContent[StuffContentType]:
- """
- Get main stuff content as ListContent with items of type StuffContentType.
+ def main_stuff_as_list(self, item_type: type[StuffContentType]) -> ListContent[StuffContentType]:
+ """Get main stuff content as ListContent with items of type StuffContentType.
If the items are of possibly various types, use item_type=StuffContent.
"""
return self.get_stuff_as_list(name=MAIN_STUFF_NAME, item_type=item_type)
- def main_list_stuff_first_item_as(self, item_type: Type[StuffContentType]) -> StuffContentType:
+ def main_list_stuff_first_item_as(self, item_type: type[StuffContentType]) -> StuffContentType:
"""Get main stuff content as first list item of type StuffContentType."""
return self.get_list_stuff_first_item_as(name=MAIN_STUFF_NAME, item_type=item_type)
diff --git a/pipelex/core/memory/working_memory_factory.py b/pipelex/core/memory/working_memory_factory.py
index a0f8f80f2..67f93973e 100644
--- a/pipelex/core/memory/working_memory_factory.py
+++ b/pipelex/core/memory/working_memory_factory.py
@@ -1,4 +1,4 @@
-from typing import Any, Dict, List, Optional, cast
+from typing import Any, cast
import shortuuid
from polyfactory.factories.pydantic_factory import ModelFactory
@@ -23,7 +23,7 @@ class WorkingMemoryFactory(BaseModel):
cls,
text: str,
concept_string: str = SpecialDomain.NATIVE + "." + NativeConceptEnum.TEXT,
- name: Optional[str] = "text",
+ name: str | None = "text",
) -> WorkingMemory:
ConceptBlueprint.validate_concept_string(concept_string=concept_string)
return cls.make_from_single_stuff(
@@ -31,7 +31,7 @@ class WorkingMemoryFactory(BaseModel):
concept=get_required_concept(concept_string=concept_string),
content=TextContent(text=text),
name=name,
- )
+ ),
)
@classmethod
@@ -39,7 +39,7 @@ class WorkingMemoryFactory(BaseModel):
cls,
image_url: str,
concept_string: str = SpecialDomain.NATIVE + "." + NativeConceptEnum.IMAGE,
- name: Optional[str] = "image",
+ name: str | None = "image",
) -> WorkingMemory:
# TODO: validate that the concept is compatible with an image concept
ConceptBlueprint.validate_concept_string(concept_string=concept_string)
@@ -55,7 +55,7 @@ class WorkingMemoryFactory(BaseModel):
cls,
pdf_url: str,
concept_string: str = SpecialDomain.NATIVE + "." + NativeConceptEnum.PDF,
- name: Optional[str] = "pdf",
+ name: str | None = "pdf",
) -> WorkingMemory:
ConceptBlueprint.validate_concept_string(concept_string=concept_string)
return cls.make_from_single_stuff(
@@ -63,21 +63,22 @@ class WorkingMemoryFactory(BaseModel):
concept=get_required_concept(concept_string=concept_string),
content=PDFContent(url=pdf_url),
name=name,
- )
+ ),
)
@classmethod
def make_from_single_stuff(cls, stuff: Stuff) -> WorkingMemory:
if not stuff.stuff_name:
- raise WorkingMemoryFactoryError(f"Cannot make_from_single_stuff because stuff has no name: {stuff}")
+ msg = f"Cannot make_from_single_stuff because stuff has no name: {stuff}"
+ raise WorkingMemoryFactoryError(msg)
stuff_dict: StuffDict = {stuff.stuff_name: stuff}
return WorkingMemory(root=stuff_dict, aliases={MAIN_STUFF_NAME: stuff.stuff_name})
@classmethod
def make_from_multiple_stuffs(
cls,
- stuff_list: List[Stuff],
- main_name: Optional[str] = None,
+ stuff_list: list[Stuff],
+ main_name: str | None = None,
is_ignore_unnamed: bool = False,
) -> WorkingMemory:
stuff_dict: StuffDict = {}
@@ -86,19 +87,19 @@ class WorkingMemoryFactory(BaseModel):
if not name:
if is_ignore_unnamed:
continue
- else:
- raise WorkingMemoryFactoryError(f"Stuff {stuff} has no name")
+ msg = f"Stuff {stuff} has no name"
+ raise WorkingMemoryFactoryError(msg)
stuff_dict[name] = stuff
- aliases: Dict[str, str] = {}
+ aliases: dict[str, str] = {}
if stuff_dict:
if main_name:
aliases[MAIN_STUFF_NAME] = main_name
else:
- aliases[MAIN_STUFF_NAME] = list(stuff_dict.keys())[0]
+ aliases[MAIN_STUFF_NAME] = next(iter(stuff_dict.keys()))
return WorkingMemory(root=stuff_dict, aliases=aliases)
@classmethod
- def make_from_strings_from_dict(cls, input_dict: Dict[str, Any]) -> WorkingMemory:
+ def make_from_strings_from_dict(cls, input_dict: dict[str, Any]) -> WorkingMemory:
# TODO: Add unit tests for this method
stuff_dict: StuffDict = {}
for name, content in input_dict.items():
@@ -121,9 +122,9 @@ class WorkingMemoryFactory(BaseModel):
def make_from_compact_memory(
cls,
compact_memory: CompactMemory,
- search_domains: Optional[List[str]] = None,
+ search_domains: list[str] | None = None,
) -> WorkingMemory:
- implicit_memory = cast(ImplicitMemory, compact_memory)
+ implicit_memory = cast("ImplicitMemory", compact_memory)
return cls.make_from_implicit_memory(
implicit_memory=implicit_memory,
search_domains=search_domains,
@@ -133,16 +134,17 @@ class WorkingMemoryFactory(BaseModel):
def make_from_implicit_memory(
cls,
implicit_memory: ImplicitMemory,
- search_domains: Optional[List[str]] = None,
+ search_domains: list[str] | None = None,
) -> WorkingMemory:
- """
- Create a WorkingMemory from a compact memory dictionary.
+ """Create a WorkingMemory from a implicit memory dictionary.
Args:
- compact_memory: Dictionary in the format from API serialization
+ implicit_memory: Dictionary in the format from API serialization
+ search_domains: List of search domains to use when making stuff
Returns:
- WorkingMemory object reconstructed from the compact format
+ WorkingMemory object reconstructed from the implicit format
+
"""
working_memory = cls.make_empty()
@@ -162,34 +164,34 @@ class WorkingMemoryFactory(BaseModel):
"""Helper method to create mock content for a requirement."""
if requirement.structure_class:
# Create mock object using polyfactory
- class MockFactory(ModelFactory[requirement.structure_class]): # type: ignore
+ class MockFactory(ModelFactory[requirement.structure_class]): # type: ignore[name-defined]
__model__ = requirement.structure_class
__check_model__ = True
__use_examples__ = True
__allow_none_optionals__ = False # Ensure Optional fields always get values
- return MockFactory.build(factory_use_construct=True) # type: ignore
- else:
- # Fallback to text content
- return TextContent(text=f"DRY RUN: Mock content for '{requirement.variable_name}' ({requirement.concept.code})")
+ return MockFactory.build(factory_use_construct=True) # type: ignore[no-any-return]
+
+ # Fallback to text content
+ return TextContent(text=f"DRY RUN: Mock content for '{requirement.variable_name}' ({requirement.concept.code})")
@classmethod
- def make_for_dry_run(cls, needed_inputs: List[TypedNamedInputRequirement]) -> "WorkingMemory":
- """
- Create a WorkingMemory with mock objects for dry run mode.
+ def make_for_dry_run(cls, needed_inputs: list[TypedNamedInputRequirement]) -> "WorkingMemory":
+ """Create a WorkingMemory with mock objects for dry run mode.
Args:
needed_inputs: List of tuples (stuff_name, concept_code, structure_class)
Returns:
WorkingMemory with mock objects for each needed input
+
"""
working_memory = cls.make_empty()
for requirement in needed_inputs:
log.debug(
f"Creating dry run mock for '{requirement.variable_name}' with concept "
- f"'{requirement.concept.code}' and class '{requirement.structure_class.__name__}'"
+ f"'{requirement.concept.code}' and class '{requirement.structure_class.__name__}'",
)
try:
@@ -214,7 +216,7 @@ class WorkingMemoryFactory(BaseModel):
else:
nb_stuffs = requirement.multiplicity
- items: List[StuffContent] = []
+ items: list[StuffContent] = []
for _ in range(nb_stuffs):
item_mock_content = cls.create_mock_content(requirement)
items.append(item_mock_content)
@@ -233,7 +235,7 @@ class WorkingMemoryFactory(BaseModel):
except Exception as exc:
log.warning(
- f"Failed to create mock for '{requirement.variable_name}' ({requirement.concept.code}): {exc}. Using fallback text content."
+ f"Failed to create mock for '{requirement.variable_name}' ({requirement.concept.code}): {exc}. Using fallback text content.",
)
# Create fallback text content
fallback_content = TextContent(text=f"DRY RUN: Fallback mock for '{requirement.variable_name}' ({requirement.concept.code})")
diff --git a/pipelex/core/pipes/__init__.py b/pipelex/core/pipes/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/pipelex/core/pipes/exceptions.py b/pipelex/core/pipes/exceptions.py
index baf2346d9..0fae11bce 100644
--- a/pipelex/core/pipes/exceptions.py
+++ b/pipelex/core/pipes/exceptions.py
@@ -1,4 +1,2 @@
class PipeBlueprintError(Exception):
"""Exception raised for errors in the PipeBlueprint class."""
-
- pass
diff --git a/pipelex/core/pipes/pipe_abstract.py b/pipelex/core/pipes/pipe_abstract.py
index 3a7948bea..68705a742 100644
--- a/pipelex/core/pipes/pipe_abstract.py
+++ b/pipelex/core/pipes/pipe_abstract.py
@@ -1,5 +1,5 @@
from abc import ABC, abstractmethod
-from typing import Any, List, Optional, Set, Type
+from typing import Any
from pydantic import BaseModel, ConfigDict, Field, field_validator
@@ -20,36 +20,30 @@ class PipeAbstract(ABC, BaseModel):
type: Any # Any so that subclasses can put a Literal
code: str
domain: str
- definition: Optional[str] = None
+ definition: str | None = None
inputs: PipeInputSpec = Field(default_factory=PipeInputSpec)
output: Concept
@field_validator("code", mode="before")
- def validate_pipe_code_syntax(cls, code: str) -> str:
+ @staticmethod
+ def validate_pipe_code_syntax(code: str) -> str:
PipeBlueprint.validate_pipe_code_syntax(pipe_code=code)
return code
@abstractmethod
def validate_output(self):
- """
- Validate the output for the pipe.
- """
- pass
+ """Validate the output for the pipe."""
@property
def class_name(self) -> str:
return self.__class__.__name__
def validate_with_libraries(self):
- """
- Validate the pipe with the libraries, after the static validation
- """
- pass
+ """Validate the pipe with the libraries, after the static validation"""
@abstractmethod
- def required_variables(self) -> Set[str]:
- """
- Return the variables that are required for the pipe to run.
+ def required_variables(self) -> set[str]:
+ """Return the variables that are required for the pipe to run.
The required variables are only the list:
# 1 - The inputs of dependency pipes
# 2 - The variables in the pipe definition
@@ -57,12 +51,10 @@ class PipeAbstract(ABC, BaseModel):
- PipeBatch : Variables in the batch_params
- PipeLLM : Variables in the prompt
"""
- pass
@abstractmethod
- def needed_inputs(self, visited_pipes: Optional[Set[str]] = None) -> PipeInputSpec:
- """
- Return the inputs that are needed for the pipe to run.
+ def needed_inputs(self, visited_pipes: set[str] | None = None) -> PipeInputSpec:
+ """Return the inputs that are needed for the pipe to run.
Args:
visited_pipes: Set of pipe codes currently being processed to prevent infinite recursion.
@@ -70,22 +62,20 @@ class PipeAbstract(ABC, BaseModel):
Returns:
PipeInputSpec containing all needed inputs for this pipe
- """
- pass
- def pipe_dependencies(self) -> Set[str]:
"""
- Return the pipes that are dependencies of the pipe.
- - PipeBatch : The pipe that is being batched
- - PipeCondition : The pipes in the pipe_map
- - PipeSequence : The pipes in the steps
+
+ def pipe_dependencies(self) -> set[str]:
+ """Return the pipes that are dependencies of the pipe.
+ - PipeBatch : The pipe that is being batched
+ - PipeCondition : The pipes in the pipe_map
+ - PipeSequence : The pipes in the steps
"""
return set()
- def concept_dependencies(self) -> List[Concept]:
- required_concepts: List[Concept] = [self.output]
- for concept in self.inputs.concepts:
- required_concepts.append(concept)
+ def concept_dependencies(self) -> list[Concept]:
+ required_concepts: list[Concept] = [self.output]
+ required_concepts.extend(self.inputs.concepts)
required_concepts.append(self.output)
return required_concepts
@@ -95,8 +85,8 @@ class PipeAbstract(ABC, BaseModel):
job_metadata: JobMetadata,
working_memory: WorkingMemory,
pipe_run_params: PipeRunParams,
- output_name: Optional[str] = None,
- print_intermediate_outputs: Optional[bool] = False,
+ output_name: str | None = None,
+ print_intermediate_outputs: bool | None = False,
) -> PipeOutput:
pass
@@ -104,7 +94,8 @@ class PipeAbstract(ABC, BaseModel):
pipe_stack = pipe_run_params.pipe_stack
limit = pipe_run_params.pipe_stack_limit
if len(pipe_stack) > limit:
- raise PipeStackOverflowError(f"Exceeded pipe stack limit of {limit}. You can raise that limit in the config. Stack:\n{pipe_stack}")
+ msg = f"Exceeded pipe stack limit of {limit}. You can raise that limit in the config. Stack:\n{pipe_stack}"
+ raise PipeStackOverflowError(msg)
-PipeAbstractType = Type[PipeAbstract]
+PipeAbstractType = type[PipeAbstract]
diff --git a/pipelex/core/pipes/pipe_blueprint.py b/pipelex/core/pipes/pipe_blueprint.py
index b776a3b52..d684c82ed 100644
--- a/pipelex/core/pipes/pipe_blueprint.py
+++ b/pipelex/core/pipes/pipe_blueprint.py
@@ -1,4 +1,4 @@
-from typing import Any, Dict, List, Optional, Union
+from typing import Any
from pydantic import BaseModel, field_validator
@@ -14,8 +14,8 @@ class AllowedPipeCategories(StrEnum):
PIPE_CONTROLLER = "PipeController"
@classmethod
- def value_list(cls) -> List[str]:
- return [value for value in cls]
+ def value_list(cls) -> list[str]:
+ return list(cls)
class AllowedPipeTypes(StrEnum):
@@ -32,38 +32,44 @@ class AllowedPipeTypes(StrEnum):
PIPE_SEQUENCE = "PipeSequence"
@classmethod
- def value_list(cls) -> List[str]:
- return [value for value in cls]
+ def value_list(cls) -> list[str]:
+ return list(cls)
class PipeBlueprint(BaseModel):
category: Any
type: Any # TODO: Find a better way to handle this.
- definition: Optional[str] = None
- inputs: Optional[Dict[str, Union[str, InputRequirementBlueprint]]] = None
+ definition: str | None = None
+ inputs: dict[str, str | InputRequirementBlueprint] | None = None
output: str
@field_validator("type", mode="after")
- def validate_pipe_type(cls, value: Any) -> Any:
+ @staticmethod
+ def validate_pipe_type(value: Any) -> Any:
"""Validate that the pipe type is one of the allowed values."""
if value not in AllowedPipeTypes.value_list():
- raise PipeBlueprintError(f"Invalid pipe type '{value}'. Must be one of: {AllowedPipeTypes.value_list()}")
+ msg = f"Invalid pipe type '{value}'. Must be one of: {AllowedPipeTypes.value_list()}"
+ raise PipeBlueprintError(msg)
return value
@field_validator("category", mode="after")
- def validate_pipe_category(cls, value: Any) -> Any:
+ @staticmethod
+ def validate_pipe_category(value: Any) -> Any:
"""Validate that the pipe category is one of the allowed values."""
if value not in AllowedPipeCategories.value_list():
- raise PipeBlueprintError(f"Invalid pipe category '{value}'. Must be one of: {AllowedPipeCategories.value_list()}")
+ msg = f"Invalid pipe category '{value}'. Must be one of: {AllowedPipeCategories.value_list()}"
+ raise PipeBlueprintError(msg)
return value
@field_validator("output", mode="before")
- def validate_concept_string_or_code(cls, output: str) -> str:
+ @staticmethod
+ def validate_concept_string_or_code(output: str) -> str:
ConceptBlueprint.validate_concept_string_or_code(concept_string_or_code=output)
return output
@classmethod
def validate_pipe_code_syntax(cls, pipe_code: str) -> str:
if not is_snake_case(pipe_code):
- raise PipeBlueprintError(f"Invalid pipe code syntax '{pipe_code}'. Must be in snake_case.")
+ msg = f"Invalid pipe code syntax '{pipe_code}'. Must be in snake_case."
+ raise PipeBlueprintError(msg)
return pipe_code
diff --git a/pipelex/core/pipes/pipe_factory.py b/pipelex/core/pipes/pipe_factory.py
index d4a3ba12b..1b9bfc37d 100644
--- a/pipelex/core/pipes/pipe_factory.py
+++ b/pipelex/core/pipes/pipe_factory.py
@@ -1,4 +1,4 @@
-from typing import Any, List, Optional, Protocol, Type, TypeVar
+from typing import Any, Protocol, TypeVar
from kajson.exceptions import ClassRegistryInheritanceError, ClassRegistryNotFoundError
from kajson.kajson_manager import KajsonManager
@@ -20,7 +20,7 @@ class PipeFactoryProtocol(Protocol[PipeBlueprintType, PipeType]):
domain: str,
pipe_code: str,
blueprint: PipeBlueprintType,
- concept_codes_from_the_same_domain: Optional[List[str]] = None,
+ concept_codes_from_the_same_domain: list[str] | None = None,
) -> PipeType: ...
@@ -32,23 +32,21 @@ class PipeFactory(PipeFactoryProtocol[PipeBlueprint, PipeAbstract]):
domain: str,
pipe_code: str,
blueprint: PipeBlueprint,
- concept_codes_from_the_same_domain: Optional[List[str]] = None,
+ concept_codes_from_the_same_domain: list[str] | None = None,
) -> PipeAbstract:
# The factory class name for that specific type of Pipe is the pipe class name with "Factory" suffix
factory_class_name = f"{blueprint.type}Factory"
try:
- pipe_factory: Type[PipeFactoryProtocol[Any, Any]] = KajsonManager.get_class_registry().get_required_subclass(
+ pipe_factory: type[PipeFactoryProtocol[Any, Any]] = KajsonManager.get_class_registry().get_required_subclass(
name=factory_class_name,
base_class=PipeFactoryProtocol,
)
except ClassRegistryNotFoundError as factory_not_found_error:
- raise PipeFactoryError(
- f"Pipe '{pipe_code}' couldn't be created: factory '{factory_class_name}' not found: {factory_not_found_error}"
- ) from factory_not_found_error
+ msg = f"Pipe '{pipe_code}' couldn't be created: factory '{factory_class_name}' not found: {factory_not_found_error}"
+ raise PipeFactoryError(msg) from factory_not_found_error
except ClassRegistryInheritanceError as factory_inheritance_error:
- raise PipeFactoryError(
- f"Pipe '{pipe_code}' couldn't be created: factory '{factory_class_name}' is not a subclass of {type(PipeFactoryProtocol)}."
- ) from factory_inheritance_error
+ msg = f"Pipe '{pipe_code}' couldn't be created: factory '{factory_class_name}' is not a subclass of {type(PipeFactoryProtocol)}."
+ raise PipeFactoryError(msg) from factory_inheritance_error
pipe_from_blueprint: PipeAbstract = pipe_factory.make_from_blueprint(
domain=domain,
diff --git a/pipelex/core/pipes/pipe_input.py b/pipelex/core/pipes/pipe_input.py
index db2ca2071..1aa6b86fe 100644
--- a/pipelex/core/pipes/pipe_input.py
+++ b/pipelex/core/pipes/pipe_input.py
@@ -1,4 +1,4 @@
-from typing import Callable, Dict, List, Optional, Tuple, Type
+from collections.abc import Callable
from pydantic import BaseModel, Field, RootModel, field_validator
@@ -11,27 +11,27 @@ from pipelex.exceptions import PipeInputNotFoundError
class InputRequirement(BaseModel):
concept: Concept
- multiplicity: Optional[PipeOutputMultiplicity] = None
+ multiplicity: PipeOutputMultiplicity | None = None
class NamedInputRequirement(InputRequirement):
variable_name: str
- requirement_expression: Optional[str] = None
+ requirement_expression: str | None = None
class TypedNamedInputRequirement(NamedInputRequirement):
- structure_class: Type[StuffContent]
+ structure_class: type[StuffContent]
@classmethod
def make_from_named(
cls,
named: NamedInputRequirement,
- structure_class: Type[StuffContent],
+ structure_class: type[StuffContent],
) -> "TypedNamedInputRequirement":
return cls(**named.model_dump(), structure_class=structure_class)
-PipeInputSpecRoot = Dict[str, InputRequirement]
+PipeInputSpecRoot = dict[str, InputRequirement]
class PipeInputSpec(RootModel[PipeInputSpecRoot]):
@@ -41,14 +41,14 @@ class PipeInputSpec(RootModel[PipeInputSpecRoot]):
@classmethod
def validate_concept_codes(
cls,
- input_value: Dict[str, InputRequirement],
- handler: Callable[[Dict[str, InputRequirement]], Dict[str, InputRequirement]],
- ) -> Dict[str, InputRequirement]:
+ input_value: dict[str, InputRequirement],
+ handler: Callable[[dict[str, InputRequirement]], dict[str, InputRequirement]],
+ ) -> dict[str, InputRequirement]:
# First let Pydantic handle the basic type validation
- validated_dict: Dict[str, InputRequirement] = handler(input_value)
+ validated_dict: dict[str, InputRequirement] = handler(input_value)
# Now we can transform and validate the keys and values
- transformed_dict: Dict[str, InputRequirement] = {}
+ transformed_dict: dict[str, InputRequirement] = {}
for required_input, requirement in validated_dict.items():
# in case of sub-attribute, the variable name is the object name, before the 1st dot
transformed_key: str = required_input.split(".", 1)[0]
@@ -57,7 +57,7 @@ class PipeInputSpec(RootModel[PipeInputSpecRoot]):
if transformed_key in transformed_dict and transformed_dict[transformed_key] != requirement:
log.verbose(
- f"Variable {transformed_key} already exists with a different concept code: {transformed_dict[transformed_key]} -> {requirement}"
+ f"Variable {transformed_key} already exists with a different concept code: {transformed_dict[transformed_key]} -> {requirement}",
)
transformed_dict[transformed_key] = InputRequirement(concept=requirement.concept, multiplicity=requirement.multiplicity)
@@ -73,39 +73,40 @@ class PipeInputSpec(RootModel[PipeInputSpecRoot]):
def get_required_input_requirement(self, variable_name: str) -> InputRequirement:
requirement = self.root.get(variable_name)
if not requirement:
- raise PipeInputNotFoundError(f"Variable '{variable_name}' not found in input spec")
+ msg = f"Variable '{variable_name}' not found in input spec"
+ raise PipeInputNotFoundError(msg)
return requirement
- def add_requirement(self, variable_name: str, concept: Concept, multiplicity: Optional[PipeOutputMultiplicity] = None):
+ def add_requirement(self, variable_name: str, concept: Concept, multiplicity: PipeOutputMultiplicity | None = None):
self.root[variable_name] = InputRequirement(concept=concept, multiplicity=multiplicity)
@property
- def items(self) -> List[Tuple[str, InputRequirement]]:
+ def items(self) -> list[tuple[str, InputRequirement]]:
return list(self.root.items())
@property
- def concepts(self) -> List[Concept]:
- all_concepts: List[Concept] = []
+ def concepts(self) -> list[Concept]:
+ all_concepts: list[Concept] = []
for requirement in self.root.values():
if requirement.concept.concept_string not in [c.concept_string for c in all_concepts]:
all_concepts.append(requirement.concept)
return all_concepts
@property
- def variables(self) -> List[str]:
+ def variables(self) -> list[str]:
return list(self.root.keys())
@property
- def required_names(self) -> List[str]:
- the_required_names: List[str] = []
- for requirement_expression in self.root.keys():
+ def required_names(self) -> list[str]:
+ the_required_names: list[str] = []
+ for requirement_expression in self.root:
required_variable_name = requirement_expression.split(".", 1)[0]
the_required_names.append(required_variable_name)
return the_required_names
@property
- def named_input_requirements(self) -> List[NamedInputRequirement]:
- the_requirements: List[NamedInputRequirement] = []
+ def named_input_requirements(self) -> list[NamedInputRequirement]:
+ the_requirements: list[NamedInputRequirement] = []
for requirement_expression, requirement in self.root.items():
required_variable_name = requirement_expression.split(".", 1)[0]
# TODO: refactor this with a proper class like InputRequirement
@@ -115,6 +116,6 @@ class PipeInputSpec(RootModel[PipeInputSpecRoot]):
requirement_expression=requirement_expression,
concept=requirement.concept,
multiplicity=requirement.multiplicity,
- )
+ ),
)
return the_requirements
diff --git a/pipelex/core/pipes/pipe_input_blueprint.py b/pipelex/core/pipes/pipe_input_blueprint.py
index 13da21156..4af759768 100644
--- a/pipelex/core/pipes/pipe_input_blueprint.py
+++ b/pipelex/core/pipes/pipe_input_blueprint.py
@@ -1,5 +1,3 @@
-from typing import Optional
-
from pydantic import BaseModel, field_validator
from pipelex.core.concepts.concept_blueprint import ConceptBlueprint
@@ -8,7 +6,7 @@ from pipelex.core.pipes.pipe_run_params import PipeOutputMultiplicity
class InputRequirementBlueprint(BaseModel):
concept: str
- multiplicity: Optional[PipeOutputMultiplicity] = None
+ multiplicity: PipeOutputMultiplicity | None = None
@field_validator("concept", mode="before")
@classmethod
diff --git a/pipelex/core/pipes/pipe_input_factory.py b/pipelex/core/pipes/pipe_input_factory.py
index e5ce3bf8b..351f4006f 100644
--- a/pipelex/core/pipes/pipe_input_factory.py
+++ b/pipelex/core/pipes/pipe_input_factory.py
@@ -1,5 +1,3 @@
-from typing import Dict, List, Optional, Union
-
from pipelex.core.concepts.concept_blueprint import ConceptBlueprint
from pipelex.core.concepts.concept_factory import ConceptFactory
from pipelex.core.pipes.pipe_input import InputRequirement, PipeInputSpec
@@ -14,9 +12,12 @@ class PipeInputSpecFactory:
@classmethod
def make_from_blueprint(
- cls, domain: str, blueprint: Dict[str, Union[str, InputRequirementBlueprint]], concept_codes_from_the_same_domain: Optional[List[str]] = None
+ cls,
+ domain: str,
+ blueprint: dict[str, str | InputRequirementBlueprint],
+ concept_codes_from_the_same_domain: list[str] | None = None,
) -> PipeInputSpec:
- inputs: Dict[str, InputRequirement] = {}
+ inputs: dict[str, InputRequirement] = {}
for var_name, input_requirement_blueprint in blueprint.items():
if isinstance(input_requirement_blueprint, str):
input_requirement_blueprint = InputRequirementBlueprint(concept=input_requirement_blueprint)
@@ -32,8 +33,9 @@ class PipeInputSpecFactory:
inputs[var_name] = InputRequirement(
concept=get_concept_provider().get_required_concept(
concept_string=ConceptFactory.construct_concept_string_with_domain(
- domain=input_domain_and_code.domain, concept_code=input_domain_and_code.concept_code
- )
+ domain=input_domain_and_code.domain,
+ concept_code=input_domain_and_code.concept_code,
+ ),
),
multiplicity=input_requirement_blueprint.multiplicity,
)
diff --git a/pipelex/core/pipes/pipe_library.py b/pipelex/core/pipes/pipe_library.py
index 9ece942fb..ebccc3279 100644
--- a/pipelex/core/pipes/pipe_library.py
+++ b/pipelex/core/pipes/pipe_library.py
@@ -1,18 +1,18 @@
from itertools import groupby
-from typing import Dict, List, Optional
from pydantic import RootModel
from rich import box
from rich.table import Table
-from typing_extensions import Self, override
+from typing_extensions import override
from pipelex import pretty_print
from pipelex.core.pipes.pipe_abstract import PipeAbstract
from pipelex.core.pipes.pipe_provider_abstract import PipeProviderAbstract
from pipelex.exceptions import ConceptError, ConceptLibraryConceptNotFoundError, PipeLibraryError, PipeLibraryPipeNotFoundError
from pipelex.hub import get_concept_provider
+from pipelex.types import Self
-PipeLibraryRoot = Dict[str, PipeAbstract]
+PipeLibraryRoot = dict[str, PipeAbstract]
class PipeLibrary(RootModel[PipeLibraryRoot], PipeProviderAbstract):
@@ -26,14 +26,14 @@ class PipeLibrary(RootModel[PipeLibraryRoot], PipeProviderAbstract):
try:
concept_provider.get_required_concept(concept_string=concept.concept_string)
except ConceptError as concept_error:
- raise PipeLibraryError(
- f"Error validating pipe '{pipe.code}' dependency concept '{concept.concept_string}' because of: {concept_error}"
- ) from concept_error
+ msg = f"Error validating pipe '{pipe.code}' dependency concept '{concept.concept_string}' because of: {concept_error}"
+ raise PipeLibraryError(msg) from concept_error
for pipe_code in pipe.pipe_dependencies():
self.get_required_pipe(pipe_code=pipe_code)
pipe.validate_with_libraries()
except (ConceptLibraryConceptNotFoundError, PipeLibraryPipeNotFoundError) as not_found_error:
- raise PipeLibraryError(f"Missing dependency for pipe '{pipe.code}': {not_found_error}") from not_found_error
+ msg = f"Missing dependency for pipe '{pipe.code}': {not_found_error}"
+ raise PipeLibraryError(msg) from not_found_error
@classmethod
def make_empty(cls) -> Self:
@@ -42,11 +42,12 @@ class PipeLibrary(RootModel[PipeLibraryRoot], PipeProviderAbstract):
@override
def add_new_pipe(self, pipe: PipeAbstract):
if pipe.code in self.root:
- raise PipeLibraryError(f"Pipe '{pipe.code}' already exists in the library")
+ msg = f"Pipe '{pipe.code}' already exists in the library"
+ raise PipeLibraryError(msg)
self.root[pipe.code] = pipe
@override
- def add_pipes(self, pipes: List[PipeAbstract]):
+ def add_pipes(self, pipes: list[PipeAbstract]):
for pipe in pipes:
self.add_new_pipe(pipe=pipe)
@@ -58,27 +59,26 @@ class PipeLibrary(RootModel[PipeLibraryRoot], PipeProviderAbstract):
self.root[name] = pipe
@override
- def get_optional_pipe(self, pipe_code: str) -> Optional[PipeAbstract]:
+ def get_optional_pipe(self, pipe_code: str) -> PipeAbstract | None:
return self.root.get(pipe_code)
@override
def get_required_pipe(self, pipe_code: str) -> PipeAbstract:
the_pipe = self.get_optional_pipe(pipe_code=pipe_code)
if not the_pipe:
- raise PipeLibraryPipeNotFoundError(
- f"Pipe '{pipe_code}' not found. Check for typos and make sure it is declared in a library listed in the config."
- )
+ msg = f"Pipe '{pipe_code}' not found. Check for typos and make sure it is declared in a library listed in the config."
+ raise PipeLibraryPipeNotFoundError(msg)
return the_pipe
@override
- def get_pipes(self) -> List[PipeAbstract]:
+ def get_pipes(self) -> list[PipeAbstract]:
return list(self.root.values())
@override
- def get_pipes_dict(self) -> Dict[str, PipeAbstract]:
+ def get_pipes_dict(self) -> dict[str, PipeAbstract]:
return self.root
- def remove_pipes_by_codes(self, pipe_codes: List[str]) -> None:
+ def remove_pipes_by_codes(self, pipe_codes: list[str]) -> None:
# TODO: We should create a separate library, that copies the original one, and then removes the pipes from it
# Then run the dry run + validation to see if removing those pipe has not broken any other pipe.
# If validated, it should update the real library.
@@ -92,7 +92,7 @@ class PipeLibrary(RootModel[PipeLibraryRoot], PipeProviderAbstract):
@override
def pretty_list_pipes(self) -> None:
- def _format_concept_code(concept_code: Optional[str], current_domain: str) -> str:
+ def _format_concept_code(concept_code: str | None, current_domain: str) -> str:
"""Format concept code by removing domain prefix if it matches current domain."""
if not concept_code:
return ""
@@ -107,7 +107,7 @@ class PipeLibrary(RootModel[PipeLibraryRoot], PipeProviderAbstract):
ordered_items = sorted(pipes, key=lambda pipe: (pipe.domain or "", pipe.code or ""))
# Create dictionary for return value
- pipes_dict: Dict[str, Dict[str, Dict[str, str]]] = {}
+ pipes_dict: dict[str, dict[str, dict[str, str]]] = {}
# Group by domain and create separate tables
for domain, domain_pipes in groupby(ordered_items, key=lambda pipe: pipe.domain):
diff --git a/pipelex/core/pipes/pipe_output.py b/pipelex/core/pipes/pipe_output.py
index 1a5143d52..73e8669bb 100644
--- a/pipelex/core/pipes/pipe_output.py
+++ b/pipelex/core/pipes/pipe_output.py
@@ -1,4 +1,4 @@
-from typing import List, Type, TypeVar
+from typing import TypeVar
from pydantic import BaseModel, Field
@@ -25,23 +25,20 @@ class PipeOutput(BaseModel):
def main_stuff(self) -> Stuff:
return self.working_memory.get_main_stuff()
- def main_stuff_as_list(self, item_type: Type[StuffContentType]) -> ListContent[StuffContentType]:
- """
- Get main stuff content as ListContent with items of type StuffContentType.
+ def main_stuff_as_list(self, item_type: type[StuffContentType]) -> ListContent[StuffContentType]:
+ """Get main stuff content as ListContent with items of type StuffContentType.
If the items are of possibly various types, use item_type=StuffContent.
"""
return self.working_memory.main_stuff_as_list(item_type=item_type)
- def main_stuff_as_items(self, item_type: Type[StuffContentType]) -> List[StuffContentType]:
- """
- Get main stuff content as ListContent with items of type StuffContentType.
+ def main_stuff_as_items(self, item_type: type[StuffContentType]) -> list[StuffContentType]:
+ """Get main stuff content as ListContent with items of type StuffContentType.
Return the actual items
"""
return self.working_memory.main_stuff_as_list(item_type=item_type).items
- def main_stuff_as(self, content_type: Type[StuffContentType]) -> StuffContentType:
- """
- Get main stuff content as StuffContentType.
+ def main_stuff_as(self, content_type: type[StuffContentType]) -> StuffContentType:
+ """Get main stuff content as StuffContentType.
If the items are of possibly various types, use item_type=StuffContent.
"""
return self.working_memory.main_stuff_as(content_type=content_type)
diff --git a/pipelex/core/pipes/pipe_provider_abstract.py b/pipelex/core/pipes/pipe_provider_abstract.py
index 48fca9e43..89456a6f8 100644
--- a/pipelex/core/pipes/pipe_provider_abstract.py
+++ b/pipelex/core/pipes/pipe_provider_abstract.py
@@ -1,9 +1,8 @@
from abc import ABC, abstractmethod
-from typing import Dict, List, Optional
from pipelex.core.pipes.pipe_abstract import PipeAbstract
-PipeLibraryRoot = Dict[str, PipeAbstract]
+PipeLibraryRoot = dict[str, PipeAbstract]
class PipeProviderAbstract(ABC):
@@ -16,15 +15,15 @@ class PipeProviderAbstract(ABC):
pass
@abstractmethod
- def get_optional_pipe(self, pipe_code: str) -> Optional[PipeAbstract]:
+ def get_optional_pipe(self, pipe_code: str) -> PipeAbstract | None:
pass
@abstractmethod
- def get_pipes(self) -> List[PipeAbstract]:
+ def get_pipes(self) -> list[PipeAbstract]:
pass
@abstractmethod
- def get_pipes_dict(self) -> Dict[str, PipeAbstract]:
+ def get_pipes_dict(self) -> dict[str, PipeAbstract]:
pass
@abstractmethod
@@ -40,5 +39,5 @@ class PipeProviderAbstract(ABC):
pass
@abstractmethod
- def add_pipes(self, pipes: List[PipeAbstract]) -> None:
+ def add_pipes(self, pipes: list[PipeAbstract]) -> None:
pass
diff --git a/pipelex/core/pipes/pipe_run_params.py b/pipelex/core/pipes/pipe_run_params.py
index d4d424e6d..2a71d40d4 100644
--- a/pipelex/core/pipes/pipe_run_params.py
+++ b/pipelex/core/pipes/pipe_run_params.py
@@ -1,11 +1,12 @@
-from typing import Any, Dict, List, Optional, Union
+from __future__ import annotations
+
+from typing import Any, Union
from pydantic import BaseModel, Field, field_validator
-from typing_extensions import Self
from pipelex import log
from pipelex.core.memory.working_memory import BATCH_ITEM_STUFF_NAME, MAIN_STUFF_NAME
-from pipelex.types import StrEnum
+from pipelex.types import Self, StrEnum
class PipeRunParamKey(StrEnum):
@@ -24,23 +25,21 @@ PipeOutputMultiplicity = Union[bool, int]
class OutputMultiplicityResolution(BaseModel):
- """
- Result of resolving output multiplicity settings between base and override values.
+ """Result of resolving output multiplicity settings between base and override values.
This model provides a clear, structured representation of how output multiplicity
should be applied, replacing the previous tuple-based return format.
"""
- resolved_multiplicity: Optional[PipeOutputMultiplicity] = Field(description="The final multiplicity value to use after resolution")
+ resolved_multiplicity: PipeOutputMultiplicity | None = Field(description="The final multiplicity value to use after resolution")
enable_multiple_outputs: bool = Field(description="Whether multiple outputs should be generated")
- specific_output_count: Optional[int] = Field(default=None, description="Exact number of outputs to generate, if specified")
+ specific_output_count: int | None = Field(default=None, description="Exact number of outputs to generate, if specified")
-def make_output_multiplicity(nb_output: Optional[int], multiple_output: Optional[bool]) -> Optional[PipeOutputMultiplicity]:
- """
- This function takes two mutually exclusive parameters that control how many outputs
+def make_output_multiplicity(nb_output: int | None, multiple_output: bool | None) -> PipeOutputMultiplicity | None:
+ """This function takes two mutually exclusive parameters that control how many outputs
a pipe should generate and converts them into a single PipeOutputMultiplicity type.
Args:
@@ -58,8 +57,9 @@ def make_output_multiplicity(nb_output: Optional[int], multiple_output: Optional
None
>>> make_output_multiplicity(nb_output=0, multiple_output=True)
True
+
"""
- output_multiplicity: Optional[PipeOutputMultiplicity]
+ output_multiplicity: PipeOutputMultiplicity | None
if nb_output:
output_multiplicity = nb_output
elif multiple_output:
@@ -70,11 +70,10 @@ def make_output_multiplicity(nb_output: Optional[int], multiple_output: Optional
def output_multiplicity_to_apply(
- base_multiplicity: Optional[PipeOutputMultiplicity],
- override_multiplicity: Optional[PipeOutputMultiplicity],
+ base_multiplicity: PipeOutputMultiplicity | None,
+ override_multiplicity: PipeOutputMultiplicity | None,
) -> OutputMultiplicityResolution:
- """
- Resolve output multiplicity settings by combining base configuration with override.
+ """Resolve output multiplicity settings by combining base configuration with override.
This function implements a priority system where override values take precedence over
base values, with clear logic to handle different type combinations and return
@@ -113,6 +112,7 @@ def output_multiplicity_to_apply(
>>> result = output_multiplicity_to_apply(3, None)
>>> (result.resolved_multiplicity, result.enable_multiple_outputs, result.specific_output_count)
(3, True, 3)
+
"""
log.debug(f"base_multiplicity = {base_multiplicity}")
log.debug(f"override_multiplicity = {override_multiplicity}")
@@ -124,12 +124,16 @@ def output_multiplicity_to_apply(
if isinstance(base_multiplicity, bool):
log.debug("base_multiplicity is bool")
return OutputMultiplicityResolution(
- resolved_multiplicity=base_multiplicity, enable_multiple_outputs=base_multiplicity, specific_output_count=None
+ resolved_multiplicity=base_multiplicity,
+ enable_multiple_outputs=base_multiplicity,
+ specific_output_count=None,
)
elif isinstance(base_multiplicity, int):
log.debug("base_multiplicity is int")
return OutputMultiplicityResolution(
- resolved_multiplicity=base_multiplicity, enable_multiple_outputs=True, specific_output_count=base_multiplicity
+ resolved_multiplicity=base_multiplicity,
+ enable_multiple_outputs=True,
+ specific_output_count=base_multiplicity,
)
else:
log.debug("base_multiplicity is None")
@@ -156,11 +160,13 @@ def output_multiplicity_to_apply(
log.debug("override_multiplicity is False - forcing single output")
return OutputMultiplicityResolution(resolved_multiplicity=False, enable_multiple_outputs=False, specific_output_count=None)
- # Case 3: Override is an integer
else:
+ # Case 3: Override is an integer
log.debug("override_multiplicity is int - using override value")
return OutputMultiplicityResolution(
- resolved_multiplicity=override_multiplicity, enable_multiple_outputs=True, specific_output_count=override_multiplicity
+ resolved_multiplicity=override_multiplicity,
+ enable_multiple_outputs=True,
+ specific_output_count=override_multiplicity,
)
@@ -171,10 +177,10 @@ class BatchParams(BaseModel):
@classmethod
def make_optional_batch_params(
cls,
- input_list_name: Union[bool, str],
- input_item_name: Optional[str] = None,
- ) -> Optional["BatchParams"]:
- the_batch_params: Optional[BatchParams] = None
+ input_list_name: bool | str,
+ input_item_name: str | None = None,
+ ) -> BatchParams | None:
+ the_batch_params: BatchParams | None = None
if input_list_name or input_item_name:
input_list_stuff_name: str
if isinstance(input_list_name, str):
@@ -189,7 +195,7 @@ class BatchParams(BaseModel):
return the_batch_params
@classmethod
- def make_default(cls) -> "BatchParams":
+ def make_default(cls) -> BatchParams:
return BatchParams(
input_list_stuff_name=MAIN_STUFF_NAME,
input_item_stuff_name=BATCH_ITEM_STUFF_NAME,
@@ -198,26 +204,27 @@ class BatchParams(BaseModel):
class PipeRunParams(BaseModel):
run_mode: PipeRunMode = PipeRunMode.LIVE
- final_stuff_code: Optional[str] = None
- is_with_preliminary_text: Optional[bool] = None
- output_multiplicity: Optional[PipeOutputMultiplicity] = None
- dynamic_output_concept_code: Optional[str] = None
- batch_params: Optional[BatchParams] = None
- params: Dict[str, Any] = Field(default_factory=dict)
+ final_stuff_code: str | None = None
+ is_with_preliminary_text: bool | None = None
+ output_multiplicity: PipeOutputMultiplicity | None = None
+ dynamic_output_concept_code: str | None = None
+ batch_params: BatchParams | None = None
+ params: dict[str, Any] = Field(default_factory=dict)
pipe_stack_limit: int
- pipe_stack: List[str] = Field(default_factory=list)
- pipe_layers: List[str] = Field(default_factory=list)
+ pipe_stack: list[str] = Field(default_factory=list)
+ pipe_layers: list[str] = Field(default_factory=list)
@field_validator("params")
@classmethod
- def validate_param_keys(cls, value: Dict[str, Any]) -> Dict[str, Any]:
+ def validate_param_keys(cls, value: dict[str, Any]) -> dict[str, Any]:
for key in value:
if key == PipeRunParamKey.DYNAMIC_OUTPUT_CONCEPT:
# TODO: validate the concept code
pass
if not key.startswith("_"):
- raise ValueError(f"Parameter key '{key}' must start with an underscore '_'")
+ msg = f"Parameter key '{key}' must start with an underscore '_'"
+ raise ValueError(msg)
return value
def make_deep_copy(self) -> Self:
@@ -230,11 +237,10 @@ class PipeRunParams(BaseModel):
def copy_by_injecting_multiplicity(
cls,
pipe_run_params: Self,
- applied_output_multiplicity: Optional[PipeOutputMultiplicity],
- is_with_preliminary_text: Optional[bool] = None,
+ applied_output_multiplicity: PipeOutputMultiplicity | None,
+ is_with_preliminary_text: bool | None = None,
) -> Self:
- """
- Copy the run params the nb_output into the params, and remove the attribute.
+ """Copy the run params the nb_output into the params, and remove the attribute.
This is useful to make a single prompt with multiple outputs.
"""
new_run_params = pipe_run_params.model_copy()
diff --git a/pipelex/core/pipes/pipe_run_params_factory.py b/pipelex/core/pipes/pipe_run_params_factory.py
index 65b006529..75665c7ed 100644
--- a/pipelex/core/pipes/pipe_run_params_factory.py
+++ b/pipelex/core/pipes/pipe_run_params_factory.py
@@ -1,4 +1,4 @@
-from typing import Any, Dict, Optional
+from typing import Any
from pipelex.config import get_config
from pipelex.core.pipes.pipe_run_params import BatchParams, PipeOutputMultiplicity, PipeRunMode, PipeRunParams
@@ -9,11 +9,11 @@ class PipeRunParamsFactory:
def make_run_params(
cls,
pipe_run_mode: PipeRunMode = PipeRunMode.LIVE,
- pipe_stack_limit: Optional[int] = None,
- output_multiplicity: Optional[PipeOutputMultiplicity] = None,
- dynamic_output_concept_code: Optional[str] = None,
- batch_params: Optional[BatchParams] = None,
- params: Optional[Dict[str, Any]] = None,
+ pipe_stack_limit: int | None = None,
+ output_multiplicity: PipeOutputMultiplicity | None = None,
+ dynamic_output_concept_code: str | None = None,
+ batch_params: BatchParams | None = None,
+ params: dict[str, Any] | None = None,
) -> PipeRunParams:
pipe_stack_limit = pipe_stack_limit or get_config().pipelex.pipe_run_config.pipe_stack_limit
return PipeRunParams(
diff --git a/pipelex/core/registry_models.py b/pipelex/core/registry_models.py
index 413e79663..644b8795b 100644
--- a/pipelex/core/registry_models.py
+++ b/pipelex/core/registry_models.py
@@ -1,4 +1,4 @@
-from typing import Any, ClassVar, List
+from typing import Any, ClassVar
from pipelex.core.pipes.pipe_abstract import PipeAbstractType
from pipelex.core.pipes.pipe_factory import PipeFactoryProtocol
@@ -38,9 +38,9 @@ from pipelex.tools.registry_models import ModelType, RegistryModels
class PipelexRegistryModels(RegistryModels):
- FIELD_EXTRACTION: ClassVar[List[ModelType]] = []
+ FIELD_EXTRACTION: ClassVar[list[ModelType]] = []
- PIPE_OPERATORS: ClassVar[List[PipeAbstractType]] = [
+ PIPE_OPERATORS: ClassVar[list[PipeAbstractType]] = [
PipeFunc,
PipeImgGen,
PipeJinja2,
@@ -48,7 +48,7 @@ class PipelexRegistryModels(RegistryModels):
PipeOcr,
]
- PIPE_OPERATORS_FACTORY: ClassVar[List[PipeFactoryProtocol[Any, Any]]] = [
+ PIPE_OPERATORS_FACTORY: ClassVar[list[PipeFactoryProtocol[Any, Any]]] = [
PipeFuncFactory,
PipeImgGenFactory,
PipeJinja2Factory,
@@ -56,21 +56,21 @@ class PipelexRegistryModels(RegistryModels):
PipeOcrFactory,
]
- PIPE_CONTROLLERS: ClassVar[List[PipeAbstractType]] = [
+ PIPE_CONTROLLERS: ClassVar[list[PipeAbstractType]] = [
PipeBatch,
PipeCondition,
PipeParallel,
PipeSequence,
]
- PIPE_CONTROLLERS_FACTORY: ClassVar[List[PipeFactoryProtocol[Any, Any]]] = [
+ PIPE_CONTROLLERS_FACTORY: ClassVar[list[PipeFactoryProtocol[Any, Any]]] = [
PipeBatchFactory,
PipeConditionFactory,
PipeParallelFactory,
PipeSequenceFactory,
]
- STUFF: ClassVar[List[ModelType]] = [
+ STUFF: ClassVar[list[ModelType]] = [
TextContent,
NumberContent,
ImageContent,
@@ -84,6 +84,6 @@ class PipelexRegistryModels(RegistryModels):
PageContent,
]
- EXPERIMENTAL: ClassVar[List[ModelType]] = [
+ EXPERIMENTAL: ClassVar[list[ModelType]] = [
DynamicContent,
]
diff --git a/pipelex/core/stuffs/__init__.py b/pipelex/core/stuffs/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/pipelex/core/stuffs/stuff.py b/pipelex/core/stuffs/stuff.py
index 354ed7235..29abdce1c 100644
--- a/pipelex/core/stuffs/stuff.py
+++ b/pipelex/core/stuffs/stuff.py
@@ -1,4 +1,4 @@
-from typing import Any, Dict, Optional, Type, Union
+from typing import Any, cast
from pydantic import ConfigDict, ValidationError
from typing_extensions import override
@@ -27,23 +27,22 @@ class Stuff(CustomBaseModel):
model_config = ConfigDict(extra="ignore", strict=True)
stuff_code: str
- stuff_name: Optional[str] = None
+ stuff_name: str | None = None
concept: Concept
content: StuffContent
def make_artefact(self) -> StuffArtefact:
- artefact_dict: Dict[str, Any] = self.content.model_dump(serialize_as_any=True)
+ artefact_dict: dict[str, Any] = self.content.model_dump(serialize_as_any=True)
- def set_artefact_field(key: str, value: Optional[Union[str, StuffContent]]):
+ def set_artefact_field(key: str, value: str | StuffContent | None):
if value is None:
return
if key in artefact_dict:
stuff_name = self.stuff_name or f"unnamed using concept code {self.concept.code}"
- raise StuffError(
- f"""Cannot create stuff artefact for stuff '{stuff_name}' of concept '{self.concept.code}' because reserved field '{key}'
+ msg = f"""Cannot create stuff artefact for stuff '{stuff_name}' of concept '{self.concept.code}' because reserved field '{key}'
in the structured output '{self.content.__class__.__name__}' already exists in the stuff content.
Forbidden fields are: 'stuff_name', 'content_class', 'concept_code', 'stuff_code', 'content'."""
- )
+ raise StuffError(msg)
artefact_dict[key] = value
set_artefact_field("stuff_name", self.stuff_name)
@@ -101,12 +100,12 @@ Forbidden fields are: 'stuff_name', 'content_class', 'concept_code', 'stuff_code
def is_number(self) -> bool:
return isinstance(self.content, NumberContent)
- def content_as(self, content_type: Type[StuffContentType]) -> StuffContentType:
+ def content_as(self, content_type: type[StuffContentType]) -> StuffContentType:
"""Get content with proper typing if it's of the expected type."""
return self.verify_content_type(self.content, content_type)
@classmethod
- def verify_content_type(cls, content: StuffContent, content_type: Type[StuffContentType]) -> StuffContentType:
+ def verify_content_type(cls, content: StuffContent, content_type: type[StuffContentType]) -> StuffContentType:
"""Verify and convert content to the expected type."""
# First try the direct isinstance check for performance
if isinstance(content, content_type):
@@ -123,18 +122,20 @@ Forbidden fields are: 'stuff_name', 'content_class', 'concept_code', 'stuff_code
except ValidationError as exc:
formatted_error = format_pydantic_validation_error(exc)
raise StuffContentValidationError(
- original_type=type(content).__name__, target_type=content_type.__name__, validation_error=formatted_error
+ original_type=type(content).__name__,
+ target_type=content_type.__name__,
+ validation_error=formatted_error,
) from exc
- raise TypeError(f"Content is of type '{type(content)}', instead of the expected '{content_type}'")
+ msg = f"Content is of type '{type(content)}', instead of the expected '{content_type}'"
+ raise TypeError(msg)
def as_list_content(self) -> ListContent: # pyright: ignore[reportMissingTypeArgument, reportUnknownParameterType]
"""Get content as ListContent with items of any type."""
return self.content_as(content_type=ListContent) # pyright: ignore[reportUnknownVariableType]
- def as_list_of_fixed_content_type(self, item_type: Type[StuffContentType]) -> ListContent[StuffContentType]:
- """
- Get content as ListContent with items of type T.
+ def as_list_of_fixed_content_type(self, item_type: type[StuffContentType]) -> ListContent[StuffContentType]:
+ """Get content as ListContent with items of type T.
Args:
item_type: The expected type of items in the list.
@@ -144,8 +145,9 @@ Forbidden fields are: 'stuff_name', 'content_class', 'concept_code', 'stuff_code
Raises:
TypeError: If content is not ListContent or items don't match expected type
+
"""
- list_content: ListContent[StuffContentType] = self.content_as(content_type=ListContent)
+ list_content = cast("ListContent[StuffContentType]", self.content_as(content_type=ListContent))
# Validate all items are of the expected type
for item in list_content.items:
diff --git a/pipelex/core/stuffs/stuff_artefact.py b/pipelex/core/stuffs/stuff_artefact.py
index e1cc2e771..cf494b212 100644
--- a/pipelex/core/stuffs/stuff_artefact.py
+++ b/pipelex/core/stuffs/stuff_artefact.py
@@ -1,4 +1,4 @@
-from typing import Any, Dict, Optional, Tuple
+from typing import Any
from jinja2.runtime import Context
from pydantic import RootModel
@@ -10,7 +10,7 @@ from pipelex.tools.templating.jinja2_models import Jinja2ContextKey, Jinja2Tagga
from pipelex.tools.templating.templating_models import TextFormat
-class StuffArtefact(RootModel[Dict[str, Any]], Jinja2TaggableAbstract):
+class StuffArtefact(RootModel[dict[str, Any]], Jinja2TaggableAbstract):
"""A flattened representation of Stuff and its content as a dictionary.
This RootModel implementation allows for subscript access to the underlying dictionary
@@ -45,12 +45,12 @@ class StuffArtefact(RootModel[Dict[str, Any]], Jinja2TaggableAbstract):
def rendered_str(self, text_format: TextFormat) -> str:
content = self.root["content"]
if not isinstance(content, StuffContent):
- raise StuffArtefactError(f"StuffArtefact has no StuffContent, content: {self}")
- rendered_str = content.rendered_str(text_format=text_format)
- return rendered_str
+ msg = f"StuffArtefact has no StuffContent, content: {self}"
+ raise StuffArtefactError(msg)
+ return content.rendered_str(text_format=text_format)
@override
- def render_tagged_for_jinja2(self, context: Context, tag_name: Optional[str] = None) -> Tuple[Any, Optional[str]]:
+ def render_tagged_for_jinja2(self, context: Context, tag_name: str | None = None) -> tuple[Any, str | None]:
# TODO: factorize the text formatting with the jinja2 "text_format" filter
text_format = context.get(Jinja2ContextKey.TEXT_FORMAT, default=TextFormat.PLAIN)
rendered_str = self.rendered_str(text_format=text_format)
diff --git a/pipelex/core/stuffs/stuff_content.py b/pipelex/core/stuffs/stuff_content.py
index 43f54ada9..0da0eea6a 100644
--- a/pipelex/core/stuffs/stuff_content.py
+++ b/pipelex/core/stuffs/stuff_content.py
@@ -2,14 +2,14 @@ import base64
import json
from abc import ABC, abstractmethod
from io import BytesIO
-from typing import Any, Dict, Generic, List, Optional, Type, TypeVar, Union
+from typing import Any, Generic, TypeVar
import markdown
from json2html import json2html
from kajson import kajson
from PIL import Image
from pydantic import BaseModel
-from typing_extensions import Self, override
+from typing_extensions import override
from yattag import Doc
from pipelex.cogt.ocr.ocr_output import ExtractedImage
@@ -20,6 +20,7 @@ from pipelex.tools.misc.markdown_utils import convert_to_markdown
from pipelex.tools.misc.path_utils import InterpretedPathOrUrl, interpret_path_or_url
from pipelex.tools.templating.templating_models import TextFormat
from pipelex.tools.typing.pydantic_utils import CustomBaseModel, clean_model_to_dict
+from pipelex.types import Self
ObjectContentType = TypeVar("ObjectContentType", bound=BaseModel)
StuffContentType = TypeVar("StuffContentType", bound="StuffContent")
@@ -32,7 +33,7 @@ class StuffContent(ABC, CustomBaseModel):
def short_desc(self) -> str:
return f"some {self.__class__.__name__}"
- def smart_dump(self) -> Union[str, Dict[str, Any], List[str], List[Dict[str, Any]]]:
+ def smart_dump(self) -> str | dict[str, Any] | list[str] | list[dict[str, Any]]:
return self.model_dump(serialize_as_any=True)
@override
@@ -81,7 +82,7 @@ class TextContent(StuffContentInitableFromStr):
text: str
@override
- def smart_dump(self) -> Union[str, Dict[str, Any], List[str], List[Dict[str, Any]]]:
+ def smart_dump(self) -> str | dict[str, Any] | list[str] | list[dict[str, Any]]:
return self.text
@property
@@ -105,8 +106,7 @@ class TextContent(StuffContentInitableFromStr):
@override
def rendered_html(self) -> str:
# Convert a markdown string to HTML and return HTML as a Unicode string.
- html = markdown.markdown(self.text)
- return html
+ return markdown.markdown(self.text)
@override
def rendered_markdown(self, level: int = 1, is_pretty: bool = False) -> str:
@@ -138,10 +138,10 @@ class DynamicContent(StuffContent):
class NumberContent(StuffContentInitableFromStr):
- number: Union[int, float]
+ number: int | float
@override
- def smart_dump(self) -> Union[str, Dict[str, Any], List[str], List[Dict[str, Any]]]:
+ def smart_dump(self) -> str | dict[str, Any] | list[str] | list[dict[str, Any]]:
return str(self.number)
@property
@@ -182,9 +182,9 @@ class NumberContent(StuffContentInitableFromStr):
class ImageContent(StuffContentInitableFromStr):
url: str
- source_prompt: Optional[str] = None
- caption: Optional[str] = None
- base_64: Optional[str] = None
+ source_prompt: str | None = None
+ caption: str | None = None
+ base_64: str | None = None
@property
@override
@@ -234,27 +234,26 @@ class ImageContent(StuffContentInitableFromStr):
base_64=base_64,
)
- def save_to_directory(self, directory: str, base_name: Optional[str] = None, extension: Optional[str] = None):
+ def save_to_directory(self, directory: str, base_name: str | None = None, extension: str | None = None):
ensure_directory_exists(directory)
base_name = base_name or "img"
- if base_64 := self.base_64:
- if not extension:
- match interpret_path_or_url(path_or_uri=self.url):
- case InterpretedPathOrUrl.FILE_NAME:
- parts = self.url.rsplit(".", 1)
- base_name = parts[0]
- extension = parts[1]
- case _:
- file_type = detect_file_type_from_base64(b64=base_64)
- base_name = base_name or "img"
- extension = file_type.extension
- file_path = get_incremental_file_path(
- base_path=directory,
- base_name=base_name,
- extension=extension,
- avoid_suffix_if_possible=True,
- )
- save_base64_to_binary_file(b64=base_64, file_path=file_path)
+ if (base_64 := self.base_64) and not extension:
+ match interpret_path_or_url(path_or_uri=self.url):
+ case InterpretedPathOrUrl.FILE_NAME:
+ parts = self.url.rsplit(".", 1)
+ base_name = parts[0]
+ extension = parts[1]
+ case InterpretedPathOrUrl.FILE_PATH | InterpretedPathOrUrl.FILE_URI | InterpretedPathOrUrl.URL | InterpretedPathOrUrl.BASE_64:
+ file_type = detect_file_type_from_base64(b64=base_64)
+ base_name = base_name or "img"
+ extension = file_type.extension
+ file_path = get_incremental_file_path(
+ base_path=directory,
+ base_name=base_name,
+ extension=extension,
+ avoid_suffix_if_possible=True,
+ )
+ save_base64_to_binary_file(b64=base_64, file_path=file_path)
if caption := self.caption:
caption_file_path = get_incremental_file_path(
@@ -399,13 +398,13 @@ class StructuredContent(StuffContent):
class ListContent(StuffContent, Generic[StuffContentType]):
- items: List[StuffContentType]
+ items: list[StuffContentType]
@property
def nb_items(self) -> int:
return len(self.items)
- def get_items(self, item_type: Type[StuffContent]) -> List[StuffContent]:
+ def get_items(self, item_type: type[StuffContent]) -> list[StuffContent]:
return [item for item in self.items if isinstance(item, item_type)]
@property
@@ -414,22 +413,21 @@ class ListContent(StuffContent, Generic[StuffContentType]):
nb_items = len(self.items)
if nb_items == 0:
return "empty list"
- elif nb_items == 1:
+ if nb_items == 1:
return f"list of 1 {self.items[0].__class__.__name__}"
+ item_classes: list[str] = [item.__class__.__name__ for item in self.items]
+ item_classes_set = set(item_classes)
+ nb_classes = len(item_classes_set)
+ if nb_classes == 1:
+ return f"list of {len(self.items)} {item_classes[0]}s"
+ elif nb_items == nb_classes:
+ return f"list of {len(self.items)} items of different types"
else:
- item_classes: List[str] = [item.__class__.__name__ for item in self.items]
- item_classes_set = set(item_classes)
- nb_classes = len(item_classes_set)
- if nb_classes == 1:
- return f"list of {len(self.items)} {item_classes[0]}s"
- elif nb_items == nb_classes:
- return f"list of {len(self.items)} items of different types"
- else:
- return f"list of {len(self.items)} items of {nb_classes} different types"
+ return f"list of {len(self.items)} items of {nb_classes} different types"
@property
- def _single_class_name(self) -> Optional[str]:
- item_classes: List[str] = [item.__class__.__name__ for item in self.items]
+ def _single_class_name(self) -> str | None:
+ item_classes: list[str] = [item.__class__.__name__ for item in self.items]
item_classes_set = set(item_classes)
nb_classes = len(item_classes_set)
if nb_classes == 1:
@@ -473,8 +471,8 @@ class ListContent(StuffContent, Generic[StuffContentType]):
class TextAndImagesContent(StuffContent):
- text: Optional[TextContent]
- images: Optional[List[ImageContent]]
+ text: TextContent | None
+ images: list[ImageContent] | None
@property
@override
@@ -510,7 +508,7 @@ class TextAndImagesContent(StuffContent):
class PageContent(StructuredContent):
text_and_images: TextAndImagesContent
- page_view: Optional[ImageContent] = None
+ page_view: ImageContent | None = None
def save_to_directory(self, directory: str):
ensure_directory_exists(directory)
diff --git a/pipelex/core/stuffs/stuff_factory.py b/pipelex/core/stuffs/stuff_factory.py
index b2d202367..18a33be98 100644
--- a/pipelex/core/stuffs/stuff_factory.py
+++ b/pipelex/core/stuffs/stuff_factory.py
@@ -1,4 +1,4 @@
-from typing import Any, Dict, List, Optional, Type, cast
+from typing import Any, cast
import shortuuid
from pydantic import BaseModel, ValidationError, field_validator
@@ -26,10 +26,11 @@ class StuffFactoryError(PipelexError):
class StuffBlueprint(BaseModel):
stuff_name: str
concept_string: str
- content: Dict[str, Any] | str
+ content: dict[str, Any] | str
@field_validator("concept_string")
- def validate_concept_string(cls, concept_string: str) -> str:
+ @staticmethod
+ def validate_concept_string(concept_string: str) -> str:
ConceptBlueprint.validate_concept_string(concept_string)
return concept_string
@@ -62,8 +63,8 @@ class StuffFactory:
cls,
concept: Concept,
content: StuffContent,
- name: Optional[str] = None,
- code: Optional[str] = None,
+ name: str | None = None,
+ code: str | None = None,
) -> Stuff:
if not name:
name = cls.make_stuff_name(concept=concept)
@@ -78,10 +79,10 @@ class StuffFactory:
def make_stuff_using_concept_name_and_search_domains(
cls,
concept_name: str,
- search_domains: List[str],
+ search_domains: list[str],
content: StuffContent,
- name: Optional[str] = None,
- code: Optional[str] = None,
+ name: str | None = None,
+ code: str | None = None,
) -> Stuff:
# TODO: Add unit tests for this method
concept_provider = get_concept_provider()
@@ -90,7 +91,8 @@ class StuffFactory:
search_domains=search_domains,
)
if not concept:
- raise StuffFactoryError(f"Could not find a concept named '{concept_name}' in domains {search_domains}")
+ msg = f"Could not find a concept named '{concept_name}' in domains {search_domains}"
+ raise StuffFactoryError(msg)
return cls.make_stuff(concept=concept, content=content, name=name, code=code)
@classmethod
@@ -121,20 +123,17 @@ class StuffFactory:
def combine_stuffs(
cls,
concept: Concept,
- stuff_contents: Dict[str, StuffContent],
- name: Optional[str] = None,
+ stuff_contents: dict[str, StuffContent],
+ name: str | None = None,
) -> Stuff:
# TODO: Add unit tests for this method
- """
- Combine a dictionary of stuffs into a single stuff.
- """
+ """Combine a dictionary of stuffs into a single stuff."""
the_subclass = get_class_registry().get_required_subclass(name=concept.structure_class_name, base_class=StuffContent)
try:
the_stuff_content = the_subclass.model_validate(obj=stuff_contents)
except ValidationError as exc:
- raise StuffFactoryError(
- f"Error combining stuffs for concept {concept.code}, stuff named `{name}`: {format_pydantic_validation_error(exc=exc)}"
- ) from exc
+ msg = f"Error combining stuffs for concept {concept.code}, stuff named `{name}`: {format_pydantic_validation_error(exc=exc)}"
+ raise StuffFactoryError(msg) from exc
return cls.make_stuff(
concept=concept,
content=the_stuff_content,
@@ -146,16 +145,17 @@ class StuffFactory:
cls,
name: str,
stuff_content_or_data: StuffContentOrData,
- search_domains: List[str],
- stuff_code: Optional[str] = None,
+ search_domains: list[str],
+ stuff_code: str | None = None,
) -> Stuff:
# TODO: Add unit tests for this method
content: StuffContent
concept_name: str
if isinstance(stuff_content_or_data, ListContent):
- content = cast(ListContent[Any], stuff_content_or_data)
+ content = cast("ListContent[Any]", stuff_content_or_data)
if len(content.items) == 0:
- raise StuffFactoryError("ListContent in compact memory has no items")
+ msg = "ListContent in compact memory has no items"
+ raise StuffFactoryError(msg)
concept_name = type(content.items[0]).__name__
try:
return cls.make_stuff_using_concept_name_and_search_domains(
@@ -166,7 +166,8 @@ class StuffFactory:
code=stuff_code,
)
except StuffFactoryError as exc:
- raise StuffFactoryError(f"Could not make stuff for ListContent '{name}': {exc}") from exc
+ msg = f"Could not make stuff for ListContent '{name}': {exc}"
+ raise StuffFactoryError(msg) from exc
elif isinstance(stuff_content_or_data, StuffContent):
content = stuff_content_or_data
concept_class_name = type(content).__name__
@@ -191,11 +192,13 @@ class StuffFactory:
code=stuff_code,
)
except StuffFactoryError as exc:
- raise StuffFactoryError(f"Could not make stuff for StuffContent '{name}': {exc}") from exc
+ msg = f"Could not make stuff for StuffContent '{name}': {exc}"
+ raise StuffFactoryError(msg) from exc
elif isinstance(stuff_content_or_data, list):
items = stuff_content_or_data
if len(items) == 0:
- raise StuffFactoryError("List in compact memory has no items")
+ msg = "List in compact memory has no items"
+ raise StuffFactoryError(msg)
first_item = items[0]
concept_name = type(first_item).__name__
content = ListContent[Any](items=items)
@@ -208,7 +211,8 @@ class StuffFactory:
code=stuff_code,
)
except StuffFactoryError as exc:
- raise StuffFactoryError(f"Could not make stuff for list of StuffContent '{name}': {exc}") from exc
+ msg = f"Could not make stuff for list of StuffContent '{name}': {exc}"
+ raise StuffFactoryError(msg) from exc
elif isinstance(stuff_content_or_data, str):
str_stuff: str = stuff_content_or_data
return StuffFactory.make_stuff(
@@ -217,11 +221,12 @@ class StuffFactory:
name=name,
)
else:
- stuff_content_dict: Dict[str, Any] = stuff_content_or_data
+ stuff_content_dict: dict[str, Any] = stuff_content_or_data
try:
concept_code = stuff_content_dict.get("concept") or stuff_content_dict.get("concept_code")
if not concept_code:
- raise StuffFactoryError("Stuff content data dict is badly formed: no concept code")
+ msg = "Stuff content data dict is badly formed: no concept code"
+ raise StuffFactoryError(msg)
content_value = stuff_content_dict["content"]
if NativeConceptManager.is_native_concept(concept_string_or_code=concept_code):
concept = ConceptFactory.make_native_concept(native_concept_data=NATIVE_CONCEPTS_DATA[NativeConceptEnum(concept_code)])
@@ -236,7 +241,8 @@ class StuffFactory:
code=stuff_code,
)
except KeyError as exc:
- raise StuffFactoryError(f"Stuff content data dict is badly formed: {exc}") from exc
+ msg = f"Stuff content data dict is badly formed: {exc}"
+ raise StuffFactoryError(msg) from exc
if isinstance(content_value, StuffContent):
return StuffFactory.make_stuff(
@@ -245,17 +251,16 @@ class StuffFactory:
content=content_value,
code=stuff_code,
)
- else:
- content = StuffContentFactory.make_stuff_content_from_concept_with_fallback(
- concept=get_concept_provider().get_required_concept(concept_string=concept_code),
- value=content_value,
- )
- return StuffFactory.make_stuff(
- concept=get_concept_provider().get_required_concept(concept_string=concept_code),
- name=name,
- content=content,
- code=stuff_code,
- )
+ content = StuffContentFactory.make_stuff_content_from_concept_with_fallback(
+ concept=get_concept_provider().get_required_concept(concept_string=concept_code),
+ value=content_value,
+ )
+ return StuffFactory.make_stuff(
+ concept=get_concept_provider().get_required_concept(concept_string=concept_code),
+ name=name,
+ content=content,
+ code=stuff_code,
+ )
class StuffContentFactoryError(PipelexError):
@@ -264,15 +269,14 @@ class StuffContentFactoryError(PipelexError):
class StuffContentFactory:
@classmethod
- def make_content_from_value(cls, stuff_content_subclass: Type[StuffContent], value: Dict[str, Any] | str) -> StuffContent:
+ def make_content_from_value(cls, stuff_content_subclass: type[StuffContent], value: dict[str, Any] | str) -> StuffContent:
if isinstance(value, str) and stuff_content_subclass == TextContent:
return TextContent(text=value)
return stuff_content_subclass.model_validate(obj=value)
@classmethod
- def make_stuff_content_from_concept_required(cls, concept: Concept, value: Dict[str, Any] | str) -> StuffContent:
- """
- Create StuffContent from concept code, requiring the concept to be linked to a class in the registry.
+ def make_stuff_content_from_concept_required(cls, concept: Concept, value: dict[str, Any] | str) -> StuffContent:
+ """Create StuffContent from concept code, requiring the concept to be linked to a class in the registry.
Raises StuffContentFactoryError if no registry class is found.
"""
the_subclass_name = concept.structure_class_name
@@ -280,16 +284,15 @@ class StuffContentFactory:
return cls.make_content_from_value(stuff_content_subclass=the_subclass, value=value)
@classmethod
- def make_stuff_content_from_concept_with_fallback(cls, concept: Concept, value: Dict[str, Any] | str) -> StuffContent:
- """
- Create StuffContent from concept code, falling back to TextContent if no registry class is found.
- """
+ def make_stuff_content_from_concept_with_fallback(cls, concept: Concept, value: dict[str, Any] | str) -> StuffContent:
+ """Create StuffContent from concept code, falling back to TextContent if no registry class is found."""
the_structure_class = get_class_registry().get_class(name=concept.structure_class_name)
if the_structure_class is None:
return cls.make_content_from_value(stuff_content_subclass=TextContent, value=value)
if not issubclass(the_structure_class, StuffContent):
- raise StuffContentFactoryError(f"Concept '{concept.code}', subclass '{the_structure_class}' is not a subclass of StuffContent")
+ msg = f"Concept '{concept.code}', subclass '{the_structure_class}' is not a subclass of StuffContent"
+ raise StuffContentFactoryError(msg)
return cls.make_content_from_value(stuff_content_subclass=the_structure_class, value=value)
diff --git a/pipelex/exceptions.py b/pipelex/exceptions.py
index 320aa600f..1b83b4d3e 100644
--- a/pipelex/exceptions.py
+++ b/pipelex/exceptions.py
@@ -1,5 +1,3 @@
-from typing import List, Optional
-
from click import ClickException
from typing_extensions import override
@@ -24,12 +22,12 @@ class StaticValidationError(Exception):
self,
error_type: StaticValidationErrorType,
domain: str,
- pipe_code: Optional[str] = None,
- variable_names: Optional[List[str]] = None,
- required_concept_codes: Optional[List[str]] = None,
- provided_concept_code: Optional[str] = None,
- file_path: Optional[str] = None,
- explanation: Optional[str] = None,
+ pipe_code: str | None = None,
+ variable_names: list[str] | None = None,
+ required_concept_codes: list[str] | None = None,
+ provided_concept_code: str | None = None,
+ file_path: str | None = None,
+ explanation: str | None = None,
):
self.error_type = error_type
self.domain = domain
@@ -93,8 +91,6 @@ class WorkingMemoryStuffNotFoundError(WorkingMemoryVariableError):
class PipelexCLIError(PipelexError, ClickException):
"""Raised when there's an error in CLI usage or operation."""
- pass
-
class PipelexConfigError(PipelexError):
pass
@@ -185,7 +181,7 @@ class PipeStackOverflowError(PipeExecutionError):
class DryRunError(PipeExecutionError):
"""Raised when a dry run fails due to missing inputs or other validation issues."""
- def __init__(self, message: str, missing_inputs: Optional[List[str]] = None, pipe_code: Optional[str] = None):
+ def __init__(self, message: str, missing_inputs: list[str] | None = None, pipe_code: str | None = None):
self.missing_inputs = missing_inputs or []
self.pipe_code = pipe_code
super().__init__(message)
@@ -206,32 +202,22 @@ class PipeRunParamsError(PipelexError):
class PipeBatchError(PipelexError):
"""Base class for all PipeBatch-related errors."""
- pass
-
class PipeBatchRecursionError(PipeBatchError):
"""Raised when a PipeBatch attempts to run itself recursively."""
- pass
-
class PipeBatchInputError(PipeBatchError):
"""Raised when the input to a PipeBatch is not a ListContent or is invalid."""
- pass
-
class PipeBatchOutputError(PipeBatchError):
"""Raised when there's an error with the output structure of a PipeBatch operation."""
- pass
-
class PipeBatchBranchError(PipeBatchError):
"""Raised when there's an error with a branch pipe execution in PipeBatch."""
- pass
-
class JobHistoryError(PipelexError):
pass
@@ -272,10 +258,8 @@ class PipeInputDetailsError(PipelexError):
class ApiSerializationError(Exception):
"""Exception raised when API serialization fails."""
- pass
-
-class StartPipelineException(Exception):
+class StartPipelineError(Exception):
pass
diff --git a/pipelex/hub.py b/pipelex/hub.py
index d6348390d..8f1364d1e 100644
--- a/pipelex/hub.py
+++ b/pipelex/hub.py
@@ -1,4 +1,4 @@
-from typing import ClassVar, Optional, Type
+from typing import ClassVar, Optional
from kajson.class_registry_abstract import ClassRegistryAbstract
@@ -35,8 +35,7 @@ from pipelex.tools.templating.template_provider_abstract import TemplateProvider
class PipelexHub:
- """
- PipelexHub serves as a central dependency manager to break cyclic imports between components.
+ """PipelexHub serves as a central dependency manager to break cyclic imports between components.
It provides access to core providers and factories through a singleton instance,
allowing components to retrieve dependencies based on protocols without direct imports that could create cycles.
"""
@@ -45,30 +44,30 @@ class PipelexHub:
def __init__(self):
# tools
- self._config: Optional[ConfigRoot] = None
- self._secrets_provider: Optional[SecretsProviderAbstract] = None
- self._template_provider: Optional[TemplateProviderAbstract] = None
- self._class_registry: Optional[ClassRegistryAbstract] = None
- self._storage_provider: Optional[StorageProviderAbstract] = None
+ self._config: ConfigRoot | None = None
+ self._secrets_provider: SecretsProviderAbstract | None = None
+ self._template_provider: TemplateProviderAbstract | None = None
+ self._class_registry: ClassRegistryAbstract | None = None
+ self._storage_provider: StorageProviderAbstract | None = None
# cogt
- self._models_manager: Optional[ModelManagerAbstract] = None
- self._plugin_manager: Optional[PluginManager] = None
+ self._models_manager: ModelManagerAbstract | None = None
+ self._plugin_manager: PluginManager | None = None
self._inference_manager: InferenceManagerProtocol
self._report_delegate: ReportingProtocol
- self._content_generator: Optional[ContentGeneratorProtocol] = None
+ self._content_generator: ContentGeneratorProtocol | None = None
# pipelex
- self._domain_provider: Optional[DomainProviderAbstract] = None
- self._concept_provider: Optional[ConceptProviderAbstract] = None
- self._pipe_provider: Optional[PipeProviderAbstract] = None
- self._pipe_router: Optional[PipeRouterProtocol] = None
- self._library_manager: Optional[LibraryManagerAbstract] = None
+ self._domain_provider: DomainProviderAbstract | None = None
+ self._concept_provider: ConceptProviderAbstract | None = None
+ self._pipe_provider: PipeProviderAbstract | None = None
+ self._pipe_router: PipeRouterProtocol | None = None
+ self._library_manager: LibraryManagerAbstract | None = None
# pipeline
- self._pipeline_tracker: Optional[PipelineTrackerProtocol] = None
- self._pipeline_manager: Optional[PipelineManagerAbstract] = None
- self._activity_manager: Optional[ActivityManagerProtocol] = None
- self._observer_provider: Optional[ObserverProtocol] = None
+ self._pipeline_tracker: PipelineTrackerProtocol | None = None
+ self._pipeline_manager: PipelineManagerAbstract | None = None
+ self._activity_manager: ActivityManagerProtocol | None = None
+ self._observer_provider: ObserverProtocol | None = None
############################################################
# Class methods for singleton management
@@ -77,7 +76,8 @@ class PipelexHub:
@classmethod
def get_instance(cls) -> "PipelexHub":
if cls._instance is None:
- raise RuntimeError("PipelexHub is not initialized")
+ msg = "PipelexHub is not initialized"
+ raise RuntimeError(msg)
return cls._instance
@classmethod
@@ -90,9 +90,8 @@ class PipelexHub:
# tools
- def setup_config(self, config_cls: Type[ConfigRoot], specific_config_path: Optional[str] = None):
- """
- Set the global configuration instance.
+ def setup_config(self, config_cls: type[ConfigRoot], specific_config_path: str | None = None):
+ """Set the global configuration instance.
# Args:
# config (Config): The configuration instance to set.
@@ -108,9 +107,7 @@ class PipelexHub:
self._config = config
def reset_config(self) -> None:
- """
- Reset the global configuration instance and the config manager.
- """
+ """Reset the global configuration instance and the config manager."""
self._config = None
log.reset()
@@ -179,8 +176,7 @@ class PipelexHub:
# tools
def get_required_config(self) -> ConfigRoot:
- """
- Get the current configuration instance as an instance of a particular subclass of ConfigRoot. This should be used only from pipelex.tools.
+ """Get the current configuration instance as an instance of a particular subclass of ConfigRoot. This should be used only from pipelex.tools.
when getting the config from other projects, use their own project.get_config() method to get the Config
with the proper subclass which is required for proper type checking.
@@ -189,41 +185,49 @@ class PipelexHub:
Raises:
RuntimeError: If the configuration has not been set.
+
"""
if self._config is None:
- raise RuntimeError("Config instance is not set. You must initialize Pipelex first.")
+ msg = "Config instance is not set. You must initialize Pipelex first."
+ raise RuntimeError(msg)
return self._config
def get_required_secrets_provider(self) -> SecretsProviderAbstract:
if self._secrets_provider is None:
- raise RuntimeError("Secrets provider is not set. You must initialize Pipelex first.")
+ msg = "Secrets provider is not set. You must initialize Pipelex first."
+ raise RuntimeError(msg)
return self._secrets_provider
def get_required_template_provider(self) -> TemplateProviderAbstract:
if self._template_provider is None:
- raise RuntimeError("Template provider is not set. You must initialize Pipelex first.")
+ msg = "Template provider is not set. You must initialize Pipelex first."
+ raise RuntimeError(msg)
return self._template_provider
def get_required_class_registry(self) -> ClassRegistryAbstract:
if self._class_registry is None:
- raise RuntimeError("ClassRegistry is not initialized")
+ msg = "ClassRegistry is not initialized"
+ raise RuntimeError(msg)
return self._class_registry
def get_storage_provider(self) -> StorageProviderAbstract:
if self._storage_provider is None:
- raise RuntimeError("StorageProvider is not initialized")
+ msg = "StorageProvider is not initialized"
+ raise RuntimeError(msg)
return self._storage_provider
# cogt
def get_required_models_manager(self) -> ModelManagerAbstract:
if self._models_manager is None:
- raise RuntimeError("ModelsManager is not initialized")
+ msg = "ModelsManager is not initialized"
+ raise RuntimeError(msg)
return self._models_manager
def get_plugin_manager(self) -> PluginManager:
if self._plugin_manager is None:
- raise RuntimeError("PluginManager2 is not initialized")
+ msg = "PluginManager2 is not initialized"
+ raise RuntimeError(msg)
return self._plugin_manager
def get_inference_manager(self) -> InferenceManagerProtocol:
@@ -234,63 +238,73 @@ class PipelexHub:
def get_required_content_generator(self) -> ContentGeneratorProtocol:
if self._content_generator is None:
- raise RuntimeError("ContentGenerator is not initialized")
+ msg = "ContentGenerator is not initialized"
+ raise RuntimeError(msg)
return self._content_generator
# pipelex
def get_required_domain_provider(self) -> DomainProviderAbstract:
if self._domain_provider is None:
- raise RuntimeError("DomainProvider is not initialized")
+ msg = "DomainProvider is not initialized"
+ raise RuntimeError(msg)
return self._domain_provider
- def get_optional_domain_provider(self) -> Optional[DomainProviderAbstract]:
+ def get_optional_domain_provider(self) -> DomainProviderAbstract | None:
return self._domain_provider
def get_required_concept_provider(self) -> ConceptProviderAbstract:
if self._concept_provider is None:
- raise RuntimeError("ConceptProvider is not initialized")
+ msg = "ConceptProvider is not initialized"
+ raise RuntimeError(msg)
return self._concept_provider
- def get_optional_concept_provider(self) -> Optional[ConceptProviderAbstract]:
+ def get_optional_concept_provider(self) -> ConceptProviderAbstract | None:
return self._concept_provider
def get_required_pipe_provider(self) -> PipeProviderAbstract:
if self._pipe_provider is None:
- raise RuntimeError("PipeProvider is not initialized")
+ msg = "PipeProvider is not initialized"
+ raise RuntimeError(msg)
return self._pipe_provider
def get_required_pipe_router(self) -> PipeRouterProtocol:
if self._pipe_router is None:
- raise RuntimeError("PipeRouter is not initialized")
+ msg = "PipeRouter is not initialized"
+ raise RuntimeError(msg)
return self._pipe_router
def get_pipeline_tracker(self) -> PipelineTrackerProtocol:
if self._pipeline_tracker is None:
- raise RuntimeError("PipelineTracker is not initialized")
+ msg = "PipelineTracker is not initialized"
+ raise RuntimeError(msg)
return self._pipeline_tracker
def get_required_pipeline_manager(self) -> PipelineManagerAbstract:
if self._pipeline_manager is None:
- raise RuntimeError("PipelineManager is not initialized")
+ msg = "PipelineManager is not initialized"
+ raise RuntimeError(msg)
return self._pipeline_manager
def get_activity_manager(self) -> ActivityManagerProtocol:
if self._activity_manager is None:
- raise RuntimeError("Activity manager is not set. You must initialize Pipelex first.")
+ msg = "Activity manager is not set. You must initialize Pipelex first."
+ raise RuntimeError(msg)
return self._activity_manager
def get_required_library_manager(self) -> LibraryManagerAbstract:
if self._library_manager is None:
- raise RuntimeError("Library manager is not set. You must initialize Pipelex first.")
+ msg = "Library manager is not set. You must initialize Pipelex first."
+ raise RuntimeError(msg)
return self._library_manager
- def get_optional_library_manager(self) -> Optional[LibraryManagerAbstract]:
+ def get_optional_library_manager(self) -> LibraryManagerAbstract | None:
return self._library_manager
def get_observer_provider(self) -> ObserverProtocol:
if self._observer_provider is None:
- raise RuntimeError("Observer is not set. You must initialize Pipelex first.")
+ msg = "Observer is not set. You must initialize Pipelex first."
+ raise RuntimeError(msg)
return self._observer_provider
@@ -390,11 +404,10 @@ def get_required_domain(domain: str) -> Domain:
return get_pipelex_hub().get_required_domain_provider().get_required_domain(domain=domain)
-def get_optional_domain(domain: str) -> Optional[Domain]:
+def get_optional_domain(domain: str) -> Domain | None:
if domain_provider := get_pipelex_hub().get_optional_domain_provider():
return domain_provider.get_domain(domain=domain)
- else:
- return None
+ return None
def get_pipe_provider() -> PipeProviderAbstract:
@@ -405,7 +418,7 @@ def get_required_pipe(pipe_code: str) -> PipeAbstract:
return get_pipelex_hub().get_required_pipe_provider().get_required_pipe(pipe_code=pipe_code)
-def get_optional_pipe(pipe_code: str) -> Optional[PipeAbstract]:
+def get_optional_pipe(pipe_code: str) -> PipeAbstract | None:
return get_pipelex_hub().get_required_pipe_provider().get_optional_pipe(pipe_code=pipe_code)
@@ -413,7 +426,7 @@ def get_concept_provider() -> ConceptProviderAbstract:
return get_pipelex_hub().get_required_concept_provider()
-def get_optional_concept_provider() -> Optional[ConceptProviderAbstract]:
+def get_optional_concept_provider() -> ConceptProviderAbstract | None:
return get_pipelex_hub().get_optional_concept_provider()
diff --git a/pipelex/libraries/library_config.py b/pipelex/libraries/library_config.py
index 3bc048d19..bfb5650b4 100644
--- a/pipelex/libraries/library_config.py
+++ b/pipelex/libraries/library_config.py
@@ -1,4 +1,4 @@
-from typing import ClassVar, List
+from typing import ClassVar
from pipelex.tools.config.config_model import ConfigModel
from pipelex.tools.misc.file_utils import copy_file_from_package, copy_folder_from_package, find_files_in_dir
@@ -27,10 +27,10 @@ class LibraryConfig(ConfigModel):
return "tests/test_pipelines"
@property
- def failing_pipelines_file_paths(self) -> List[str]:
+ def failing_pipelines_file_paths(self) -> list[str]:
return ["tests/test_pipelines/failing_pipelines.plx"]
- def get_templates_paths(self) -> List[str]:
+ def get_templates_paths(self) -> list[str]:
return [str(path) for path in find_files_in_dir(dir_path=self.templates_dir_path, pattern="*.toml", is_recursive=True)]
def export_libraries(self, overwrite: bool = False) -> None:
diff --git a/pipelex/libraries/library_manager.py b/pipelex/libraries/library_manager.py
index 6a0ad2922..164fb76b5 100644
--- a/pipelex/libraries/library_manager.py
+++ b/pipelex/libraries/library_manager.py
@@ -1,6 +1,6 @@
import os
from pathlib import Path
-from typing import ClassVar, List, Optional, Type
+from typing import ClassVar
from pydantic import ValidationError
from typing_extensions import override
@@ -42,7 +42,7 @@ class LibraryComponent(StrEnum):
PIPE = "pipe"
@property
- def error_class(self) -> Type[LibraryError]:
+ def error_class(self) -> type[LibraryError]:
match self:
case LibraryComponent.CONCEPT:
return ConceptLibraryError
@@ -51,7 +51,7 @@ class LibraryComponent(StrEnum):
class LibraryManager(LibraryManagerAbstract):
- allowed_root_attributes: ClassVar[List[str]] = [
+ allowed_root_attributes: ClassVar[list[str]] = [
"domain",
"definition",
"system_prompt",
@@ -90,8 +90,9 @@ class LibraryManager(LibraryManagerAbstract):
try:
validate_toml_file(template_path)
except TOMLValidationError as exc:
- log.error(f"PLX formatting issues in template file '{template_path}': {exc}")
- raise LibraryError(f"PLX validation failed for template file '{template_path}': {exc}") from exc
+ msg = f"PLX validation failed for template file '{template_path}': {exc}"
+ log.error(msg)
+ raise LibraryError(msg) from exc
@override
def setup(self) -> None:
@@ -108,20 +109,20 @@ class LibraryManager(LibraryManagerAbstract):
self.teardown()
self.setup()
- def _get_pipeline_library_dirs(self) -> List[Path]:
+ def _get_pipeline_library_dirs(self) -> list[Path]:
library_dirs = [Path(self.library_config.pipelines_dir_path)]
if runtime_manager.is_unit_testing:
log.debug("Registering test pipeline structures for unit testing")
library_dirs += [Path(self.library_config.test_pipelines_dir_path)]
return library_dirs
- def _get_pipelex_plx_files_from_dirs(self, dirs: List[Path]) -> List[Path]:
+ def _get_pipelex_plx_files_from_dirs(self, dirs: list[Path]) -> list[Path]:
"""Get all valid Pipelex PLX files from the given directories."""
-
- all_plx_paths: List[Path] = []
+ all_plx_paths: list[Path] = []
for dir_path in dirs:
if not dir_path.exists():
- raise LibraryError(f"Directory does not exist: {dir_path}")
+ msg = f"Directory does not exist: {dir_path}"
+ raise LibraryError(msg)
# Find all TOML files in the directory
plx_files = find_files_in_dir(
@@ -143,13 +144,14 @@ class LibraryManager(LibraryManagerAbstract):
def load_from_file(self, plx_path: Path) -> None:
"""Load a single file - this method is kept for compatibility."""
if not PipelexInterpreter.is_pipelex_file(plx_path):
- raise LibraryError(f"File is not a valid Pipelex PLX file: {plx_path}")
+ msg = f"File is not a valid Pipelex PLX file: {plx_path}"
+ raise LibraryError(msg)
blueprint = PipelexInterpreter(file_path=plx_path).make_pipelex_bundle_blueprint()
self.load_from_blueprint(blueprint)
@override
- def load_from_blueprint(self, blueprint: PipelexBundleBlueprint) -> List[PipeAbstract]:
+ def load_from_blueprint(self, blueprint: PipelexBundleBlueprint) -> list[PipeAbstract]:
"""Load a blueprint."""
# Create and load domain
domain = self._load_domain_from_blueprint(blueprint)
@@ -174,7 +176,7 @@ class LibraryManager(LibraryManagerAbstract):
if blueprint.concept is not None:
concept_codes_to_remove = [
ConceptFactory.construct_concept_string_with_domain(domain=blueprint.domain, concept_code=concept_code)
- for concept_code in blueprint.concept.keys()
+ for concept_code in blueprint.concept
]
self.concept_library.remove_concepts_by_codes(concept_codes=concept_codes_to_remove)
@@ -188,11 +190,11 @@ class LibraryManager(LibraryManagerAbstract):
system_prompt=blueprint.system_prompt,
system_prompt_to_structure=blueprint.system_prompt_to_structure,
prompt_template_to_structure=blueprint.prompt_template_to_structure,
- )
+ ),
)
- def _load_concepts_from_blueprint(self, blueprint: PipelexBundleBlueprint) -> List[Concept]:
- concepts: List[Concept] = []
+ def _load_concepts_from_blueprint(self, blueprint: PipelexBundleBlueprint) -> list[Concept]:
+ concepts: list[Concept] = []
if blueprint.concept is not None:
for concept_code, concept_blueprint_or_str in blueprint.concept.items():
@@ -207,8 +209,8 @@ class LibraryManager(LibraryManagerAbstract):
concepts.append(concept)
return concepts
- def _load_pipes_from_blueprint(self, blueprint: PipelexBundleBlueprint) -> List[PipeAbstract]:
- pipes: List[PipeAbstract] = []
+ def _load_pipes_from_blueprint(self, blueprint: PipelexBundleBlueprint) -> list[PipeAbstract]:
+ pipes: list[PipeAbstract] = []
if blueprint.pipe is not None:
for pipe_name, pipe_blueprint in blueprint.pipe.items():
pipe = PipeFactory.make_from_blueprint(
@@ -223,11 +225,11 @@ class LibraryManager(LibraryManagerAbstract):
@override
def load_libraries(
self,
- library_dirs: Optional[List[Path]] = None,
- library_file_paths: Optional[List[Path]] = None,
+ library_dirs: list[Path] | None = None,
+ library_file_paths: list[Path] | None = None,
) -> None:
- dirs_to_use: List[Path] = self._get_pipeline_library_dirs()
- all_plx_paths: List[Path] = self._get_pipelex_plx_files_from_dirs(dirs_to_use)
+ dirs_to_use: list[Path] = self._get_pipeline_library_dirs()
+ all_plx_paths: list[Path] = self._get_pipelex_plx_files_from_dirs(dirs_to_use)
# Remove failing pipelines from the list
failing_pipelines_file_paths = get_config().pipelex.library_config.failing_pipelines_file_paths
@@ -246,19 +248,21 @@ class LibraryManager(LibraryManagerAbstract):
all_plx_paths = library_file_paths
# Parse all blueprints first
- blueprints: List[PipelexBundleBlueprint] = []
+ blueprints: list[PipelexBundleBlueprint] = []
for plx_file_path in all_plx_paths:
try:
blueprint = PipelexInterpreter(file_path=plx_file_path).make_pipelex_bundle_blueprint()
except FileNotFoundError as exc:
- raise PipelexSetupError(f"Could not find PLX blueprint at '{plx_file_path}'") from exc
+ msg = f"Could not find PLX blueprint at '{plx_file_path}'"
+ raise PipelexSetupError(msg) from exc
except ValidationError as exc:
- error_msg = format_pydantic_validation_error(exc)
- raise PipelexSetupError(f"Could not load PLX blueprint from '{plx_file_path}' because of: {error_msg}") from exc
+ formatted_error_msg = format_pydantic_validation_error(exc)
+ msg = f"Could not load PLX blueprint from '{plx_file_path}' because of: {formatted_error_msg}"
+ raise PipelexSetupError(msg) from exc
blueprints.append(blueprint)
# Load all domains first
- all_domains: List[Domain] = []
+ all_domains: list[Domain] = []
for blueprint in blueprints:
domain = self._load_domain_from_blueprint(blueprint)
all_domains.append(domain)
@@ -266,14 +270,14 @@ class LibraryManager(LibraryManagerAbstract):
self.domain_library.add_domain(domain=domain)
# Load all concepts second
- all_concepts: List[Concept] = []
+ all_concepts: list[Concept] = []
for blueprint in blueprints:
concepts = self._load_concepts_from_blueprint(blueprint)
all_concepts.extend(concepts)
self.concept_library.add_concepts(concepts=all_concepts)
# Load all pipes third
- all_pipes: List[PipeAbstract] = []
+ all_pipes: list[PipeAbstract] = []
for blueprint in blueprints:
pipes = self._load_pipes_from_blueprint(blueprint)
all_pipes.extend(pipes)
diff --git a/pipelex/libraries/library_manager_abstract.py b/pipelex/libraries/library_manager_abstract.py
index 37e77c21b..3f93d87ed 100644
--- a/pipelex/libraries/library_manager_abstract.py
+++ b/pipelex/libraries/library_manager_abstract.py
@@ -1,6 +1,5 @@
from abc import ABC, abstractmethod
from pathlib import Path
-from typing import List, Optional
from pipelex.core.bundles.pipelex_bundle_blueprint import PipelexBundleBlueprint
from pipelex.core.pipes.pipe_abstract import PipeAbstract
@@ -24,7 +23,7 @@ class LibraryManagerAbstract(ABC):
pass
@abstractmethod
- def load_libraries(self, library_dirs: Optional[List[Path]] = None, library_file_paths: Optional[List[Path]] = None) -> None:
+ def load_libraries(self, library_dirs: list[Path] | None = None, library_file_paths: list[Path] | None = None) -> None:
pass
@abstractmethod
@@ -32,7 +31,7 @@ class LibraryManagerAbstract(ABC):
pass
@abstractmethod
- def load_from_blueprint(self, blueprint: PipelexBundleBlueprint) -> List[PipeAbstract]:
+ def load_from_blueprint(self, blueprint: PipelexBundleBlueprint) -> list[PipeAbstract]:
pass
@abstractmethod
diff --git a/pipelex/libraries/library_manager_factory.py b/pipelex/libraries/library_manager_factory.py
index 686b31f29..c63ff4412 100644
--- a/pipelex/libraries/library_manager_factory.py
+++ b/pipelex/libraries/library_manager_factory.py
@@ -24,10 +24,13 @@ class LibraryManagerFactory:
@classmethod
def make(
- cls, domain_library: DomainLibrary, concept_library: ConceptLibrary, pipe_library: PipeLibrary, config_dir_path: str
+ cls,
+ domain_library: DomainLibrary,
+ concept_library: ConceptLibrary,
+ pipe_library: PipeLibrary,
+ config_dir_path: str,
) -> "LibraryManager":
"""Create a LibraryManager with provided libraries."""
-
library_config = LibraryConfig(config_dir_path=config_dir_path)
return LibraryManager(
diff --git a/pipelex/libraries/pipelines/builder/__init__.py b/pipelex/libraries/pipelines/builder/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/pipelex/libraries/pipelines/builder/builder.py b/pipelex/libraries/pipelines/builder/builder.py
index 89b7bc6da..e3ad8d3fe 100644
--- a/pipelex/libraries/pipelines/builder/builder.py
+++ b/pipelex/libraries/pipelines/builder/builder.py
@@ -1,4 +1,4 @@
-from typing import Annotated, Dict, List, Optional, Union, cast
+from typing import Annotated, cast
from pydantic import ConfigDict, Field, ValidationError, field_validator
@@ -20,7 +20,7 @@ from pipelex.libraries.pipelines.builder.pipe.pipe_parallel_spec import PipePara
from pipelex.libraries.pipelines.builder.pipe.pipe_sequence_spec import PipeSequenceSpec
from pipelex.libraries.pipelines.builder.pipe.pipe_signature import PipeSignature
from pipelex.pipe_works.pipe_dry import dry_run_pipes
-from pipelex.tools.typing.pydantic_utils import format_pydantic_validation_error
+from pipelex.tools.typing.pydantic_utils import empty_list_factory_of, format_pydantic_validation_error
from pipelex.types import StrEnum
@@ -39,25 +39,21 @@ class PipelexBundleSpecDraft(StructuredContent):
domain: str = Field(description="The domain of the pipeline library.")
definition: str = Field(description="The definition of the pipeline library.")
- concept: Dict[str, ConceptSpec] = Field(default_factory=dict, description="The concepts of the pipeline library.")
+ concept: dict[str, ConceptSpec] = Field(default_factory=dict, description="The concepts of the pipeline library.")
- pipe: Dict[str, PipeSignature] = Field(default_factory=dict, description="The pipes of the pipeline library.")
+ pipe: dict[str, PipeSignature] = Field(default_factory=dict, description="The pipes of the pipeline library.")
PipeSpecUnion = Annotated[
- Union[
- # Pipe operators
- PipeFuncSpec,
- PipeImgGenSpec,
- PipeJinja2Spec,
- PipeLLMSpec,
- PipeOcrSpec,
- # Pipe controllers
- PipeBatchSpec,
- PipeConditionSpec,
- PipeParallelSpec,
- PipeSequenceSpec,
- ],
+ PipeFuncSpec
+ | PipeImgGenSpec
+ | PipeJinja2Spec
+ | PipeLLMSpec
+ | PipeOcrSpec
+ | PipeBatchSpec
+ | PipeConditionSpec
+ | PipeParallelSpec
+ | PipeSequenceSpec,
Field(discriminator="type"),
]
@@ -92,19 +88,20 @@ class PipelexBundleSpec(StructuredContent):
3. Pipe keys must be in snake_case format.
4. Extra fields are forbidden (strict mode).
5. Pipe types must match their blueprint discriminator.
+
"""
model_config = ConfigDict(extra="forbid")
domain: str
- definition: Optional[str] = None
- system_prompt: Optional[str] = None
- system_prompt_to_structure: Optional[str] = None
- prompt_template_to_structure: Optional[str] = None
+ definition: str | None = None
+ system_prompt: str | None = None
+ system_prompt_to_structure: str | None = None
+ prompt_template_to_structure: str | None = None
- concept: Optional[Dict[str, Union[ConceptSpec, str]]] = Field(default_factory=dict)
+ concept: dict[str, ConceptSpec | str] | None = Field(default_factory=dict)
- pipe: Optional[Dict[str, PipeSpecUnion]] = Field(default_factory=dict)
+ pipe: dict[str, PipeSpecUnion] | None = Field(default_factory=dict)
@field_validator("domain", mode="before")
@classmethod
@@ -113,7 +110,7 @@ class PipelexBundleSpec(StructuredContent):
return domain
def to_blueprint(self) -> PipelexBundleBlueprint:
- concept: Optional[Dict[str, Union[ConceptBlueprint, str]]] = None
+ concept: dict[str, ConceptBlueprint | str] | None = None
if self.concept:
concept = {}
@@ -123,7 +120,7 @@ class PipelexBundleSpec(StructuredContent):
else:
concept[concept_code] = ConceptBlueprint(definition=concept_code, structure=concept_blueprint)
- pipe: Optional[Dict[str, PipeBlueprintUnion]] = None
+ pipe: dict[str, PipeBlueprintUnion] | None = None
if self.pipe:
pipe = {}
for pipe_code, pipe_blueprint in self.pipe.items():
@@ -142,7 +139,7 @@ class PipelexBundleSpec(StructuredContent):
# TODO: Put this in a factory. Investigate why it is necessary.
def _convert_pipe_spec(pipe_spec: PipeSpecUnion) -> PipeSpecUnion:
- pipe_type_to_class: Dict[str, type] = {
+ pipe_type_to_class: dict[str, type] = {
"PipeFunc": PipeFuncSpec,
"PipeImgGen": PipeImgGenSpec,
"PipeJinja2": PipeJinja2Spec,
@@ -158,7 +155,7 @@ def _convert_pipe_spec(pipe_spec: PipeSpecUnion) -> PipeSpecUnion:
if pipe_class is None:
msg = f"Unknown pipe type: {pipe_spec.type}"
raise PipeBuilderError(msg)
- return cast(PipeSpecUnion, pipe_class(**pipe_spec.model_dump(serialize_as_any=True)))
+ return cast("PipeSpecUnion", pipe_class(**pipe_spec.model_dump(serialize_as_any=True)))
async def compile_in_pipelex_bundle_spec(working_memory: WorkingMemory) -> PipelexBundleSpec:
@@ -169,6 +166,7 @@ async def compile_in_pipelex_bundle_spec(working_memory: WorkingMemory) -> Pipel
Returns:
PipelexBundleSpec: The constructed pipeline spec.
+
"""
# The working memory actually contains ConceptSpec objects (not ConceptSpecDraft)
# but they may have been deserialized incorrectly
@@ -177,11 +175,11 @@ async def compile_in_pipelex_bundle_spec(working_memory: WorkingMemory) -> Pipel
item_type=ConceptSpec,
)
- pipe_specs = cast(ListContent[PipeSpecUnion], working_memory.get_stuff(name="pipe_specs").content)
+ pipe_specs = cast("ListContent[PipeSpecUnion]", working_memory.get_stuff(name="pipe_specs").content)
domain_information = working_memory.get_stuff_as(name="domain_information", content_type=DomainInformation)
# Properly validate and reconstruct concept specs to ensure proper Pydantic validation
- validated_concepts: Dict[str, Union[ConceptSpec, str]] = {}
+ validated_concepts: dict[str, ConceptSpec | str] = {}
for concept_spec in concept_specs.items:
try:
# Re-create the ConceptSpec to ensure proper Pydantic validation
@@ -200,6 +198,7 @@ async def compile_in_pipelex_bundle_spec(working_memory: WorkingMemory) -> Pipel
)
+# TODO: rename or merge with PipeDry.DryRunStatus
class DryRunStatus(StrEnum):
SUCCESS = "SUCCESS"
FAILURE = "FAILURE"
@@ -208,8 +207,6 @@ class DryRunStatus(StrEnum):
class ValidateDryRunError(Exception):
"""Raised when validating the dry run of a pipelex bundle blueprint."""
- pass
-
class PipeFailure(StructuredContent):
"""Details of a single pipe failure during dry run."""
@@ -222,7 +219,10 @@ class DryRunResult(StructuredContent):
"""A result of a dry run of a pipelex bundle blueprint."""
status: DryRunStatus
- failed_pipes: List[PipeFailure] = Field(default_factory=list, description="List of pipes that failed during dry run")
+ failed_pipes: list[PipeFailure] = Field(
+ default_factory=empty_list_factory_of(PipeFailure),
+ description="List of pipes that failed during dry run",
+ )
async def validate_dry_run(working_memory: WorkingMemory) -> ListContent[PipeFailure]:
@@ -247,29 +247,28 @@ async def validate_dry_run(working_memory: WorkingMemory) -> ListContent[PipeFai
"PipeSequence": PipeSequenceSpec,
}
- failed_pipes: List[PipeFailure] = []
+ failed_pipes: list[PipeFailure] = []
for pipe_code, dry_run_output in dry_run_result.items():
- if dry_run_output.status == DryRunStatus.FAILURE:
- if pipelex_bundle_spec.pipe and pipe_code in pipelex_bundle_spec.pipe:
- pipe_spec = pipelex_bundle_spec.pipe[pipe_code]
- spec_class = pipe_type_to_spec_class.get(pipe_spec.type)
- if not spec_class:
- msg = f"Unknown pipe type: {pipe_spec.type}"
- raise ValidateDryRunError(msg)
- pipe_spec = spec_class(**pipe_spec.model_dump(serialize_as_any=True))
- failed_pipes.append(
- PipeFailure(
- pipe=pipe_spec,
- error_message=dry_run_output.error_message or "",
- )
- )
+ if dry_run_output.status.is_failure and pipelex_bundle_spec.pipe and pipe_code in pipelex_bundle_spec.pipe:
+ pipe_spec = pipelex_bundle_spec.pipe[pipe_code]
+ spec_class = pipe_type_to_spec_class.get(pipe_spec.type)
+ if not spec_class:
+ msg = f"Unknown pipe type: {pipe_spec.type}"
+ raise ValidateDryRunError(msg)
+ pipe_spec = spec_class(**pipe_spec.model_dump(serialize_as_any=True))
+ failed_pipes.append(
+ PipeFailure(
+ pipe=pipe_spec,
+ error_message=dry_run_output.error_message or "",
+ ),
+ )
return ListContent[PipeFailure](items=failed_pipes)
async def reconstruct_bundle_with_all_fixes(working_memory: WorkingMemory) -> PipelexBundleSpec:
pipelex_bundle_spec = working_memory.get_stuff_as(name="pipelex_bundle_spec", content_type=PipelexBundleSpec)
- fixed_pipes_list = cast(ListContent[PipeSpecUnion], working_memory.get_stuff(name="fixed_pipes").content)
+ fixed_pipes_list = cast("ListContent[PipeSpecUnion]", working_memory.get_stuff(name="fixed_pipes").content)
if not pipelex_bundle_spec.pipe:
msg = "No pipes section found in bundle spec"
diff --git a/pipelex/libraries/pipelines/builder/concept/__init__.py b/pipelex/libraries/pipelines/builder/concept/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/pipelex/libraries/pipelines/builder/concept/concept_spec.py b/pipelex/libraries/pipelines/builder/concept/concept_spec.py
index 78e22f008..7e0751fc9 100644
--- a/pipelex/libraries/pipelines/builder/concept/concept_spec.py
+++ b/pipelex/libraries/pipelines/builder/concept/concept_spec.py
@@ -1,8 +1,7 @@
from datetime import datetime
-from typing import Any, Dict, Optional, Union, cast
+from typing import Any, cast
from pydantic import ConfigDict, Field, field_validator, model_validator
-from typing_extensions import Self
from pipelex.core.concepts.concept_blueprint import (
ConceptBlueprint,
@@ -18,7 +17,7 @@ from pipelex.core.domains.domain_blueprint import DomainBlueprint
from pipelex.core.memory.working_memory import WorkingMemory
from pipelex.core.stuffs.stuff_content import StructuredContent
from pipelex.tools.misc.string_utils import is_pascal_case
-from pipelex.types import StrEnum
+from pipelex.types import Self, StrEnum
class ConceptStructureSpecFieldType(StrEnum):
@@ -54,13 +53,14 @@ class ConceptStructureSpec(StructuredContent):
- For typed fields: type must be specified and default_value must match that type
- Type validation includes: text (str), integer (int), boolean (bool),
number (int/float), dict (dict)
+
"""
the_field_name: str = Field(description="Field name. Must be snake_case.")
definition: str
- type: Optional[ConceptStructureSpecFieldType] = Field(default=None, description="The type of the field.")
- required: Optional[bool] = True
- default_value: Optional[Any] = None
+ type: ConceptStructureSpecFieldType | None = Field(default=None, description="The type of the field.")
+ required: bool | None = True
+ default_value: Any | None = None
@model_validator(mode="after")
def validate_structure_blueprint(self) -> Self:
@@ -95,13 +95,10 @@ class ConceptStructureSpec(StructuredContent):
self._raise_type_mismatch_error("date", type(self.default_value).__name__)
def _raise_type_mismatch_error(self, expected_type_name: str, actual_type_name: str) -> None:
- """Raise a type mismatch error with consistent formatting."""
- raise ConceptStructureBlueprintError(
- f"default_value type mismatch: expected {expected_type_name} for type '{self.type}', but got {actual_type_name}"
- )
+ msg = f"default_value type mismatch: expected {expected_type_name} for type '{self.type}', but got {actual_type_name}"
+ raise ConceptStructureBlueprintError(msg)
def to_blueprint(self) -> ConceptStructureBlueprint:
- """Convert this ConceptStructureBlueprint to the core ConceptStructureBlueprint."""
# Convert the type enum value - self.type is already a ConceptStructureBlueprintFieldType enum
# We need to get the corresponding value in the core enum
core_type = None
@@ -121,9 +118,9 @@ class ConceptSpecDraft(StructuredContent):
the_concept_code: str = Field(description="Concept code. Must be PascalCase.")
definition: str = Field(description="Description of the concept, in natural language.")
structure: str = Field(
- description="A description of a dict with fieldnames as keys, and values being a dict with: definition, type, required, default_value"
+ description="A description of a dict with fieldnames as keys, and values being a dict with: definition, type, required, default_value",
)
- refines: Optional[str] = Field(
+ refines: str | None = Field(
default=None,
description="The native concept this concept extends (Text, Image, PDF, TextAndImages, Number, Page) "
"in PascalCase format. Cannot be used together with 'structure'.",
@@ -158,37 +155,38 @@ class ConceptSpec(StructuredContent):
5. Native concepts: When refining, must be one of the valid native concepts.
6. Structure values: In structure dict, values must be either valid concept strings
or ConceptStructureBlueprint instances.
+
"""
model_config = ConfigDict(extra="forbid")
the_concept_code: str = Field(description="Concept code. Must be PascalCase.")
definition: str
- structure: Optional[Union[str, Dict[str, Union[str, ConceptStructureSpec]]]] = None
- refines: Optional[str] = None
+ structure: str | dict[str, str | ConceptStructureSpec] | None = None
+ refines: str | None = None
@classmethod
def is_native_concept_code(cls, concept_code: str) -> bool:
ConceptSpec.validate_concept_code(concept_code=concept_code)
- return concept_code in [native_concept for native_concept in [native_concept for native_concept in NativeConceptEnum]]
+ return concept_code in NativeConceptEnum.values_list()
@classmethod
def validate_concept_code(cls, concept_code: str) -> None:
if not is_pascal_case(concept_code):
- raise ConceptCodeError(
- f"Concept code '{concept_code}' must be PascalCase (letters and numbers only, starting with uppercase, without `.`)"
- )
+ msg = f"Concept code '{concept_code}' must be PascalCase (letters and numbers only, starting with uppercase, without `.`)"
+ raise ConceptCodeError(msg)
@classmethod
def validate_concept_string_or_code(cls, concept_string_or_code: str) -> None:
if concept_string_or_code.count(".") > 1:
- raise ConceptStringOrConceptCodeError(
+ msg = (
f"concept_string_or_code '{concept_string_or_code}' is invalid. "
"It should either contain a domain in snake_case and a concept code in PascalCase separated by one dot, "
"or be a concept code in PascalCase."
)
+ raise ConceptStringOrConceptCodeError(msg)
- elif concept_string_or_code.count(".") == 1:
+ if concept_string_or_code.count(".") == 1:
domain, concept_code = concept_string_or_code.split(".")
DomainBlueprint.validate_domain_code(code=domain)
cls.validate_concept_code(concept_code=concept_code)
@@ -198,70 +196,68 @@ class ConceptSpec(StructuredContent):
@staticmethod
def validate_concept_string(concept_string: str) -> None:
"""Validate that a concept code follows PascalCase convention."""
- if "." not in concept_string:
- raise ConceptStringError(
- f"Concept string '{concept_string}' is invalid. It should contain a domain in snake_case "
- "and a concept code in PascalCase separated by one dot."
- )
- elif concept_string.count(".") > 1:
- raise ConceptStringError(
+ if "." not in concept_string or concept_string.count(".") > 1:
+ msg = (
f"Concept string '{concept_string}' is invalid. It should contain a domain in snake_case "
"and a concept code in PascalCase separated by one dot."
)
- else:
- domain, concept_code = concept_string.split(".", 1)
+ raise ConceptStringError(msg)
+ domain, concept_code = concept_string.split(".", 1)
DomainBlueprint.validate_domain_code(domain)
if not is_pascal_case(concept_code):
- raise ConceptCodeError(
- f"Concept code '{concept_code}' must be PascalCase (letters and numbers only, starting with uppercase, without `.`)"
- )
+ msg = f"Concept code '{concept_code}' must be PascalCase (letters and numbers only, starting with uppercase, without `.`)"
+ raise ConceptCodeError(msg)
# Validate that if the concept code is among the native concepts, the domain MUST be native.
- if concept_code in [concept.value for concept in [native_concept for native_concept in NativeConceptEnum]]:
- if domain != SpecialDomain.NATIVE:
- raise ConceptStringError(
- f"Concept string '{concept_string}' is invalid. "
- f"Concept code '{concept_code}' is a native concept, so the domain must be '{SpecialDomain.NATIVE}', "
- f"or nothing, but not '{domain}'"
- )
+ if not SpecialDomain.is_native(domain=domain) and concept_code in NativeConceptEnum.values_list():
+ msg = (
+ f"Concept string '{concept_string}' is invalid. "
+ f"Concept code '{concept_code}' is a native concept, so the domain must be '{SpecialDomain.NATIVE}', "
+ f"or nothing, but not '{domain}'"
+ )
+ raise ConceptStringError(msg)
# Validate that if the domain is native, the concept code is a native concept
if SpecialDomain.is_native(domain=domain):
- if concept_code not in [native_concept for native_concept in NativeConceptEnum]:
- raise ConceptStringError(
+ if concept_code not in NativeConceptEnum.values_list():
+ msg = (
f"Concept string '{concept_string}' is invalid. "
f"Concept code '{concept_code}' is not a native concept, so the domain must not be '{SpecialDomain.NATIVE}'."
)
+ raise ConceptStringError(msg)
@field_validator("refines", mode="before")
@classmethod
- def validate_refines(cls, refines: Optional[str] = None) -> Optional[str]:
+ def validate_refines(cls, refines: str | None = None) -> str | None:
if refines is not None:
if not NativeConceptManager.is_native_concept(refines):
- raise ConceptBlueprintError(f"Forbidden to refine a non-native concept: '{refines}'. Refining non-native concepts will come soon.")
+ msg = f"Forbidden to refine a non-native concept: '{refines}'. Refining non-native concepts will come soon."
+ raise ConceptBlueprintError(msg)
cls.validate_concept_string_or_code(concept_string_or_code=refines)
return refines
@model_validator(mode="before")
- def model_validate_spec(cls, values: Dict[str, Any]) -> Dict[str, Any]:
+ @classmethod
+ def model_validate_spec(cls, values: dict[str, Any]) -> dict[str, Any]:
if values.get("refines") and values.get("structure"):
- raise ConceptSpecError(
+ msg = (
f"Forbidden to have refines and structure at the same time: `{values.get('refines')}` "
- f"and `{values.get('structure')}` for concept that has the definition `{values.get('definition')}`"
+ f"and `{values.get('structure')}` for concept that has the definition `{values.get('definition')}`",
)
+ raise ConceptSpecError(msg)
return values
def to_blueprint(self) -> ConceptBlueprint:
"""Convert this ConceptBlueprint to the original core ConceptBlueprint."""
# TODO: Clarify concept structure blueprint
- converted_structure: Optional[Union[str, Dict[str, Union[str, ConceptStructureBlueprint]]]] = None
+ converted_structure: str | dict[str, str | ConceptStructureBlueprint] | None = None
if self.structure:
converted_structure = {}
if isinstance(self.structure, str):
converted_structure = self.structure
else:
- for field_name, field_spec in cast(Dict[str, ConceptStructureSpec], self.structure).items():
+ for field_name, field_spec in cast("dict[str, ConceptStructureSpec]", self.structure).items():
converted_structure[field_name] = field_spec.to_blueprint()
return ConceptBlueprint(definition=self.definition, structure=converted_structure, refines=self.refines)
@@ -271,7 +267,7 @@ async def create_concept_spec(working_memory: WorkingMemory) -> ConceptSpec:
concept_spec_draft = working_memory.get_stuff_as(name="concept_spec_draft", content_type=ConceptSpecDraft)
concept_spec_structures_stuff = working_memory.get_stuff_as_list(name="concept_spec_structures", item_type=ConceptStructureSpec)
- structure_dict: Dict[str, Union[str, ConceptStructureSpec]] = {}
+ structure_dict: dict[str, str | ConceptStructureSpec] = {}
for structure_item in concept_spec_structures_stuff.items:
structure_spec = ConceptStructureSpec(
the_field_name=structure_item.the_field_name,
diff --git a/pipelex/libraries/pipelines/builder/pipe/__init__.py b/pipelex/libraries/pipelines/builder/pipe/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/pipelex/libraries/pipelines/builder/pipe/inputs_spec.py b/pipelex/libraries/pipelines/builder/pipe/inputs_spec.py
index 43c6f0b64..159ad2d6b 100644
--- a/pipelex/libraries/pipelines/builder/pipe/inputs_spec.py
+++ b/pipelex/libraries/pipelines/builder/pipe/inputs_spec.py
@@ -1,5 +1,3 @@
-from typing import Optional
-
from pydantic import Field, field_validator
from pipelex.core.concepts.concept_blueprint import ConceptBlueprint
@@ -25,10 +23,11 @@ class InputRequirementSpec(StructuredContent):
1. Concept must be a valid concept code (PascalCase) or concept string (domain.ConceptCode).
2. Domain and concept code are separated by a dot when using full concept strings.
3. Concept validation is performed using ConceptBlueprint.validate_concept_string_or_code.
+
"""
concept: str = Field(description="Concept code or concept string in PascalCase format")
- multiplicity: Optional[PipeOutputMultiplicity] = None
+ multiplicity: PipeOutputMultiplicity | None = None
@field_validator("concept", mode="before")
@classmethod
diff --git a/pipelex/libraries/pipelines/builder/pipe/pipe_batch_spec.py b/pipelex/libraries/pipelines/builder/pipe/pipe_batch_spec.py
index 558ba5aec..46741432b 100644
--- a/pipelex/libraries/pipelines/builder/pipe/pipe_batch_spec.py
+++ b/pipelex/libraries/pipelines/builder/pipe/pipe_batch_spec.py
@@ -1,4 +1,4 @@
-from typing import Literal, Optional
+from typing import Literal
from pydantic import Field
from typing_extensions import override
@@ -32,14 +32,15 @@ class PipeBatchSpec(PipeSpec):
1. branch_pipe_code must reference an existing pipe in the pipeline.
2. When input_list_name is specified, it must reference a list in context.
3. The branch pipe should be designed to process single items.
+
"""
type: Literal["PipeBatch"] = "PipeBatch"
category: Literal["PipeController"] = "PipeController"
the_pipe_code: str = Field(description="Pipe code. Must be snake_case.")
branch_pipe_code: str
- input_list_name: Optional[str] = None
- input_item_name: Optional[str] = None
+ input_list_name: str | None = None
+ input_item_name: str | None = None
@override
def to_blueprint(self) -> PipeBatchBlueprint:
diff --git a/pipelex/libraries/pipelines/builder/pipe/pipe_condition_spec.py b/pipelex/libraries/pipelines/builder/pipe/pipe_condition_spec.py
index 0127a4719..65d745fba 100644
--- a/pipelex/libraries/pipelines/builder/pipe/pipe_condition_spec.py
+++ b/pipelex/libraries/pipelines/builder/pipe/pipe_condition_spec.py
@@ -1,4 +1,4 @@
-from typing import Dict, Literal, Optional
+from typing import Literal
from pydantic import Field, RootModel
from typing_extensions import override
@@ -6,7 +6,7 @@ from typing_extensions import override
from pipelex.libraries.pipelines.builder.pipe.pipe_signature import PipeSpec
from pipelex.pipe_controllers.condition.pipe_condition_blueprint import PipeConditionBlueprint, PipeConditionPipeMapBlueprint
-PipeConditionPipeMapRoot = Dict[str, str]
+PipeConditionPipeMapRoot = dict[str, str]
class PipeConditionPipeMapSpec(RootModel[PipeConditionPipeMapRoot]):
@@ -18,6 +18,7 @@ class PipeConditionPipeMapSpec(RootModel[PipeConditionPipeMapRoot]):
root: Dictionary mapping condition results (keys) to pipe codes (values).
Each key represents a possible condition outcome, and its value
is the pipe code to execute when that condition is met.
+
"""
root: PipeConditionPipeMapRoot = Field(default_factory=dict)
@@ -48,16 +49,17 @@ class PipeConditionSpec(PipeSpec):
1. Either expression or expression_template should be provided, not both.
2. pipe_map keys must be strings representing possible condition outcomes.
3. All pipe codes in pipe_map and default_pipe_code must be valid pipe references.
+
"""
type: Literal["PipeCondition"] = "PipeCondition"
category: Literal["PipeController"] = "PipeController"
the_pipe_code: str = Field(description="Pipe code. Must be snake_case.")
- expression_template: Optional[str] = None
- expression: Optional[str] = None
+ expression_template: str | None = None
+ expression: str | None = None
pipe_map: PipeConditionPipeMapSpec = Field(default_factory=PipeConditionPipeMapSpec)
- default_pipe_code: Optional[str] = None
- add_alias_from_expression_to: Optional[str] = None
+ default_pipe_code: str | None = None
+ add_alias_from_expression_to: str | None = None
@override
def to_blueprint(self) -> PipeConditionBlueprint:
diff --git a/pipelex/libraries/pipelines/builder/pipe/pipe_func_spec.py b/pipelex/libraries/pipelines/builder/pipe/pipe_func_spec.py
index 48773f9d4..08e39c20e 100644
--- a/pipelex/libraries/pipelines/builder/pipe/pipe_func_spec.py
+++ b/pipelex/libraries/pipelines/builder/pipe/pipe_func_spec.py
@@ -8,14 +8,14 @@ from pipelex.pipe_operators.func.pipe_func_blueprint import PipeFuncBlueprint
class PipeFuncSpec(PipeSpec):
- """
- PipeFunc enables calling custom functions in the Pipelex framework.
+ """PipeFunc enables calling custom functions in the Pipelex framework.
Attributes:
the_pipe_code: Pipe code. Must be snake_case.
type: Fixed to "PipeFunc" for this pipe type.
category: Fixed to "PipeOperator" for this pipe category.
function_name: The name of the function to call.
+
"""
type: Literal["PipeFunc"] = "PipeFunc"
diff --git a/pipelex/libraries/pipelines/builder/pipe/pipe_img_spec.py b/pipelex/libraries/pipelines/builder/pipe/pipe_img_spec.py
index 331b3dbae..a0c4f83b3 100644
--- a/pipelex/libraries/pipelines/builder/pipe/pipe_img_spec.py
+++ b/pipelex/libraries/pipelines/builder/pipe/pipe_img_spec.py
@@ -1,4 +1,4 @@
-from typing import Literal, Optional, Union
+from typing import Literal
from pydantic import Field
from typing_extensions import override
@@ -44,24 +44,25 @@ class PipeImgGenSpec(PipeSpec):
Raises:
PipeDefinitionError: When validation rules are violated or mutually exclusive
fields are set simultaneously.
+
"""
type: Literal["PipeImgGen"] = "PipeImgGen"
category: Literal["PipeOperator"] = "PipeOperator"
the_pipe_code: str = Field(description="Pipe code. Must be snake_case.")
- img_gen_prompt: Optional[str] = None
- img_gen_prompt_var_name: Optional[str] = None
+ img_gen_prompt: str | None = None
+ img_gen_prompt_var_name: str | None = None
# New ImgGenChoice pattern (like LLM)
- img_gen: Optional[ImgGenChoice] = None
+ img_gen: ImgGenChoice | None = None
# One-time settings (not in ImgGenSetting)
- aspect_ratio: Optional[AspectRatio] = Field(default=None, strict=False)
- is_raw: Optional[bool] = None
- seed: Optional[Union[int, Literal["auto"]]] = None
- nb_output: Optional[int] = Field(default=None, ge=1)
- background: Optional[Background] = Field(default=None, strict=False)
- output_format: Optional[OutputFormat] = Field(default=None, strict=False)
+ aspect_ratio: AspectRatio | None = Field(default=None, strict=False)
+ is_raw: bool | None = None
+ seed: int | Literal["auto"] | None = None
+ nb_output: int | None = Field(default=None, ge=1)
+ background: Background | None = Field(default=None, strict=False)
+ output_format: OutputFormat | None = Field(default=None, strict=False)
@override
def to_blueprint(self) -> PipeImgGenBlueprint:
diff --git a/pipelex/libraries/pipelines/builder/pipe/pipe_jinja2_spec.py b/pipelex/libraries/pipelines/builder/pipe/pipe_jinja2_spec.py
index 4b6fc56dd..83a9d1cc8 100644
--- a/pipelex/libraries/pipelines/builder/pipe/pipe_jinja2_spec.py
+++ b/pipelex/libraries/pipelines/builder/pipe/pipe_jinja2_spec.py
@@ -1,4 +1,4 @@
-from typing import Any, Dict, Literal, Optional
+from typing import Any, Literal
from pydantic import Field
from typing_extensions import override
@@ -16,14 +16,14 @@ class PromptingStyleSpec(StructuredContent):
class Jinja2Spec(StructuredContent):
- jinja2_name: Optional[str] = Field(default=None, description="Name of the Jinja2 template to use")
- jinja2: Optional[str] = Field(default=None, description="Raw Jinja2 template string")
- prompting_style: Optional[PromptingStyleSpec] = Field(default=None, description="Style of prompting to use (typically for different LLMs)")
+ jinja2_name: str | None = Field(default=None, description="Name of the Jinja2 template to use")
+ jinja2: str | None = Field(default=None, description="Raw Jinja2 template string")
+ prompting_style: PromptingStyleSpec | None = Field(default=None, description="Style of prompting to use (typically for different LLMs)")
template_category: Jinja2TemplateCategory = Field(
default=Jinja2TemplateCategory.LLM_PROMPT,
description="Category of the template (could also be HTML, MARKDOWN, MERMAID, etc.), influences Jinja2 rendering environment config",
)
- extra_context: Optional[Dict[str, Any]] = Field(default=None, description="Additional context variables for template rendering")
+ extra_context: dict[str, Any] | None = Field(default=None, description="Additional context variables for template rendering")
class PipeJinja2Spec(PipeSpec, Jinja2Spec):
diff --git a/pipelex/libraries/pipelines/builder/pipe/pipe_llm_spec.py b/pipelex/libraries/pipelines/builder/pipe/pipe_llm_spec.py
index b3d8d8ce9..c193f884f 100644
--- a/pipelex/libraries/pipelines/builder/pipe/pipe_llm_spec.py
+++ b/pipelex/libraries/pipelines/builder/pipe/pipe_llm_spec.py
@@ -1,7 +1,7 @@
-from typing import Literal, Optional, Union
+from typing import Literal, Union
from pydantic import Field, field_validator, model_validator
-from typing_extensions import Self, override
+from typing_extensions import override
from pipelex.cogt.llm.llm_setting import LLMChoice, LLMSetting
from pipelex.core.stuffs.stuff_content import StructuredContent
@@ -9,21 +9,20 @@ from pipelex.exceptions import PipeDefinitionError
from pipelex.libraries.pipelines.builder.pipe.pipe_signature import PipeSpec
from pipelex.pipe_operators.llm.pipe_llm_blueprint import PipeLLMBlueprint, StructuringMethod
from pipelex.tools.typing.validation_utils import has_more_than_one_among_attributes_from_lists
+from pipelex.types import Self
class LLMSettingSpec(StructuredContent):
llm_handle: str
temperature: float = Field(..., ge=0, le=1)
- max_tokens: Optional[int] = None
+ max_tokens: int | None = None
@field_validator("max_tokens", mode="before")
@classmethod
- def validate_max_tokens(cls, value: Union[int, Literal["auto"], None]) -> Optional[int]:
- if value is None:
+ def validate_max_tokens(cls, value: int | Literal["auto"] | None) -> int | None:
+ if value is None or (isinstance(value, str) and value == "auto"):
return None
- elif isinstance(value, str) and value == "auto":
- return None
- elif isinstance(value, int): # pyright: ignore[reportUnnecessaryIsInstance]
+ if isinstance(value, int): # pyright: ignore[reportUnnecessaryIsInstance]
return value
@@ -73,35 +72,38 @@ class PipeLLMSpec(PipeSpec):
3. Output cardinality: nb_output and multiple_output are mutually exclusive.
4. nb_output must be greater than 0 when specified.
5. Structuring method must be 'direct' or 'preliminary_text' when specified.
+
"""
type: Literal["PipeLLM"] = "PipeLLM"
category: Literal["PipeOperator"] = "PipeOperator"
the_pipe_code: str = Field(description="Pipe code. Must be snake_case.")
- system_prompt_template: Optional[str] = None
- system_prompt_template_name: Optional[str] = None
- system_prompt_name: Optional[str] = None
- system_prompt: Optional[str] = None
+ system_prompt_template: str | None = None
+ system_prompt_template_name: str | None = None
+ system_prompt_name: str | None = None
+ system_prompt: str | None = None
- prompt_template: Optional[str] = None
- template_name: Optional[str] = None
- prompt_name: Optional[str] = None
- prompt: Optional[str] = None
+ prompt_template: str | None = None
+ template_name: str | None = None
+ prompt_name: str | None = None
+ prompt: str | None = None
- llm: Optional[LLMChoiceSpec] = None
- llm_to_structure: Optional[LLMChoiceSpec] = None
+ llm: LLMChoiceSpec | None = None
+ llm_to_structure: LLMChoiceSpec | None = None
- structuring_method: Optional[StructuringMethod] = None
- prompt_template_to_structure: Optional[str] = None
- system_prompt_to_structure: Optional[str] = None
+ structuring_method: StructuringMethod | None = None
+ prompt_template_to_structure: str | None = None
+ system_prompt_to_structure: str | None = None
- nb_output: Optional[int] = None
- multiple_output: Optional[bool] = None
+ nb_output: int | None = None
+ multiple_output: bool | None = None
@field_validator("nb_output", mode="after")
- def validate_nb_output(cls, value: Optional[int] = None) -> Optional[int]:
+ @staticmethod
+ def validate_nb_output(value: int | None = None) -> int | None:
if value and value < 1:
- raise PipeDefinitionError("PipeLLMBlueprint nb_output must be greater than 0")
+ msg = "PipeLLMBlueprint nb_output must be greater than 0"
+ raise PipeDefinitionError(msg)
return value
@model_validator(mode="after")
@@ -114,19 +116,20 @@ class PipeLLMSpec(PipeSpec):
["prompt", "prompt_name", "prompt_template", "template_name"],
],
):
- raise PipeDefinitionError(f"PipeLLMBlueprint should have no more than one of {excess_attributes_list} among them")
+ msg = f"PipeLLMBlueprint should have no more than one of {excess_attributes_list} among them"
+ raise PipeDefinitionError(msg)
return self
@override
def to_blueprint(self) -> PipeLLMBlueprint:
base_blueprint = super().to_blueprint()
- llm: Optional[LLMChoice] = None
+ llm: LLMChoice | None = None
if isinstance(self.llm, LLMSettingSpec):
llm = LLMSetting(llm_handle=self.llm.llm_handle, temperature=self.llm.temperature, max_tokens=self.llm.max_tokens)
elif isinstance(self.llm, str):
llm = self.llm
- llm_to_structure: Optional[LLMChoice] = None
+ llm_to_structure: LLMChoice | None = None
if isinstance(self.llm_to_structure, LLMSettingSpec):
llm_to_structure = LLMSetting(
llm_handle=self.llm_to_structure.llm_handle,
diff --git a/pipelex/libraries/pipelines/builder/pipe/pipe_ocr_spec.py b/pipelex/libraries/pipelines/builder/pipe/pipe_ocr_spec.py
index 299197280..2a377d064 100644
--- a/pipelex/libraries/pipelines/builder/pipe/pipe_ocr_spec.py
+++ b/pipelex/libraries/pipelines/builder/pipe/pipe_ocr_spec.py
@@ -1,4 +1,4 @@
-from typing import Literal, Optional
+from typing import Literal
from pydantic import Field
from typing_extensions import override
@@ -34,16 +34,17 @@ class PipeOcrSpec(PipeSpec):
1. OCR model must be "base_ocr_mistral".
2. Boolean flags (page_images, page_image_captions, page_views) are optional.
3. page_views_dpi should be a positive integer when specified.
+
"""
type: Literal["PipeOcr"] = "PipeOcr"
category: Literal["PipeOperator"] = "PipeOperator"
the_pipe_code: str = Field(description="Pipe code. Must be snake_case.")
ocr: str = "base_ocr_mistral"
- page_images: Optional[bool] = None
- page_image_captions: Optional[bool] = None
- page_views: Optional[bool] = None
- page_views_dpi: Optional[int] = None
+ page_images: bool | None = None
+ page_image_captions: bool | None = None
+ page_views: bool | None = None
+ page_views_dpi: int | None = None
@override
def to_blueprint(self) -> PipeOcrBlueprint:
diff --git a/pipelex/libraries/pipelines/builder/pipe/pipe_parallel_spec.py b/pipelex/libraries/pipelines/builder/pipe/pipe_parallel_spec.py
index ec6180e59..2afdf8b6e 100644
--- a/pipelex/libraries/pipelines/builder/pipe/pipe_parallel_spec.py
+++ b/pipelex/libraries/pipelines/builder/pipe/pipe_parallel_spec.py
@@ -1,4 +1,4 @@
-from typing import List, Literal, Optional
+from typing import Literal
from pydantic import Field, field_validator
from typing_extensions import override
@@ -31,17 +31,19 @@ class PipeParallelSpec(PipeSpec):
2. Each parallel step must be a valid SubPipeSpec.
3. combined_output, when specified, must be a valid concept string or code.
4. Pipe codes in parallels must reference existing pipes.
+
"""
type: Literal["PipeParallel"] = "PipeParallel"
category: Literal["PipeController"] = "PipeController"
the_pipe_code: str = Field(description="Pipe code. Must be snake_case.")
- parallels: List[SubPipeSpec]
+ parallels: list[SubPipeSpec]
add_each_output: bool = True
- combined_output: Optional[str] = None
+ combined_output: str | None = None
@field_validator("combined_output", mode="before")
- def validate_combined_output(cls, combined_output: str) -> str:
+ @staticmethod
+ def validate_combined_output(combined_output: str) -> str:
if combined_output:
ConceptBlueprint.validate_concept_string_or_code(concept_string_or_code=combined_output)
return combined_output
diff --git a/pipelex/libraries/pipelines/builder/pipe/pipe_sequence_spec.py b/pipelex/libraries/pipelines/builder/pipe/pipe_sequence_spec.py
index bda5a78a4..2921c0a2e 100644
--- a/pipelex/libraries/pipelines/builder/pipe/pipe_sequence_spec.py
+++ b/pipelex/libraries/pipelines/builder/pipe/pipe_sequence_spec.py
@@ -1,4 +1,4 @@
-from typing import List, Literal
+from typing import Literal
from pydantic import Field
from typing_extensions import override
@@ -26,12 +26,13 @@ class PipeSequenceSpec(PipeSpec):
1. Steps list must not be empty.
2. Each step must be a valid SubPipeBlueprint instance.
3. Pipe codes referenced in steps must exist in the pipeline.
+
"""
type: Literal["PipeSequence"] = "PipeSequence"
category: Literal["PipeController"] = "PipeController"
the_pipe_code: str = Field(description="Pipe code. Must be snake_case.")
- steps: List[SubPipeSpec]
+ steps: list[SubPipeSpec]
@override
def to_blueprint(self) -> PipeSequenceBlueprint:
diff --git a/pipelex/libraries/pipelines/builder/pipe/pipe_signature.py b/pipelex/libraries/pipelines/builder/pipe/pipe_signature.py
index f8424f81a..90a5132ca 100644
--- a/pipelex/libraries/pipelines/builder/pipe/pipe_signature.py
+++ b/pipelex/libraries/pipelines/builder/pipe/pipe_signature.py
@@ -1,4 +1,4 @@
-from typing import Any, Dict, Optional, Union
+from typing import Any
from pydantic import Field, field_validator
@@ -16,10 +16,10 @@ class PipeSignature(StructuredContent):
type: AllowedPipeTypes = Field(description="Pipe type.")
category: AllowedPipeCategories = Field(description="Pipe category.")
definition: str = Field(description="What the pipe does")
- inputs: Dict[str, ConceptSpecDraft] = Field(description="Pipe inputs: key is the concept code in pascal Case.")
+ inputs: dict[str, ConceptSpecDraft] = Field(description="Pipe inputs: key is the concept code in pascal Case.")
result: str = Field(description="The name of the result of the pipe. Must be snake_case. It will be used in the inputs of the next pipes.")
output: ConceptSpecDraft = Field(description="Concept as output")
- important_features: Optional[Dict[str, Any]] = Field(
+ important_features: dict[str, Any] | None = Field(
default=None,
description="Important features specific to this pipe type "
"(e.g., referenced pipe codes for controllers, specific configuration for operators)",
@@ -51,40 +51,44 @@ class PipeSpec(StructuredContent):
2. Output concept: Must be valid concept string or code in PascalCase.
3. Input concepts: When provided, must use PascalCase for concept references.
4. Pipe codes: When validating pipe codes, must be in snake_case format.
+
"""
type: Any = Field(description=f"Pipe type. Must be one of: {AllowedPipeTypes}")
category: Any = Field(description=f"Pipe category. Must be one of: {AllowedPipeCategories}")
- definition: Optional[str] = Field(description="Natural language description of what the pipe does.")
- inputs: Optional[Dict[str, Union[str, InputRequirementSpec]]] = Field(
+ definition: str | None = Field(description="Natural language description of what the pipe does.")
+ inputs: dict[str, str | InputRequirementSpec] | None = Field(
description=(
"Input concept specifications. Can be either: "
"InputRequirementSpec with additional constraints"
"Dictionary keys are input names, values are concept specifications. If Its the concept itself, use the concept code in PascalCase."
- )
+ ),
)
output: str = Field(description="Output concept code in PascalCase format!! Very important")
@field_validator("type", mode="after")
- def validate_pipe_type(cls, value: Any) -> Any:
- allowed_types = [_type.value for _type in AllowedPipeTypes]
- if value not in allowed_types:
- raise PipeBlueprintError(f"Invalid pipe type '{value}'. Must be one of: {allowed_types}")
+ @staticmethod
+ def validate_pipe_type(value: Any) -> Any:
+ if value not in AllowedPipeTypes.value_list():
+ msg = f"Invalid pipe type '{value}'. Must be one of: {AllowedPipeTypes.value_list()}"
+ raise PipeBlueprintError(msg)
return value
@field_validator("output", mode="before")
- def validate_concept_string_or_code(cls, output: str) -> str:
+ @staticmethod
+ def validate_concept_string_or_code(output: str) -> str:
ConceptBlueprint.validate_concept_string_or_code(concept_string_or_code=output)
return output
@classmethod
def validate_pipe_code_syntax(cls, pipe_code: str) -> str:
if not is_snake_case(pipe_code):
- raise PipeBlueprintError(f"Invalid pipe code syntax '{pipe_code}'. Must be in snake_case.")
+ msg = f"Invalid pipe code syntax '{pipe_code}'. Must be in snake_case."
+ raise PipeBlueprintError(msg)
return pipe_code
def to_blueprint(self) -> PipeBlueprint:
- converted_inputs: Optional[Dict[str, Union[str, InputRequirementBlueprint]]] = None
+ converted_inputs: dict[str, str | InputRequirementBlueprint] | None = None
if self.inputs is not None:
converted_inputs = {}
for input_name, input_spec in self.inputs.items():
diff --git a/pipelex/libraries/pipelines/builder/pipe/sub_pipe_spec.py b/pipelex/libraries/pipelines/builder/pipe/sub_pipe_spec.py
index aafc03094..42423bd3b 100644
--- a/pipelex/libraries/pipelines/builder/pipe/sub_pipe_spec.py
+++ b/pipelex/libraries/pipelines/builder/pipe/sub_pipe_spec.py
@@ -1,12 +1,10 @@
-from typing import Optional, Union
-
from pydantic import ConfigDict, model_validator
-from typing_extensions import Self
from pipelex.core.stuffs.stuff_content import StructuredContent
from pipelex.exceptions import PipeDefinitionError
from pipelex.pipe_controllers.sub_pipe_blueprint import SubPipeBlueprint
from pipelex.tools.typing.validation_utils import has_more_than_one_among_attributes_from_list
+from pipelex.types import Self
class SubPipeSpec(StructuredContent):
@@ -35,21 +33,23 @@ class SubPipeSpec(StructuredContent):
2. batch_over and batch_as must be specified together (both or neither).
3. pipe must reference a valid pipe code.
4. result, when specified, should follow naming conventions.
+
"""
model_config = ConfigDict(extra="forbid")
the_pipe_code: str
- result: Optional[str] = None
- nb_output: Optional[int] = None
- multiple_output: Optional[bool] = None
- batch_over: Union[bool, str] = False
- batch_as: Optional[str] = None
+ result: str | None = None
+ nb_output: int | None = None
+ multiple_output: bool | None = None
+ batch_over: bool | str = False
+ batch_as: str | None = None
@model_validator(mode="after")
def validate_multiple_output(self) -> Self:
if has_more_than_one_among_attributes_from_list(self, attributes_list=["nb_output", "multiple_output"]):
- raise PipeDefinitionError("PipeStepBlueprint should have no more than '1' of nb_output or multiple_output")
+ msg = "PipeStepBlueprint should have no more than '1' of nb_output or multiple_output"
+ raise PipeDefinitionError(msg)
return self
@model_validator(mode="after")
@@ -58,10 +58,12 @@ class SubPipeSpec(StructuredContent):
batch_as_is_specified = self.batch_as is not None and self.batch_as != ""
if batch_over_is_specified and not batch_as_is_specified:
- raise PipeDefinitionError(f"In pipe '{self.the_pipe_code}': When 'batch_over' is specified, 'batch_as' must also be provided")
+ msg = f"In pipe '{self.the_pipe_code}': When 'batch_over' is specified, 'batch_as' must also be provided"
+ raise PipeDefinitionError(msg)
if batch_as_is_specified and not batch_over_is_specified:
- raise PipeDefinitionError(f"In pipe '{self.the_pipe_code}': When 'batch_as' is specified, 'batch_over' must also be provided")
+ msg = f"In pipe '{self.the_pipe_code}': When 'batch_as' is specified, 'batch_over' must also be provided"
+ raise PipeDefinitionError(msg)
return self
diff --git a/pipelex/migration/migrate_v0_1_0_to_v0_2_0.py b/pipelex/migration/migrate_v0_1_0_to_v0_2_0.py
index 0d536e3d4..e9ec77e70 100644
--- a/pipelex/migration/migrate_v0_1_0_to_v0_2_0.py
+++ b/pipelex/migration/migrate_v0_1_0_to_v0_2_0.py
@@ -3,7 +3,8 @@
import re
import shutil
from pathlib import Path
-from typing import Any, ClassVar, Dict, List, Match
+from re import Match
+from typing import Any, ClassVar
from pipelex.migration.migration_result import MigrationResult
@@ -36,10 +37,11 @@ class TOMLMigrator:
re.MULTILINE,
)
- def find_toml_files(self, directory: Path) -> List[Path]:
+ def find_toml_files(self, directory: Path) -> list[Path]:
"""Find all TOML files in directory and subdirectories."""
if not directory.exists():
- raise FileNotFoundError(f"Directory not found: {directory}")
+ msg = f"Directory not found: {directory}"
+ raise FileNotFoundError(msg)
return list(directory.glob("**/*.toml"))
@@ -65,15 +67,11 @@ class TOMLMigrator:
# Check pipe patterns
pipe_matches = list(self.pipe_pattern.finditer(content))
- for match in pipe_matches:
- if not self._is_line_inside_multiline_string(content, match.start()):
- return True
-
- return False
+ return any(not self._is_line_inside_multiline_string(content, match.start()) for match in pipe_matches)
- def get_migration_preview(self, content: str) -> List[Dict[str, Any]]:
+ def get_migration_preview(self, content: str) -> list[dict[str, Any]]:
"""Get preview of changes that would be made."""
- changes: List[Dict[str, Any]] = []
+ changes: list[dict[str, Any]] = []
# Handle concept migrations
concept_matches = list(self.concept_pattern.finditer(content))
@@ -117,46 +115,43 @@ class TOMLMigrator:
if self._is_line_inside_multiline_string(content, match.start()):
# Return the original text unchanged
return match.group(0)
- else:
- # Apply the normal replacement: Concept = -> definition =
- return f"{match.group(1)}definition{match.group(3)}{match.group(4)}"
+ # Apply the normal replacement: Concept = -> definition =
+ return f"{match.group(1)}definition{match.group(3)}{match.group(4)}"
def pipe_replacement_function(match: Match[str]) -> str:
# Check if this match is inside a multiline string
if self._is_line_inside_multiline_string(content, match.start()):
# Return the original text unchanged
return match.group(0)
- else:
- # Apply pipe replacement: PipeClassName = "desc" -> type = "PipeClassName"\ndefinition = "desc"
- leading_whitespace = match.group(1)
- pipe_class_name = match.group(2)
- definition_value = match.group(4)
- trailing_whitespace = match.group(5)
+ # Apply pipe replacement: PipeClassName = "desc" -> type = "PipeClassName"\ndefinition = "desc"
+ leading_whitespace = match.group(1)
+ pipe_class_name = match.group(2)
+ definition_value = match.group(4)
+ trailing_whitespace = match.group(5)
- type_line = f'{leading_whitespace}type = "{pipe_class_name}"'
- definition_line = f"{leading_whitespace}definition = {definition_value}"
+ type_line = f'{leading_whitespace}type = "{pipe_class_name}"'
+ definition_line = f"{leading_whitespace}definition = {definition_value}"
- return f"{type_line}\n{definition_line}{trailing_whitespace}"
+ return f"{type_line}\n{definition_line}{trailing_whitespace}"
# Apply concept migrations first
migrated_content = self.concept_pattern.sub(concept_replacement_function, content)
# Then apply pipe migrations
- migrated_content = self.pipe_pattern.sub(pipe_replacement_function, migrated_content)
-
- return migrated_content
+ return self.pipe_pattern.sub(pipe_replacement_function, migrated_content)
def migrate_file(self, file_path: Path, create_backup: bool = True) -> int:
- """
- Migrate a single file from old to new syntax.
+ """Migrate a single file from old to new syntax.
Returns:
Number of changes made in the file
+
"""
try:
- with open(file_path, "r", encoding="utf-8") as f:
+ with open(file_path, encoding="utf-8") as f:
original_content = f.read()
except Exception as e:
- raise IOError(f"Failed to read file {file_path}: {e}")
+ msg = f"Failed to read file {file_path}: {e}"
+ raise OSError(msg) from e
if not self.needs_migration(original_content):
return 0
@@ -175,19 +170,20 @@ class TOMLMigrator:
try:
shutil.copy2(file_path, backup_path)
except Exception as e:
- raise IOError(f"Failed to create backup for {file_path}: {e}")
+ msg = f"Failed to create backup for {file_path}: {e}"
+ raise OSError(msg) from e
try:
with open(file_path, "w", encoding="utf-8") as f:
f.write(migrated_content)
except Exception as e:
- raise IOError(f"Failed to write migrated content to {file_path}: {e}")
+ msg = f"Failed to write migrated content to {file_path}: {e}"
+ raise OSError(msg) from e
return changes_count
def migrate_directory(self, directory: Path, create_backups: bool = True, dry_run: bool = False) -> MigrationResult:
- """
- Migrate all TOML files in a directory.
+ """Migrate all TOML files in a directory.
Args:
directory: Directory containing TOML files
@@ -196,17 +192,18 @@ class TOMLMigrator:
Returns:
MigrationResult with statistics and details
+
"""
toml_files = self.find_toml_files(directory)
files_processed = len(toml_files)
files_modified = 0
total_changes = 0
- modified_files: List[Path] = []
- errors: List[str] = []
+ modified_files: list[Path] = []
+ errors: list[str] = []
for toml_file in toml_files:
try:
- with open(toml_file, "r", encoding="utf-8") as f:
+ with open(toml_file, encoding="utf-8") as f:
content = f.read()
if not self.needs_migration(content):
@@ -244,8 +241,7 @@ class TOMLMigrator:
def migrate_concept_syntax(directory: Path, create_backups: bool = True, dry_run: bool = False) -> MigrationResult:
- """
- Convenience function to migrate TOML files from Concept = to definition = syntax.
+ """Convenience function to migrate TOML files from Concept = to definition = syntax.
This function accepts either a directory (the historical behavior) or a path to a single TOML file.
@@ -256,6 +252,7 @@ def migrate_concept_syntax(directory: Path, create_backups: bool = True, dry_run
Returns:
MigrationResult with migration statistics
+
"""
migrator = TOMLMigrator()
@@ -270,7 +267,7 @@ def migrate_concept_syntax(directory: Path, create_backups: bool = True, dry_run
return result
try:
- with open(input_path, "r", encoding="utf-8") as f:
+ with open(input_path, encoding="utf-8") as f:
content = f.read()
except Exception as e:
result.errors.append(f"Failed to read file {input_path}: {e}")
diff --git a/pipelex/migration/migration_result.py b/pipelex/migration/migration_result.py
index 935f82806..184000741 100644
--- a/pipelex/migration/migration_result.py
+++ b/pipelex/migration/migration_result.py
@@ -1,10 +1,11 @@
"""Shared migration result types."""
from pathlib import Path
-from typing import List
from pydantic import BaseModel, Field
+from pipelex.tools.typing.pydantic_utils import empty_list_factory_of
+
class MigrationResult(BaseModel):
"""Result of migration operation."""
@@ -12,5 +13,5 @@ class MigrationResult(BaseModel):
files_processed: int = 0
files_modified: int = 0
total_changes: int = 0
- modified_files: List[Path] = Field(default_factory=list)
- errors: List[str] = Field(default_factory=list)
+ modified_files: list[Path] = Field(default_factory=empty_list_factory_of(Path))
+ errors: list[str] = Field(default_factory=list)
diff --git a/pipelex/observer/local_observer.py b/pipelex/observer/local_observer.py
index 831ce50aa..506b77922 100644
--- a/pipelex/observer/local_observer.py
+++ b/pipelex/observer/local_observer.py
@@ -1,5 +1,4 @@
import os
-from typing import Optional
import kajson
from typing_extensions import override
@@ -9,7 +8,7 @@ from pipelex.observer.observer_protocol import ObserverProtocol, PayloadType
class LocalObserver(ObserverProtocol):
- def __init__(self, storage_dir: Optional[str] = None) -> None:
+ def __init__(self, storage_dir: str | None = None) -> None:
self.storage_dir = storage_dir or get_config().pipelex.observer_config.observer_dir
os.makedirs(self.storage_dir, exist_ok=True)
diff --git a/pipelex/observer/observer_protocol.py b/pipelex/observer/observer_protocol.py
index b0bd33c85..293b4daea 100644
--- a/pipelex/observer/observer_protocol.py
+++ b/pipelex/observer/observer_protocol.py
@@ -1,6 +1,6 @@
-from typing import Any, Dict, Protocol
+from typing import Any, Protocol
-PayloadType = Dict[str, Any]
+PayloadType = dict[str, Any]
class ObserverProtocol(Protocol):
diff --git a/pipelex/pipe_controllers/batch/__init__.py b/pipelex/pipe_controllers/batch/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/pipelex/pipe_controllers/batch/pipe_batch.py b/pipelex/pipe_controllers/batch/pipe_batch.py
index 4ca87ad49..5bdb437c5 100644
--- a/pipelex/pipe_controllers/batch/pipe_batch.py
+++ b/pipelex/pipe_controllers/batch/pipe_batch.py
@@ -1,9 +1,9 @@
import asyncio
-from typing import Any, Coroutine, List, Literal, Optional, Set, cast
+from typing import TYPE_CHECKING, Any, Literal, cast
import shortuuid
from pydantic import model_validator
-from typing_extensions import Self, override
+from typing_extensions import override
from pipelex import log
from pipelex.config import get_config
@@ -11,7 +11,6 @@ from pipelex.core.memory.working_memory import MAIN_STUFF_NAME, WorkingMemory
from pipelex.core.pipes.pipe_input import PipeInputSpec
from pipelex.core.pipes.pipe_output import PipeOutput
from pipelex.core.pipes.pipe_run_params import BatchParams, PipeRunMode, PipeRunParams
-from pipelex.core.stuffs.stuff import Stuff
from pipelex.core.stuffs.stuff_content import ListContent, StuffContent
from pipelex.core.stuffs.stuff_factory import StuffFactory
from pipelex.exceptions import (
@@ -22,17 +21,23 @@ from pipelex.exceptions import (
from pipelex.hub import get_pipeline_tracker, get_required_pipe
from pipelex.pipe_controllers.pipe_controller import PipeController
from pipelex.pipeline.job_metadata import JobMetadata
+from pipelex.types import Self
+
+if TYPE_CHECKING:
+ from collections.abc import Coroutine
+
+ from pipelex.core.stuffs.stuff import Stuff
class PipeBatch(PipeController):
type: Literal["PipeBatch"] = "PipeBatch"
branch_pipe_code: str
- batch_params: Optional[BatchParams] = None
+ batch_params: BatchParams | None = None
@override
- def pipe_dependencies(self) -> Set[str]:
- return set([self.branch_pipe_code])
+ def pipe_dependencies(self) -> set[str]:
+ return {self.branch_pipe_code}
@override
def validate_output(self):
@@ -51,13 +56,14 @@ class PipeBatch(PipeController):
# Now check that the required vairables ARE in the inputs of the pipe
required_variables = self.required_variables()
for variable_name in required_variables:
- if variable_name not in self.inputs.root.keys():
- raise PipeInputError(f"Input '{variable_name}' of pipe '{self.code}' is not in the inputs of the pipe '{self.branch_pipe_code}'")
+ if variable_name not in self.inputs.root:
+ msg = f"Input '{variable_name}' of pipe '{self.code}' is not in the inputs of the pipe '{self.branch_pipe_code}'"
+ raise PipeInputError(msg)
return self
@override
- def required_variables(self) -> Set[str]:
- required_variables: Set[str] = set()
+ def required_variables(self) -> set[str]:
+ required_variables: set[str] = set()
# 1. Check that the inputs of the pipe branch_pipe_code are in the inputs of the pipe
pipe = get_required_pipe(pipe_code=self.branch_pipe_code)
for variable_name, _ in pipe.inputs.items:
@@ -68,7 +74,7 @@ class PipeBatch(PipeController):
return required_variables
@override
- def needed_inputs(self, visited_pipes: Optional[Set[str]] = None) -> PipeInputSpec:
+ def needed_inputs(self, visited_pipes: set[str] | None = None) -> PipeInputSpec:
return self.inputs
async def _run_batch_pipe(
@@ -76,7 +82,7 @@ class PipeBatch(PipeController):
job_metadata: JobMetadata,
working_memory: WorkingMemory,
pipe_run_params: PipeRunParams,
- output_name: Optional[str] = None,
+ output_name: str | None = None,
) -> PipeOutput:
"""Common logic for running or dry-running a pipe in batch mode."""
batch_params = pipe_run_params.batch_params or self.batch_params or BatchParams.make_default()
@@ -84,9 +90,8 @@ class PipeBatch(PipeController):
try:
input_item_concept_code = self.inputs.get_required_input_requirement(input_item_stuff_name)
except PipeInputNotFoundError as exc:
- raise PipeInputError(
- f"Batch input item stuff named '{input_item_stuff_name}' is not in this PipeBatch '{self.code}' input spec: {self.inputs}"
- ) from exc
+ msg = f"Batch input item stuff named '{input_item_stuff_name}' is not in this PipeBatch '{self.code}' input spec: {self.inputs}"
+ raise PipeInputError(msg) from exc
if pipe_run_params.final_stuff_code:
method_name = "dry_run_pipe" if pipe_run_params.run_mode == PipeRunMode.DRY else "_run_controller_pipe"
@@ -97,26 +102,27 @@ class PipeBatch(PipeController):
try:
input_stuff = working_memory.get_stuff(batch_params.input_list_stuff_name)
except WorkingMemoryStuffNotFoundError as exc:
- raise PipeInputError(
+ msg = (
f"Input list stuff '{batch_params.input_list_stuff_name}' required by this PipeBatch '{self.code}' not found in working memory: {exc}"
- ) from exc
+ )
+ raise PipeInputError(msg) from exc
+
input_stuff_code = input_stuff.stuff_code
input_content = input_stuff.content
if not isinstance(input_content, ListContent):
- raise PipeInputError(
- f"Input of PipeBatch must be ListContent, got {input_stuff.stuff_name or 'unnamed'} = {type(input_content)}. stuff: {input_stuff}"
- )
- input_content = cast(ListContent[StuffContent], input_content)
+ msg = f"Input of PipeBatch must be ListContent, got {input_stuff.stuff_name or 'unnamed'} = {type(input_content)}. stuff: {input_stuff}"
+ raise PipeInputError(msg)
+ input_content = cast("ListContent[StuffContent]", input_content)
# TODO: Make commented code work when inputing images named "a.b.c"
sub_pipe = get_required_pipe(pipe_code=self.branch_pipe_code)
nb_history_items_limit = get_config().pipelex.tracker_config.applied_nb_items_limit
batch_output_stuff_code = shortuuid.uuid()
- tasks: List[Coroutine[Any, Any, PipeOutput]] = []
- item_stuffs: List[Stuff] = []
- required_stuff_lists: List[List[Stuff]] = []
- branch_output_item_codes: List[str] = []
+ tasks: list[Coroutine[Any, Any, PipeOutput]] = []
+ item_stuffs: list[Stuff] = []
+ required_stuff_lists: list[list[Stuff]] = []
+ branch_output_item_codes: list[str] = []
for branch_index, item in enumerate(input_content.items):
branch_output_item_code = f"{batch_output_stuff_code}-branch-{branch_index}"
@@ -160,10 +166,10 @@ class PipeBatch(PipeController):
pipe_outputs = await asyncio.gather(*tasks)
- output_items: List[StuffContent] = []
- output_stuffs: List[Stuff] = []
+ output_items: list[StuffContent] = []
+ output_stuffs: list[Stuff] = []
output_stuff_code = shortuuid.uuid()[:5]
- for branch_index, pipe_output in enumerate(pipe_outputs):
+ for pipe_output in pipe_outputs:
branch_output_stuff = pipe_output.main_stuff
output_stuffs.append(branch_output_stuff)
output_items.append(branch_output_stuff.content)
@@ -181,7 +187,7 @@ class PipeBatch(PipeController):
required_stuff_list,
item_input_stuff,
item_output_stuff,
- ) in enumerate(zip(required_stuff_lists, item_stuffs, output_stuffs)):
+ ) in enumerate(zip(required_stuff_lists, item_stuffs, output_stuffs, strict=False)):
get_pipeline_tracker().add_batch_step(
from_stuff=input_stuff,
to_stuff=item_input_stuff,
@@ -225,7 +231,7 @@ class PipeBatch(PipeController):
job_metadata: JobMetadata,
working_memory: WorkingMemory,
pipe_run_params: PipeRunParams,
- output_name: Optional[str] = None,
+ output_name: str | None = None,
) -> PipeOutput:
"""Run a pipe in batch mode for each item in the input list."""
return await self._run_batch_pipe(
@@ -241,7 +247,7 @@ class PipeBatch(PipeController):
job_metadata: JobMetadata,
working_memory: WorkingMemory,
pipe_run_params: PipeRunParams,
- output_name: Optional[str] = None,
+ output_name: str | None = None,
) -> PipeOutput:
"""Dry run a pipe in batch mode for each item in the input list."""
return await self._run_batch_pipe(
diff --git a/pipelex/pipe_controllers/batch/pipe_batch_blueprint.py b/pipelex/pipe_controllers/batch/pipe_batch_blueprint.py
index 80707fdef..7f0655c0f 100644
--- a/pipelex/pipe_controllers/batch/pipe_batch_blueprint.py
+++ b/pipelex/pipe_controllers/batch/pipe_batch_blueprint.py
@@ -1,4 +1,4 @@
-from typing import Literal, Optional
+from typing import Literal
from pipelex.core.pipes.pipe_blueprint import PipeBlueprint
@@ -7,5 +7,5 @@ class PipeBatchBlueprint(PipeBlueprint):
type: Literal["PipeBatch"] = "PipeBatch"
category: Literal["PipeController"] = "PipeController"
branch_pipe_code: str
- input_list_name: Optional[str] = None
- input_item_name: Optional[str] = None
+ input_list_name: str | None = None
+ input_item_name: str | None = None
diff --git a/pipelex/pipe_controllers/batch/pipe_batch_factory.py b/pipelex/pipe_controllers/batch/pipe_batch_factory.py
index 91a2f9e6e..3eeb04bb2 100644
--- a/pipelex/pipe_controllers/batch/pipe_batch_factory.py
+++ b/pipelex/pipe_controllers/batch/pipe_batch_factory.py
@@ -1,5 +1,3 @@
-from typing import List, Optional
-
from typing_extensions import override
from pipelex.core.concepts.concept_factory import ConceptFactory
@@ -19,7 +17,7 @@ class PipeBatchFactory(PipeFactoryProtocol[PipeBatchBlueprint, PipeBatch]):
domain: str,
pipe_code: str,
blueprint: PipeBatchBlueprint,
- concept_codes_from_the_same_domain: Optional[List[str]] = None,
+ concept_codes_from_the_same_domain: list[str] | None = None,
) -> PipeBatch:
output_domain_and_code = ConceptFactory.make_domain_and_concept_code_from_concept_string_or_code(
domain=domain,
@@ -31,12 +29,15 @@ class PipeBatchFactory(PipeFactoryProtocol[PipeBatchBlueprint, PipeBatch]):
code=pipe_code,
definition=blueprint.definition,
inputs=PipeInputSpecFactory.make_from_blueprint(
- domain=domain, blueprint=blueprint.inputs or {}, concept_codes_from_the_same_domain=concept_codes_from_the_same_domain
+ domain=domain,
+ blueprint=blueprint.inputs or {},
+ concept_codes_from_the_same_domain=concept_codes_from_the_same_domain,
),
output=get_concept_provider().get_required_concept(
concept_string=ConceptFactory.construct_concept_string_with_domain(
- domain=output_domain_and_code.domain, concept_code=output_domain_and_code.concept_code
- )
+ domain=output_domain_and_code.domain,
+ concept_code=output_domain_and_code.concept_code,
+ ),
),
branch_pipe_code=blueprint.branch_pipe_code,
batch_params=BatchParams.make_optional_batch_params(
diff --git a/pipelex/pipe_controllers/condition/__init__.py b/pipelex/pipe_controllers/condition/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/pipelex/pipe_controllers/condition/pipe_condition.py b/pipelex/pipe_controllers/condition/pipe_condition.py
index f53a8940e..793a84c16 100644
--- a/pipelex/pipe_controllers/condition/pipe_condition.py
+++ b/pipelex/pipe_controllers/condition/pipe_condition.py
@@ -1,8 +1,8 @@
-from typing import Dict, List, Literal, Optional, Set, Union, cast
+from typing import TYPE_CHECKING, Literal, cast
import shortuuid
from pydantic import field_validator, model_validator
-from typing_extensions import Self, override
+from typing_extensions import override
from pipelex import log
from pipelex.config import StaticValidationReaction, get_config
@@ -34,24 +34,27 @@ from pipelex.pipe_operators.jinja2.pipe_jinja2_factory import PipeJinja2Factory
from pipelex.pipe_works.pipe_job_factory import PipeJobFactory
from pipelex.pipeline.job_metadata import JobCategory, JobMetadata
from pipelex.tools.typing.validation_utils import has_exactly_one_among_attributes_from_list
+from pipelex.types import Self
+
+if TYPE_CHECKING:
+ from pipelex.pipe_operators.jinja2.pipe_jinja2 import PipeJinja2Output
class PipeCondition(PipeController):
type: Literal["PipeCondition"] = "PipeCondition"
- expression_template: Optional[str] = None
- expression: Optional[str] = None
+ expression_template: str | None = None
+ expression: str | None = None
# TODO: rething this pipe_map.
- pipe_map: List[PipeConditionPipeMap]
- default_pipe_code: Optional[str] = None
- add_alias_from_expression_to: Optional[str] = None
+ pipe_map: list[PipeConditionPipeMap]
+ default_pipe_code: str | None = None
+ add_alias_from_expression_to: str | None = None
#########################################################################################
# Validation
#########################################################################################
@override
def validate_output(self):
- """
- Validate the output for the pipe condition.
+ """Validate the output for the pipe condition.
The output of the pipe condition should match the output of all the conditional pipes, and the default pipe.
"""
# This pipe CONTINUE enables to leave a PipeCondition and continue the sequence.
@@ -82,24 +85,23 @@ class PipeCondition(PipeController):
@field_validator("pipe_map")
@classmethod
- def validate_pipe_map(cls, pipe_map: List[PipeConditionPipeMap]) -> List[PipeConditionPipeMap]:
+ def validate_pipe_map(cls, pipe_map: list[PipeConditionPipeMap]) -> list[PipeConditionPipeMap]:
# Validate that the expressions and pipe_code are UNIQUE
expression_results = [pipe_condition_pipe_map.expression_result for pipe_condition_pipe_map in pipe_map]
pipe_codes = [pipe_condition_pipe_map.pipe_code for pipe_condition_pipe_map in pipe_map]
if len(expression_results) != len(set(expression_results)):
- raise PipeDefinitionError(
- f"PipeCondition '{cls.code}' must have a unique expression result for each pipe in pipe_map in pipe_map: {pipe_map}"
- )
+ msg = f"PipeCondition '{cls.code}' must have a unique expression result for each pipe in pipe_map in pipe_map: {pipe_map}"
+ raise PipeDefinitionError(msg)
if len(pipe_codes) != len(set(pipe_codes)):
- raise PipeDefinitionError(
- f"PipeCondition '{cls.code}' must have a unique pipe code for each expression result in pipe_map in pipe_map: {pipe_map}"
- )
+ msg = f"PipeCondition '{cls.code}' must have a unique pipe code for each expression result in pipe_map in pipe_map: {pipe_map}"
+ raise PipeDefinitionError(msg)
return pipe_map
@model_validator(mode="after")
def validate_expression(self) -> Self:
if not has_exactly_one_among_attributes_from_list(self, attributes_list=["expression_template", "expression"]):
- raise PipeDefinitionError("PipeCondition should have exactly one of expression_template or expression")
+ msg = "PipeCondition should have exactly one of expression_template or expression"
+ raise PipeDefinitionError(msg)
return self
def _make_pipe_condition_details(self, evaluated_expression: str, chosen_pipe_code: str) -> PipeConditionDetails:
@@ -116,18 +118,18 @@ class PipeCondition(PipeController):
def applied_expression_template(self) -> str:
if self.expression_template:
return self.expression_template
- elif self.expression:
+ if self.expression:
return "{{ " + self.expression + " }}"
- else:
- raise PipeExecutionError("No expression or expression_template provided")
+ msg = "No expression or expression_template provided"
+ raise PipeExecutionError(msg)
#########################################################################################
# Inputs
#########################################################################################
@override
- def required_variables(self) -> Set[str]:
- required_variables: Set[str] = set()
+ def required_variables(self) -> set[str]:
+ required_variables: set[str] = set()
# Variables from the expression/expression_template
pipe_jinja2 = PipeJinja2Factory.make_pipe_jinja2_from_template_str(
domain=self.domain,
@@ -144,11 +146,12 @@ class PipeCondition(PipeController):
def _validate_required_variables(self) -> Self:
for required_variable_name in self.required_variables():
if required_variable_name not in self.inputs.variables:
- raise PipeDefinitionError(f"Required variable '{required_variable_name}' is not in the inputs of pipe {self.code}")
+ msg = f"Required variable '{required_variable_name}' is not in the inputs of pipe {self.code}"
+ raise PipeDefinitionError(msg)
return self
@override
- def needed_inputs(self, visited_pipes: Optional[Set[str]] = None) -> PipeInputSpec:
+ def needed_inputs(self, visited_pipes: set[str] | None = None) -> PipeInputSpec:
if visited_pipes is None:
visited_pipes = set()
@@ -194,15 +197,14 @@ class PipeCondition(PipeController):
@model_validator(mode="after")
def validate_inputs(self) -> Self:
if not self.pipe_map:
- raise ValueError(f"Pipe'{self.code}'(PipeCondition) must have at least one mapping in pipe_map")
+ msg = f"Pipe'{self.code}'(PipeCondition) must have at least one mapping in pipe_map"
+ raise ValueError(msg)
# Skip validation during model creation - it will be done in validate_with_libraries()
return self
def _validate_inputs(self):
- """
- Validate that the inputs declared for this PipeCondition match what is actually needed.
- """
+ """Validate that the inputs declared for this PipeCondition match what is actually needed."""
static_validation_config = get_config().pipelex.static_validation_config
default_reaction = static_validation_config.default_reaction
reactions = static_validation_config.reactions
@@ -245,15 +247,14 @@ class PipeCondition(PipeController):
@override
def validate_with_libraries(self):
- """
- Perform full validation after all libraries are loaded.
+ """Perform full validation after all libraries are loaded.
This is called after all pipes and concepts are available.
"""
self._validate_inputs()
self._validate_required_variables()
@override
- def pipe_dependencies(self) -> Set[str]:
+ def pipe_dependencies(self) -> set[str]:
pipe_codes = [
pipe_condition_pipe_map.pipe_code
for pipe_condition_pipe_map in self.pipe_map
@@ -269,7 +270,7 @@ class PipeCondition(PipeController):
job_metadata: JobMetadata,
working_memory: WorkingMemory,
pipe_run_params: PipeRunParams,
- output_name: Optional[str] = None,
+ output_name: str | None = None,
) -> PipeOutput:
log.dev(f"{self.class_name} generating a '{self.output.code}'")
@@ -277,7 +278,7 @@ class PipeCondition(PipeController):
# pipe_run_params.push_pipe_code(pipe_code=pipe_code)
# Convert PipeInputSpec to blueprint format
- inputs_blueprint: Dict[str, Union[str, InputRequirementBlueprint]] = {}
+ inputs_blueprint: dict[str, str | InputRequirementBlueprint] = {}
for var_name, requirement in self.inputs.root.items():
inputs_blueprint[var_name] = InputRequirementBlueprint(
concept=requirement.concept.concept_string,
@@ -300,7 +301,7 @@ class PipeCondition(PipeController):
jinja2_job_metadata = job_metadata.copy_with_update(
updated_metadata=JobMetadata(
job_category=JobCategory.JINJA2_JOB,
- )
+ ),
)
log.debug(f"Jinja2 expression: {self.applied_expression_template}")
# evaluated_expression = (
@@ -312,7 +313,7 @@ class PipeCondition(PipeController):
# ).rendered_text.strip()
# TODO: restore the possibility above, without need to explicitly cast the output
pipe_jinja2_output = cast(
- PipeJinja2Output,
+ "PipeJinja2Output",
await pipe_jinja2.run_pipe(
job_metadata=jinja2_job_metadata,
working_memory=working_memory,
@@ -357,14 +358,15 @@ class PipeCondition(PipeController):
)
required_variables = pipe_jinja2.required_variables()
log.debug(required_variables, title=f"Required variables for PipeCondition '{self.code}'")
- required_stuff_names = set([required_variable for required_variable in required_variables if not required_variable.startswith("_")])
+ required_stuff_names = {required_variable for required_variable in required_variables if not required_variable.startswith("_")}
try:
required_stuffs = working_memory.get_stuffs(names=required_stuff_names)
except WorkingMemoryStuffNotFoundError as exc:
- pipe_condition_path = pipe_run_params.pipe_layers + [self.code]
+ pipe_condition_path = [*pipe_run_params.pipe_layers, self.code]
pipe_condition_path_str = ".".join(pipe_condition_path)
error_details = f"PipeCondition '{pipe_condition_path_str}', required_variables: {required_variables}, missing: '{exc.variable_name}'"
- raise PipeInputError(f"Some required stuff(s) not found: {error_details}") from exc
+ msg = f"Some required stuff(s) not found: {error_details}"
+ raise PipeInputError(msg) from exc
for required_stuff in required_stuffs:
get_pipeline_tracker().add_condition_step(
@@ -399,17 +401,16 @@ class PipeCondition(PipeController):
job_metadata: JobMetadata,
working_memory: WorkingMemory,
pipe_run_params: PipeRunParams,
- output_name: Optional[str] = None,
+ output_name: str | None = None,
) -> PipeOutput:
- """
- Dry run implementation for PipeCondition.
+ """Dry run implementation for PipeCondition.
Validates that all required inputs are present, expression is valid, and target pipes exist.
"""
log.debug(f"PipeCondition: dry run controller pipe: {self.code}")
# 1. Validate that all required inputs are present in the working memory
needed_inputs = self.needed_inputs()
- missing_input_names: List[str] = []
+ missing_input_names: list[str] = []
for named_input_requirement in needed_inputs.named_input_requirements:
if not working_memory.get_optional_stuff(named_input_requirement.variable_name):
@@ -442,14 +443,14 @@ class PipeCondition(PipeController):
message=error_msg,
missing_inputs=[],
pipe_code=self.code,
- )
+ ) from exc
# 3. Validate that all pipes in the pipe_map exist
- all_pipe_codes = set([pipe_condition_pipe_map.pipe_code for pipe_condition_pipe_map in self.pipe_map])
+ all_pipe_codes = {pipe_condition_pipe_map.pipe_code for pipe_condition_pipe_map in self.pipe_map}
if self.default_pipe_code:
all_pipe_codes.add(self.default_pipe_code)
- missing_pipes: List[str] = []
+ missing_pipes: list[str] = []
for pipe_code in all_pipe_codes:
try:
get_required_pipe(pipe_code=pipe_code)
diff --git a/pipelex/pipe_controllers/condition/pipe_condition_blueprint.py b/pipelex/pipe_controllers/condition/pipe_condition_blueprint.py
index 6898cb51c..fa5ed48a4 100644
--- a/pipelex/pipe_controllers/condition/pipe_condition_blueprint.py
+++ b/pipelex/pipe_controllers/condition/pipe_condition_blueprint.py
@@ -1,10 +1,10 @@
-from typing import Dict, Literal, Optional
+from typing import Literal
from pydantic import Field, RootModel
from pipelex.core.pipes.pipe_blueprint import PipeBlueprint
-PipeConditionPipeMapRoot = Dict[str, str]
+PipeConditionPipeMapRoot = dict[str, str]
class PipeConditionPipeMapBlueprint(RootModel[PipeConditionPipeMapRoot]):
@@ -14,8 +14,8 @@ class PipeConditionPipeMapBlueprint(RootModel[PipeConditionPipeMapRoot]):
class PipeConditionBlueprint(PipeBlueprint):
type: Literal["PipeCondition"] = "PipeCondition"
category: Literal["PipeController"] = "PipeController"
- expression_template: Optional[str] = None
- expression: Optional[str] = None
+ expression_template: str | None = None
+ expression: str | None = None
pipe_map: PipeConditionPipeMapBlueprint = Field(default_factory=PipeConditionPipeMapBlueprint)
- default_pipe_code: Optional[str] = None
- add_alias_from_expression_to: Optional[str] = None
+ default_pipe_code: str | None = None
+ add_alias_from_expression_to: str | None = None
diff --git a/pipelex/pipe_controllers/condition/pipe_condition_details.py b/pipelex/pipe_controllers/condition/pipe_condition_details.py
index f88084956..c97f513b2 100644
--- a/pipelex/pipe_controllers/condition/pipe_condition_details.py
+++ b/pipelex/pipe_controllers/condition/pipe_condition_details.py
@@ -1,5 +1,3 @@
-from typing import List, Optional
-
from pydantic import BaseModel
@@ -11,7 +9,7 @@ class PipeConditionPipeMap(BaseModel):
class PipeConditionDetails(BaseModel):
code: str
test_expression: str
- pipe_map: List[PipeConditionPipeMap]
- default_pipe_code: Optional[str] = None
+ pipe_map: list[PipeConditionPipeMap]
+ default_pipe_code: str | None = None
evaluated_expression: str
chosen_pipe_code: str
diff --git a/pipelex/pipe_controllers/condition/pipe_condition_factory.py b/pipelex/pipe_controllers/condition/pipe_condition_factory.py
index fb7a8c65f..215c7c978 100644
--- a/pipelex/pipe_controllers/condition/pipe_condition_factory.py
+++ b/pipelex/pipe_controllers/condition/pipe_condition_factory.py
@@ -1,5 +1,3 @@
-from typing import List, Optional
-
from typing_extensions import override
from pipelex.core.concepts.concept_factory import ConceptFactory
@@ -13,7 +11,7 @@ from pipelex.pipe_controllers.condition.pipe_condition_details import PipeCondit
class PipeConditionFactory(PipeFactoryProtocol[PipeConditionBlueprint, PipeCondition]):
@classmethod
- def make_pipe_condition_pipe_map(cls, pipe_map: PipeConditionPipeMapBlueprint) -> List[PipeConditionPipeMap]:
+ def make_pipe_condition_pipe_map(cls, pipe_map: PipeConditionPipeMapBlueprint) -> list[PipeConditionPipeMap]:
return [
PipeConditionPipeMap(expression_result=expression_result, pipe_code=pipe_code) for expression_result, pipe_code in pipe_map.root.items()
]
@@ -25,7 +23,7 @@ class PipeConditionFactory(PipeFactoryProtocol[PipeConditionBlueprint, PipeCondi
domain: str,
pipe_code: str,
blueprint: PipeConditionBlueprint,
- concept_codes_from_the_same_domain: Optional[List[str]] = None,
+ concept_codes_from_the_same_domain: list[str] | None = None,
) -> PipeCondition:
output_domain_and_code = ConceptFactory.make_domain_and_concept_code_from_concept_string_or_code(
domain=domain,
@@ -37,12 +35,15 @@ class PipeConditionFactory(PipeFactoryProtocol[PipeConditionBlueprint, PipeCondi
code=pipe_code,
definition=blueprint.definition,
inputs=PipeInputSpecFactory.make_from_blueprint(
- domain=domain, blueprint=blueprint.inputs or {}, concept_codes_from_the_same_domain=concept_codes_from_the_same_domain
+ domain=domain,
+ blueprint=blueprint.inputs or {},
+ concept_codes_from_the_same_domain=concept_codes_from_the_same_domain,
),
output=get_concept_provider().get_required_concept(
concept_string=ConceptFactory.construct_concept_string_with_domain(
- domain=output_domain_and_code.domain, concept_code=output_domain_and_code.concept_code
- )
+ domain=output_domain_and_code.domain,
+ concept_code=output_domain_and_code.concept_code,
+ ),
),
expression_template=blueprint.expression_template,
expression=blueprint.expression,
diff --git a/pipelex/pipe_controllers/parallel/__init__.py b/pipelex/pipe_controllers/parallel/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/pipelex/pipe_controllers/parallel/pipe_parallel.py b/pipelex/pipe_controllers/parallel/pipe_parallel.py
index 933152bcb..7afd7d605 100644
--- a/pipelex/pipe_controllers/parallel/pipe_parallel.py
+++ b/pipelex/pipe_controllers/parallel/pipe_parallel.py
@@ -1,8 +1,8 @@
import asyncio
-from typing import Any, Coroutine, Dict, List, Literal, Optional, Set
+from typing import TYPE_CHECKING, Any, Literal
from pydantic import model_validator
-from typing_extensions import Self, override
+from typing_extensions import override
from pipelex import log
from pipelex.config import StaticValidationReaction, get_config
@@ -12,29 +12,34 @@ from pipelex.core.pipes.pipe_input import PipeInputSpec
from pipelex.core.pipes.pipe_input_factory import PipeInputSpecFactory
from pipelex.core.pipes.pipe_output import PipeOutput
from pipelex.core.pipes.pipe_run_params import PipeRunMode, PipeRunParams
-from pipelex.core.stuffs.stuff import Stuff
-from pipelex.core.stuffs.stuff_content import StuffContent
from pipelex.core.stuffs.stuff_factory import StuffFactory
from pipelex.exceptions import DryRunError, PipeDefinitionError, PipeRunParamsError, StaticValidationError, StaticValidationErrorType
from pipelex.hub import get_pipeline_tracker, get_required_pipe
from pipelex.pipe_controllers.pipe_controller import PipeController
from pipelex.pipe_controllers.sub_pipe import SubPipe
from pipelex.pipeline.job_metadata import JobMetadata
+from pipelex.types import Self
+
+if TYPE_CHECKING:
+ from collections.abc import Coroutine
+
+ from pipelex.core.stuffs.stuff import Stuff
+ from pipelex.core.stuffs.stuff_content import StuffContent
class PipeParallel(PipeController):
type: Literal["PipeParallel"] = "PipeParallel"
- parallel_sub_pipes: List[SubPipe]
+ parallel_sub_pipes: list[SubPipe]
add_each_output: bool
- combined_output: Optional[Concept]
+ combined_output: Concept | None
@override
- def required_variables(self) -> Set[str]:
+ def required_variables(self) -> set[str]:
return set()
@override
- def needed_inputs(self, visited_pipes: Optional[Set[str]] = None) -> PipeInputSpec:
+ def needed_inputs(self, visited_pipes: set[str] | None = None) -> PipeInputSpec:
if visited_pipes is None:
visited_pipes = set()
@@ -69,7 +74,8 @@ class PipeParallel(PipeController):
def validate_inputs(self) -> Self:
# Validate that either add_each_output or combined_output is set
if not self.add_each_output and not self.combined_output:
- raise PipeDefinitionError(f"PipeParallel'{self.code}'requires either add_each_output or combined_output to be set")
+ msg = f"PipeParallel'{self.code}'requires either add_each_output or combined_output to be set"
+ raise PipeDefinitionError(msg)
return self
@@ -78,9 +84,7 @@ class PipeParallel(PipeController):
pass
def _validate_inputs(self):
- """
- Validate that the inputs declared for this PipeParallel match what is actually needed.
- """
+ """Validate that the inputs declared for this PipeParallel match what is actually needed."""
static_validation_config = get_config().pipelex.static_validation_config
default_reaction = static_validation_config.default_reaction
reactions = static_validation_config.reactions
@@ -123,15 +127,14 @@ class PipeParallel(PipeController):
@override
def validate_with_libraries(self):
- """
- Perform full validation after all libraries are loaded.
+ """Perform full validation after all libraries are loaded.
This is called after all pipes and concepts are available.
"""
self._validate_inputs()
@override
- def pipe_dependencies(self) -> Set[str]:
- return set(sub_pipe.pipe_code for sub_pipe in self.parallel_sub_pipes)
+ def pipe_dependencies(self) -> set[str]:
+ return {sub_pipe.pipe_code for sub_pipe in self.parallel_sub_pipes}
@override
async def _run_controller_pipe(
@@ -139,19 +142,17 @@ class PipeParallel(PipeController):
job_metadata: JobMetadata,
working_memory: WorkingMemory,
pipe_run_params: PipeRunParams,
- output_name: Optional[str] = None,
+ output_name: str | None = None,
) -> PipeOutput:
- """
- Run a list of pipes in parallel.
- """
-
+ """Run a list of pipes in parallel."""
if not self.add_each_output and not self.combined_output:
- raise PipeDefinitionError("PipeParallel requires either add_each_output or combined_output to be set")
+ msg = "PipeParallel requires either add_each_output or combined_output to be set"
+ raise PipeDefinitionError(msg)
if pipe_run_params.final_stuff_code:
log.debug(f"PipeBatch.run_pipe() final_stuff_code: {pipe_run_params.final_stuff_code}")
pipe_run_params.final_stuff_code = None
- tasks: List[Coroutine[Any, Any, PipeOutput]] = []
+ tasks: list[Coroutine[Any, Any, PipeOutput]] = []
for sub_pipe in self.parallel_sub_pipes:
tasks.append(
@@ -160,35 +161,34 @@ class PipeParallel(PipeController):
job_metadata=job_metadata,
working_memory=working_memory.make_deep_copy(),
sub_pipe_run_params=pipe_run_params.make_deep_copy(),
- )
+ ),
)
pipe_outputs = await asyncio.gather(*tasks)
- output_stuff_content_items: List[StuffContent] = []
- output_stuffs: Dict[str, Stuff] = {}
- output_stuff_contents: Dict[str, StuffContent] = {}
+ output_stuff_content_items: list[StuffContent] = []
+ output_stuffs: dict[str, Stuff] = {}
+ output_stuff_contents: dict[str, StuffContent] = {}
# TODO: refactor this to use a specific function for this that can also be used in dry run
for output_index, pipe_output in enumerate(pipe_outputs):
output_stuff = pipe_output.main_stuff
sub_pipe_output_name = self.parallel_sub_pipes[output_index].output_name
if not sub_pipe_output_name:
- raise PipeDefinitionError("PipeParallel requires a result specified for each parallel sub pipe")
+ msg = "PipeParallel requires a result specified for each parallel sub pipe"
+ raise PipeDefinitionError(msg)
if self.add_each_output:
working_memory.add_new_stuff(name=sub_pipe_output_name, stuff=output_stuff)
output_stuff_content_items.append(output_stuff.content)
if sub_pipe_output_name in output_stuffs:
# TODO: check that at the blueprint / factory level
- raise PipeDefinitionError(
- f"PipeParallel requires unique output names for each parallel sub pipe, but {sub_pipe_output_name} is already used"
- )
+ msg = f"PipeParallel requires unique output names for each parallel sub pipe, but {sub_pipe_output_name} is already used"
+ raise PipeDefinitionError(msg)
output_stuffs[sub_pipe_output_name] = output_stuff
if sub_pipe_output_name in output_stuff_contents:
# TODO: check that at the blueprint / factory level
- raise PipeDefinitionError(
- f"PipeParallel requires unique output names for each parallel sub pipe, but {sub_pipe_output_name} is already used"
- )
+ msg = f"PipeParallel requires unique output names for each parallel sub pipe, but {sub_pipe_output_name} is already used"
+ raise PipeDefinitionError(msg)
output_stuff_contents[sub_pipe_output_name] = output_stuff.content
log.debug(f"PipeParallel '{self.code}': output_stuff_contents[{sub_pipe_output_name}]: {output_stuff_contents[sub_pipe_output_name]}")
@@ -220,28 +220,29 @@ class PipeParallel(PipeController):
job_metadata: JobMetadata,
working_memory: WorkingMemory,
pipe_run_params: PipeRunParams,
- output_name: Optional[str] = None,
+ output_name: str | None = None,
) -> PipeOutput:
- """
- Dry run implementation for PipeParallel.
+ """Dry run implementation for PipeParallel.
Validates that all required inputs are present and that all parallel sub-pipes can be dry run.
"""
log.debug(f"PipeParallel: dry run controller pipe: {self.code}")
if pipe_run_params.run_mode != PipeRunMode.DRY:
- raise PipeRunParamsError(f"PipeSequence._dry_run_controller_pipe() called with run_mode = {pipe_run_params.run_mode} in pipe {self.code}")
+ msg = f"PipeSequence._dry_run_controller_pipe() called with run_mode = {pipe_run_params.run_mode} in pipe {self.code}"
+ raise PipeRunParamsError(msg)
# 1. Validate that all required inputs are present in the working memory
needed_inputs = self.needed_inputs()
- missing_input_names: List[str] = []
+ missing_input_names: list[str] = []
for named_input_requirement in needed_inputs.named_input_requirements:
if not working_memory.get_optional_stuff(named_input_requirement.variable_name):
missing_input_names.append(named_input_requirement.variable_name)
if missing_input_names:
+ msg = f"Dry run failed: missing required inputs: {missing_input_names}"
log.error(f"Dry run failed: missing required inputs: {missing_input_names}")
raise DryRunError(
- message=f"Dry run failed for pipe '{self.code}' (PipeParallel): missing required inputs: {', '.join(missing_input_names)}",
+ message=msg,
missing_inputs=missing_input_names,
pipe_code=self.code,
)
@@ -251,10 +252,11 @@ class PipeParallel(PipeController):
try:
get_required_pipe(pipe_code=sub_pipe.pipe_code)
except Exception as exc:
- raise PipeDefinitionError(f"PipeParallel'{self.code}'sub-pipe '{sub_pipe.pipe_code}' not found") from exc
+ msg = f"PipeParallel'{self.code}'sub-pipe '{sub_pipe.pipe_code}' not found"
+ raise PipeDefinitionError(msg) from exc
# 3. Run all sub-pipes in dry mode
- tasks: List[Coroutine[Any, Any, PipeOutput]] = []
+ tasks: list[Coroutine[Any, Any, PipeOutput]] = []
for sub_pipe in self.parallel_sub_pipes:
tasks.append(
@@ -263,29 +265,31 @@ class PipeParallel(PipeController):
job_metadata=job_metadata,
working_memory=working_memory.make_deep_copy(),
sub_pipe_run_params=pipe_run_params.make_deep_copy(),
- )
+ ),
)
try:
pipe_outputs = await asyncio.gather(*tasks)
except Exception as exc:
- log.error(f"Dry run failed: parallel sub-pipe execution failed: {exc}")
+ msg = f"Dry run failed: parallel sub-pipe execution failed: {exc}"
+ log.error(msg)
raise DryRunError(
- message=f"Dry run failed for pipe '{self.code}' (PipeParallel): parallel sub-pipe execution failed: {exc}",
+ message=msg,
missing_inputs=[],
pipe_code=self.code,
- )
+ ) from exc
# 4. Process outputs as in the regular run
- output_stuffs: Dict[str, Stuff] = {}
- output_stuff_contents: Dict[str, StuffContent] = {}
+ output_stuffs: dict[str, Stuff] = {}
+ output_stuff_contents: dict[str, StuffContent] = {}
for output_index, pipe_output in enumerate(pipe_outputs):
output_stuff = pipe_output.main_stuff
sub_pipe_output_name = self.parallel_sub_pipes[output_index].output_name
if not sub_pipe_output_name:
+ msg = f"Dry run failed for pipe '{self.code}' (PipeParallel): sub-pipe output name not specified"
raise DryRunError(
- message=f"Dry run failed for pipe '{self.code}' (PipeParallel): sub-pipe output name not specified",
+ message=msg,
missing_inputs=[],
pipe_code=self.code,
)
@@ -294,8 +298,9 @@ class PipeParallel(PipeController):
working_memory.add_new_stuff(name=sub_pipe_output_name, stuff=output_stuff)
if sub_pipe_output_name in output_stuffs:
+ msg = f"Dry run failed for pipe '{self.code}' (PipeParallel): duplicate output name '{sub_pipe_output_name}'"
raise DryRunError(
- message=f"Dry run failed for pipe '{self.code}' (PipeParallel): duplicate output name '{sub_pipe_output_name}'",
+ message=msg,
missing_inputs=[],
pipe_code=self.code,
)
diff --git a/pipelex/pipe_controllers/parallel/pipe_parallel_blueprint.py b/pipelex/pipe_controllers/parallel/pipe_parallel_blueprint.py
index 1d810ddb4..f9f5599a0 100644
--- a/pipelex/pipe_controllers/parallel/pipe_parallel_blueprint.py
+++ b/pipelex/pipe_controllers/parallel/pipe_parallel_blueprint.py
@@ -1,4 +1,4 @@
-from typing import List, Literal, Optional
+from typing import Literal
from pydantic import field_validator
@@ -10,12 +10,13 @@ from pipelex.pipe_controllers.sub_pipe_blueprint import SubPipeBlueprint
class PipeParallelBlueprint(PipeBlueprint):
type: Literal["PipeParallel"] = "PipeParallel"
category: Literal["PipeController"] = "PipeController"
- parallels: List[SubPipeBlueprint]
+ parallels: list[SubPipeBlueprint]
add_each_output: bool = True
- combined_output: Optional[str] = None
+ combined_output: str | None = None
@field_validator("combined_output", mode="before")
- def validate_combined_output(cls, combined_output: str) -> str:
+ @staticmethod
+ def validate_combined_output(combined_output: str) -> str:
if combined_output:
ConceptBlueprint.validate_concept_string_or_code(concept_string_or_code=combined_output)
return combined_output
diff --git a/pipelex/pipe_controllers/parallel/pipe_parallel_factory.py b/pipelex/pipe_controllers/parallel/pipe_parallel_factory.py
index 012556b05..af322d650 100644
--- a/pipelex/pipe_controllers/parallel/pipe_parallel_factory.py
+++ b/pipelex/pipe_controllers/parallel/pipe_parallel_factory.py
@@ -1,4 +1,4 @@
-from typing import List, Optional
+from typing import TYPE_CHECKING
from typing_extensions import override
@@ -9,9 +9,11 @@ from pipelex.exceptions import PipeDefinitionError
from pipelex.hub import get_concept_provider
from pipelex.pipe_controllers.parallel.pipe_parallel import PipeParallel
from pipelex.pipe_controllers.parallel.pipe_parallel_blueprint import PipeParallelBlueprint
-from pipelex.pipe_controllers.sub_pipe import SubPipe
from pipelex.pipe_controllers.sub_pipe_factory import SubPipeFactory
+if TYPE_CHECKING:
+ from pipelex.pipe_controllers.sub_pipe import SubPipe
+
class PipeParallelFactory(PipeFactoryProtocol[PipeParallelBlueprint, PipeParallel]):
@classmethod
@@ -21,16 +23,18 @@ class PipeParallelFactory(PipeFactoryProtocol[PipeParallelBlueprint, PipeParalle
domain: str,
pipe_code: str,
blueprint: PipeParallelBlueprint,
- concept_codes_from_the_same_domain: Optional[List[str]] = None,
+ concept_codes_from_the_same_domain: list[str] | None = None,
) -> PipeParallel:
- parallel_sub_pipes: List[SubPipe] = []
+ parallel_sub_pipes: list[SubPipe] = []
for sub_pipe_blueprint in blueprint.parallels:
if not sub_pipe_blueprint.result:
- raise PipeDefinitionError("PipeParallel requires a result specified for each parallel sub pipe")
+ msg = "PipeParallel requires a result specified for each parallel sub pipe"
+ raise PipeDefinitionError(msg)
sub_pipe = SubPipeFactory.make_from_blueprint(sub_pipe_blueprint, concept_codes_from_the_same_domain=concept_codes_from_the_same_domain)
parallel_sub_pipes.append(sub_pipe)
if not blueprint.add_each_output and not blueprint.combined_output:
- raise PipeDefinitionError("PipeParallel requires either add_each_output or combined_output to be set")
+ msg = "PipeParallel requires either add_each_output or combined_output to be set"
+ raise PipeDefinitionError(msg)
if blueprint.combined_output:
combined_output_domain_and_code = ConceptFactory.make_domain_and_concept_code_from_concept_string_or_code(
@@ -40,8 +44,9 @@ class PipeParallelFactory(PipeFactoryProtocol[PipeParallelBlueprint, PipeParalle
)
combined_output = get_concept_provider().get_required_concept(
concept_string=ConceptFactory.construct_concept_string_with_domain(
- domain=combined_output_domain_and_code.domain, concept_code=combined_output_domain_and_code.concept_code
- )
+ domain=combined_output_domain_and_code.domain,
+ concept_code=combined_output_domain_and_code.concept_code,
+ ),
)
else:
combined_output = None
@@ -56,12 +61,15 @@ class PipeParallelFactory(PipeFactoryProtocol[PipeParallelBlueprint, PipeParalle
code=pipe_code,
definition=blueprint.definition,
inputs=PipeInputSpecFactory.make_from_blueprint(
- domain=domain, blueprint=blueprint.inputs or {}, concept_codes_from_the_same_domain=concept_codes_from_the_same_domain
+ domain=domain,
+ blueprint=blueprint.inputs or {},
+ concept_codes_from_the_same_domain=concept_codes_from_the_same_domain,
),
output=get_concept_provider().get_required_concept(
concept_string=ConceptFactory.construct_concept_string_with_domain(
- domain=output_domain_and_code.domain, concept_code=output_domain_and_code.concept_code
- )
+ domain=output_domain_and_code.domain,
+ concept_code=output_domain_and_code.concept_code,
+ ),
),
parallel_sub_pipes=parallel_sub_pipes,
add_each_output=blueprint.add_each_output,
diff --git a/pipelex/pipe_controllers/pipe_controller.py b/pipelex/pipe_controllers/pipe_controller.py
index 5cb192290..8211f7449 100644
--- a/pipelex/pipe_controllers/pipe_controller.py
+++ b/pipelex/pipe_controllers/pipe_controller.py
@@ -1,5 +1,5 @@
from abc import abstractmethod
-from typing import Literal, Optional
+from typing import Literal
from typing_extensions import override
@@ -20,8 +20,8 @@ class PipeController(PipeAbstract):
job_metadata: JobMetadata,
working_memory: WorkingMemory,
pipe_run_params: PipeRunParams,
- output_name: Optional[str] = None,
- print_intermediate_outputs: Optional[bool] = False,
+ output_name: str | None = None,
+ print_intermediate_outputs: bool | None = False,
) -> PipeOutput:
pipe_run_params.push_pipe_to_stack(pipe_code=self.code)
self.monitor_pipe_stack(pipe_run_params=pipe_run_params)
@@ -67,7 +67,7 @@ class PipeController(PipeAbstract):
job_metadata: JobMetadata,
working_memory: WorkingMemory,
pipe_run_params: PipeRunParams,
- output_name: Optional[str] = None,
+ output_name: str | None = None,
) -> PipeOutput:
pass
@@ -77,6 +77,6 @@ class PipeController(PipeAbstract):
job_metadata: JobMetadata,
working_memory: WorkingMemory,
pipe_run_params: PipeRunParams,
- output_name: Optional[str] = None,
+ output_name: str | None = None,
) -> PipeOutput:
pass
diff --git a/pipelex/pipe_controllers/sequence/__init__.py b/pipelex/pipe_controllers/sequence/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/pipelex/pipe_controllers/sequence/pipe_sequence.py b/pipelex/pipe_controllers/sequence/pipe_sequence.py
index 38e008ca1..7b6e157c6 100644
--- a/pipelex/pipe_controllers/sequence/pipe_sequence.py
+++ b/pipelex/pipe_controllers/sequence/pipe_sequence.py
@@ -1,7 +1,7 @@
-from typing import List, Literal, Optional, Set
+from typing import Literal
from pydantic import model_validator
-from typing_extensions import Self, override
+from typing_extensions import override
from pipelex import log
from pipelex.config import StaticValidationReaction, get_config
@@ -21,14 +21,15 @@ from pipelex.pipe_controllers.pipe_controller import PipeController
from pipelex.pipe_controllers.sequence.exceptions import PipeSequenceError
from pipelex.pipe_controllers.sub_pipe import SubPipe
from pipelex.pipeline.job_metadata import JobMetadata
+from pipelex.types import Self
class PipeSequence(PipeController):
type: Literal["PipeSequence"] = "PipeSequence"
- sequential_sub_pipes: List[SubPipe]
+ sequential_sub_pipes: list[SubPipe]
@override
- def needed_inputs(self, visited_pipes: Optional[Set[str]] = None) -> PipeInputSpec:
+ def needed_inputs(self, visited_pipes: set[str] | None = None) -> PipeInputSpec:
if visited_pipes is None:
visited_pipes = set()
@@ -40,33 +41,30 @@ class PipeSequence(PipeController):
visited_pipes_with_current = visited_pipes | {self.code}
needed_inputs = PipeInputSpecFactory.make_empty()
- generated_outputs: Set[str] = set()
+ generated_outputs: set[str] = set()
for sequential_sub_pipe in self.sequential_sub_pipes:
sub_pipe = get_required_pipe(pipe_code=sequential_sub_pipe.pipe_code)
# Use the centralized recursion detection
sub_pipe_needed_inputs = sub_pipe.needed_inputs(visited_pipes_with_current)
- if isinstance(sub_pipe, PipeParallel):
- if sub_pipe.add_each_output:
- for sub_parallel_pipe in sub_pipe.parallel_sub_pipes:
- if sub_pipe.add_each_output and sub_parallel_pipe.output_name:
- generated_outputs.add(sub_parallel_pipe.output_name)
- elif sub_parallel_pipe.output_name:
- generated_outputs.add(sub_parallel_pipe.output_name)
+ if isinstance(sub_pipe, PipeParallel) and sub_pipe.add_each_output:
+ for sub_parallel_pipe in sub_pipe.parallel_sub_pipes:
+ if (sub_pipe.add_each_output and sub_parallel_pipe.output_name) or sub_parallel_pipe.output_name:
+ generated_outputs.add(sub_parallel_pipe.output_name)
if sequential_sub_pipe.batch_params:
if sequential_sub_pipe.batch_params.input_list_stuff_name not in generated_outputs:
needed_inputs.add_requirement(
variable_name=sequential_sub_pipe.batch_params.input_list_stuff_name,
concept=sub_pipe_needed_inputs.get_required_input_requirement(
- variable_name=sequential_sub_pipe.batch_params.input_item_stuff_name
+ variable_name=sequential_sub_pipe.batch_params.input_item_stuff_name,
).concept,
multiplicity=True,
)
- for input_name, requirement in sub_pipe_needed_inputs.items:
- if input_name != sequential_sub_pipe.batch_params.input_item_stuff_name and input_name not in generated_outputs:
- needed_inputs.add_requirement(input_name, requirement.concept, requirement.multiplicity)
+ for input_name, requirement in sub_pipe_needed_inputs.items:
+ if input_name != sequential_sub_pipe.batch_params.input_item_stuff_name and input_name not in generated_outputs:
+ needed_inputs.add_requirement(input_name, requirement.concept, requirement.multiplicity)
else:
for input_name, requirement in sub_pipe_needed_inputs.items:
if input_name not in generated_outputs:
@@ -78,40 +76,38 @@ class PipeSequence(PipeController):
return needed_inputs
@override
- def required_variables(self) -> Set[str]:
+ def required_variables(self) -> set[str]:
return set()
@override
def validate_output(self):
- """
- Validate the output for the pipe sequence.
+ """Validate the output for the pipe sequence.
The output of the pipe sequence should match the output of the last step.
"""
last_step_output_pipe = get_required_pipe(pipe_code=self.sequential_sub_pipes[-1].pipe_code)
if self.output.concept_string != last_step_output_pipe.output.concept_string:
- raise PipeSequenceError(
+ msg = (
f"The output concept code '{self.output.concept_string}' of the pipe '{self.code}' is "
f"not matching the output concept code '{last_step_output_pipe.output.concept_string}' "
- f"of the last step '{self.sequential_sub_pipes[-1].pipe_code}'"
+ f"of the last step '{self.sequential_sub_pipes[-1].pipe_code}'",
)
+ raise PipeSequenceError(msg)
@model_validator(mode="after")
def validate_inputs(self) -> Self:
if len(self.sequential_sub_pipes) == 0:
- raise ValueError(f"Pipe'{self.code}'(PipeSequence) must have at least 1 step")
+ msg = f"Pipe'{self.code}'(PipeSequence) must have at least 1 step"
+ raise ValueError(msg)
return self
def _validate_output_multiplicity_support(self, pipe_run_params: PipeRunParams) -> None:
"""Validate that the pipe supports the requested output multiplicity."""
if pipe_run_params.is_multiple_output_required:
- raise PipeRunParamsError(
- f"{self.__class__.__name__} does not support multiple outputs, got output_multiplicity = {pipe_run_params.output_multiplicity}"
- )
+ msg = f"{self.__class__.__name__} does not support multiple outputs, got output_multiplicity = {pipe_run_params.output_multiplicity}"
+ raise PipeRunParamsError(msg)
def _validate_inputs(self):
- """
- Validate that the inputs declared for this PipeSequence match what is actually needed.
- """
+ """Validate that the inputs declared for this PipeSequence match what is actually needed."""
static_validation_config = get_config().pipelex.static_validation_config
default_reaction = static_validation_config.default_reaction
reactions = static_validation_config.reactions
@@ -159,15 +155,14 @@ class PipeSequence(PipeController):
@override
def validate_with_libraries(self):
- """
- Perform full validation after all libraries are loaded.
+ """Perform full validation after all libraries are loaded.
This is called after all pipes and concepts are available.
"""
self._validate_inputs()
@override
- def pipe_dependencies(self) -> Set[str]:
- return set(sub_pipe.pipe_code for sub_pipe in self.sequential_sub_pipes)
+ def pipe_dependencies(self) -> set[str]:
+ return {sub_pipe.pipe_code for sub_pipe in self.sequential_sub_pipes}
@override
async def _run_controller_pipe(
@@ -175,7 +170,7 @@ class PipeSequence(PipeController):
job_metadata: JobMetadata,
working_memory: WorkingMemory,
pipe_run_params: PipeRunParams,
- output_name: Optional[str] = None,
+ output_name: str | None = None,
) -> PipeOutput:
pipe_run_params.push_pipe_layer(pipe_code=self.code)
self._validate_output_multiplicity_support(pipe_run_params)
@@ -206,18 +201,20 @@ class PipeSequence(PipeController):
job_metadata: JobMetadata,
working_memory: WorkingMemory,
pipe_run_params: PipeRunParams,
- output_name: Optional[str] = None,
+ output_name: str | None = None,
) -> PipeOutput:
if pipe_run_params.run_mode != PipeRunMode.DRY:
- raise PipeRunParamsError(f"PipeSequence._dry_run_controller_pipe() called with run_mode = {pipe_run_params.run_mode} in pipe {self.code}")
+ msg = f"PipeSequence._dry_run_controller_pipe() called with run_mode = {pipe_run_params.run_mode} in pipe {self.code}"
+ raise PipeRunParamsError(msg)
log.debug(f"PipeSequence._dry_run_controller_pipe() called with {self.code=} {pipe_run_params=}")
# Verify the output of this pipe is matching the output of the last step.
concept_string_of_last_step = get_required_pipe(pipe_code=self.sequential_sub_pipes[-1].pipe_code).output.concept_string
if self.output.concept_string != concept_string_of_last_step:
- raise ValueError(
+ msg = (
f"The output concept code '{self.output.concept_string}' of the pipe '{self.code}' is "
f"not matching the output concept code '{concept_string_of_last_step}' of the last step '{self.sequential_sub_pipes[-1].pipe_code}'"
)
+ raise ValueError(msg)
return await self._run_controller_pipe(
job_metadata=job_metadata,
working_memory=working_memory,
diff --git a/pipelex/pipe_controllers/sequence/pipe_sequence_blueprint.py b/pipelex/pipe_controllers/sequence/pipe_sequence_blueprint.py
index f20fe2582..47fe17dda 100644
--- a/pipelex/pipe_controllers/sequence/pipe_sequence_blueprint.py
+++ b/pipelex/pipe_controllers/sequence/pipe_sequence_blueprint.py
@@ -1,4 +1,4 @@
-from typing import List, Literal
+from typing import Literal
from pipelex.core.pipes.pipe_blueprint import PipeBlueprint
from pipelex.pipe_controllers.sub_pipe_blueprint import SubPipeBlueprint
@@ -7,4 +7,4 @@ from pipelex.pipe_controllers.sub_pipe_blueprint import SubPipeBlueprint
class PipeSequenceBlueprint(PipeBlueprint):
type: Literal["PipeSequence"] = "PipeSequence"
category: Literal["PipeController"] = "PipeController"
- steps: List[SubPipeBlueprint]
+ steps: list[SubPipeBlueprint]
diff --git a/pipelex/pipe_controllers/sequence/pipe_sequence_factory.py b/pipelex/pipe_controllers/sequence/pipe_sequence_factory.py
index 7ecf86457..a91d084cd 100644
--- a/pipelex/pipe_controllers/sequence/pipe_sequence_factory.py
+++ b/pipelex/pipe_controllers/sequence/pipe_sequence_factory.py
@@ -1,5 +1,3 @@
-from typing import List, Optional
-
from typing_extensions import override
from pipelex.core.concepts.concept_factory import ConceptFactory
@@ -19,7 +17,7 @@ class PipeSequenceFactory(PipeFactoryProtocol[PipeSequenceBlueprint, PipeSequenc
domain: str,
pipe_code: str,
blueprint: PipeSequenceBlueprint,
- concept_codes_from_the_same_domain: Optional[List[str]] = None,
+ concept_codes_from_the_same_domain: list[str] | None = None,
) -> PipeSequence:
output_domain_and_code = ConceptFactory.make_domain_and_concept_code_from_concept_string_or_code(
domain=domain,
@@ -31,12 +29,15 @@ class PipeSequenceFactory(PipeFactoryProtocol[PipeSequenceBlueprint, PipeSequenc
code=pipe_code,
definition=blueprint.definition,
inputs=PipeInputSpecFactory.make_from_blueprint(
- domain=domain, blueprint=blueprint.inputs or {}, concept_codes_from_the_same_domain=concept_codes_from_the_same_domain
+ domain=domain,
+ blueprint=blueprint.inputs or {},
+ concept_codes_from_the_same_domain=concept_codes_from_the_same_domain,
),
output=get_concept_provider().get_required_concept(
concept_string=ConceptFactory.construct_concept_string_with_domain(
- domain=output_domain_and_code.domain, concept_code=output_domain_and_code.concept_code
- )
+ domain=output_domain_and_code.domain,
+ concept_code=output_domain_and_code.concept_code,
+ ),
),
sequential_sub_pipes=[
SubPipeFactory.make_from_blueprint(blueprint=step, concept_codes_from_the_same_domain=concept_codes_from_the_same_domain)
diff --git a/pipelex/pipe_controllers/sub_pipe.py b/pipelex/pipe_controllers/sub_pipe.py
index eb18970f0..1ddff16f8 100644
--- a/pipelex/pipe_controllers/sub_pipe.py
+++ b/pipelex/pipe_controllers/sub_pipe.py
@@ -1,5 +1,3 @@
-from typing import List, Optional
-
from pydantic import BaseModel
from pipelex import log
@@ -19,10 +17,10 @@ from pipelex.pipeline.job_metadata import JobMetadata
class SubPipe(BaseModel):
pipe_code: str
- output_name: Optional[str] = None
- output_multiplicity: Optional[PipeOutputMultiplicity] = None
- batch_params: Optional[BatchParams] = None
- concept_codes_from_the_same_domain: Optional[List[str]] = None
+ output_name: str | None = None
+ output_multiplicity: PipeOutputMultiplicity | None = None
+ batch_params: BatchParams | None = None
+ concept_codes_from_the_same_domain: list[str] | None = None
async def run_pipe(
self,
@@ -44,10 +42,11 @@ class SubPipe(BaseModel):
try:
working_memory.get_typed_object_or_attribute(name=batch_params.input_list_stuff_name, wanted_type=ListContent)
except WorkingMemoryStuffNotFoundError as exc:
- raise PipeInputError(
+ msg = (
f"Input list stuff named '{batch_params.input_list_stuff_name}' required by sub_pipe '{self.pipe_code}' "
f"of pipe '{calling_pipe_code}' not found in working memory: {exc}"
- ) from exc
+ )
+ raise PipeInputError(msg) from exc
pipe_batch_blueprint = PipeBatchBlueprint(
definition=f"Batch processing for {self.pipe_code}",
@@ -57,7 +56,7 @@ class SubPipe(BaseModel):
input_item_name=batch_params.input_item_stuff_name,
inputs={
batch_params.input_item_stuff_name: InputRequirementBlueprint(
- concept=sub_pipe.inputs.root[batch_params.input_item_stuff_name].concept.concept_string
+ concept=sub_pipe.inputs.root[batch_params.input_item_stuff_name].concept.concept_string,
),
},
)
@@ -111,14 +110,15 @@ class SubPipe(BaseModel):
# Case 3: Normal processing
required_variables = sub_pipe.required_variables()
log.debug(required_variables, title=f"Required variables for {self.pipe_code}")
- required_stuff_names = set([required_variable for required_variable in required_variables if not required_variable.startswith("_")])
+ required_stuff_names = {rv for rv in required_variables if not rv.startswith("_")}
try:
required_stuffs = working_memory.get_stuffs(names=required_stuff_names)
except WorkingMemoryStuffNotFoundError as exc:
- sub_pipe_path = sub_pipe_run_params.pipe_layers + [self.pipe_code]
+ sub_pipe_path = [*sub_pipe_run_params.pipe_layers, self.pipe_code]
sub_pipe_path_str = ".".join(sub_pipe_path)
error_details = f"SubPipe '{sub_pipe_path_str}', required_variables: {required_variables}, missing: '{exc.variable_name}'"
- raise PipeInputError(f"Some required stuff(s) not found: {error_details}") from exc
+ msg = f"Some required stuff(s) not found: {error_details}"
+ raise PipeInputError(msg) from exc
log.verbose(required_stuffs, title=f"Required stuffs for {self.pipe_code}")
# This is the only line that changes between run and dry_run
if sub_pipe_run_params.run_mode == PipeRunMode.DRY:
diff --git a/pipelex/pipe_controllers/sub_pipe_blueprint.py b/pipelex/pipe_controllers/sub_pipe_blueprint.py
index 256251d78..73ee2d1ac 100644
--- a/pipelex/pipe_controllers/sub_pipe_blueprint.py
+++ b/pipelex/pipe_controllers/sub_pipe_blueprint.py
@@ -1,26 +1,25 @@
-from typing import Optional, Union
-
from pydantic import BaseModel, ConfigDict, model_validator
-from typing_extensions import Self
from pipelex.exceptions import PipeDefinitionError
from pipelex.tools.typing.validation_utils import has_more_than_one_among_attributes_from_list
+from pipelex.types import Self
class SubPipeBlueprint(BaseModel):
model_config = ConfigDict(extra="forbid")
pipe: str
- result: Optional[str] = None
- nb_output: Optional[int] = None
- multiple_output: Optional[bool] = None
- batch_over: Union[bool, str] = False
- batch_as: Optional[str] = None
+ result: str | None = None
+ nb_output: int | None = None
+ multiple_output: bool | None = None
+ batch_over: bool | str = False
+ batch_as: str | None = None
@model_validator(mode="after")
def validate_multiple_output(self) -> Self:
if has_more_than_one_among_attributes_from_list(self, attributes_list=["nb_output", "multiple_output"]):
- raise PipeDefinitionError("PipeStepBlueprint should have no more than '1' of nb_output or multiple_output")
+ msg = "PipeStepBlueprint should have no more than '1' of nb_output or multiple_output"
+ raise PipeDefinitionError(msg)
return self
@model_validator(mode="after")
@@ -29,9 +28,11 @@ class SubPipeBlueprint(BaseModel):
batch_as_is_specified = self.batch_as is not None and self.batch_as != ""
if batch_over_is_specified and not batch_as_is_specified:
- raise PipeDefinitionError(f"In pipe '{self.pipe}': When 'batch_over' is specified, 'batch_as' must also be provided")
+ msg = f"In pipe '{self.pipe}': When 'batch_over' is specified, 'batch_as' must also be provided"
+ raise PipeDefinitionError(msg)
if batch_as_is_specified and not batch_over_is_specified:
- raise PipeDefinitionError(f"In pipe '{self.pipe}': When 'batch_as' is specified, 'batch_over' must also be provided")
+ msg = f"In pipe '{self.pipe}': When 'batch_as' is specified, 'batch_over' must also be provided"
+ raise PipeDefinitionError(msg)
return self
diff --git a/pipelex/pipe_controllers/sub_pipe_factory.py b/pipelex/pipe_controllers/sub_pipe_factory.py
index 39987175d..aa7a33e84 100644
--- a/pipelex/pipe_controllers/sub_pipe_factory.py
+++ b/pipelex/pipe_controllers/sub_pipe_factory.py
@@ -1,5 +1,3 @@
-from typing import List, Optional
-
from pipelex.core.pipes.pipe_run_params import BatchParams, make_output_multiplicity
from pipelex.pipe_controllers.sub_pipe import SubPipe
from pipelex.pipe_controllers.sub_pipe_blueprint import SubPipeBlueprint
@@ -10,7 +8,7 @@ class SubPipeFactory:
def make_from_blueprint(
cls,
blueprint: SubPipeBlueprint,
- concept_codes_from_the_same_domain: Optional[List[str]] = None,
+ concept_codes_from_the_same_domain: list[str] | None = None,
) -> SubPipe:
"""Create a SubPipe from a SubPipeBlueprint."""
output_multiplicity = make_output_multiplicity(
diff --git a/pipelex/pipe_operators/func/__init__.py b/pipelex/pipe_operators/func/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/pipelex/pipe_operators/func/pipe_func.py b/pipelex/pipe_operators/func/pipe_func.py
index ac65f2e0d..932925e3e 100644
--- a/pipelex/pipe_operators/func/pipe_func.py
+++ b/pipelex/pipe_operators/func/pipe_func.py
@@ -1,4 +1,4 @@
-from typing import List, Literal, Optional, Set, cast, get_type_hints
+from typing import Literal, cast, get_type_hints
from typing_extensions import override
@@ -26,11 +26,11 @@ class PipeFunc(PipeOperator[PipeFuncOutput]):
function_name: str
@override
- def required_variables(self) -> Set[str]:
+ def required_variables(self) -> set[str]:
return set()
@override
- def needed_inputs(self, visited_pipes: Optional[Set[str]] = None) -> PipeInputSpec:
+ def needed_inputs(self, visited_pipes: set[str] | None = None) -> PipeInputSpec:
return self.inputs
@override
@@ -43,25 +43,27 @@ class PipeFunc(PipeOperator[PipeFuncOutput]):
job_metadata: JobMetadata,
working_memory: WorkingMemory,
pipe_run_params: PipeRunParams,
- output_name: Optional[str] = None,
+ output_name: str | None = None,
) -> PipeFuncOutput:
log.debug(f"Applying function '{self.function_name}'")
function = func_registry.get_required_function(self.function_name)
if not callable(function):
- raise ValueError(f"Function '{self.function_name}' is not callable")
+ msg = f"Function '{self.function_name}' is not callable"
+ raise TypeError(msg)
func_output_object = await function(working_memory=working_memory)
the_content: StuffContent
if isinstance(func_output_object, StuffContent):
the_content = func_output_object
elif isinstance(func_output_object, list):
- func_result_list = cast(List[StuffContent], func_output_object)
+ func_result_list = cast("list[StuffContent]", func_output_object)
the_content = ListContent(items=func_result_list)
elif isinstance(func_output_object, str):
the_content = TextContent(text=func_output_object)
else:
- raise ValueError(f"Function '{self.function_name}' must return a StuffContent or a list, got {type(func_output_object)}")
+ msg = f"Function '{self.function_name}' must return a StuffContent or a list, got {type(func_output_object)}"
+ raise TypeError(msg)
output_stuff = StuffFactory.make_stuff(
name=output_name,
@@ -74,11 +76,10 @@ class PipeFunc(PipeOperator[PipeFuncOutput]):
name=output_name,
)
- pipe_output = PipeFuncOutput(
+ return PipeFuncOutput(
working_memory=working_memory,
pipeline_run_id=job_metadata.pipeline_run_id,
)
- return pipe_output
@override
async def _dry_run_operator_pipe(
@@ -86,46 +87,48 @@ class PipeFunc(PipeOperator[PipeFuncOutput]):
job_metadata: JobMetadata,
working_memory: WorkingMemory,
pipe_run_params: PipeRunParams,
- output_name: Optional[str] = None,
+ output_name: str | None = None,
) -> PipeFuncOutput:
log.debug(f"Dry run for PipeFunc '{self.function_name}'")
function = func_registry.get_required_function(self.function_name)
if not callable(function):
- raise ValueError(f"Function '{self.function_name}' is not callable")
+ msg = f"Function '{self.function_name}' is not callable"
+ raise TypeError(msg)
# Check that all needed inputs are present in working memory
needed_inputs = self.needed_inputs()
for input_name, _ in needed_inputs.items:
if input_name not in working_memory.root:
- raise DryRunError(
- f"Required input '{input_name}' not found in working memory for function '{self.function_name}' in pipe '{self.code}'"
- )
+ msg = f"Required input '{input_name}' not found in working memory for function '{self.function_name}' in pipe '{self.code}'"
+ raise DryRunError(msg)
try:
return_type = get_type_hints(function).get("return")
if return_type is None:
- raise DryRunError(f"Function '{self.function_name}' has no return type annotation")
- else:
- if not issubclass(return_type, StuffContent):
- raise DryRunError(f"Function '{self.function_name}' return type {return_type} is not a subclass of StuffContent")
-
- requirement = TypedNamedInputRequirement(
- variable_name="mock_output",
- concept=ConceptFactory.make(
- concept_code=self.output.code,
- domain="generic",
- definition="Lorem Ipsum",
- structure_class_name=self.output.structure_class_name,
- ),
- structure_class=return_type,
- multiplicity=False,
- )
- mock_content = WorkingMemoryFactory.create_mock_content(requirement)
+ msg = f"Function '{self.function_name}' has no return type annotation"
+ raise DryRunError(msg)
+ if not issubclass(return_type, StuffContent):
+ msg = f"Function '{self.function_name}' return type {return_type} is not a subclass of StuffContent"
+ raise DryRunError(msg)
+
+ requirement = TypedNamedInputRequirement(
+ variable_name="mock_output",
+ concept=ConceptFactory.make(
+ concept_code=self.output.code,
+ domain="generic",
+ definition="Lorem Ipsum",
+ structure_class_name=self.output.structure_class_name,
+ ),
+ structure_class=return_type,
+ multiplicity=False,
+ )
+ mock_content = WorkingMemoryFactory.create_mock_content(requirement)
except Exception as exc:
- raise DryRunError(f"Failed to get type hints for function '{self.function_name}' in pipe '{self.code}': {exc}")
+ msg = f"Failed to get type hints for function '{self.function_name}' in pipe '{self.code}': {exc}"
+ raise DryRunError(msg) from exc
output_stuff = StuffFactory.make_stuff(
name=output_name,
diff --git a/pipelex/pipe_operators/func/pipe_func_factory.py b/pipelex/pipe_operators/func/pipe_func_factory.py
index 773ff4e64..685423dc6 100644
--- a/pipelex/pipe_operators/func/pipe_func_factory.py
+++ b/pipelex/pipe_operators/func/pipe_func_factory.py
@@ -1,5 +1,3 @@
-from typing import List, Optional
-
from typing_extensions import override
from pipelex.core.concepts.concept_factory import ConceptFactory
@@ -18,7 +16,7 @@ class PipeFuncFactory(PipeFactoryProtocol[PipeFuncBlueprint, PipeFunc]):
domain: str,
pipe_code: str,
blueprint: PipeFuncBlueprint,
- concept_codes_from_the_same_domain: Optional[List[str]] = None,
+ concept_codes_from_the_same_domain: list[str] | None = None,
) -> PipeFunc:
output_domain_and_code = ConceptFactory.make_domain_and_concept_code_from_concept_string_or_code(
domain=domain,
@@ -30,12 +28,15 @@ class PipeFuncFactory(PipeFactoryProtocol[PipeFuncBlueprint, PipeFunc]):
code=pipe_code,
definition=blueprint.definition,
inputs=PipeInputSpecFactory.make_from_blueprint(
- domain=domain, blueprint=blueprint.inputs or {}, concept_codes_from_the_same_domain=concept_codes_from_the_same_domain
+ domain=domain,
+ blueprint=blueprint.inputs or {},
+ concept_codes_from_the_same_domain=concept_codes_from_the_same_domain,
),
output=get_concept_provider().get_required_concept(
concept_string=ConceptFactory.construct_concept_string_with_domain(
- domain=output_domain_and_code.domain, concept_code=output_domain_and_code.concept_code
- )
+ domain=output_domain_and_code.domain,
+ concept_code=output_domain_and_code.concept_code,
+ ),
),
function_name=blueprint.function_name,
)
diff --git a/pipelex/pipe_operators/img_gen/__init__.py b/pipelex/pipe_operators/img_gen/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/pipelex/pipe_operators/img_gen/pipe_img_gen.py b/pipelex/pipe_operators/img_gen/pipe_img_gen.py
index 5ec4f3f1c..be9036289 100644
--- a/pipelex/pipe_operators/img_gen/pipe_img_gen.py
+++ b/pipelex/pipe_operators/img_gen/pipe_img_gen.py
@@ -1,7 +1,7 @@
-from typing import List, Literal, Optional, Set, Union
+from typing import Literal
from pydantic import Field, field_validator, model_validator
-from typing_extensions import Self, override
+from typing_extensions import override
from pipelex import log
from pipelex.cogt.content_generation.content_generator_dry import ContentGeneratorDry
@@ -33,12 +33,13 @@ from pipelex.exceptions import (
from pipelex.hub import get_concept_provider, get_content_generator, get_model_deck
from pipelex.pipe_operators.pipe_operator import PipeOperator
from pipelex.pipeline.job_metadata import JobMetadata
+from pipelex.types import Self
class PipeImgGenOutput(PipeOutput):
@property
- def image_urls(self) -> List[str]:
- the_urls: List[str] = []
+ def image_urls(self) -> list[str]:
+ the_urls: list[str] = []
content = self.main_stuff.content
if isinstance(content, ListContent):
items = self.main_stuff_as_items(item_type=ImageContent)
@@ -46,7 +47,8 @@ class PipeImgGenOutput(PipeOutput):
elif isinstance(content, ImageContent):
the_urls = [content.url]
else:
- raise PipeRunParamsError(f"PipeImgGen output should be a ListContent or an ImageContent, got {type(content)}")
+ msg = f"PipeImgGen output should be a ListContent or an ImageContent, got {type(content)}"
+ raise PipeRunParamsError(msg)
return the_urls
@@ -55,33 +57,34 @@ DEFAULT_PROMPT_VAR_NAME = "prompt"
class PipeImgGen(PipeOperator[PipeImgGenOutput]):
type: Literal["PipeImgGen"] = "PipeImgGen"
- img_gen_prompt: Optional[str] = None
- img_gen_prompt_var_name: Optional[str] = None
+ img_gen_prompt: str | None = None
+ img_gen_prompt_var_name: str | None = None
# New ImgGenChoice pattern (like LLM)
- img_gen: Optional[ImgGenChoice] = None
+ img_gen: ImgGenChoice | None = None
# Legacy individual settings (for backwards compatibility)
- img_gen_handle: Optional[str] = None
- quality: Optional[Quality] = Field(default=None, strict=False)
- nb_steps: Optional[int] = Field(default=None, gt=0)
- guidance_scale: Optional[float] = Field(default=None, gt=0)
- is_moderated: Optional[bool] = None
- safety_tolerance: Optional[int] = Field(default=None, ge=1, le=6)
+ img_gen_handle: str | None = None
+ quality: Quality | None = Field(default=None, strict=False)
+ nb_steps: int | None = Field(default=None, gt=0)
+ guidance_scale: float | None = Field(default=None, gt=0)
+ is_moderated: bool | None = None
+ safety_tolerance: int | None = Field(default=None, ge=1, le=6)
# One-time settings (not in ImgGenSetting)
- aspect_ratio: Optional[AspectRatio] = Field(default=None, strict=False)
- is_raw: Optional[bool] = None
- seed: Optional[Union[int, Literal["auto"]]] = None
- background: Optional[Background] = Field(default=None, strict=False)
- output_format: Optional[OutputFormat] = Field(default=None, strict=False)
+ aspect_ratio: AspectRatio | None = Field(default=None, strict=False)
+ is_raw: bool | None = None
+ seed: int | Literal["auto"] | None = None
+ background: Background | None = Field(default=None, strict=False)
+ output_format: OutputFormat | None = Field(default=None, strict=False)
output_multiplicity: PipeOutputMultiplicity
@field_validator("img_gen_prompt_var_name")
@classmethod
- def validate_input_var_name_not_provided_as_attribute(cls, value: Optional[str]) -> Optional[str]:
+ def validate_input_var_name_not_provided_as_attribute(cls, value: str | None) -> str | None:
if value is not None:
- raise PipeDefinitionError("img_gen_prompt_var_name must be None before input validation")
+ msg = "img_gen_prompt_var_name must be None before input validation"
+ raise PipeDefinitionError(msg)
return value
@model_validator(mode="after")
@@ -102,13 +105,14 @@ class PipeImgGen(PipeOperator[PipeImgGenOutput]):
wanted_concept=get_concept_provider().get_native_concept(native_concept=NativeConceptEnum.IMAGE),
strict=True,
):
- raise PipeDefinitionError(
+ msg = (
f"The output of a ImgGen pipe must be compatible with the Image concept. "
f"In the pipe '{self.code}' the output is '{self.output.concept_string}'"
)
+ raise PipeDefinitionError(msg)
@override
- def needed_inputs(self, visited_pipes: Optional[Set[str]] = None) -> PipeInputSpec:
+ def needed_inputs(self, visited_pipes: set[str] | None = None) -> PipeInputSpec:
needed_inputs = PipeInputSpecFactory.make_empty()
if self.img_gen_prompt:
needed_inputs.add_requirement(
@@ -123,11 +127,10 @@ class PipeImgGen(PipeOperator[PipeImgGenOutput]):
return needed_inputs
@override
- def required_variables(self) -> Set[str]:
+ def required_variables(self) -> set[str]:
if self.img_gen_prompt_var_name:
return {self.img_gen_prompt_var_name}
- else:
- return {DEFAULT_PROMPT_VAR_NAME}
+ return {DEFAULT_PROMPT_VAR_NAME}
def _validate_inputs(self):
concept_provider = get_concept_provider()
@@ -137,12 +140,12 @@ class PipeImgGen(PipeOperator[PipeImgGenOutput]):
# check that we have either an img_gen_prompt passed as attribute or as a single text input
if self.img_gen_prompt:
if self.inputs.items:
- raise PipeDefinitionError("img_gen_prompt_var_name must be None if img_gen_prompt is provided")
- else:
- # we're good with the prompt provided as attribute
- return
+ msg = "img_gen_prompt_var_name must be None if img_gen_prompt is provided"
+ raise PipeDefinitionError(msg)
+ # we're good with the prompt provided as attribute
+ return
- candidate_prompt_var_names: List[str] = []
+ candidate_prompt_var_names: list[str] = []
for input_name, requirement in self.inputs.items:
log.debug(f"Validating input '{input_name}' with concept code '{requirement.concept.code}'")
if concept_provider.is_compatible(
@@ -205,8 +208,8 @@ class PipeImgGen(PipeOperator[PipeImgGenOutput]):
job_metadata: JobMetadata,
working_memory: WorkingMemory,
pipe_run_params: PipeRunParams,
- output_name: Optional[str] = None,
- content_generator: Optional[ContentGeneratorProtocol] = None,
+ output_name: str | None = None,
+ content_generator: ContentGeneratorProtocol | None = None,
) -> PipeImgGenOutput:
content_generator = content_generator or get_content_generator()
@@ -223,9 +226,11 @@ class PipeImgGen(PipeOperator[PipeImgGenOutput]):
try:
img_gen_prompt_text = working_memory.get_stuff_as_str(stuff_name)
except WorkingMemoryStuffNotFoundError as exc:
- raise PipeInputError(f"Could not find a valid user image named '{stuff_name}' in the working_memory: {exc}") from exc
+ msg = f"Could not find a valid user image named '{stuff_name}' in the working_memory: {exc}"
+ raise PipeInputError(msg) from exc
else:
- raise UnexpectedPipeDefinitionError("You must provide an image gen prompt either as attribute of the pipe or as a single text input")
+ msg = "You must provide an image gen prompt either as attribute of the pipe or as a single text input"
+ raise UnexpectedPipeDefinitionError(msg)
img_gen_config = get_config().cogt.img_gen_config
img_gen_param_defaults = img_gen_config.img_gen_param_defaults
@@ -252,13 +257,13 @@ class PipeImgGen(PipeOperator[PipeImgGenOutput]):
# Process one-time settings
seed_setting = self.seed or img_gen_param_defaults.seed
- seed: Optional[int]
+ seed: int | None
if isinstance(seed_setting, str) and seed_setting == "auto":
seed = None
else:
seed = seed_setting
- # Build ImggJobParams from ImgGenSetting + one-time settings
+ # Build ImgGenJobParams from ImgGenSetting + one-time settings
img_gen_job_params = ImgGenJobParams(
aspect_ratio=self.aspect_ratio or img_gen_param_defaults.aspect_ratio,
background=self.background or img_gen_param_defaults.background,
@@ -276,7 +281,7 @@ class PipeImgGen(PipeOperator[PipeImgGenOutput]):
log.debug(f"Using img_gen handle: {img_gen_handle}")
the_content: StuffContent
- image_urls: List[str] = []
+ image_urls: list[str] = []
nb_images: int
if isinstance(applied_output_multiplicity, bool):
if self.output_multiplicity:
@@ -285,8 +290,7 @@ class PipeImgGen(PipeOperator[PipeImgGenOutput]):
msg += f" and pipe_run_params.output_multiplicity = {pipe_run_params.output_multiplicity}."
msg += f" Tried to apply applied_output_multiplicity = {applied_output_multiplicity}."
raise PipeRunParamsError(msg)
- else:
- nb_images = 1
+ nb_images = 1
elif isinstance(applied_output_multiplicity, int):
nb_images = applied_output_multiplicity
else:
@@ -303,13 +307,13 @@ class PipeImgGen(PipeOperator[PipeImgGenOutput]):
img_gen_job_params=img_gen_job_params,
img_gen_job_config=img_gen_config.img_gen_job_config,
)
- image_content_items: List[StuffContent] = []
+ image_content_items: list[StuffContent] = []
for generated_image in generated_image_list:
image_content_items.append(
ImageContent(
url=generated_image.url,
source_prompt=img_gen_prompt_text,
- )
+ ),
)
image_urls.append(generated_image.url)
the_content = ListContent(
@@ -347,11 +351,10 @@ class PipeImgGen(PipeOperator[PipeImgGenOutput]):
name=output_name,
)
- pipe_output = PipeImgGenOutput(
+ return PipeImgGenOutput(
working_memory=working_memory,
pipeline_run_id=job_metadata.pipeline_run_id,
)
- return pipe_output
@override
async def _dry_run_operator_pipe(
@@ -359,15 +362,14 @@ class PipeImgGen(PipeOperator[PipeImgGenOutput]):
job_metadata: JobMetadata,
working_memory: WorkingMemory,
pipe_run_params: PipeRunParams,
- output_name: Optional[str] = None,
+ output_name: str | None = None,
) -> PipeImgGenOutput:
log.debug(f"PipeImgGen: dry run operator pipe: {self.code}")
content_generator_dry = ContentGeneratorDry()
- pipe_output = await self._run_operator_pipe(
+ return await self._run_operator_pipe(
job_metadata=job_metadata,
working_memory=working_memory,
pipe_run_params=pipe_run_params or PipeRunParamsFactory.make_run_params(pipe_run_mode=PipeRunMode.DRY),
output_name=output_name,
content_generator=content_generator_dry,
)
- return pipe_output
diff --git a/pipelex/pipe_operators/img_gen/pipe_img_gen_blueprint.py b/pipelex/pipe_operators/img_gen/pipe_img_gen_blueprint.py
index 27e39ba0d..938ce84d4 100644
--- a/pipelex/pipe_operators/img_gen/pipe_img_gen_blueprint.py
+++ b/pipelex/pipe_operators/img_gen/pipe_img_gen_blueprint.py
@@ -1,4 +1,4 @@
-from typing import Literal, Optional, Union
+from typing import Literal
from pydantic import Field
@@ -10,16 +10,16 @@ from pipelex.core.pipes.pipe_blueprint import PipeBlueprint
class PipeImgGenBlueprint(PipeBlueprint):
type: Literal["PipeImgGen"] = "PipeImgGen"
category: Literal["PipeOperator"] = "PipeOperator"
- img_gen_prompt: Optional[str] = None
- img_gen_prompt_var_name: Optional[str] = None
+ img_gen_prompt: str | None = None
+ img_gen_prompt_var_name: str | None = None
# New ImgGenChoice pattern (like LLM)
- img_gen: Optional[ImgGenChoice] = None
+ img_gen: ImgGenChoice | None = None
# One-time settings (not in ImgGenSetting)
- aspect_ratio: Optional[AspectRatio] = Field(default=None, strict=False)
- is_raw: Optional[bool] = None
- seed: Optional[Union[int, Literal["auto"]]] = None
- nb_output: Optional[int] = Field(default=None, ge=1)
- background: Optional[Background] = Field(default=None, strict=False)
- output_format: Optional[OutputFormat] = Field(default=None, strict=False)
+ aspect_ratio: AspectRatio | None = Field(default=None, strict=False)
+ is_raw: bool | None = None
+ seed: int | Literal["auto"] | None = None
+ nb_output: int | None = Field(default=None, ge=1)
+ background: Background | None = Field(default=None, strict=False)
+ output_format: OutputFormat | None = Field(default=None, strict=False)
diff --git a/pipelex/pipe_operators/img_gen/pipe_img_gen_factory.py b/pipelex/pipe_operators/img_gen/pipe_img_gen_factory.py
index ad09adc6a..de36dde65 100644
--- a/pipelex/pipe_operators/img_gen/pipe_img_gen_factory.py
+++ b/pipelex/pipe_operators/img_gen/pipe_img_gen_factory.py
@@ -1,5 +1,3 @@
-from typing import List, Optional
-
from typing_extensions import override
from pipelex.core.concepts.concept_factory import ConceptFactory
@@ -18,7 +16,7 @@ class PipeImgGenFactory(PipeFactoryProtocol[PipeImgGenBlueprint, PipeImgGen]):
domain: str,
pipe_code: str,
blueprint: PipeImgGenBlueprint,
- concept_codes_from_the_same_domain: Optional[List[str]] = None,
+ concept_codes_from_the_same_domain: list[str] | None = None,
) -> PipeImgGen:
output_domain_and_code = ConceptFactory.make_domain_and_concept_code_from_concept_string_or_code(
domain=domain,
@@ -30,12 +28,15 @@ class PipeImgGenFactory(PipeFactoryProtocol[PipeImgGenBlueprint, PipeImgGen]):
code=pipe_code,
definition=blueprint.definition,
inputs=PipeInputSpecFactory.make_from_blueprint(
- domain=domain, blueprint=blueprint.inputs or {}, concept_codes_from_the_same_domain=concept_codes_from_the_same_domain
+ domain=domain,
+ blueprint=blueprint.inputs or {},
+ concept_codes_from_the_same_domain=concept_codes_from_the_same_domain,
),
output=get_concept_provider().get_required_concept(
concept_string=ConceptFactory.construct_concept_string_with_domain(
- domain=output_domain_and_code.domain, concept_code=output_domain_and_code.concept_code
- )
+ domain=output_domain_and_code.domain,
+ concept_code=output_domain_and_code.concept_code,
+ ),
),
output_multiplicity=blueprint.nb_output or 1,
img_gen_prompt=blueprint.img_gen_prompt,
diff --git a/pipelex/pipe_operators/jinja2/__init__.py b/pipelex/pipe_operators/jinja2/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/pipelex/pipe_operators/jinja2/pipe_jinja2.py b/pipelex/pipe_operators/jinja2/pipe_jinja2.py
index a37ab4174..342b4a74d 100644
--- a/pipelex/pipe_operators/jinja2/pipe_jinja2.py
+++ b/pipelex/pipe_operators/jinja2/pipe_jinja2.py
@@ -1,8 +1,8 @@
-from typing import Any, ClassVar, Dict, Literal, Optional, Set
+from typing import Any, ClassVar, Literal
from jinja2 import TemplateSyntaxError
from pydantic import ConfigDict, model_validator
-from typing_extensions import Self, override
+from typing_extensions import override
from pipelex import log
from pipelex.cogt.content_generation.content_generator_dry import ContentGeneratorDry
@@ -29,6 +29,7 @@ from pipelex.tools.templating.jinja2_required_variables import detect_jinja2_req
from pipelex.tools.templating.jinja2_template_category import Jinja2TemplateCategory
from pipelex.tools.templating.templating_models import PromptingStyle
from pipelex.tools.typing.validation_utils import has_exactly_one_among_attributes_from_list
+from pipelex.types import Self
class PipeJinja2Output(PipeOutput):
@@ -46,21 +47,23 @@ class PipeJinja2(PipeOperator[PipeJinja2Output]):
native_concept_data=NATIVE_CONCEPTS_DATA[NativeConceptEnum.TEXT],
)
- jinja2_name: Optional[str] = None
- jinja2: Optional[str] = None
- prompting_style: Optional[PromptingStyle] = None
+ jinja2_name: str | None = None
+ jinja2: str | None = None
+ prompting_style: PromptingStyle | None = None
template_category: Jinja2TemplateCategory = Jinja2TemplateCategory.LLM_PROMPT
- extra_context: Optional[Dict[str, Any]] = None
+ extra_context: dict[str, Any] | None = None
@model_validator(mode="after")
def validate_jinja2(self) -> Self:
if not has_exactly_one_among_attributes_from_list(self, attributes_list=["jinja2_name", "jinja2"]):
- raise PipeDefinitionError("PipeJinja2 should have exactly one of jinja2_name or jinja2")
+ msg = "PipeJinja2 should have exactly one of jinja2_name or jinja2"
+ raise PipeDefinitionError(msg)
if self.jinja2:
try:
check_jinja2_parsing(jinja2_template_source=self.jinja2, template_category=self.template_category)
except TemplateSyntaxError as exc:
- raise Jinja2TemplateError(f"Could not parse Jinja2 template included in PipeJinja2: {exc}") from exc
+ msg = f"Could not parse Jinja2 template included in PipeJinja2: {exc}"
+ raise Jinja2TemplateError(msg) from exc
return self
@model_validator(mode="after")
@@ -73,7 +76,8 @@ class PipeJinja2(PipeOperator[PipeJinja2Output]):
required_variables = self.required_variables()
for required_variable_name in required_variables:
if required_variable_name not in self.inputs.variables:
- raise PipeDefinitionError(f"Required variable '{required_variable_name}' is not in the inputs of pipe {self.code}")
+ msg = f"Required variable '{required_variable_name}' is not in the inputs of pipe {self.code}"
+ raise PipeDefinitionError(msg)
return self
@override
@@ -87,7 +91,7 @@ class PipeJinja2(PipeOperator[PipeJinja2Output]):
log.debug(f"Validated jinja2 template '{self.jinja2_name}':\n{the_template}")
@override
- def needed_inputs(self, visited_pipes: Optional[Set[str]] = None) -> PipeInputSpec:
+ def needed_inputs(self, visited_pipes: set[str] | None = None) -> PipeInputSpec:
needed_inputs = PipeInputSpecFactory.make_empty()
for input_name, requirement in self.inputs.root.items():
needed_inputs.add_requirement(variable_name=input_name, concept=requirement.concept)
@@ -103,7 +107,7 @@ class PipeJinja2(PipeOperator[PipeJinja2Output]):
return "Jinja2 template not defined"
@override
- def required_variables(self) -> Set[str]:
+ def required_variables(self) -> set[str]:
required_variables = detect_jinja2_required_variables(
template_category=self.template_category,
template_provider=get_template_provider(),
@@ -113,7 +117,7 @@ class PipeJinja2(PipeOperator[PipeJinja2Output]):
return {
variable_name
for variable_name in required_variables
- if not variable_name.startswith("_") and variable_name != "preliminary_text" and variable_name != "place_holder"
+ if not variable_name.startswith("_") and variable_name not in ("preliminary_text", "place_holder")
}
@override
@@ -122,16 +126,15 @@ class PipeJinja2(PipeOperator[PipeJinja2Output]):
job_metadata: JobMetadata,
working_memory: WorkingMemory,
pipe_run_params: PipeRunParams,
- output_name: Optional[str] = None,
- content_generator: Optional[ContentGeneratorProtocol] = None,
+ output_name: str | None = None,
+ content_generator: ContentGeneratorProtocol | None = None,
) -> PipeJinja2Output:
content_generator = content_generator or get_content_generator()
if pipe_run_params.is_multiple_output_required:
- raise PipeRunParamsError(
- f"PipeJinja2 does not suppport multiple outputs, got output_multiplicity = {pipe_run_params.output_multiplicity}"
- )
+ msg = f"PipeJinja2 does not suppport multiple outputs, got output_multiplicity = {pipe_run_params.output_multiplicity}"
+ raise PipeRunParamsError(msg)
- context: Dict[str, Any] = working_memory.generate_context()
+ context: dict[str, Any] = working_memory.generate_context()
if pipe_run_params:
context.update(**pipe_run_params.params)
if self.extra_context:
@@ -155,20 +158,18 @@ class PipeJinja2(PipeOperator[PipeJinja2Output]):
name=output_name,
)
- pipe_output = PipeJinja2Output(
+ return PipeJinja2Output(
working_memory=working_memory,
pipeline_run_id=job_metadata.pipeline_run_id,
)
- return pipe_output
-
@override
async def _dry_run_operator_pipe(
self,
job_metadata: JobMetadata,
working_memory: WorkingMemory,
pipe_run_params: PipeRunParams,
- output_name: Optional[str] = None,
+ output_name: str | None = None,
) -> PipeJinja2Output:
content_generator_used: ContentGeneratorProtocol
if get_config().pipelex.dry_run_config.apply_to_jinja2_rendering:
@@ -178,11 +179,10 @@ class PipeJinja2(PipeOperator[PipeJinja2Output]):
log.debug(f"PipeJinja2: using regular operator pipe for jinja2 rendering (dry run not applied to jinja2): {self.code}")
content_generator_used = get_content_generator()
- pipe_output = await self._run_operator_pipe(
+ return await self._run_operator_pipe(
job_metadata=job_metadata,
working_memory=working_memory,
pipe_run_params=pipe_run_params or PipeRunParamsFactory.make_run_params(pipe_run_mode=PipeRunMode.DRY),
output_name=output_name,
content_generator=content_generator_used,
)
- return pipe_output
diff --git a/pipelex/pipe_operators/jinja2/pipe_jinja2_factory.py b/pipelex/pipe_operators/jinja2/pipe_jinja2_factory.py
index e0b26a6e9..2ce9d7436 100644
--- a/pipelex/pipe_operators/jinja2/pipe_jinja2_factory.py
+++ b/pipelex/pipe_operators/jinja2/pipe_jinja2_factory.py
@@ -1,5 +1,3 @@
-from typing import List, Optional
-
from typing_extensions import override
from pipelex.core.concepts.concept_factory import ConceptFactory
@@ -23,9 +21,9 @@ class PipeJinja2Factory(PipeFactoryProtocol[PipeJinja2Blueprint, PipeJinja2]):
domain: str,
pipe_code: str,
blueprint: PipeJinja2Blueprint,
- concept_codes_from_the_same_domain: Optional[List[str]] = None,
+ concept_codes_from_the_same_domain: list[str] | None = None,
) -> PipeJinja2:
- preprocessed_template: Optional[str] = None
+ preprocessed_template: str | None = None
if blueprint.jinja2:
preprocessed_template = preprocess_template(blueprint.jinja2)
check_jinja2_parsing(
@@ -45,12 +43,15 @@ class PipeJinja2Factory(PipeFactoryProtocol[PipeJinja2Blueprint, PipeJinja2]):
code=pipe_code,
definition=blueprint.definition,
inputs=PipeInputSpecFactory.make_from_blueprint(
- domain=domain, blueprint=blueprint.inputs or {}, concept_codes_from_the_same_domain=concept_codes_from_the_same_domain
+ domain=domain,
+ blueprint=blueprint.inputs or {},
+ concept_codes_from_the_same_domain=concept_codes_from_the_same_domain,
),
output=get_concept_provider().get_required_concept(
concept_string=ConceptFactory.construct_concept_string_with_domain(
- domain=output_domain_and_code.domain, concept_code=output_domain_and_code.concept_code
- )
+ domain=output_domain_and_code.domain,
+ concept_code=output_domain_and_code.concept_code,
+ ),
),
jinja2_name=blueprint.jinja2_name,
jinja2=preprocessed_template,
@@ -63,9 +64,9 @@ class PipeJinja2Factory(PipeFactoryProtocol[PipeJinja2Blueprint, PipeJinja2]):
def make_pipe_jinja2_from_template_str(
cls,
domain: str,
- inputs: Optional[PipeInputSpec] = None,
- template_str: Optional[str] = None,
- template_name: Optional[str] = None,
+ inputs: PipeInputSpec | None = None,
+ template_str: str | None = None,
+ template_name: str | None = None,
) -> PipeJinja2:
if template_str:
preprocessed_template = preprocess_template(template_str)
@@ -79,12 +80,12 @@ class PipeJinja2Factory(PipeFactoryProtocol[PipeJinja2Blueprint, PipeJinja2]):
jinja2=preprocessed_template,
inputs=inputs or PipeInputSpecFactory.make_empty(),
)
- elif template_name:
+ if template_name:
return PipeJinja2(
domain=domain,
code="adhoc_pipe_jinja2_from_template_name",
jinja2_name=template_name,
inputs=inputs or PipeInputSpecFactory.make_empty(),
)
- else:
- raise PipeDefinitionError("Either template_str or template_name must be provided to make_pipe_jinja2_from_template_str")
+ msg = "Either template_str or template_name must be provided to make_pipe_jinja2_from_template_str"
+ raise PipeDefinitionError(msg)
diff --git a/pipelex/pipe_operators/llm/__init__.py b/pipelex/pipe_operators/llm/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/pipelex/pipe_operators/llm/pipe_llm.py b/pipelex/pipe_operators/llm/pipe_llm.py
index 64c892878..3204c4503 100644
--- a/pipelex/pipe_operators/llm/pipe_llm.py
+++ b/pipelex/pipe_operators/llm/pipe_llm.py
@@ -1,7 +1,7 @@
-from typing import List, Literal, Optional, Set, Type, cast
+from typing import Literal, cast
from pydantic import model_validator
-from typing_extensions import Self, override
+from typing_extensions import override
from pipelex import log
from pipelex.cogt.content_generation.content_generator_dry import ContentGeneratorDry
@@ -50,6 +50,7 @@ from pipelex.pipe_operators.llm.pipe_llm_blueprint import StructuringMethod
from pipelex.pipe_operators.pipe_operator import PipeOperator
from pipelex.pipeline.job_metadata import JobMetadata
from pipelex.tools.typing.structure_printer import StructurePrinter
+from pipelex.types import Self
class PipeLLMOutput(PipeOutput):
@@ -59,11 +60,11 @@ class PipeLLMOutput(PipeOutput):
class PipeLLM(PipeOperator[PipeLLMOutput]):
type: Literal["PipeLLM"] = "PipeLLM"
llm_prompt_spec: LLMPromptSpec
- llm_choices: Optional[LLMSettingChoices] = None
- structuring_method: Optional[StructuringMethod] = None
- prompt_template_to_structure: Optional[str] = None
- system_prompt_to_structure: Optional[str] = None
- output_multiplicity: Optional[PipeOutputMultiplicity] = None
+ llm_choices: LLMSettingChoices | None = None
+ structuring_method: StructuringMethod | None = None
+ prompt_template_to_structure: str | None = None
+ system_prompt_to_structure: str | None = None
+ output_multiplicity: PipeOutputMultiplicity | None = None
@model_validator(mode="after")
def validate_inputs(self) -> Self:
@@ -73,12 +74,12 @@ class PipeLLM(PipeOperator[PipeLLMOutput]):
@model_validator(mode="after")
def validate_output_concept_consistency(self) -> Self:
- if self.structuring_method is not None:
- if self.output.structure_class_name == NativeConceptEnum.TEXT:
- raise PipeDefinitionError(
- f"Output concept '{self.output.code}' is considered a Text concept, "
- f"so it cannot be structured. Maybe you forgot to add '{NativeConceptEnum.TEXT}' to the class registry?"
- )
+ if self.structuring_method is not None and self.output.structure_class_name == NativeConceptEnum.TEXT:
+ msg = (
+ f"Output concept '{self.output.code}' is considered a Text concept, "
+ f"so it cannot be structured. Maybe you forgot to add '{NativeConceptEnum.TEXT}' to the class registry?"
+ )
+ raise PipeDefinitionError(msg)
return self
@override
@@ -99,13 +100,14 @@ class PipeLLM(PipeOperator[PipeLLMOutput]):
tested_concept=self.output,
wanted_concept=get_concept_provider().get_native_concept(native_concept=NativeConceptEnum.IMAGE),
):
- raise PipeDefinitionError(
+ msg = (
f"The output of a LLM pipe cannot be compatible with the Image concept. In the "
f"pipe '{self.code}' the output is '{self.output.concept_string}'"
)
+ raise PipeDefinitionError(msg)
@override
- def needed_inputs(self, visited_pipes: Optional[Set[str]] = None) -> PipeInputSpec:
+ def needed_inputs(self, visited_pipes: set[str] | None = None) -> PipeInputSpec:
"""Needed inputs are the inputs needed to run the pipe, specified in the inputs attribute of the pipe"""
# The images are not tagged in the prompt_template.
# Therefore if an image is provided in the inputs, it becomes a needed input.
@@ -115,7 +117,8 @@ class PipeLLM(PipeOperator[PipeLLMOutput]):
for input_name, requirement in self.inputs.items:
if concept_provider.is_image_concept(concept=requirement.concept):
needed_inputs.add_requirement(
- variable_name=input_name, concept=concept_provider.get_native_concept(native_concept=NativeConceptEnum.IMAGE)
+ variable_name=input_name,
+ concept=concept_provider.get_native_concept(native_concept=NativeConceptEnum.IMAGE),
)
else:
needed_inputs.add_requirement(variable_name=input_name, concept=requirement.concept)
@@ -123,19 +126,19 @@ class PipeLLM(PipeOperator[PipeLLMOutput]):
return needed_inputs
@override
- def required_variables(self) -> Set[str]:
+ def required_variables(self) -> set[str]:
"""Required variables are the variables that are used in the current prompt template or system prompt"""
- required_variables: Set[str] = set()
+ required_variables: set[str] = set()
required_variables.update(self.llm_prompt_spec.required_variables())
- required_variables = {variable_name for variable_name in required_variables if not variable_name.startswith("_")}
- return required_variables
+ return {variable_name for variable_name in required_variables if not variable_name.startswith("_")}
def _validate_required_variables(self) -> Self:
"""This method checks that all required variables are in the inputs"""
required_variables = self.required_variables()
for required_variable_name in required_variables:
if required_variable_name not in self.inputs.variables:
- raise PipeDefinitionError(f"Required variable '{required_variable_name}' is not in the inputs of pipe {self.code}")
+ msg = f"Required variable '{required_variable_name}' is not in the inputs of pipe {self.code}"
+ raise PipeDefinitionError(msg)
return self
def _validate_inputs(self):
@@ -166,12 +169,11 @@ class PipeLLM(PipeOperator[PipeLLMOutput]):
if named_input_requirement.concept == concept_provider.get_native_concept(native_concept=NativeConceptEnum.IMAGE):
try:
input_requirement_of_declared_input = self.inputs.get_required_input_requirement(
- variable_name=named_input_requirement.variable_name
+ variable_name=named_input_requirement.variable_name,
)
except PipeInputNotFoundError as exc:
- raise PipeInputError(
- f"Input variable '{named_input_requirement.variable_name}' is not in this PipeLLM '{self.code}' input spec: {self.inputs}"
- ) from exc
+ msg = f"Input variable '{named_input_requirement.variable_name}' is not in this PipeLLM '{self.code}' input spec: {self.inputs}"
+ raise PipeInputError(msg) from exc
if not concept_provider.is_compatible(
tested_concept=input_requirement_of_declared_input.concept,
wanted_concept=named_input_requirement.concept,
@@ -230,8 +232,8 @@ class PipeLLM(PipeOperator[PipeLLMOutput]):
job_metadata: JobMetadata,
working_memory: WorkingMemory,
pipe_run_params: PipeRunParams,
- output_name: Optional[str] = None,
- content_generator: Optional[ContentGeneratorProtocol] = None,
+ output_name: str | None = None,
+ content_generator: ContentGeneratorProtocol | None = None,
) -> PipeLLMOutput:
content_generator = content_generator or get_content_generator()
# interpret / unwrap the arguments
@@ -246,7 +248,7 @@ class PipeLLM(PipeOperator[PipeLLMOutput]):
output_concept_code = SpecialDomain.NATIVE + "." + NativeConceptEnum.TEXT
else:
output_concept = get_concept_provider().get_required_concept(
- concept_string=ConceptFactory.construct_concept_string_with_domain(domain=self.domain, concept_code=output_concept_code)
+ concept_string=ConceptFactory.construct_concept_string_with_domain(domain=self.domain, concept_code=output_concept_code),
)
multiplicity_resolution = output_multiplicity_to_apply(
@@ -258,8 +260,8 @@ class PipeLLM(PipeOperator[PipeLLMOutput]):
fixed_nb_output = multiplicity_resolution.specific_output_count
# Collect what LLM settings we have for this particular PipeLLM
- llm_for_text_choice: Optional[LLMChoice] = None
- llm_for_object_choice: Optional[LLMChoice] = None
+ llm_for_text_choice: LLMChoice | None = None
+ llm_for_object_choice: LLMChoice | None = None
if self.llm_choices:
llm_for_text_choice = self.llm_choices.for_text
llm_for_object_choice = self.llm_choices.for_object
@@ -295,10 +297,10 @@ class PipeLLM(PipeOperator[PipeLLMOutput]):
self.structuring_method == StructuringMethod.PRELIMINARY_TEXT
) or get_config().pipelex.structure_config.is_default_text_then_structure
log.verbose(
- f"is_with_preliminary_text: {is_with_preliminary_text} for pipe {self.code} because the structuring_method is {self.structuring_method}"
+ f"is_with_preliminary_text: {is_with_preliminary_text} for pipe {self.code} because the structuring_method is {self.structuring_method}",
)
# Append output structure prompt if needed
- output_structure_prompt: Optional[str] = PipeLLM.get_output_structure_prompt(
+ output_structure_prompt: str | None = PipeLLM.get_output_structure_prompt(
concept_string=pipe_run_params.dynamic_output_concept_code or output_concept.concept_string,
is_with_preliminary_text=is_with_preliminary_text,
)
@@ -337,9 +339,9 @@ class PipeLLM(PipeOperator[PipeLLMOutput]):
# TODO: we need a better solution for structuring_method (text then object), meanwhile,
# we acknowledge the code here with llm_prompt_1 and llm_prompt_2 is overly complex and should be refactored.
- llm_prompt_2_factory: Optional[LLMPromptFactoryAbstract]
+ llm_prompt_2_factory: LLMPromptFactoryAbstract | None
if self.structuring_method:
- structuring_method = cast(StructuringMethod, self.structuring_method)
+ structuring_method = cast("StructuringMethod", self.structuring_method)
log.debug(f"PipeLLM pipe_code is '{self.code}' and structuring_method is '{structuring_method}'")
match structuring_method:
case StructuringMethod.DIRECT:
@@ -409,25 +411,24 @@ class PipeLLM(PipeOperator[PipeLLMOutput]):
name=output_name,
)
- pipe_output = PipeLLMOutput(
+ return PipeLLMOutput(
working_memory=working_memory,
pipeline_run_id=job_metadata.pipeline_run_id,
)
- return pipe_output
async def _llm_gen_object_stuff_content(
self,
job_metadata: JobMetadata,
is_multiple_output: bool,
- fixed_nb_output: Optional[int],
+ fixed_nb_output: int | None,
output_class_name: str,
llm_setting_main: LLMSetting,
llm_setting_for_object: LLMSetting,
llm_prompt_1: LLMPrompt,
- llm_prompt_2_factory: Optional[LLMPromptFactoryAbstract],
+ llm_prompt_2_factory: LLMPromptFactoryAbstract | None,
content_generator: ContentGeneratorProtocol,
) -> StuffContent:
- content_class: Type[StuffContent] = get_class_registry().get_required_subclass(name=output_class_name, base_class=StuffContent)
+ content_class: type[StuffContent] = get_class_registry().get_required_subclass(name=output_class_name, base_class=StuffContent)
task_desc: str
the_content: StuffContent
@@ -438,7 +439,7 @@ class PipeLLM(PipeOperator[PipeLLMOutput]):
else:
task_desc = f"{self.class_name}_gen_list_{content_class.__class__.__name__}"
log.dev(task_desc)
- generated_objects: List[StuffContent]
+ generated_objects: list[StuffContent]
if llm_prompt_2_factory is not None:
# We're generating a list of objects using preliminary text
method_desc = "text_then_object"
@@ -504,17 +505,16 @@ class PipeLLM(PipeOperator[PipeLLMOutput]):
job_metadata: JobMetadata,
working_memory: WorkingMemory,
pipe_run_params: PipeRunParams,
- output_name: Optional[str] = None,
+ output_name: str | None = None,
) -> PipeLLMOutput:
content_generator_dry = ContentGeneratorDry()
- pipe_output = await self._run_operator_pipe(
+ return await self._run_operator_pipe(
job_metadata=job_metadata,
working_memory=working_memory,
pipe_run_params=pipe_run_params,
output_name=output_name,
content_generator=content_generator_dry,
)
- return pipe_output
@staticmethod
def get_output_structure_prompt(concept_string: str, is_with_preliminary_text: bool) -> str:
diff --git a/pipelex/pipe_operators/llm/pipe_llm_blueprint.py b/pipelex/pipe_operators/llm/pipe_llm_blueprint.py
index 84605a5b3..a7c15ee59 100644
--- a/pipelex/pipe_operators/llm/pipe_llm_blueprint.py
+++ b/pipelex/pipe_operators/llm/pipe_llm_blueprint.py
@@ -1,13 +1,12 @@
-from typing import Literal, Optional
+from typing import Literal
from pydantic import field_validator, model_validator
-from typing_extensions import Self
from pipelex.cogt.llm.llm_setting import LLMChoice
from pipelex.core.pipes.pipe_blueprint import PipeBlueprint
from pipelex.exceptions import PipeDefinitionError
from pipelex.tools.typing.validation_utils import has_more_than_one_among_attributes_from_lists
-from pipelex.types import StrEnum
+from pipelex.types import Self, StrEnum
class StructuringMethod(StrEnum):
@@ -18,30 +17,32 @@ class StructuringMethod(StrEnum):
class PipeLLMBlueprint(PipeBlueprint):
type: Literal["PipeLLM"] = "PipeLLM"
category: Literal["PipeOperator"] = "PipeOperator"
- system_prompt_template: Optional[str] = None
- system_prompt_template_name: Optional[str] = None
- system_prompt_name: Optional[str] = None
- system_prompt: Optional[str] = None
+ system_prompt_template: str | None = None
+ system_prompt_template_name: str | None = None
+ system_prompt_name: str | None = None
+ system_prompt: str | None = None
- prompt_template: Optional[str] = None
- template_name: Optional[str] = None
- prompt_name: Optional[str] = None
- prompt: Optional[str] = None
+ prompt_template: str | None = None
+ template_name: str | None = None
+ prompt_name: str | None = None
+ prompt: str | None = None
- llm: Optional[LLMChoice] = None
- llm_to_structure: Optional[LLMChoice] = None
+ llm: LLMChoice | None = None
+ llm_to_structure: LLMChoice | None = None
- structuring_method: Optional[StructuringMethod] = None
- prompt_template_to_structure: Optional[str] = None
- system_prompt_to_structure: Optional[str] = None
+ structuring_method: StructuringMethod | None = None
+ prompt_template_to_structure: str | None = None
+ system_prompt_to_structure: str | None = None
- nb_output: Optional[int] = None
- multiple_output: Optional[bool] = None
+ nb_output: int | None = None
+ multiple_output: bool | None = None
@field_validator("nb_output", mode="after")
- def validate_nb_output(cls, value: Optional[int] = None) -> Optional[int]:
+ @staticmethod
+ def validate_nb_output(value: int | None = None) -> int | None:
if value and value < 1:
- raise PipeDefinitionError("PipeLLMBlueprint nb_output must be greater than 0")
+ msg = "PipeLLMBlueprint nb_output must be greater than 0"
+ raise PipeDefinitionError(msg)
return value
@model_validator(mode="after")
@@ -54,5 +55,6 @@ class PipeLLMBlueprint(PipeBlueprint):
["prompt", "prompt_name", "prompt_template", "template_name"],
],
):
- raise PipeDefinitionError(f"PipeLLMBlueprint should have no more than one of {excess_attributes_list} among them")
+ msg = f"PipeLLMBlueprint should have no more than one of {excess_attributes_list} among them"
+ raise PipeDefinitionError(msg)
return self
diff --git a/pipelex/pipe_operators/llm/pipe_llm_factory.py b/pipelex/pipe_operators/llm/pipe_llm_factory.py
index e1479f74e..60410525b 100644
--- a/pipelex/pipe_operators/llm/pipe_llm_factory.py
+++ b/pipelex/pipe_operators/llm/pipe_llm_factory.py
@@ -1,5 +1,3 @@
-from typing import List, Optional
-
from typing_extensions import override
from pipelex.cogt.llm.llm_prompt_spec import LLMPromptSpec
@@ -26,10 +24,10 @@ class PipeLLMFactory(PipeFactoryProtocol[PipeLLMBlueprint, PipeLLM]):
domain: str,
pipe_code: str,
blueprint: PipeLLMBlueprint,
- concept_codes_from_the_same_domain: Optional[List[str]] = None,
+ concept_codes_from_the_same_domain: list[str] | None = None,
) -> PipeLLM:
- system_prompt_jinja2_blueprint: Optional[Jinja2Blueprint] = None
- system_prompt: Optional[str] = None
+ system_prompt_jinja2_blueprint: Jinja2Blueprint | None = None
+ system_prompt: str | None = None
if blueprint.system_prompt_template or blueprint.system_prompt_template_name:
try:
system_prompt_jinja2_blueprint = Jinja2Blueprint(
@@ -48,7 +46,7 @@ class PipeLLMFactory(PipeFactoryProtocol[PipeLLMBlueprint, PipeLLM]):
if domain_obj := get_optional_domain(domain=domain):
system_prompt = domain_obj.system_prompt
- user_text_jinja2_blueprint: Optional[Jinja2Blueprint] = None
+ user_text_jinja2_blueprint: Jinja2Blueprint | None = None
if blueprint.prompt_template or blueprint.template_name:
try:
user_text_jinja2_blueprint = Jinja2Blueprint(
@@ -72,12 +70,15 @@ class PipeLLMFactory(PipeFactoryProtocol[PipeLLMBlueprint, PipeLLM]):
error_msg = f"Jinja2 template not found for pipe '{pipe_code}' in domain '{domain}': {exc}."
raise PipeDefinitionError(error_msg) from exc
- user_images: List[str] = []
+ user_images: list[str] = []
if blueprint.inputs:
for stuff_name, requirement in blueprint.inputs.items():
if isinstance(requirement, str):
- requirement = InputRequirementBlueprint(concept=requirement)
- concept_string = requirement.concept
+ input_requirement_blueprint = InputRequirementBlueprint(concept=requirement)
+ else:
+ input_requirement_blueprint = requirement
+
+ concept_string = input_requirement_blueprint.concept
domain_and_code = ConceptFactory.make_domain_and_concept_code_from_concept_string_or_code(
domain=domain,
concept_string_or_code=concept_string,
@@ -85,8 +86,9 @@ class PipeLLMFactory(PipeFactoryProtocol[PipeLLMBlueprint, PipeLLM]):
)
concept = get_concept_provider().get_required_concept(
concept_string=ConceptFactory.construct_concept_string_with_domain(
- domain=domain_and_code.domain, concept_code=domain_and_code.concept_code
- )
+ domain=domain_and_code.domain,
+ concept_code=domain_and_code.concept_code,
+ ),
)
if get_concept_provider().is_image_concept(concept=concept):
@@ -125,10 +127,12 @@ class PipeLLMFactory(PipeFactoryProtocol[PipeLLMBlueprint, PipeLLM]):
code=pipe_code,
definition=blueprint.definition,
inputs=PipeInputSpecFactory.make_from_blueprint(
- domain=domain, blueprint=blueprint.inputs or {}, concept_codes_from_the_same_domain=concept_codes_from_the_same_domain
+ domain=domain,
+ blueprint=blueprint.inputs or {},
+ concept_codes_from_the_same_domain=concept_codes_from_the_same_domain,
),
output=get_concept_provider().get_required_concept(
- concept_string=ConceptFactory.construct_concept_string_with_domain(domain=output_concept_domain, concept_code=output_concept_code)
+ concept_string=ConceptFactory.construct_concept_string_with_domain(domain=output_concept_domain, concept_code=output_concept_code),
),
llm_prompt_spec=llm_prompt_spec,
llm_choices=llm_choices,
diff --git a/pipelex/pipe_operators/ocr/__init__.py b/pipelex/pipe_operators/ocr/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/pipelex/pipe_operators/ocr/pipe_ocr.py b/pipelex/pipe_operators/ocr/pipe_ocr.py
index 6530fb618..34af1cf67 100644
--- a/pipelex/pipe_operators/ocr/pipe_ocr.py
+++ b/pipelex/pipe_operators/ocr/pipe_ocr.py
@@ -1,7 +1,7 @@
-from typing import List, Literal, Optional, Set
+from typing import Literal
from pydantic import model_validator
-from typing_extensions import Self, override
+from typing_extensions import override
from pipelex import log
from pipelex.cogt.content_generation.content_generator_dry import ContentGeneratorDry
@@ -33,6 +33,7 @@ from pipelex.hub import (
from pipelex.pipe_operators.pipe_operator import PipeOperator
from pipelex.pipeline.job_metadata import JobMetadata
from pipelex.tools.pdf.pypdfium2_renderer import pypdfium2_renderer
+from pipelex.types import Self
class PipeOcrOutput(PipeOutput):
@@ -44,14 +45,14 @@ PIPE_OCR_INPUT_NAME = "ocr_input"
class PipeOcr(PipeOperator[PipeOcrOutput]):
type: Literal["PipeOcr"] = "PipeOcr"
- ocr_choice: Optional[OcrChoice]
+ ocr_choice: OcrChoice | None
should_caption_images: bool
should_include_images: bool
should_include_page_views: bool
page_views_dpi: int
- image_stuff_name: Optional[str] = None
- pdf_stuff_name: Optional[str] = None
+ image_stuff_name: str | None = None
+ pdf_stuff_name: str | None = None
@model_validator(mode="after")
def validate_inputs(self) -> Self:
@@ -65,13 +66,14 @@ class PipeOcr(PipeOperator[PipeOcrOutput]):
check_ocr_choice_with_deck(ocr_choice=self.ocr_choice)
@override
- def required_variables(self) -> Set[str]:
+ def required_variables(self) -> set[str]:
return {PIPE_OCR_INPUT_NAME}
@override
def validate_output(self):
if self.output != get_concept_provider().get_native_concept(native_concept=NativeConceptEnum.PAGE):
- raise PipeDefinitionError(f"PipeOcr output should be a Page concept, but is {self.output.concept_string}")
+ msg = f"PipeOcr output should be a Page concept, but is {self.output.concept_string}"
+ raise PipeDefinitionError(msg)
def _validate_inputs(self):
concept_provider = get_concept_provider()
@@ -80,7 +82,7 @@ class PipeOcr(PipeOperator[PipeOcrOutput]):
reactions = static_validation_config.reactions
# check that we have either an image or a pdf in inputs, at most one of them and nothing else
- candidate_prompt_var_names: List[str] = []
+ candidate_prompt_var_names: list[str] = []
for input_name, requirement in self.inputs.items:
log.debug(f"{input_name=}")
log.debug(f"{requirement=}")
@@ -147,7 +149,7 @@ class PipeOcr(PipeOperator[PipeOcrOutput]):
raise missing_input_var_error
@override
- def needed_inputs(self, visited_pipes: Optional[Set[str]] = None) -> PipeInputSpec:
+ def needed_inputs(self, visited_pipes: set[str] | None = None) -> PipeInputSpec:
return PipeInputSpecFactory.make_from_blueprint(
domain=self.domain,
blueprint={PIPE_OCR_INPUT_NAME: InputRequirementBlueprint(concept=self.inputs.root[PIPE_OCR_INPUT_NAME].concept.concept_string)},
@@ -159,13 +161,13 @@ class PipeOcr(PipeOperator[PipeOcrOutput]):
job_metadata: JobMetadata,
working_memory: WorkingMemory,
pipe_run_params: PipeRunParams,
- output_name: Optional[str] = None,
- content_generator: Optional[ContentGeneratorProtocol] = None,
+ output_name: str | None = None,
+ content_generator: ContentGeneratorProtocol | None = None,
) -> PipeOcrOutput:
content_generator = content_generator or get_content_generator()
- image_uri: Optional[str] = None
- pdf_uri: Optional[str] = None
+ image_uri: str | None = None
+ pdf_uri: str | None = None
if self.image_stuff_name:
image_stuff = working_memory.get_stuff_as_image(name=self.image_stuff_name)
image_uri = image_stuff.url
@@ -173,7 +175,8 @@ class PipeOcr(PipeOperator[PipeOcrOutput]):
pdf_stuff = working_memory.get_stuff_as_pdf(name=self.pdf_stuff_name)
pdf_uri = pdf_stuff.url
else:
- raise PipeDefinitionError("PipeOcr should have a non-None image_stuff_name or pdf_stuff_name")
+ msg = "PipeOcr should have a non-None image_stuff_name or pdf_stuff_name"
+ raise PipeDefinitionError(msg)
ocr_choice: OcrChoice = self.ocr_choice or get_model_deck().ocr_choice_default
ocr_setting: OcrSetting = get_model_deck().get_ocr_setting(ocr_choice=ocr_choice)
@@ -199,13 +202,13 @@ class PipeOcr(PipeOperator[PipeOcrOutput]):
)
# Build the output stuff, which is a list of page contents
- page_view_contents: List[ImageContent] = []
+ page_view_contents: list[ImageContent] = []
if self.should_include_page_views:
log.debug(f"should_include_page_views: {self.should_include_page_views}, pdf_uri: {pdf_uri}, image_uri: {image_uri}")
if pdf_uri:
- for page in ocr_output.pages.values():
- if page.page_view:
- page_view_contents.append(ImageContent.make_from_extracted_image(extracted_image=page.page_view))
+ page_view_contents.extend(
+ ImageContent.make_from_extracted_image(extracted_image=page.page_view) for page in ocr_output.pages.values() if page.page_view
+ )
log.debug(f"page_view_contents: {page_view_contents}")
needs_to_generate_page_views: bool
if len(page_view_contents) == 0:
@@ -224,7 +227,7 @@ class PipeOcr(PipeOperator[PipeOcrOutput]):
elif image_uri:
page_view_contents = [ImageContent.make_from_str(str_value=image_uri)]
- page_contents: List[PageContent] = []
+ page_contents: list[PageContent] = []
for page_index, page in ocr_output.pages.items():
images = [ImageContent.make_from_extracted_image(extracted_image=img) for img in page.extracted_images]
log.debug(f"images: {images}, page_view_contents: {page_view_contents}, index: {page_index}")
@@ -236,7 +239,7 @@ class PipeOcr(PipeOperator[PipeOcrOutput]):
images=images,
),
page_view=page_view,
- )
+ ),
)
content: ListContent[PageContent] = ListContent(items=page_contents)
@@ -252,11 +255,10 @@ class PipeOcr(PipeOperator[PipeOcrOutput]):
name=output_name,
)
- pipe_output = PipeOcrOutput(
+ return PipeOcrOutput(
working_memory=working_memory,
pipeline_run_id=job_metadata.pipeline_run_id,
)
- return pipe_output
@override
async def _dry_run_operator_pipe(
@@ -264,18 +266,18 @@ class PipeOcr(PipeOperator[PipeOcrOutput]):
job_metadata: JobMetadata,
working_memory: WorkingMemory,
pipe_run_params: PipeRunParams,
- output_name: Optional[str] = None,
+ output_name: str | None = None,
) -> PipeOcrOutput:
log.debug(f"PipeOcr: dry run operator pipe: {self.code}")
if pipe_run_params.run_mode != PipeRunMode.DRY:
- raise PipeDefinitionError(f"Running pipe '{self.code}' (PipeOcr) _dry_run_operator_pipe() in non-dry mode is not allowed.")
+ msg = f"Running pipe '{self.code}' (PipeOcr) _dry_run_operator_pipe() in non-dry mode is not allowed."
+ raise PipeDefinitionError(msg)
content_generator_dry = ContentGeneratorDry()
- pipe_output = await self._run_operator_pipe(
+ return await self._run_operator_pipe(
job_metadata=job_metadata,
working_memory=working_memory,
pipe_run_params=pipe_run_params,
output_name=output_name,
content_generator=content_generator_dry,
)
- return pipe_output
diff --git a/pipelex/pipe_operators/ocr/pipe_ocr_blueprint.py b/pipelex/pipe_operators/ocr/pipe_ocr_blueprint.py
index 610eb5e96..b09179f33 100644
--- a/pipelex/pipe_operators/ocr/pipe_ocr_blueprint.py
+++ b/pipelex/pipe_operators/ocr/pipe_ocr_blueprint.py
@@ -1,4 +1,4 @@
-from typing import Literal, Optional
+from typing import Literal
from pipelex.cogt.ocr.ocr_setting import OcrChoice
from pipelex.core.pipes.pipe_blueprint import PipeBlueprint
@@ -7,8 +7,8 @@ from pipelex.core.pipes.pipe_blueprint import PipeBlueprint
class PipeOcrBlueprint(PipeBlueprint):
type: Literal["PipeOcr"] = "PipeOcr"
category: Literal["PipeOperator"] = "PipeOperator"
- ocr: Optional[OcrChoice] = None
- page_images: Optional[bool] = None
- page_image_captions: Optional[bool] = None
- page_views: Optional[bool] = None
- page_views_dpi: Optional[int] = None
+ ocr: OcrChoice | None = None
+ page_images: bool | None = None
+ page_image_captions: bool | None = None
+ page_views: bool | None = None
+ page_views_dpi: int | None = None
diff --git a/pipelex/pipe_operators/ocr/pipe_ocr_factory.py b/pipelex/pipe_operators/ocr/pipe_ocr_factory.py
index 80251a98a..65fd09980 100644
--- a/pipelex/pipe_operators/ocr/pipe_ocr_factory.py
+++ b/pipelex/pipe_operators/ocr/pipe_ocr_factory.py
@@ -1,5 +1,3 @@
-from typing import List, Optional
-
from typing_extensions import override
from pipelex.config import get_config
@@ -19,7 +17,7 @@ class PipeOcrFactory(PipeFactoryProtocol[PipeOcrBlueprint, PipeOcr]):
domain: str,
pipe_code: str,
blueprint: PipeOcrBlueprint,
- concept_codes_from_the_same_domain: Optional[List[str]] = None,
+ concept_codes_from_the_same_domain: list[str] | None = None,
) -> PipeOcr:
output_domain_and_code = ConceptFactory.make_domain_and_concept_code_from_concept_string_or_code(
domain=domain,
@@ -33,11 +31,14 @@ class PipeOcrFactory(PipeFactoryProtocol[PipeOcrBlueprint, PipeOcr]):
definition=blueprint.definition,
output=get_concept_provider().get_required_concept(
concept_string=ConceptFactory.construct_concept_string_with_domain(
- domain=output_domain_and_code.domain, concept_code=output_domain_and_code.concept_code
- )
+ domain=output_domain_and_code.domain,
+ concept_code=output_domain_and_code.concept_code,
+ ),
),
inputs=PipeInputSpecFactory.make_from_blueprint(
- domain=domain, blueprint=blueprint.inputs or {}, concept_codes_from_the_same_domain=concept_codes_from_the_same_domain
+ domain=domain,
+ blueprint=blueprint.inputs or {},
+ concept_codes_from_the_same_domain=concept_codes_from_the_same_domain,
),
ocr_choice=blueprint.ocr,
should_include_images=blueprint.page_images or False,
diff --git a/pipelex/pipe_operators/pipe_operator.py b/pipelex/pipe_operators/pipe_operator.py
index b0f650534..75ad892f0 100644
--- a/pipelex/pipe_operators/pipe_operator.py
+++ b/pipelex/pipe_operators/pipe_operator.py
@@ -1,5 +1,5 @@
from abc import abstractmethod
-from typing import Generic, Literal, Optional, TypeVar
+from typing import Generic, Literal, TypeVar
from typing_extensions import override
@@ -25,8 +25,8 @@ class PipeOperator(PipeAbstract, Generic[PipeOperatorOutputType]):
job_metadata: JobMetadata,
working_memory: WorkingMemory,
pipe_run_params: PipeRunParams,
- output_name: Optional[str] = None,
- print_intermediate_outputs: Optional[bool] = False,
+ output_name: str | None = None,
+ print_intermediate_outputs: bool | None = False,
) -> PipeOutput:
pipe_run_params.push_pipe_to_stack(pipe_code=self.code)
self.monitor_pipe_stack(pipe_run_params=pipe_run_params)
@@ -70,7 +70,7 @@ class PipeOperator(PipeAbstract, Generic[PipeOperatorOutputType]):
activity_report=ActivityReport(
job_metadata=job_metadata,
content=pipe_output.main_stuff,
- )
+ ),
)
pipe_run_params.pop_pipe_from_stack(pipe_code=self.code)
@@ -83,7 +83,7 @@ class PipeOperator(PipeAbstract, Generic[PipeOperatorOutputType]):
job_metadata: JobMetadata,
working_memory: WorkingMemory,
pipe_run_params: PipeRunParams,
- output_name: Optional[str] = None,
+ output_name: str | None = None,
) -> PipeOperatorOutputType:
pass
@@ -93,6 +93,6 @@ class PipeOperator(PipeAbstract, Generic[PipeOperatorOutputType]):
job_metadata: JobMetadata,
working_memory: WorkingMemory,
pipe_run_params: PipeRunParams,
- output_name: Optional[str] = None,
+ output_name: str | None = None,
) -> PipeOperatorOutputType:
pass
diff --git a/pipelex/pipe_works/__init__.py b/pipelex/pipe_works/__init__.py
index 8b1378917..e69de29bb 100644
--- a/pipelex/pipe_works/__init__.py
+++ b/pipelex/pipe_works/__init__.py
@@ -1 +0,0 @@
-
diff --git a/pipelex/pipe_works/pipe_dry.py b/pipelex/pipe_works/pipe_dry.py
index eb7695b9f..f5285278a 100644
--- a/pipelex/pipe_works/pipe_dry.py
+++ b/pipelex/pipe_works/pipe_dry.py
@@ -2,7 +2,6 @@ import asyncio
import functools
import time
from concurrent.futures import ThreadPoolExecutor
-from typing import Dict, List, Optional
from pydantic import BaseModel
@@ -22,26 +21,32 @@ from pipelex.types import StrEnum
class DryRunError(Exception):
"""Raised when a dry run fails due to missing inputs or other validation issues."""
- pass
-
class DryRunStatus(StrEnum):
SUCCESS = "SUCCESS"
FAILURE = "FAILURE"
WARNING = "WARNING"
+ @property
+ def is_failure(self) -> bool:
+ match self:
+ case DryRunStatus.FAILURE:
+ return True
+ case DryRunStatus.SUCCESS:
+ return False
+ case DryRunStatus.WARNING:
+ return False
+
class DryRunOutput(BaseModel):
pipe_code: str
status: DryRunStatus
- error_message: Optional[str] = None
- warning_message: Optional[str] = None
+ error_message: str | None = None
+ warning_message: str | None = None
async def dry_run_pipe(pipe: PipeAbstract, raise_on_failure: bool = False) -> DryRunOutput:
- """
- Dry run a single pipe directly without parallelization.
- """
+ """Dry run a single pipe directly without parallelization."""
allowed_to_fail_pipes = get_config().pipelex.dry_run_config.allowed_to_fail_pipes
needed_inputs_for_factory = _convert_to_working_memory_format(needed_inputs_spec=pipe.needed_inputs())
@@ -58,23 +63,24 @@ async def dry_run_pipe(pipe: PipeAbstract, raise_on_failure: bool = False) -> Dr
warning_message = f"Allowed to fail dry run for pipe '{pipe.code}': {exc}"
log.warning(warning_message)
return DryRunOutput(pipe_code=pipe.code, status=DryRunStatus.WARNING, warning_message=warning_message)
- elif raise_on_failure:
- raise exc
- else:
- error_message = f"Dry run failed for pipe '{pipe.code}': {exc}"
- log.error(error_message)
- return DryRunOutput(pipe_code=pipe.code, status=DryRunStatus.FAILURE, error_message=error_message)
+
+ if raise_on_failure:
+ raise
+
+ error_message = f"Dry run failed for pipe '{pipe.code}': {exc}"
+ log.error(error_message)
+ return DryRunOutput(pipe_code=pipe.code, status=DryRunStatus.FAILURE, error_message=error_message)
log.info(f"Pipe '{pipe.code}' dry run completed successfully")
return DryRunOutput(pipe_code=pipe.code, status=DryRunStatus.SUCCESS)
-async def dry_run_pipes(pipes: List[PipeAbstract], run_in_parallel: bool = True, raise_on_failure: bool = True) -> Dict[str, DryRunOutput]:
- """
- Dry run pipes with optional parallelization.
+async def dry_run_pipes(pipes: list[PipeAbstract], run_in_parallel: bool = True, raise_on_failure: bool = True) -> dict[str, DryRunOutput]:
+ """Dry run pipes with optional parallelization.
Args:
pipes: List of pipes to dry run
run_in_parallel: If True, run pipes in parallel using ThreadPoolExecutor. If False, run sequentially.
+ raise_on_failure: If True, raise an exception if any pipe fails.
For each pipe, this method:
1. Gets the pipe's needed inputs
@@ -83,10 +89,10 @@ async def dry_run_pipes(pipes: List[PipeAbstract], run_in_parallel: bool = True,
Returns:
Dict mapping pipe codes to their dry run status ("SUCCESS" or error message)
- """
+ """
start_time = time.time()
- results: Dict[str, DryRunOutput] = {}
+ results: dict[str, DryRunOutput] = {}
allowed_to_fail_pipes = get_config().pipelex.dry_run_config.allowed_to_fail_pipes
if run_in_parallel:
@@ -113,30 +119,30 @@ async def dry_run_pipes(pipes: List[PipeAbstract], run_in_parallel: bool = True,
log.info(
f"Dry run completed: '{len(successful_pipes)}' successful, '{len(failed_pipes)}' failed, "
- f"'{len(allowed_to_fail_pipes)}' allowed to fail, in '{time.time() - start_time:.2f}' seconds"
+ f"'{len(allowed_to_fail_pipes)}' allowed to fail, in '{time.time() - start_time:.2f}' seconds",
)
if unexpected_failures:
unexpected_failures_details = "\n".join([f"'{pipe_code}': {results[pipe_code]}" for pipe_code in unexpected_failures])
if raise_on_failure:
- raise DryRunError(f"Dry run failed with '{len(unexpected_failures)}' unexpected pipe failures:\n{unexpected_failures_details}")
- else:
- log.error(f"Dry run failed with '{len(unexpected_failures)}' unexpected pipe failures:\n{unexpected_failures_details}")
- return results
+ msg = f"Dry run failed with '{len(unexpected_failures)}' unexpected pipe failures:\n{unexpected_failures_details}"
+ raise DryRunError(msg)
+ log.error(f"Dry run failed with '{len(unexpected_failures)}' unexpected pipe failures:\n{unexpected_failures_details}")
+ return results
return results
-def _convert_to_working_memory_format(needed_inputs_spec: PipeInputSpec) -> List[TypedNamedInputRequirement]:
- """
- Convert PipeInputSpec to the format needed by WorkingMemoryFactory.make_for_dry_run.
+def _convert_to_working_memory_format(needed_inputs_spec: PipeInputSpec) -> list[TypedNamedInputRequirement]:
+ """Convert PipeInputSpec to the format needed by WorkingMemoryFactory.make_for_dry_run.
Args:
needed_inputs_spec: PipeInputSpec with detailed_requirements
Returns:
List of tuples (variable_name, concept_code, structure_class)
+
"""
- needed_inputs_for_factory: List[TypedNamedInputRequirement] = []
+ needed_inputs_for_factory: list[TypedNamedInputRequirement] = []
class_registry = get_class_registry()
for named_input_requirement in needed_inputs_spec.named_input_requirements:
@@ -158,7 +164,7 @@ def _convert_to_working_memory_format(needed_inputs_spec: PipeInputSpec) -> List
# Fallback to TextContent if we can't get the proper class
log.warning(
f"Could not get structure class '{structure_class_name}' for "
- f"concept '{named_input_requirement.concept.code}', falling back to TextContent"
+ f"concept '{named_input_requirement.concept.code}', falling back to TextContent",
)
text_typed_named_input_requirement = TypedNamedInputRequirement.make_from_named(
named=named_input_requirement,
diff --git a/pipelex/pipe_works/pipe_job.py b/pipelex/pipe_works/pipe_job.py
index d25b0f61a..172b44ce9 100644
--- a/pipelex/pipe_works/pipe_job.py
+++ b/pipelex/pipe_works/pipe_job.py
@@ -1,5 +1,3 @@
-from typing import Optional
-
from pydantic import BaseModel, Field
from pipelex.core.memory.working_memory import WorkingMemory
@@ -13,4 +11,4 @@ class PipeJob(BaseModel):
working_memory: WorkingMemory = Field(default_factory=WorkingMemory)
pipe_run_params: PipeRunParams
job_metadata: JobMetadata
- output_name: Optional[str] = None
+ output_name: str | None = None
diff --git a/pipelex/pipe_works/pipe_job_factory.py b/pipelex/pipe_works/pipe_job_factory.py
index 73340757e..8f6f2f185 100644
--- a/pipelex/pipe_works/pipe_job_factory.py
+++ b/pipelex/pipe_works/pipe_job_factory.py
@@ -1,5 +1,3 @@
-from typing import Optional
-
from pipelex.core.memory.working_memory import WorkingMemory
from pipelex.core.memory.working_memory_factory import WorkingMemoryFactory
from pipelex.core.pipes.pipe_abstract import PipeAbstract
@@ -14,10 +12,10 @@ class PipeJobFactory:
def make_pipe_job(
cls,
pipe: PipeAbstract,
- pipe_run_params: Optional[PipeRunParams] = None,
- working_memory: Optional[WorkingMemory] = None,
- job_metadata: Optional[JobMetadata] = None,
- output_name: Optional[str] = None,
+ pipe_run_params: PipeRunParams | None = None,
+ working_memory: WorkingMemory | None = None,
+ job_metadata: JobMetadata | None = None,
+ output_name: str | None = None,
) -> PipeJob:
job_metadata = job_metadata or JobMetadata()
working_memory = working_memory or WorkingMemoryFactory.make_empty()
diff --git a/pipelex/pipe_works/pipe_router_protocol.py b/pipelex/pipe_works/pipe_router_protocol.py
index ee1e3793f..2977bd4f6 100644
--- a/pipelex/pipe_works/pipe_router_protocol.py
+++ b/pipelex/pipe_works/pipe_router_protocol.py
@@ -35,6 +35,7 @@ class PipeRouterProtocol(Protocol):
) -> None:
payload: PayloadType = {
"pipe_job": pipe_job,
+ "error": error,
}
await self.observer_provider.observe_after_failing_run(payload)
@@ -48,7 +49,7 @@ class PipeRouterProtocol(Protocol):
pipe_output = await self._run_pipe_job(pipe_job)
except Exception as exc:
await self._after_failing_run(pipe_job, exc)
- raise exc
+ raise
await self._after_successful_run(pipe_job, pipe_output)
diff --git a/pipelex/pipelex.py b/pipelex/pipelex.py
index e9926031e..4a66d11f1 100644
--- a/pipelex/pipelex.py
+++ b/pipelex/pipelex.py
@@ -1,14 +1,13 @@
import inspect
import os
from importlib.metadata import metadata
-from typing import Optional, Type, cast
+from typing import cast
from kajson.class_registry import ClassRegistry
from kajson.class_registry_abstract import ClassRegistryAbstract
from kajson.kajson_manager import KajsonManager
from kajson.singleton import MetaSingleton
from pydantic import ValidationError
-from typing_extensions import Self
from pipelex import log
from pipelex.cogt.content_generation.content_generator import ContentGenerator
@@ -53,6 +52,7 @@ from pipelex.tools.secrets.secrets_provider_abstract import SecretsProviderAbstr
from pipelex.tools.storage.storage_provider_abstract import StorageProviderAbstract
from pipelex.tools.templating.template_library import TemplateLibrary
from pipelex.tools.typing.pydantic_utils import format_pydantic_validation_error
+from pipelex.types import Self
PACKAGE_NAME = __name__.split(".", maxsplit=1)[0]
PACKAGE_VERSION = metadata(PACKAGE_NAME)["Version"]
@@ -63,16 +63,16 @@ class Pipelex(metaclass=MetaSingleton):
self,
config_dir_path: str,
# Dependency injection
- pipelex_hub: Optional[PipelexHub] = None,
- config_cls: Optional[Type[ConfigRoot]] = None,
- class_registry: Optional[ClassRegistryAbstract] = None,
- template_provider: Optional[TemplateLibrary] = None,
- models_manager: Optional[ModelManagerAbstract] = None,
- inference_manager: Optional[InferenceManager] = None,
- pipeline_manager: Optional[PipelineManager] = None,
- pipeline_tracker: Optional[PipelineTracker] = None,
- activity_manager: Optional[ActivityManagerProtocol] = None,
- reporting_delegate: Optional[ReportingProtocol] = None,
+ pipelex_hub: PipelexHub | None = None,
+ config_cls: type[ConfigRoot] | None = None,
+ class_registry: ClassRegistryAbstract | None = None,
+ template_provider: TemplateLibrary | None = None,
+ models_manager: ModelManagerAbstract | None = None,
+ inference_manager: InferenceManager | None = None,
+ pipeline_manager: PipelineManager | None = None,
+ pipeline_tracker: PipelineTracker | None = None,
+ activity_manager: ActivityManagerProtocol | None = None,
+ reporting_delegate: ReportingProtocol | None = None,
) -> None:
self.config_dir_path = config_dir_path
self.pipelex_hub = pipelex_hub or PipelexHub()
@@ -82,8 +82,9 @@ class Pipelex(metaclass=MetaSingleton):
try:
self.pipelex_hub.setup_config(config_cls=config_cls or PipelexConfig)
except ValidationError as exc:
- error_msg = format_pydantic_validation_error(exc)
- raise PipelexConfigError(f"Could not setup config because of: {error_msg}") from exc
+ formatted_error_msg = format_pydantic_validation_error(exc)
+ msg = f"Could not setup config because of: {formatted_error_msg}"
+ raise PipelexConfigError(msg) from exc
log.configure(
project_name=get_config().project_name or "unknown_project",
@@ -157,10 +158,10 @@ class Pipelex(metaclass=MetaSingleton):
def setup(
self,
- secrets_provider: Optional[SecretsProviderAbstract] = None,
- content_generator: Optional[ContentGeneratorProtocol] = None,
- pipe_router: Optional[PipeRouterProtocol] = None,
- storage_provider: Optional[StorageProviderAbstract] = None,
+ secrets_provider: SecretsProviderAbstract | None = None,
+ content_generator: ContentGeneratorProtocol | None = None,
+ pipe_router: PipeRouterProtocol | None = None,
+ storage_provider: StorageProviderAbstract | None = None,
):
# tools
self.pipelex_hub.set_secrets_provider(secrets_provider or EnvSecretsProvider())
@@ -170,9 +171,8 @@ class Pipelex(metaclass=MetaSingleton):
try:
self.models_manager.setup()
except RoutingProfileLibraryNotFoundError as routing_profile_library_exc:
- raise PipelexSetupError(
- "The routing library could not be found, please call `pipelex init config` to create it"
- ) from routing_profile_library_exc
+ msg = "The routing library could not be found, please call `pipelex init config` to create it"
+ raise PipelexSetupError(msg) from routing_profile_library_exc
except InferenceBackendCredentialsError as credentials_exc:
backend_name = credentials_exc.backend_name
var_name = credentials_exc.key_name
@@ -219,16 +219,18 @@ class Pipelex(metaclass=MetaSingleton):
self.library_manager.setup()
self.library_manager.load_libraries()
except ValidationError as exc:
- error_msg = format_pydantic_validation_error(exc)
- raise PipelexSetupError(f"Could not setup libraries because of: {error_msg}") from exc
+ formatted_error_msg = format_pydantic_validation_error(exc)
+ msg = f"Could not setup libraries because of: {formatted_error_msg}"
+ raise PipelexSetupError(msg) from exc
log.debug(f"{PACKAGE_NAME} version {PACKAGE_VERSION} setup libraries done for {get_config().project_name}")
def validate_libraries(self):
try:
self.library_manager.validate_libraries()
except ValidationError as exc:
- error_msg = format_pydantic_validation_error(exc)
- raise PipelexSetupError(f"Could not validate libraries because of: {error_msg}") from exc
+ formatted_error_msg = format_pydantic_validation_error(exc)
+ msg = f"Could not validate libraries because of: {formatted_error_msg}"
+ raise PipelexSetupError(msg) from exc
log.debug(f"{PACKAGE_NAME} version {PACKAGE_VERSION} validate libraries done for {get_config().project_name}")
def teardown(self):
@@ -259,9 +261,9 @@ class Pipelex(metaclass=MetaSingleton):
@classmethod
def make(
cls,
- relative_config_folder_path: Optional[str] = None,
- absolute_config_folder_path: Optional[str] = None,
- from_file: Optional[bool] = True,
+ relative_config_folder_path: str | None = None,
+ absolute_config_folder_path: str | None = None,
+ from_file: bool | None = True,
) -> Self:
"""Create and initialize a Pipelex instance.
@@ -283,17 +285,21 @@ class Pipelex(metaclass=MetaSingleton):
Note:
If neither path is provided, defaults to "./pipelex_libraries".
+
"""
if relative_config_folder_path is not None and absolute_config_folder_path is not None:
- raise PipelexSetupError("Cannot specify both relative_config_folder_path and absolute_config_folder_path")
+ msg = "Cannot specify both relative_config_folder_path and absolute_config_folder_path"
+ raise PipelexSetupError(msg)
if relative_config_folder_path is not None:
if from_file:
current_frame = inspect.currentframe()
if current_frame is None:
- raise PipelexSetupError("Could not find relative config folder path because of: Failed to get current frame")
+ msg = "Could not find relative config folder path because of: Failed to get current frame"
+ raise PipelexSetupError(msg)
if current_frame.f_back is None:
- raise PipelexSetupError("Could not find relative config folder path because of: Failed to get caller frame")
+ msg = "Could not find relative config folder path because of: Failed to get caller frame"
+ raise PipelexSetupError(msg)
caller_file = current_frame.f_back.f_code.co_filename
caller_dir = os.path.dirname(os.path.abspath(caller_file))
config_dir_path = os.path.abspath(os.path.join(caller_dir, relative_config_folder_path))
@@ -310,13 +316,14 @@ class Pipelex(metaclass=MetaSingleton):
return pipelex_instance
@classmethod
- def get_optional_instance(cls) -> Optional[Self]:
+ def get_optional_instance(cls) -> Self | None:
instance = MetaSingleton.instances.get(cls)
- return cast(Optional[Self], instance)
+ return cast("Self | None", instance)
@classmethod
def get_instance(cls) -> Self:
instance = MetaSingleton.instances.get(cls)
if instance is None:
- raise RuntimeError("Pipelex is not initialized")
- return cast(Self, instance)
+ msg = "Pipelex is not initialized"
+ raise RuntimeError(msg)
+ return cast("Self", instance)
diff --git a/pipelex/pipeline/__init__.py b/pipelex/pipeline/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/pipelex/pipeline/activity/activity_handler.py b/pipelex/pipeline/activity/activity_handler.py
index 9b896d5ae..68b74da7e 100644
--- a/pipelex/pipeline/activity/activity_handler.py
+++ b/pipelex/pipeline/activity/activity_handler.py
@@ -1,5 +1,5 @@
import os
-from typing import Set, cast
+from typing import cast
from pipelex import log
from pipelex.config import get_config
@@ -28,7 +28,7 @@ class ActivityHandlerForResultFiles:
img_gen_config = get_config().cogt.img_gen_config
img_gen_param_defaults = img_gen_config.img_gen_param_defaults
self.image_output_format = img_gen_param_defaults.output_format
- self.already_handled_stuff: Set[str] = set()
+ self.already_handled_stuff: set[str] = set()
def _generate_stuff_id(self, stuff: Stuff) -> str:
# Use name if available, otherwise just use code
@@ -66,7 +66,7 @@ class ActivityHandlerForResultFiles:
self._handle_structured_content(content=stuff.content, stuff_id=stuff_id)
elif isinstance(stuff.content, ListContent):
# TODO: check that all items are StuffContent
- self._handle_list_content(content=cast(ListContent[StuffContent], stuff.content), stuff_id=stuff_id) # type: ignore
+ self._handle_list_content(content=cast("ListContent[StuffContent]", stuff.content), stuff_id=stuff_id) # pyright: ignore[reportUnknownMemberType]
else:
log.error(f"Unhandled stuff content type: {type(stuff.content)}")
@@ -88,7 +88,7 @@ class ActivityHandlerForResultFiles:
# Save the image
image_path = os.path.join(self.images_dir_path, f"{stuff_id}.{self.image_output_format}")
if content.url.startswith("http"):
- image_bytes: bytes = fetch_file_from_url_httpx(url=content.url, timeout=10)
+ image_bytes: bytes = fetch_file_from_url_httpx(url=content.url, request_timeout=10)
with open(image_path, "wb") as image_file:
image_file.write(image_bytes)
else:
diff --git a/pipelex/pipeline/activity/activity_manager.py b/pipelex/pipeline/activity/activity_manager.py
index b2ba80029..662b74deb 100644
--- a/pipelex/pipeline/activity/activity_manager.py
+++ b/pipelex/pipeline/activity/activity_manager.py
@@ -1,5 +1,3 @@
-from typing import Dict
-
from pydantic import Field
from typing_extensions import override
@@ -15,7 +13,7 @@ class ActivityManagerError(RootException):
class ActivityManager(ActivityManagerProtocol):
def __init__(self) -> None:
- self.activity_callbacks: Dict[str, ActivityCallback] = Field(default_factory=dict)
+ self.activity_callbacks: dict[str, ActivityCallback] = Field(default_factory=dict)
@override
def setup(self) -> None:
diff --git a/pipelex/pipeline/activity/activity_models.py b/pipelex/pipeline/activity/activity_models.py
index afc8a37b9..fdb878e74 100644
--- a/pipelex/pipeline/activity/activity_models.py
+++ b/pipelex/pipeline/activity/activity_models.py
@@ -1,4 +1,5 @@
-from typing import Any, Callable
+from collections.abc import Callable
+from typing import Any
from pydantic import BaseModel
diff --git a/pipelex/pipeline/execute.py b/pipelex/pipeline/execute.py
index a83e83c4c..8cb52f1ba 100644
--- a/pipelex/pipeline/execute.py
+++ b/pipelex/pipeline/execute.py
@@ -1,5 +1,3 @@
-from typing import List, Optional
-
from pipelex.client.protocol import ImplicitMemory
from pipelex.core.memory.working_memory import WorkingMemory
from pipelex.core.memory.working_memory_factory import WorkingMemoryFactory
@@ -24,13 +22,13 @@ from pipelex.tools.environment import get_optional_env
async def execute_pipeline(
pipe_code: str,
- working_memory: Optional[WorkingMemory] = None,
- input_memory: Optional[ImplicitMemory] = None,
- search_domains: Optional[List[str]] = None,
- output_name: Optional[str] = None,
- output_multiplicity: Optional[PipeOutputMultiplicity] = None,
- dynamic_output_concept_code: Optional[str] = None,
- pipe_run_mode: Optional[PipeRunMode] = None,
+ working_memory: WorkingMemory | None = None,
+ input_memory: ImplicitMemory | None = None,
+ search_domains: list[str] | None = None,
+ output_name: str | None = None,
+ output_multiplicity: PipeOutputMultiplicity | None = None,
+ dynamic_output_concept_code: str | None = None,
+ pipe_run_mode: PipeRunMode | None = None,
) -> PipeOutput:
"""Execute a pipeline and wait for its completion.
@@ -57,11 +55,14 @@ async def execute_pipeline(
If not specified, the pipe run mode is inferred from the environment variable
``PIPELEX_FORCE_DRY_RUN_MODE``. If the environment variable is not set,
the pipe run mode is ``PipeRunMode.LIVE``.
+ search_domains:
+ List of domains to search for pipes.
- Returns
+ Returns:
-------
Tuple[PipeOutput, str]
A tuple containing the pipe output and the pipeline run ID.
+
"""
search_domains = search_domains or []
pipe = get_required_pipe(pipe_code=pipe_code)
@@ -70,8 +71,9 @@ async def execute_pipeline(
# Can be either working_memory or compact_memory or neither, but not both
if working_memory and input_memory:
- raise PipelineInputError(f"Cannot pass both working_memory and input_memory to `execute_pipeline` {pipe_code=}")
- elif input_memory:
+ msg = f"Cannot pass both working_memory and input_memory to `execute_pipeline` {pipe_code=}"
+ raise PipelineInputError(msg)
+ if input_memory:
working_memory = WorkingMemoryFactory.make_from_implicit_memory(
implicit_memory=input_memory,
search_domains=search_domains,
diff --git a/pipelex/pipeline/job_metadata.py b/pipelex/pipeline/job_metadata.py
index f3f94b4dc..60b214c3e 100644
--- a/pipelex/pipeline/job_metadata.py
+++ b/pipelex/pipeline/job_metadata.py
@@ -1,5 +1,4 @@
from datetime import datetime
-from typing import List, Optional
from pydantic import BaseModel, Field
@@ -24,19 +23,19 @@ class UnitJobId(StrEnum):
class JobMetadata(BaseModel):
- job_name: Optional[str] = None
+ job_name: str | None = None
pipeline_run_id: str = Field(default=SpecialPipelineId.UNTITLED)
- pipe_job_ids: Optional[List[str]] = None
+ pipe_job_ids: list[str] | None = None
- content_generation_job_id: Optional[str] = None
- unit_job_id: Optional[str] = None
- job_category: Optional[JobCategory] = None
+ content_generation_job_id: str | None = None
+ unit_job_id: str | None = None
+ job_category: JobCategory | None = None
- started_at: Optional[datetime] = Field(default_factory=lambda: datetime.now())
- completed_at: Optional[datetime] = None
+ started_at: datetime | None = Field(default_factory=lambda: datetime.now())
+ completed_at: datetime | None = None
@property
- def duration(self) -> Optional[float]:
+ def duration(self) -> float | None:
if self.started_at is not None and self.completed_at is not None:
return (self.completed_at - self.started_at).total_seconds()
return None
diff --git a/pipelex/pipeline/pipeline_manager.py b/pipelex/pipeline/pipeline_manager.py
index 99600a1c0..919ff4fce 100644
--- a/pipelex/pipeline/pipeline_manager.py
+++ b/pipelex/pipeline/pipeline_manager.py
@@ -1,5 +1,3 @@
-from typing import Dict, Optional
-
from pydantic import Field, RootModel
from typing_extensions import override
@@ -8,7 +6,7 @@ from pipelex.pipeline.pipeline import Pipeline
from pipelex.pipeline.pipeline_factory import PipelineFactory
from pipelex.pipeline.pipeline_manager_abstract import PipelineManagerAbstract
-PipelineManagerRoot = Dict[str, Pipeline]
+PipelineManagerRoot = dict[str, Pipeline]
class PipelineManager(PipelineManagerAbstract, RootModel[PipelineManagerRoot]):
@@ -23,14 +21,15 @@ class PipelineManager(PipelineManagerAbstract, RootModel[PipelineManagerRoot]):
self.root.clear()
@override
- def get_optional_pipeline(self, pipeline_run_id: str) -> Optional[Pipeline]:
+ def get_optional_pipeline(self, pipeline_run_id: str) -> Pipeline | None:
return self.root.get(pipeline_run_id)
@override
def get_pipeline(self, pipeline_run_id: str) -> Pipeline:
pipeline = self.get_optional_pipeline(pipeline_run_id=pipeline_run_id)
if pipeline is None:
- raise PipelineManagerNotFoundError(f"Pipeline {pipeline_run_id} not found")
+ msg = f"Pipeline {pipeline_run_id} not found"
+ raise PipelineManagerNotFoundError(msg)
return pipeline
def _set_pipeline(self, pipeline_run_id: str, pipeline: Pipeline) -> Pipeline:
diff --git a/pipelex/pipeline/pipeline_manager_abstract.py b/pipelex/pipeline/pipeline_manager_abstract.py
index 1c7e13878..9e3e7a972 100644
--- a/pipelex/pipeline/pipeline_manager_abstract.py
+++ b/pipelex/pipeline/pipeline_manager_abstract.py
@@ -1,5 +1,4 @@
from abc import ABC, abstractmethod
-from typing import Optional
from pipelex.pipeline.pipeline import Pipeline
@@ -14,7 +13,7 @@ class PipelineManagerAbstract(ABC):
pass
@abstractmethod
- def get_optional_pipeline(self, pipeline_run_id: str) -> Optional[Pipeline]:
+ def get_optional_pipeline(self, pipeline_run_id: str) -> Pipeline | None:
pass
@abstractmethod
diff --git a/pipelex/pipeline/start.py b/pipelex/pipeline/start.py
index 1de22d885..67b92b1dc 100644
--- a/pipelex/pipeline/start.py
+++ b/pipelex/pipeline/start.py
@@ -1,5 +1,4 @@
import asyncio
-from typing import Optional
from pipelex.client.protocol import CompactMemory
from pipelex.core.memory.working_memory import WorkingMemory
@@ -7,7 +6,7 @@ from pipelex.core.memory.working_memory_factory import WorkingMemoryFactory
from pipelex.core.pipes.pipe_output import PipeOutput
from pipelex.core.pipes.pipe_run_params import PipeOutputMultiplicity, PipeRunMode
from pipelex.core.pipes.pipe_run_params_factory import PipeRunParamsFactory
-from pipelex.exceptions import StartPipelineException
+from pipelex.exceptions import StartPipelineError
from pipelex.hub import get_pipe_router, get_pipeline_manager, get_report_delegate, get_required_pipe
from pipelex.pipe_works.pipe_job_factory import PipeJobFactory
from pipelex.pipeline.job_metadata import JobMetadata
@@ -15,11 +14,11 @@ from pipelex.pipeline.job_metadata import JobMetadata
async def start_pipeline(
pipe_code: str,
- working_memory: Optional[WorkingMemory] = None,
- input_memory: Optional[CompactMemory] = None,
- output_name: Optional[str] = None,
- output_multiplicity: Optional[PipeOutputMultiplicity] = None,
- dynamic_output_concept_code: Optional[str] = None,
+ working_memory: WorkingMemory | None = None,
+ input_memory: CompactMemory | None = None,
+ output_name: str | None = None,
+ output_multiplicity: PipeOutputMultiplicity | None = None,
+ dynamic_output_concept_code: str | None = None,
pipe_run_mode: PipeRunMode = PipeRunMode.LIVE,
) -> asyncio.Task[PipeOutput]:
"""Start a pipeline in the background.
@@ -45,15 +44,17 @@ async def start_pipeline(
Override the dynamic output concept code.
pipe_run_mode:
Pipe run mode: ``PipeRunMode.LIVE`` or ``PipeRunMode.DRY``.
- Returns
+
+ Returns:
-------
Tuple[str, asyncio.Task[PipeOutput]]
The ``pipeline_run_id`` of the newly started pipeline and a task that
can be awaited to get the pipe output.
- """
+ """
if working_memory and input_memory:
- raise StartPipelineException(f"Cannot pass both working_memory and input_memory to `start_pipeline` {pipe_code=}")
+ msg = f"Cannot pass both working_memory and input_memory to `start_pipeline` {pipe_code=}"
+ raise StartPipelineError(msg)
if input_memory:
working_memory = WorkingMemoryFactory.make_from_compact_memory(input_memory)
diff --git a/pipelex/pipeline/track/flow_chart.py b/pipelex/pipeline/track/flow_chart.py
index 6736fb75e..77e145114 100644
--- a/pipelex/pipeline/track/flow_chart.py
+++ b/pipelex/pipeline/track/flow_chart.py
@@ -3,7 +3,7 @@
# pyright: reportUnknownMemberType=false
# pyright: reportUnknownParameterType=false
# pyright: reportMissingTypeArgument=false
-from typing import Any, Dict, List, Optional, Tuple
+from typing import Any
import networkx as nx
import yaml
@@ -37,7 +37,7 @@ class PipelineFlowChart:
self._tracker_config = tracker_config
self.is_debug_mode = tracker_config.is_debug_mode
self.start_node = start_node
- self.sub_graph_class_defs: List[SubGraphClassDef] = []
+ self.sub_graph_class_defs: list[SubGraphClassDef] = []
for sub_graph_index, sub_graph_color in enumerate(self._tracker_config.sub_graph_colors):
class_def_letter = chr(ord("a") + sub_graph_index)
class_def_name = f"sub_{class_def_letter}"
@@ -45,14 +45,15 @@ class PipelineFlowChart:
def generate_mermaid_flowchart(
self,
- title: Optional[str] = None,
- subtitle: Optional[str] = None,
- ) -> Tuple[str, str]:
+ title: str | None = None,
+ subtitle: str | None = None,
+ ) -> tuple[str, str]:
nb_nodes = len(self.nx_graph.nodes)
if nb_nodes == 0:
- raise JobHistoryError("Graph has no nodes")
+ msg = "Graph has no nodes"
+ raise JobHistoryError(msg)
log.debug(f"Generating mermaid flowchart for the whole graph which holds {nb_nodes} nodes")
- mermaid_settings: Dict[str, Any] = {}
+ mermaid_settings: dict[str, Any] = {}
if title:
mermaid_settings["title"] = title
mermaid_settings["config"] = {}
@@ -73,12 +74,14 @@ class PipelineFlowChart:
for node in self.nx_graph.nodes:
node_attributes = self.nx_graph.nodes[node]
if not node_attributes:
- raise JobHistoryError(f"Node attributes are empty for node '{node}'")
+ msg = f"Node attributes are empty for node '{node}'"
+ raise JobHistoryError(msg)
node_pipe_layer = node_attributes.get(NodeAttributeKey.SUBGRAPH)
if not node_pipe_layer:
node_pipe_layer = "Unknown"
elif not isinstance(node_pipe_layer, str):
- raise JobHistoryError(f"Node '{node}' has no pipe stack: {node_attributes}")
+ msg = f"Node '{node}' has no pipe stack: {node_attributes}"
+ raise JobHistoryError(msg)
sub_graph = node_pipe_layer or "root"
# Split sub_graph by "-" and keep the last token
@@ -98,7 +101,8 @@ class PipelineFlowChart:
if subtitle:
# this is a hack to add something that looks like a subtitle, but it's actually a node with no stroke and no visible link
# if self.start_node is None:
- # raise JobHistoryError("Start node is not set")
+ # msg = "Start node is not set"
+ # raise JobHistoryError(msg)
mermaid_code += f"""
classDef subtitleNodeClass fill:transparent,stroke:#333,stroke-width:0px;
__subtitle__["{subtitle}"]
@@ -123,7 +127,8 @@ class PipelineFlowChart:
edge_tag = nice_edge_tag(pipe_code)
mermaid_code += f" {source} -- {edge_tag} {self._tracker_config.pipe_edge_style} {target}\n"
else:
- raise JobHistoryError(f"Pipe edge missing pipe code: {edge_data}")
+ msg = f"Pipe edge missing pipe code: {edge_data}"
+ raise JobHistoryError(msg)
case EdgeCategory.BATCH:
mermaid_code += f" {source} {self._tracker_config.branch_edge_style} {target}\n"
case EdgeCategory.AGGREGATE:
@@ -131,35 +136,38 @@ class PipelineFlowChart:
case EdgeCategory.CONDITION:
condition_expression = edge_data.get(EdgeAttributeKey.CONDITION_EXPRESSION)
if not condition_expression:
- raise JobHistoryError(f"Condition edge missing condition expression: {edge_data}")
+ msg = f"Condition edge missing condition expression: {edge_data}"
+ raise JobHistoryError(msg)
edge_tag = nice_edge_tag(condition_expression)
mermaid_code += f" {source} {self._tracker_config.condition_edge_style} {target}\n"
case EdgeCategory.CHOICE:
chosen_pipe_code = edge_data.get(EdgeAttributeKey.CHOSEN_PIPE)
if not chosen_pipe_code:
- raise JobHistoryError(f"No chosen pipe code set for edge {source} --- {target}")
+ msg = f"No chosen pipe code set for edge {source} --- {target}"
+ raise JobHistoryError(msg)
edge_tag = nice_edge_tag(chosen_pipe_code)
mermaid_code += f" {source} -- {edge_tag} {self._tracker_config.choice_edge_style} {target}\n"
url = make_mermaid_url(mermaid_code)
return mermaid_code, url
- def generate_subgraph_lines(self, graph_tree: GraphTree) -> List[str]:
- subgraph_lines: List[str] = []
- subgraph_class_lines: List[str] = []
+ def generate_subgraph_lines(self, graph_tree: GraphTree) -> list[str]:
+ subgraph_lines: list[str] = []
+ subgraph_class_lines: list[str] = []
- cycle = 0
- for subgraph_name, nodes in graph_tree.nodes_by_subgraph.items():
- node_lines: List[str] = []
+ for cycle, (subgraph_name, nodes) in enumerate(graph_tree.nodes_by_subgraph.items()):
+ node_lines: list[str] = []
for node in nodes:
# log.debug(f"generate_subgraph_lines for node '{node}'")
node_attributes = self.nx_graph.nodes[node]
if not node_attributes:
- raise JobHistoryError(f"Node attributes are empty for node '{node}'")
+ msg = f"Node attributes are empty for node '{node}'"
+ raise JobHistoryError(msg)
node_category = NodeCategory(node_attributes[NodeAttributeKey.CATEGORY])
node_tag = node_attributes[NodeAttributeKey.TAG]
if not node_tag:
- raise JobHistoryError(f"Node tag is empty for node '{node}'")
+ msg = f"Node tag is empty for node '{node}'"
+ raise JobHistoryError(msg)
node_text = node_tag
if self.is_debug_mode:
if node_comment := node_attributes.get(NodeAttributeKey.COMMENT):
@@ -175,29 +183,27 @@ class PipelineFlowChart:
node_lines.append(f'{node}["{node_text}"]')
case NodeCategory.CONDITION:
node_lines.append(f'{node}{{"{node_text}"}}')
- if self._tracker_config.is_include_interactivity:
- if node_description := node_attributes.get(NodeAttributeKey.DESCRIPTION):
- if not isinstance(node_description, str):
- raise JobHistoryError(f"Node description is not a string: {node_description}")
- node_description = clean_str_for_mermaid_node_title(node_description)
- node_lines.append(f'click {node} stuff_node_callback "{node_description}"')
+ if node_description := node_attributes.get(NodeAttributeKey.DESCRIPTION) and self._tracker_config.is_include_interactivity:
+ if not isinstance(node_description, str):
+ msg = f"Node description is not a string: {node_description}"
+ raise JobHistoryError(msg)
+ node_description = clean_str_for_mermaid_node_title(node_description)
+ node_lines.append(f'click {node} stuff_node_callback "{node_description}"')
if not node_lines:
- raise JobHistoryError(f"No node lines found for subgraph '{subgraph_name}'")
+ msg = f"No node lines found for subgraph '{subgraph_name}'"
+ raise JobHistoryError(msg)
if subgraph_name == "root":
- for mermaid_line in node_lines:
- subgraph_lines.append(_indent_line(mermaid_line, 2))
+ subgraph_lines.extend(_indent_line(mermaid_line, 2) for mermaid_line in node_lines)
else:
subgraph_lines.append(_indent_line(f'subgraph "{subgraph_name}"', 1))
subgraph_lines.append(_indent_line("direction LR", 1))
- for mermaid_line in node_lines:
- subgraph_lines.append(_indent_line(mermaid_line, 2))
+ subgraph_lines.extend(_indent_line(mermaid_line, 2) for mermaid_line in node_lines)
subgraph_lines.append(_indent_line("end", 1))
class_def = self.sub_graph_class_defs[cycle % len(self.sub_graph_class_defs)]
subgraph_class_lines.append(f"class {subgraph_name} {class_def.name};")
- cycle += 1
subgraph_lines.extend(subgraph_class_lines)
diff --git a/pipelex/pipeline/track/pipeline_tracker.py b/pipelex/pipeline/track/pipeline_tracker.py
index af4cb3a81..9ae145f36 100644
--- a/pipelex/pipeline/track/pipeline_tracker.py
+++ b/pipelex/pipeline/track/pipeline_tracker.py
@@ -2,7 +2,7 @@
# pyright: reportUnknownArgumentType=false
# pyright: reportUnknownMemberType=false
# pyright: reportMissingTypeArgument=false
-from typing import Any, Dict, List, Optional
+from typing import Any
import networkx as nx
from typing_extensions import override
@@ -33,7 +33,7 @@ class PipelineTracker(PipelineTrackerProtocol):
self._is_debug_mode = tracker_config.is_debug_mode
self.is_active: bool = False
self.nx_graph: nx.DiGraph = nx.DiGraph()
- self.start_node: Optional[str] = None
+ self.start_node: str | None = None
@override
def setup(self):
@@ -50,20 +50,20 @@ class PipelineTracker(PipelineTrackerProtocol):
self.teardown()
self.setup()
- def _get_node_name(self, node: str) -> Optional[str]:
+ def _get_node_name(self, node: str) -> str | None:
node_attributes = self.nx_graph.nodes[node]
node_name = node_attributes[NodeAttributeKey.NAME]
if isinstance(node_name, str):
return node_name
- else:
- raise JobHistoryError(f"Node name is not a string: {node_name}")
+ msg = f"Node name is not a string: {node_name}"
+ raise JobHistoryError(msg)
- def _pipe_layer_to_subgraph_name(self, pipe_layer: List[str]) -> str:
+ def _pipe_layer_to_subgraph_name(self, pipe_layer: list[str]) -> str:
return "-".join(pipe_layer)
def _add_start_node(self) -> str:
node = SpecialNodeName.START
- node_attributes: Dict[str, Any] = {
+ node_attributes: dict[str, Any] = {
NodeAttributeKey.CATEGORY: NodeCategory.SPECIAL,
NodeAttributeKey.TAG: "Start",
NodeAttributeKey.NAME: "Start",
@@ -74,7 +74,7 @@ class PipelineTracker(PipelineTrackerProtocol):
def _make_stuff_node_tag(
self,
stuff: Stuff,
- as_item_index: Optional[int] = None,
+ as_item_index: int | None = None,
) -> str:
concept_display = Concept.sentence_from_concept(concept=stuff.concept)
log.debug(f"Concept display: {stuff.concept.code} -> {concept_display}")
@@ -82,18 +82,18 @@ class PipelineTracker(PipelineTrackerProtocol):
concept_display = f"List of [{concept_display}]"
if as_item_index is not None:
return f"**{concept_display}** #{as_item_index + 1}"
- else:
- name = stuff.stuff_name
- if not name:
- raise JobHistoryError(f"Stuff name is empty for stuff {stuff}")
- return f"{name}:<br>**{concept_display}**"
+ name = stuff.stuff_name
+ if not name:
+ msg = f"Stuff name is empty for stuff {stuff}"
+ raise JobHistoryError(msg)
+ return f"{name}:<br>**{concept_display}**"
def _add_stuff_node(
self,
stuff: Stuff,
- pipe_layer: List[str],
+ pipe_layer: list[str],
comment: str,
- as_item_index: Optional[int] = None,
+ as_item_index: int | None = None,
) -> str:
node = stuff.stuff_code
is_existing = self.nx_graph.has_node(node)
@@ -116,7 +116,7 @@ class PipelineTracker(PipelineTrackerProtocol):
if stuff.is_text and self._tracker_config.is_include_text_preview:
node_tag += f"<br/>{stuff_content_rendered[:100]}"
pipe_layer_str = self._pipe_layer_to_subgraph_name(pipe_layer)
- node_attributes: Dict[str, Any] = {
+ node_attributes: dict[str, Any] = {
NodeAttributeKey.CATEGORY: NodeCategory.STUFF,
NodeAttributeKey.TAG: node_tag,
NodeAttributeKey.NAME: stuff.stuff_name,
@@ -133,19 +133,23 @@ class PipelineTracker(PipelineTrackerProtocol):
from_node: str,
to_node: str,
edge_category: EdgeCategory,
- attributes: Optional[Dict[str, Any]] = None,
+ attributes: dict[str, Any] | None = None,
):
# Ensure both nodes exist with attributes
if not self.nx_graph.has_node(from_node):
- raise JobHistoryError(f"Source node '{from_node}' does not exist")
+ msg = f"Source node '{from_node}' does not exist"
+ raise JobHistoryError(msg)
if not self.nx_graph.has_node(to_node):
- raise JobHistoryError(f"Target node '{to_node}' does not exist")
+ msg = f"Target node '{to_node}' does not exist"
+ raise JobHistoryError(msg)
if not self.nx_graph.nodes[from_node]:
- raise JobHistoryError(f"Source node '{from_node}' exists but has no attributes")
+ msg = f"Source node '{from_node}' exists but has no attributes"
+ raise JobHistoryError(msg)
if not self.nx_graph.nodes[to_node]:
- raise JobHistoryError(f"Target node '{to_node}' exists but has no attributes")
+ msg = f"Target node '{to_node}' exists but has no attributes"
+ raise JobHistoryError(msg)
- edge_attributes: Dict[str, Any] = {
+ edge_attributes: dict[str, Any] = {
EdgeAttributeKey.EDGE_CATEGORY: edge_category,
}
if attributes:
@@ -155,12 +159,12 @@ class PipelineTracker(PipelineTrackerProtocol):
@override
def add_pipe_step(
self,
- from_stuff: Optional[Stuff],
+ from_stuff: Stuff | None,
to_stuff: Stuff,
pipe_code: str,
comment: str,
- pipe_layer: List[str],
- as_item_index: Optional[int] = None,
+ pipe_layer: list[str],
+ as_item_index: int | None = None,
is_with_edge: bool = True,
):
if not self.is_active:
@@ -186,7 +190,7 @@ class PipelineTracker(PipelineTrackerProtocol):
edge_caption = pipe_code
if self._is_debug_mode:
edge_caption += f" ({comment})"
- edge_attributes: Dict[str, Any] = {
+ edge_attributes: dict[str, Any] = {
EdgeAttributeKey.PIPE_CODE: edge_caption,
}
if is_with_edge:
@@ -200,10 +204,10 @@ class PipelineTracker(PipelineTrackerProtocol):
@override
def add_batch_step(
self,
- from_stuff: Optional[Stuff],
+ from_stuff: Stuff | None,
to_stuff: Stuff,
to_branch_index: int,
- pipe_layer: List[str],
+ pipe_layer: list[str],
comment: str,
):
if not self.is_active:
@@ -236,7 +240,7 @@ class PipelineTracker(PipelineTrackerProtocol):
self,
from_stuff: Stuff,
to_stuff: Stuff,
- pipe_layer: List[str],
+ pipe_layer: list[str],
comment: str,
):
if not self.is_active:
@@ -257,11 +261,11 @@ class PipelineTracker(PipelineTrackerProtocol):
edge_category=EdgeCategory.AGGREGATE,
)
- def _add_condition_node(self, condition: PipeConditionDetails, pipe_layer: List[str]) -> str:
+ def _add_condition_node(self, condition: PipeConditionDetails, pipe_layer: list[str]) -> str:
node = condition.code
condition_node_tag = f"Condition:<br>**{condition.test_expression}<br>= {condition.evaluated_expression}**"
pipe_layer_str = self._pipe_layer_to_subgraph_name(pipe_layer)
- node_attributes: Dict[str, Any] = {
+ node_attributes: dict[str, Any] = {
NodeAttributeKey.CATEGORY: NodeCategory.CONDITION,
NodeAttributeKey.TAG: condition_node_tag,
NodeAttributeKey.NAME: condition.code,
@@ -276,7 +280,7 @@ class PipelineTracker(PipelineTrackerProtocol):
from_stuff: Stuff,
to_condition: PipeConditionDetails,
condition_expression: str,
- pipe_layer: List[str],
+ pipe_layer: list[str],
comment: str,
):
if not self.is_active:
@@ -287,7 +291,7 @@ class PipelineTracker(PipelineTrackerProtocol):
comment=comment,
)
to_node = self._add_condition_node(condition=to_condition, pipe_layer=pipe_layer)
- edge_attributes: Dict[str, Any] = {
+ edge_attributes: dict[str, Any] = {
EdgeAttributeKey.CONDITION_EXPRESSION: condition_expression,
}
self._add_edge(
@@ -302,7 +306,7 @@ class PipelineTracker(PipelineTrackerProtocol):
self,
from_condition: PipeConditionDetails,
to_stuff: Stuff,
- pipe_layer: List[str],
+ pipe_layer: list[str],
comment: str,
):
if not self.is_active:
@@ -312,7 +316,7 @@ class PipelineTracker(PipelineTrackerProtocol):
pipe_layer=pipe_layer,
comment=comment,
)
- edge_attributes: Dict[str, Any] = {
+ edge_attributes: dict[str, Any] = {
EdgeAttributeKey.CHOSEN_PIPE: from_condition.chosen_pipe_code,
}
self._add_edge(
@@ -322,12 +326,13 @@ class PipelineTracker(PipelineTrackerProtocol):
attributes=edge_attributes,
)
- def _print_mermaid_flowchart_code_and_url(self, title: Optional[str] = None, subtitle: Optional[str] = None):
+ def _print_mermaid_flowchart_code_and_url(self, title: str | None = None, subtitle: str | None = None):
if not self.nx_graph.nodes:
log.debug("No nodes in the pipeline tracker")
return
if self.start_node is None:
- raise JobHistoryError("Start node is not set")
+ msg = "Start node is not set"
+ raise JobHistoryError(msg)
flowchart = PipelineFlowChart(nx_graph=self.nx_graph, start_node=self.start_node, tracker_config=self._tracker_config)
mermaid_code, url = flowchart.generate_mermaid_flowchart(title=title, subtitle=subtitle)
print(mermaid_code)
@@ -336,12 +341,13 @@ class PipelineTracker(PipelineTrackerProtocol):
title_to_print += f" for {title}"
print_mermaid_url(url=url, title=title_to_print)
- def _print_mermaid_flowchart_url(self, title: Optional[str] = None, subtitle: Optional[str] = None) -> Optional[str]:
+ def _print_mermaid_flowchart_url(self, title: str | None = None, subtitle: str | None = None) -> str | None:
if not self.nx_graph.nodes:
log.debug("No nodes in the pipeline tracker")
return None
if self.start_node is None:
- raise JobHistoryError("Start node is not set")
+ msg = "Start node is not set"
+ raise JobHistoryError(msg)
flowchart = PipelineFlowChart(nx_graph=self.nx_graph, start_node=self.start_node, tracker_config=self._tracker_config)
_, url = flowchart.generate_mermaid_flowchart(title=title, subtitle=subtitle)
title_to_print = "Mermaid flowchart URL"
@@ -353,10 +359,10 @@ class PipelineTracker(PipelineTrackerProtocol):
@override
def output_flowchart(
self,
- title: Optional[str] = None,
- subtitle: Optional[str] = None,
+ title: str | None = None,
+ subtitle: str | None = None,
is_detailed: bool = False,
- ) -> Optional[str]:
+ ) -> str | None:
if is_detailed:
self._print_mermaid_flowchart_code_and_url(title=title, subtitle=subtitle)
else:
diff --git a/pipelex/pipeline/track/pipeline_tracker_protocol.py b/pipelex/pipeline/track/pipeline_tracker_protocol.py
index 374540725..55bfbfc98 100644
--- a/pipelex/pipeline/track/pipeline_tracker_protocol.py
+++ b/pipelex/pipeline/track/pipeline_tracker_protocol.py
@@ -2,7 +2,7 @@
# pyright: reportUnknownArgumentType=false
# pyright: reportUnknownMemberType=false
# pyright: reportMissingTypeArgument=false
-from typing import List, Optional, Protocol
+from typing import Protocol
from typing_extensions import override
@@ -19,21 +19,21 @@ class PipelineTrackerProtocol(Protocol):
def add_pipe_step(
self,
- from_stuff: Optional[Stuff],
+ from_stuff: Stuff | None,
to_stuff: Stuff,
pipe_code: str,
comment: str,
- pipe_layer: List[str],
- as_item_index: Optional[int] = None,
+ pipe_layer: list[str],
+ as_item_index: int | None = None,
is_with_edge: bool = True,
): ...
def add_batch_step(
self,
- from_stuff: Optional[Stuff],
+ from_stuff: Stuff | None,
to_stuff: Stuff,
to_branch_index: int,
- pipe_layer: List[str],
+ pipe_layer: list[str],
comment: str,
): ...
@@ -41,7 +41,7 @@ class PipelineTrackerProtocol(Protocol):
self,
from_stuff: Stuff,
to_stuff: Stuff,
- pipe_layer: List[str],
+ pipe_layer: list[str],
comment: str,
): ...
@@ -50,7 +50,7 @@ class PipelineTrackerProtocol(Protocol):
from_stuff: Stuff,
to_condition: PipeConditionDetails,
condition_expression: str,
- pipe_layer: List[str],
+ pipe_layer: list[str],
comment: str,
): ...
@@ -58,16 +58,16 @@ class PipelineTrackerProtocol(Protocol):
self,
from_condition: PipeConditionDetails,
to_stuff: Stuff,
- pipe_layer: List[str],
+ pipe_layer: list[str],
comment: str,
): ...
def output_flowchart(
self,
- title: Optional[str] = None,
- subtitle: Optional[str] = None,
+ title: str | None = None,
+ subtitle: str | None = None,
is_detailed: bool = False,
- ) -> Optional[str]: ...
+ ) -> str | None: ...
class PipelineTrackerNoOp(PipelineTrackerProtocol):
@@ -90,12 +90,12 @@ class PipelineTrackerNoOp(PipelineTrackerProtocol):
@override
def add_pipe_step(
self,
- from_stuff: Optional[Stuff],
+ from_stuff: Stuff | None,
to_stuff: Stuff,
pipe_code: str,
comment: str,
- pipe_layer: List[str],
- as_item_index: Optional[int] = None,
+ pipe_layer: list[str],
+ as_item_index: int | None = None,
is_with_edge: bool = True,
) -> None:
pass
@@ -103,10 +103,10 @@ class PipelineTrackerNoOp(PipelineTrackerProtocol):
@override
def add_batch_step(
self,
- from_stuff: Optional[Stuff],
+ from_stuff: Stuff | None,
to_stuff: Stuff,
to_branch_index: int,
- pipe_layer: List[str],
+ pipe_layer: list[str],
comment: str,
) -> None:
pass
@@ -116,7 +116,7 @@ class PipelineTrackerNoOp(PipelineTrackerProtocol):
self,
from_stuff: Stuff,
to_stuff: Stuff,
- pipe_layer: List[str],
+ pipe_layer: list[str],
comment: str,
) -> None:
pass
@@ -127,7 +127,7 @@ class PipelineTrackerNoOp(PipelineTrackerProtocol):
from_stuff: Stuff,
to_condition: PipeConditionDetails,
condition_expression: str,
- pipe_layer: List[str],
+ pipe_layer: list[str],
comment: str,
) -> None:
pass
@@ -137,7 +137,7 @@ class PipelineTrackerNoOp(PipelineTrackerProtocol):
self,
from_condition: PipeConditionDetails,
to_stuff: Stuff,
- pipe_layer: List[str],
+ pipe_layer: list[str],
comment: str,
) -> None:
pass
@@ -145,8 +145,8 @@ class PipelineTrackerNoOp(PipelineTrackerProtocol):
@override
def output_flowchart(
self,
- title: Optional[str] = None,
- subtitle: Optional[str] = None,
+ title: str | None = None,
+ subtitle: str | None = None,
is_detailed: bool = False,
) -> None:
pass
diff --git a/pipelex/pipeline/track/tracker_config.py b/pipelex/pipeline/track/tracker_config.py
index e74a14129..39abd0f2a 100644
--- a/pipelex/pipeline/track/tracker_config.py
+++ b/pipelex/pipeline/track/tracker_config.py
@@ -1,4 +1,4 @@
-from typing import List, Literal, Optional, Union
+from typing import Literal
from pipelex.tools.config.config_model import ConfigModel
@@ -7,11 +7,11 @@ class TrackerConfig(ConfigModel):
is_debug_mode: bool
is_include_text_preview: bool
is_include_interactivity: bool
- theme: Union[str, Literal["auto"]]
- layout: Union[str, Literal["auto"]]
- wrapping_width: Union[int, Literal["auto"]]
- nb_items_limit: Union[int, Literal["unlimited"]]
- sub_graph_colors: List[str]
+ theme: str | Literal["auto"]
+ layout: str | Literal["auto"]
+ wrapping_width: int | Literal["auto"]
+ nb_items_limit: int | Literal["unlimited"]
+ sub_graph_colors: list[str]
pipe_edge_style: str
branch_edge_style: str
aggregate_edge_style: str
@@ -19,29 +19,25 @@ class TrackerConfig(ConfigModel):
choice_edge_style: str
@property
- def applied_theme(self) -> Optional[str]:
+ def applied_theme(self) -> str | None:
if self.theme == "auto":
return None
- else:
- return self.theme
+ return self.theme
@property
- def applied_layout(self) -> Optional[str]:
+ def applied_layout(self) -> str | None:
if self.layout == "auto":
return None
- else:
- return self.layout
+ return self.layout
@property
- def applied_wrapping_width(self) -> Optional[int]:
+ def applied_wrapping_width(self) -> int | None:
if self.wrapping_width == "auto":
return None
- else:
- return self.wrapping_width
+ return self.wrapping_width
@property
- def applied_nb_items_limit(self) -> Optional[int]:
+ def applied_nb_items_limit(self) -> int | None:
if self.nb_items_limit == "unlimited":
return None
- else:
- return self.nb_items_limit
+ return self.nb_items_limit
diff --git a/pipelex/pipeline/track/tracker_models.py b/pipelex/pipeline/track/tracker_models.py
index cc1d8561a..49a2712f1 100644
--- a/pipelex/pipeline/track/tracker_models.py
+++ b/pipelex/pipeline/track/tracker_models.py
@@ -1,5 +1,3 @@
-from typing import Dict, List
-
from pydantic import BaseModel
from pipelex.types import StrEnum
@@ -41,7 +39,7 @@ class EdgeAttributeKey(StrEnum):
class GraphTree(BaseModel):
- nodes_by_subgraph: Dict[str, List[str]]
+ nodes_by_subgraph: dict[str, list[str]]
class SubGraphClassDef(BaseModel):
diff --git a/pipelex/plugins/anthropic/anthropic_factory.py b/pipelex/plugins/anthropic/anthropic_factory.py
index f7df8abc9..a59d76f56 100644
--- a/pipelex/plugins/anthropic/anthropic_factory.py
+++ b/pipelex/plugins/anthropic/anthropic_factory.py
@@ -1,11 +1,9 @@
import asyncio
-from typing import List, Optional, Union
+from typing import TYPE_CHECKING
from anthropic import AsyncAnthropic, AsyncAnthropicBedrock
from anthropic.types import Usage
-from anthropic.types.image_block_param import ImageBlockParam
from anthropic.types.message_param import MessageParam
-from anthropic.types.text_block_param import TextBlockParam
from openai.types.chat import (
ChatCompletionMessageParam,
ChatCompletionSystemMessageParam,
@@ -31,6 +29,10 @@ from pipelex.tools.misc.base_64_utils import load_binary_as_base64_async
from pipelex.tools.misc.filetype_utils import detect_file_type_from_base64
from pipelex.types import StrEnum
+if TYPE_CHECKING:
+ from anthropic.types.image_block_param import ImageBlockParam
+ from anthropic.types.text_block_param import TextBlockParam
+
class AnthropicFactoryError(CogtError):
pass
@@ -46,11 +48,12 @@ class AnthropicFactory:
def make_anthropic_client(
plugin: Plugin,
backend: InferenceBackend,
- ) -> Union[AsyncAnthropic, AsyncAnthropicBedrock]:
+ ) -> AsyncAnthropic | AsyncAnthropicBedrock:
try:
sdk_variant = AnthropicSdkVariant(plugin.sdk)
- except ValueError:
- raise AnthropicFactoryError(f"Plugin '{plugin}' is not supported by AnthropicFactory")
+ except ValueError as exc:
+ msg = f"Plugin '{plugin}' is not supported by AnthropicFactory"
+ raise AnthropicFactoryError(msg) from exc
match sdk_variant:
case AnthropicSdkVariant.ANTHROPIC:
@@ -73,7 +76,7 @@ class AnthropicFactory:
llm_job: LLMJob,
) -> MessageParam:
message: MessageParam
- content: List[Union[TextBlockParam, ImageBlockParam]] = []
+ content: list[TextBlockParam | ImageBlockParam] = []
if llm_job.llm_prompt.user_text:
text_block_param: TextBlockParam = {
@@ -93,9 +96,9 @@ class AnthropicFactory:
"type": "image",
"source": {
"type": "base64",
- "media_type": mime, # type: ignore
+ "media_type": mime, # type: ignore[typeddict-item]
"data": prepped_image.base_64.decode("utf-8"),
- }, # type: ignore
+ }, # pyright: ignore[reportAssignmentType]
}
elif isinstance(prepped_image, str): # pyright: ignore[reportUnnecessaryIsInstance]
url = prepped_image
@@ -107,7 +110,8 @@ class AnthropicFactory:
},
}
else:
- raise AnthropicFactoryError(f"Unsupported PromptImageTypedBytesOrUrl type: '{type(prepped_image).__name__}'")
+ msg = f"Unsupported PromptImageTypedBytesOrUrl type: '{type(prepped_image).__name__}'"
+ raise AnthropicFactoryError(msg)
content.append(image_block_param)
message = {
@@ -121,13 +125,13 @@ class AnthropicFactory:
@staticmethod
def openai_typed_user_message(
user_content_txt: str,
- prepped_user_images: Optional[List[PromptImageTypedUrlOrBase64]] = None,
+ prepped_user_images: list[PromptImageTypedUrlOrBase64] | None = None,
) -> ChatCompletionMessageParam:
text_block_param: TextBlockParam = {"type": "text", "text": user_content_txt}
message: MessageParam
if prepped_user_images is not None:
log.debug(prepped_user_images)
- images_block_params: List[ImageBlockParam] = []
+ images_block_params: list[ImageBlockParam] = []
for prepped_image in prepped_user_images:
image_block_param_in_loop: ImageBlockParam
if isinstance(prepped_image, PromptImageTypedBase64):
@@ -136,9 +140,9 @@ class AnthropicFactory:
"type": "image",
"source": {
"type": "base64",
- "media_type": mime, # type: ignore
+ "media_type": mime, # type: ignore[typeddict-item]
"data": prepped_image.base_64.decode("utf-8"),
- }, # type: ignore
+ }, # pyright: ignore[reportAssignmentType]
}
elif isinstance(prepped_image, str): # pyright: ignore[reportUnnecessaryIsInstance]
url = prepped_image
@@ -150,10 +154,11 @@ class AnthropicFactory:
},
}
else:
- raise AnthropicFactoryError(f"Unsupported PromptImageTypedBytesOrUrl type: '{type(prepped_image).__name__}'")
+ msg = f"Unsupported PromptImageTypedBytesOrUrl type: '{type(prepped_image).__name__}'"
+ raise AnthropicFactoryError(msg)
images_block_params.append(image_block_param_in_loop)
- content: List[Union[TextBlockParam, ImageBlockParam]] = images_block_params + [text_block_param]
+ content: list[TextBlockParam | ImageBlockParam] = [*images_block_params, text_block_param]
message = {
"role": "user",
"content": content,
@@ -165,7 +170,7 @@ class AnthropicFactory:
"content": [text_block_param],
}
- return message # type: ignore
+ return message # type: ignore[return-value, valid-type] # pyright: ignore[reportReturnType]
@classmethod
async def _prep_image_for_anthropic(
@@ -183,24 +188,23 @@ class AnthropicFactory:
b64 = await load_binary_as_base64_async(prompt_image.file_path)
typed_bytes_or_url = PromptImageTypedBase64(base_64=b64, file_type=prompt_image.get_file_type())
else:
- raise AnthropicFactoryError(f"Unsupported PromptImage type: '{type(prompt_image).__name__}'")
+ msg = f"Unsupported PromptImage type: '{type(prompt_image).__name__}'"
+ raise AnthropicFactoryError(msg)
return typed_bytes_or_url
@classmethod
async def make_simple_messages(
cls,
llm_job: LLMJob,
- ) -> List[ChatCompletionMessageParam]:
- """
- Makes a list of messages with a system message (if provided) and followed by a user message.
- """
+ ) -> list[ChatCompletionMessageParam]:
+ """Makes a list of messages with a system message (if provided) and followed by a user message."""
llm_prompt = llm_job.llm_prompt
- messages: List[ChatCompletionMessageParam] = []
+ messages: list[ChatCompletionMessageParam] = []
#### System message ####
if system_content := llm_prompt.system_text:
messages.append(ChatCompletionSystemMessageParam(role="system", content=system_content))
- prepped_user_images: Optional[List[PromptImageTypedUrlOrBase64]]
+ prepped_user_images: list[PromptImageTypedUrlOrBase64] | None
if llm_prompt.user_images:
tasks_to_prep_images = [cls._prep_image_for_anthropic(prompt_image) for prompt_image in llm_prompt.user_images]
prepped_user_images = await asyncio.gather(*tasks_to_prep_images)
@@ -212,7 +216,7 @@ class AnthropicFactory:
AnthropicFactory.openai_typed_user_message(
user_content_txt=llm_prompt.user_text if llm_prompt.user_text else "",
prepped_user_images=prepped_user_images,
- )
+ ),
)
return messages
diff --git a/pipelex/plugins/anthropic/anthropic_llm_worker.py b/pipelex/plugins/anthropic/anthropic_llm_worker.py
index 1f7c1037a..77b67798b 100644
--- a/pipelex/plugins/anthropic/anthropic_llm_worker.py
+++ b/pipelex/plugins/anthropic/anthropic_llm_worker.py
@@ -1,7 +1,7 @@
-from typing import Any, Dict, Optional, Type
+from typing import Any
import instructor
-from anthropic import NOT_GIVEN, AsyncAnthropic, AsyncAnthropicBedrock
+from anthropic import AsyncAnthropic, AsyncAnthropicBedrock, omit
from typing_extensions import override
from pipelex import log
@@ -24,29 +24,23 @@ class AnthropicExtraField(StrEnum):
class AnthropicLLMWorkerError(Exception):
"""Base exception for Anthropic LLM Worker errors."""
- pass
-
class AnthropicBadRequestError(AnthropicLLMWorkerError):
"""Raised when Anthropic API returns a BadRequestError."""
- pass
-
class AnthropicInstructorError(AnthropicLLMWorkerError):
"""Raised when Instructor encounters an error with Anthropic."""
- pass
-
class AnthropicLLMWorker(LLMWorkerInternalAbstract):
def __init__(
self,
sdk_instance: Any,
- extra_config: Dict[str, Any],
+ extra_config: dict[str, Any],
inference_model: InferenceModelSpec,
- structure_method: Optional[StructureMethod] = None,
- reporting_delegate: Optional[ReportingProtocol] = None,
+ structure_method: StructureMethod | None = None,
+ reporting_delegate: ReportingProtocol | None = None,
):
LLMWorkerInternalAbstract.__init__(
self,
@@ -54,23 +48,24 @@ class AnthropicLLMWorker(LLMWorkerInternalAbstract):
structure_method=structure_method,
reporting_delegate=reporting_delegate,
)
- self.extra_config: Dict[str, Any] = extra_config
+ self.extra_config: dict[str, Any] = extra_config
self.default_max_tokens: int = 0
if inference_model.max_tokens:
self.default_max_tokens = inference_model.max_tokens
else:
- raise AnthropicWorkerConfigurationError(
- f"No max_tokens provided for llm model '{self.inference_model.desc}', but it is required for Anthropic"
- )
+ msg = f"No max_tokens provided for llm model '{self.inference_model.desc}', but it is required for Anthropic"
+ raise AnthropicWorkerConfigurationError(msg)
# Verify if the sdk_instance is compatible with the current LLM platform
if isinstance(sdk_instance, (AsyncAnthropic, AsyncAnthropicBedrock)):
- if inference_model.sdk == AnthropicSdkVariant.ANTHROPIC and not (isinstance(sdk_instance, AsyncAnthropic)):
- raise SdkTypeError(f"Provided sdk_instance does not match LLMEngine platform:{sdk_instance}")
- elif inference_model.sdk == AnthropicSdkVariant.BEDROCK_ANTHROPIC and not (isinstance(sdk_instance, AsyncAnthropicBedrock)):
- raise SdkTypeError(f"Provided sdk_instance does not match LLMEngine platform:{sdk_instance}")
+ if (inference_model.sdk == AnthropicSdkVariant.ANTHROPIC and not (isinstance(sdk_instance, AsyncAnthropic))) or (
+ inference_model.sdk == AnthropicSdkVariant.BEDROCK_ANTHROPIC and not (isinstance(sdk_instance, AsyncAnthropicBedrock))
+ ):
+ msg = f"Provided sdk_instance does not match LLMEngine platform:{sdk_instance}"
+ raise SdkTypeError(msg)
else:
- raise SdkTypeError(f"Provided sdk_instance does not match LLMEngine platform:{sdk_instance}")
+ msg = f"Provided sdk_instance does not match LLMEngine platform:{sdk_instance}"
+ raise SdkTypeError(msg)
self.anthropic_async_client = sdk_instance
if structure_method:
@@ -85,14 +80,15 @@ class AnthropicLLMWorker(LLMWorkerInternalAbstract):
#########################################################
# TODO: implement streaming behind the scenes to avoid timeout/streaming errors with Claude 4 and high tokens
- def _adapt_max_tokens(self, max_tokens: Optional[int]) -> int:
+ def _adapt_max_tokens(self, max_tokens: int | None) -> int:
max_tokens = max_tokens or self.default_max_tokens
if (claude_4_tokens_limit := self.extra_config.get(AnthropicExtraField.CLAUDE_4_TOKENS_LIMIT)) and max_tokens > claude_4_tokens_limit:
max_tokens = claude_4_tokens_limit
log.warning(f"Max tokens is greater than the claude 4 reduced tokens limit, reducing to {max_tokens}")
if not max_tokens:
- raise AnthropicWorkerConfigurationError(f"Max tokens is None for model {self.inference_model.desc}")
+ msg = f"Max tokens is None for model {self.inference_model.desc}"
+ raise AnthropicWorkerConfigurationError(msg)
return max_tokens
@override
@@ -104,7 +100,7 @@ class AnthropicLLMWorker(LLMWorkerInternalAbstract):
max_tokens = self._adapt_max_tokens(max_tokens=llm_job.job_params.max_tokens)
response = await self.anthropic_async_client.messages.create(
messages=[message],
- system=llm_job.llm_prompt.system_text or NOT_GIVEN,
+ system=llm_job.llm_prompt.system_text or omit,
model=self.inference_model.model_id,
temperature=llm_job.job_params.temperature,
max_tokens=max_tokens,
@@ -112,12 +108,14 @@ class AnthropicLLMWorker(LLMWorkerInternalAbstract):
single_content_block = response.content[0]
if single_content_block.type != "text":
- raise LLMCompletionError(f"Unexpected content block type: {single_content_block.type}\nmodel: {self.inference_model.desc}")
+ msg = f"Unexpected content block type: {single_content_block.type}\nmodel: {self.inference_model.desc}"
+ raise LLMCompletionError(msg)
full_reply_content = single_content_block.text
single_content_block = response.content[0]
if single_content_block.type != "text":
- raise LLMCompletionError(f"Unexpected content block type: {single_content_block.type}\nmodel: {self.inference_model.desc}")
+ msg = f"Unexpected content block type: {single_content_block.type}\nmodel: {self.inference_model.desc}"
+ raise LLMCompletionError(msg)
full_reply_content = single_content_block.text
if (llm_tokens_usage := llm_job.job_report.llm_tokens_usage) and (usage := response.usage):
@@ -129,7 +127,7 @@ class AnthropicLLMWorker(LLMWorkerInternalAbstract):
async def _gen_object(
self,
llm_job: LLMJob,
- schema: Type[BaseModelTypeVar],
+ schema: type[BaseModelTypeVar],
) -> BaseModelTypeVar:
messages = await AnthropicFactory.make_simple_messages(llm_job=llm_job)
max_tokens = self._adapt_max_tokens(max_tokens=llm_job.job_params.max_tokens)
diff --git a/pipelex/plugins/anthropic/anthropic_llms.py b/pipelex/plugins/anthropic/anthropic_llms.py
index 7c9436e5f..95e20a5e7 100644
--- a/pipelex/plugins/anthropic/anthropic_llms.py
+++ b/pipelex/plugins/anthropic/anthropic_llms.py
@@ -1,5 +1,3 @@
-from typing import List
-
from anthropic import AsyncAnthropic
from anthropic.types import ModelInfo
@@ -9,22 +7,26 @@ from pipelex.plugins.anthropic.anthropic_factory import AnthropicFactory
from pipelex.plugins.plugin_sdk_registry import Plugin
-async def anthropic_list_anthropic_models(plugin: Plugin, backend: InferenceBackend) -> List[ModelInfo]:
- """
- List available Anthropic models.
+async def anthropic_list_anthropic_models(plugin: Plugin, backend: InferenceBackend) -> list[ModelInfo]:
+ """List available Anthropic models.
Returns:
List[ModelInfo]: A list of Anthropic model information objects
+
"""
anthropic_client = AnthropicFactory.make_anthropic_client(plugin=plugin, backend=backend)
if not hasattr(anthropic_client, "models"):
- raise AnthropicSDKUnsupportedError(f"{type(anthropic_client).__name__} does not support listing models")
+ msg = f"{type(anthropic_client).__name__} does not support listing models"
+ raise AnthropicSDKUnsupportedError(msg)
if not isinstance(anthropic_client, AsyncAnthropic):
- raise AnthropicSDKUnsupportedError("We only support the standard Anthropic client for listing models")
+ msg = "We only support the standard Anthropic client for listing models"
+ raise AnthropicSDKUnsupportedError(msg)
models_response = await anthropic_client.models.list()
if not models_response:
- raise AnthropicModelListingError("No models found")
+ msg = "No models found"
+ raise AnthropicModelListingError(msg)
models_list = models_response.data
if not models_list:
- raise AnthropicModelListingError("No models found")
+ msg = "No models found"
+ raise AnthropicModelListingError(msg)
return sorted(models_list, key=lambda model: model.created_at)
diff --git a/pipelex/plugins/bedrock/bedrock_client_aioboto3.py b/pipelex/plugins/bedrock/bedrock_client_aioboto3.py
index 7d0dbe647..435d20882 100644
--- a/pipelex/plugins/bedrock/bedrock_client_aioboto3.py
+++ b/pipelex/plugins/bedrock/bedrock_client_aioboto3.py
@@ -1,4 +1,4 @@
-from typing import Any, Dict, Optional, Tuple, cast
+from typing import TYPE_CHECKING, Any, cast
import aioboto3
from types_aiobotocore_bedrock_runtime.type_defs import ConverseResponseTypeDef
@@ -9,6 +9,9 @@ from pipelex.cogt.usage.token_category import NbTokensByCategoryDict, TokenCateg
from pipelex.plugins.bedrock.bedrock_client_protocol import BedrockClientProtocol
from pipelex.plugins.bedrock.bedrock_message import BedrockMessageDictList
+if TYPE_CHECKING:
+ from types_aiobotocore_bedrock_runtime.type_defs import ConverseResponseTypeDef
+
class BedrockClientAioboto3(BedrockClientProtocol):
def __init__(self, aws_region: str):
@@ -20,12 +23,12 @@ class BedrockClientAioboto3(BedrockClientProtocol):
async def chat(
self,
messages: BedrockMessageDictList,
- system_text: Optional[str],
+ system_text: str | None,
model: str,
temperature: float,
- max_tokens: Optional[int] = None,
- ) -> Tuple[str, NbTokensByCategoryDict]:
- params: Dict[str, Any] = {
+ max_tokens: int | None = None,
+ ) -> tuple[str, NbTokensByCategoryDict]:
+ params: dict[str, Any] = {
"modelId": model,
"messages": messages,
"inferenceConfig": {
@@ -36,10 +39,10 @@ class BedrockClientAioboto3(BedrockClientProtocol):
if system_text:
params["system"] = [{"text": system_text}]
- async with self.session.client("bedrock-runtime", region_name=self.aws_region) as aioboto3_session: # type: ignore
+ async with self.session.client("bedrock-runtime", region_name=self.aws_region) as aioboto3_session: # pyright: ignore[reportUnknownMemberType]
conversation_response: ConverseResponseTypeDef = await aioboto3_session.converse(**params)
- resp_dict: Dict[str, Any] = cast(Dict[str, Any], conversation_response)
- usage_dict: Dict[str, Any] = resp_dict["usage"]
+ resp_dict: dict[str, Any] = cast("dict[str, Any]", conversation_response)
+ usage_dict: dict[str, Any] = resp_dict["usage"]
nb_tokens_by_category: NbTokensByCategoryDict = {
TokenCategory.INPUT: usage_dict["inputTokens"],
TokenCategory.OUTPUT: usage_dict["outputTokens"],
diff --git a/pipelex/plugins/bedrock/bedrock_client_boto3.py b/pipelex/plugins/bedrock/bedrock_client_boto3.py
index 1a7183238..b47c62a9a 100644
--- a/pipelex/plugins/bedrock/bedrock_client_boto3.py
+++ b/pipelex/plugins/bedrock/bedrock_client_boto3.py
@@ -1,5 +1,5 @@
import asyncio
-from typing import Any, Dict, Optional, Tuple
+from typing import Any
import boto3
from typing_extensions import override
@@ -13,18 +13,18 @@ from pipelex.plugins.bedrock.bedrock_message import BedrockMessageDictList
class BedrockClientBoto3(BedrockClientProtocol):
def __init__(self, aws_region: str):
log.debug(f"Initializing BedrockClientBoto3 with region '{aws_region}'")
- self.boto3_client = boto3.client(service_name="bedrock-runtime", region_name=aws_region) # pyright: ignore
+ self.boto3_client = boto3.client(service_name="bedrock-runtime", region_name=aws_region) # pyright: ignore[reportUnknownMemberType]
@override
async def chat(
self,
messages: BedrockMessageDictList,
- system_text: Optional[str],
+ system_text: str | None,
model: str,
temperature: float,
- max_tokens: Optional[int] = None,
- ) -> Tuple[str, NbTokensByCategoryDict]:
- params: Dict[str, Any] = {
+ max_tokens: int | None = None,
+ ) -> tuple[str, NbTokensByCategoryDict]:
+ params: dict[str, Any] = {
"modelId": model,
"messages": messages,
"inferenceConfig": {
@@ -36,9 +36,9 @@ class BedrockClientBoto3(BedrockClientProtocol):
params["system"] = [{"text": system_text}]
loop: asyncio.AbstractEventLoop = asyncio.get_event_loop()
- resp_dict: Dict[str, Any] = await loop.run_in_executor(None, lambda: self.boto3_client.converse(**params)) # pyright: ignore
+ resp_dict: dict[str, Any] = await loop.run_in_executor(None, lambda: self.boto3_client.converse(**params)) # pyright: ignore[reportUnknownLambdaType, reportUnknownMemberType]
- usage_dict: Dict[str, Any] = resp_dict["usage"]
+ usage_dict: dict[str, Any] = resp_dict["usage"]
nb_tokens_by_category: NbTokensByCategoryDict = {
TokenCategory.INPUT: usage_dict["inputTokens"],
TokenCategory.OUTPUT: usage_dict["outputTokens"],
diff --git a/pipelex/plugins/bedrock/bedrock_client_protocol.py b/pipelex/plugins/bedrock/bedrock_client_protocol.py
index c3d7bd447..6ce5290b3 100644
--- a/pipelex/plugins/bedrock/bedrock_client_protocol.py
+++ b/pipelex/plugins/bedrock/bedrock_client_protocol.py
@@ -1,4 +1,4 @@
-from typing import Optional, Protocol, Tuple, runtime_checkable
+from typing import Protocol, runtime_checkable
from pipelex.cogt.usage.token_category import NbTokensByCategoryDict
from pipelex.plugins.bedrock.bedrock_message import BedrockMessageDictList
@@ -9,8 +9,8 @@ class BedrockClientProtocol(Protocol):
async def chat(
self,
messages: BedrockMessageDictList,
- system_text: Optional[str],
+ system_text: str | None,
model: str,
temperature: float,
- max_tokens: Optional[int] = None,
- ) -> Tuple[str, NbTokensByCategoryDict]: ...
+ max_tokens: int | None = None,
+ ) -> tuple[str, NbTokensByCategoryDict]: ...
diff --git a/pipelex/plugins/bedrock/bedrock_factory.py b/pipelex/plugins/bedrock/bedrock_factory.py
index f12bb584c..dd5a8691e 100644
--- a/pipelex/plugins/bedrock/bedrock_factory.py
+++ b/pipelex/plugins/bedrock/bedrock_factory.py
@@ -34,20 +34,21 @@ class BedrockFactory:
) -> BedrockClientProtocol:
try:
sdk_variant = BedrockSdkVariant(plugin.sdk)
- except ValueError:
- raise BedrockFactoryError(f"Plugin '{plugin}' is not supported by BedrockFactory")
+ except ValueError as exc:
+ msg = f"Plugin '{plugin}' is not supported by BedrockFactory"
+ raise BedrockFactoryError(msg) from exc
bedrock_async_client: BedrockClientProtocol
log.verbose(f"Using '{sdk_variant}' for BedrockClient")
match sdk_variant:
case BedrockSdkVariant.AIBOTO3:
- from pipelex.plugins.bedrock.bedrock_client_aioboto3 import BedrockClientAioboto3
+ from pipelex.plugins.bedrock.bedrock_client_aioboto3 import BedrockClientAioboto3 # noqa: PLC0415
bedrock_async_client = BedrockClientAioboto3(
aws_region=backend.extra_config[BedrockExtraField.AWS_REGION],
)
case BedrockSdkVariant.BOTO3:
- from pipelex.plugins.bedrock.bedrock_client_boto3 import BedrockClientBoto3
+ from pipelex.plugins.bedrock.bedrock_client_boto3 import BedrockClientBoto3 # noqa: PLC0415
bedrock_async_client = BedrockClientBoto3(
aws_region=backend.extra_config[BedrockExtraField.AWS_REGION],
@@ -61,13 +62,12 @@ class BedrockFactory:
@classmethod
def make_simple_message(cls, llm_job: LLMJob) -> BedrockMessage:
- """
- Makes a list of messages with a system message (if provided) and followed by a user message.
- """
+ """Makes a list of messages with a system message (if provided) and followed by a user message."""
message = BedrockMessage(role="user", content=[])
if user_text := llm_job.llm_prompt.user_text:
message.content.append(BedrockContentItem(text=user_text))
if llm_job.llm_prompt.user_images:
- raise LLMCapabilityError("BedrockFactory does not support images. Skipping images.")
+ msg = "BedrockFactory does not support images. Skipping images."
+ raise LLMCapabilityError(msg)
return message
diff --git a/pipelex/plugins/bedrock/bedrock_llm_worker.py b/pipelex/plugins/bedrock/bedrock_llm_worker.py
index da02b300b..5098dbee7 100644
--- a/pipelex/plugins/bedrock/bedrock_llm_worker.py
+++ b/pipelex/plugins/bedrock/bedrock_llm_worker.py
@@ -1,4 +1,4 @@
-from typing import Any, Optional, Type
+from typing import Any
from typing_extensions import override
@@ -23,8 +23,8 @@ class BedrockLLMWorker(LLMWorkerInternalAbstract):
self,
sdk_instance: Any,
inference_model: InferenceModelSpec,
- structure_method: Optional[StructureMethod] = None,
- reporting_delegate: Optional[ReportingProtocol] = None,
+ structure_method: StructureMethod | None = None,
+ reporting_delegate: ReportingProtocol | None = None,
):
LLMWorkerInternalAbstract.__init__(
self,
@@ -34,16 +34,14 @@ class BedrockLLMWorker(LLMWorkerInternalAbstract):
)
if not isinstance(sdk_instance, BedrockClientProtocol):
- raise SdkTypeError(
- f"Provided sdk_instance for {self.__class__.__name__} is not of type BedrockClientProtocol: it's a '{type(sdk_instance)}'"
- )
+ msg = f"Provided sdk_instance for {self.__class__.__name__} is not of type BedrockClientProtocol: it's a '{type(sdk_instance)}'"
+ raise SdkTypeError(msg)
if default_max_tokens := inference_model.max_tokens:
self.default_max_tokens = default_max_tokens
else:
- raise BedrockWorkerConfigurationError(
- f"No max_tokens provided for llm model '{self.inference_model.desc}', but it is required for Bedrock"
- )
+ msg = f"No max_tokens provided for llm model '{self.inference_model.desc}', but it is required for Bedrock"
+ raise BedrockWorkerConfigurationError(msg)
self.bedrock_client_for_text = sdk_instance
@override
@@ -70,7 +68,8 @@ class BedrockLLMWorker(LLMWorkerInternalAbstract):
async def _gen_object(
self,
llm_job: LLMJob,
- schema: Type[BaseModelTypeVar],
+ schema: type[BaseModelTypeVar],
) -> BaseModelTypeVar:
# TODO: try with the newest instructor release
- raise LLMCapabilityError(f"It is not possible to generate objects with a {self.__class__.__name__}.")
+ msg = f"It is not possible to generate objects with a {self.__class__.__name__}."
+ raise LLMCapabilityError(msg)
diff --git a/pipelex/plugins/bedrock/bedrock_message.py b/pipelex/plugins/bedrock/bedrock_message.py
index 425e59518..eb21b0163 100644
--- a/pipelex/plugins/bedrock/bedrock_message.py
+++ b/pipelex/plugins/bedrock/bedrock_message.py
@@ -1,4 +1,4 @@
-from typing import Any, Dict, List, Literal, Optional
+from typing import Any, Literal
from pydantic import BaseModel
@@ -98,8 +98,8 @@ class BedrockImage(BaseModel):
class BedrockContentItem(BaseModel):
- text: Optional[str] = None
- image: Optional[BedrockImage] = None
+ text: str | None = None
+ image: BedrockImage | None = None
# document: Optional[BedrockDocument] = None
# video: Optional[BedrockVideo] = None
# toolUse: Optional[BedrockToolUse] = None
@@ -107,13 +107,13 @@ class BedrockContentItem(BaseModel):
# guardContent: Optional[GuardContent] = None
-BedrockMessageDict = Dict[str, Any]
-BedrockMessageDictList = List[BedrockMessageDict]
+BedrockMessageDict = dict[str, Any]
+BedrockMessageDictList = list[BedrockMessageDict]
class BedrockMessage(BaseModel):
role: Literal["user", "assistant"]
- content: List[BedrockContentItem]
+ content: list[BedrockContentItem]
def to_dict(self) -> BedrockMessageDict:
return self.model_dump(exclude_none=True)
diff --git a/pipelex/plugins/fal/fal_config.py b/pipelex/plugins/fal/fal_config.py
index ea3d9e252..da52401e6 100644
--- a/pipelex/plugins/fal/fal_config.py
+++ b/pipelex/plugins/fal/fal_config.py
@@ -1,5 +1,3 @@
-from typing import Dict
-
from pydantic import field_validator
from pipelex.cogt.img_gen.img_gen_job_components import Quality
@@ -8,20 +6,23 @@ from pipelex.tools.exceptions import ConfigValidationError
class FalConfig(ConfigModel):
- flux_map_quality_to_steps: Dict[str, int]
- sdxl_lightning_map_quality_to_steps: Dict[str, int]
+ flux_map_quality_to_steps: dict[str, int]
+ sdxl_lightning_map_quality_to_steps: dict[str, int]
@field_validator("flux_map_quality_to_steps", "sdxl_lightning_map_quality_to_steps")
@classmethod
- def validate_quality_mapping(cls, value: Dict[str, int]) -> Dict[str, int]:
+ def validate_quality_mapping(cls, value: dict[str, int]) -> dict[str, int]:
valid_qualities = {quality.value for quality in Quality}
missing_qualities = valid_qualities - set(value.keys())
invalid_qualities = set(value.keys()) - valid_qualities
if missing_qualities and invalid_qualities:
- raise ConfigValidationError(f"Missing ({missing_qualities}) and invalid ({invalid_qualities}) quality levels in mapping")
- elif missing_qualities:
- raise ConfigValidationError(f"Missing quality levels in mapping: {missing_qualities}")
- elif invalid_qualities:
- raise ConfigValidationError(f"Invalid quality levels in mapping: {invalid_qualities}")
+ msg = f"Missing ({missing_qualities}) and invalid ({invalid_qualities}) quality levels in mapping"
+ raise ConfigValidationError(msg)
+ if missing_qualities:
+ msg = f"Missing quality levels in mapping: {missing_qualities}"
+ raise ConfigValidationError(msg)
+ if invalid_qualities:
+ msg = f"Invalid quality levels in mapping: {invalid_qualities}"
+ raise ConfigValidationError(msg)
return value
diff --git a/pipelex/plugins/fal/fal_factory.py b/pipelex/plugins/fal/fal_factory.py
index f94705e92..577a44c83 100644
--- a/pipelex/plugins/fal/fal_factory.py
+++ b/pipelex/plugins/fal/fal_factory.py
@@ -1,4 +1,4 @@
-from typing import Any, Dict, List, Optional
+from typing import Any
from pipelex import log
from pipelex.cogt.exceptions import ImgGenGeneratedTypeError, ImgGenParameterError
@@ -37,7 +37,8 @@ class FalFactory:
case AspectRatio.PORTRAIT_9_21:
return "portrait_21_9"
case AspectRatio.LANDSCAPE_3_2 | AspectRatio.PORTRAIT_2_3:
- raise ImgGenParameterError(f"Aspect ratio '{aspect_ratio}' is not supported by Flux-1 image generation model")
+ msg = f"Aspect ratio '{aspect_ratio}' is not supported by Flux-1 image generation model"
+ raise ImgGenParameterError(msg)
@classmethod
def make_aspect_ratio_for_flux_1_1_ultra(cls, aspect_ratio: AspectRatio) -> str:
@@ -57,7 +58,8 @@ class FalFactory:
case AspectRatio.PORTRAIT_9_21:
return "9:21"
case AspectRatio.LANDSCAPE_3_2 | AspectRatio.PORTRAIT_2_3:
- raise ImgGenParameterError(f"Aspect ratio '{aspect_ratio}' is not supported by Flux-1.1 Ultra image generation model")
+ msg = f"Aspect ratio '{aspect_ratio}' is not supported by Flux-1.1 Ultra image generation model"
+ raise ImgGenParameterError(msg)
@classmethod
def make_output_format_for_flux(cls, output_format: OutputFormat) -> str:
@@ -67,7 +69,8 @@ class FalFactory:
case OutputFormat.JPG:
return "jpeg"
case OutputFormat.WEBP:
- raise ImgGenParameterError("Output format WebP is not supported for Flux")
+ msg = "Output format WebP is not supported for Flux"
+ raise ImgGenParameterError(msg)
@classmethod
def make_fal_arguments(
@@ -75,16 +78,17 @@ class FalFactory:
fal_application: str,
img_gen_job: ImgGenJob,
nb_images: int,
- ) -> Dict[str, Any]:
+ ) -> dict[str, Any]:
params = img_gen_job.job_params
- args_dict: Dict[str, Any]
- num_inference_steps: Optional[int]
+ args_dict: dict[str, Any]
+ num_inference_steps: int | None
match fal_application:
case "fal-ai/flux-pro" | "fal-ai/flux-pro/v1.1":
num_inference_steps = params.nb_steps
if not num_inference_steps:
if not params.quality:
- raise ImgGenParameterError(f"Either nb_steps or quality must be set for image generation with '{fal_application}'")
+ msg = f"Either nb_steps or quality must be set for image generation with '{fal_application}'"
+ raise ImgGenParameterError(msg)
num_inference_steps = cls.make_nb_steps_from_quality_for_flux_pro(quality=params.quality)
args_dict = {
@@ -129,49 +133,55 @@ class FalFactory:
"sync_mode": img_gen_job.job_config.is_sync_mode,
}
case _:
- raise ImgGenParameterError(f"Invalid fal application: '{fal_application}'")
+ msg = f"Invalid fal application: '{fal_application}'"
+ raise ImgGenParameterError(msg)
return args_dict
@staticmethod
- def make_generated_image(fal_result: Dict[str, Any]) -> GeneratedImage:
+ def make_generated_image(fal_result: dict[str, Any]) -> GeneratedImage:
images = fal_result["images"]
fal_image_dict = images[0]
image_url = fal_image_dict["url"]
if not isinstance(image_url, str):
- raise ImgGenGeneratedTypeError("Image url is not a string")
+ msg = "Image url is not a string"
+ raise ImgGenGeneratedTypeError(msg)
# TODO: if the url is actual image data, send it to cloud storage?
width = fal_image_dict["width"]
if not isinstance(width, int):
- raise ImgGenGeneratedTypeError("Image width is not an integer")
+ msg = "Image width is not an integer"
+ raise ImgGenGeneratedTypeError(msg)
height = fal_image_dict["height"]
if not isinstance(height, int):
- raise ImgGenGeneratedTypeError("Image height is not an integer")
+ msg = "Image height is not an integer"
+ raise ImgGenGeneratedTypeError(msg)
- generated_image = GeneratedImage(
+ return GeneratedImage(
url=image_url,
width=width,
height=height,
)
- return generated_image
@staticmethod
- def make_generated_image_list(fal_result: Dict[str, Any]) -> List[GeneratedImage]:
+ def make_generated_image_list(fal_result: dict[str, Any]) -> list[GeneratedImage]:
fal_image_dicts = fal_result["images"]
- generated_image_list: List[GeneratedImage] = []
+ generated_image_list: list[GeneratedImage] = []
for fal_image_dict in fal_image_dicts:
image_url = fal_image_dict["url"]
if not isinstance(image_url, str):
- raise ImgGenGeneratedTypeError("Image url is not a string")
+ msg = "Image url is not a string"
+ raise ImgGenGeneratedTypeError(msg)
width = fal_image_dict["width"]
if not isinstance(width, int):
- raise ImgGenGeneratedTypeError("Image width is not an integer")
+ msg = "Image width is not an integer"
+ raise ImgGenGeneratedTypeError(msg)
height = fal_image_dict["height"]
if not isinstance(height, int):
- raise ImgGenGeneratedTypeError("Image height is not an integer")
+ msg = "Image height is not an integer"
+ raise ImgGenGeneratedTypeError(msg)
generated_image = GeneratedImage(
url=image_url,
diff --git a/pipelex/plugins/fal/fal_img_gen_worker.py b/pipelex/plugins/fal/fal_img_gen_worker.py
index 3e9df2102..cf023d174 100644
--- a/pipelex/plugins/fal/fal_img_gen_worker.py
+++ b/pipelex/plugins/fal/fal_img_gen_worker.py
@@ -1,4 +1,4 @@
-from typing import Any, List, Optional
+from typing import Any
from fal_client import AsyncClient, InProgress
from typing_extensions import override
@@ -18,12 +18,13 @@ class FalImgGenWorker(ImgGenWorkerAbstract):
self,
sdk_instance: Any,
inference_model: InferenceModelSpec,
- reporting_delegate: Optional[ReportingProtocol] = None,
+ reporting_delegate: ReportingProtocol | None = None,
):
super().__init__(inference_model=inference_model, reporting_delegate=reporting_delegate)
if not isinstance(sdk_instance, AsyncClient):
- raise SdkTypeError(f"Provided Imgg sdk_instance is not of type fal_client.AsyncClient: it's a '{type(sdk_instance)}'")
+ msg = f"Provided ImgGen sdk_instance is not of type fal_client.AsyncClient: it's a '{type(sdk_instance)}'"
+ raise SdkTypeError(msg)
self.fal_async_client = sdk_instance
@@ -64,7 +65,7 @@ class FalImgGenWorker(ImgGenWorkerAbstract):
self,
img_gen_job: ImgGenJob,
nb_images: int,
- ) -> List[GeneratedImage]:
+ ) -> list[GeneratedImage]:
application = self.inference_model.model_id
arguments = FalFactory.make_fal_arguments(
fal_application=application,
diff --git a/pipelex/plugins/google/google_factory.py b/pipelex/plugins/google/google_factory.py
index 06a77b7fe..35e17430a 100644
--- a/pipelex/plugins/google/google_factory.py
+++ b/pipelex/plugins/google/google_factory.py
@@ -1,5 +1,4 @@
import asyncio
-from typing import List, Optional
from google import genai
from google.genai import types as genai_types
@@ -38,24 +37,23 @@ class GoogleFactory:
image_bytes = prompt_image.get_decoded_bytes()
mime_type = prompt_image.get_mime_type()
return genai_types.Part.from_bytes(data=image_bytes, mime_type=mime_type)
- elif isinstance(prompt_image, PromptImagePath):
+ if isinstance(prompt_image, PromptImagePath):
image_bytes = await load_binary_async(prompt_image.file_path)
mime_type = prompt_image.get_mime_type()
return genai_types.Part.from_bytes(data=image_bytes, mime_type=mime_type)
- elif isinstance(prompt_image, PromptImageUrl):
+ if isinstance(prompt_image, PromptImageUrl):
prompt_image_binary = await PromptImageFactory.make_promptimagebinary_from_url_async(prompt_image)
image_bytes = prompt_image_binary.binary
mime_type = prompt_image_binary.get_mime_type()
return genai_types.Part.from_bytes(data=image_bytes, mime_type=mime_type)
- else:
- raise GoogleFactoryError(f"Unsupported PromptImage type: '{type(prompt_image).__name__}'")
+ msg = f"Unsupported PromptImage type: '{type(prompt_image).__name__}'"
+ raise GoogleFactoryError(msg)
@classmethod
async def prepare_user_contents(cls, llm_prompt: LLMPrompt) -> genai_types.ContentListUnion:
"""Prepare contents for Google genai API."""
-
# Build list of parts for multimodal content
- parts: List[genai_types.Part] = []
+ parts: list[genai_types.Part] = []
# Add text content if present
if llm_prompt.user_text:
@@ -71,7 +69,7 @@ class GoogleFactory:
return genai_types.Content(parts=parts, role="user")
@classmethod
- def extract_token_usage(cls, usage_metadata: Optional[genai_types.GenerateContentResponseUsageMetadata]) -> NbTokensByCategoryDict:
+ def extract_token_usage(cls, usage_metadata: genai_types.GenerateContentResponseUsageMetadata | None) -> NbTokensByCategoryDict:
"""Extract token usage from Google's usage metadata."""
if not usage_metadata:
return {}
diff --git a/pipelex/plugins/google/google_llm_worker.py b/pipelex/plugins/google/google_llm_worker.py
index 9b03e9d6e..f49d8ddd0 100644
--- a/pipelex/plugins/google/google_llm_worker.py
+++ b/pipelex/plugins/google/google_llm_worker.py
@@ -1,9 +1,11 @@
-from typing import Optional, Type, cast
+from typing import TYPE_CHECKING, cast
import instructor
from google import genai
from google.genai import types
-from openai.types.chat import ChatCompletionMessageParam
+
+if TYPE_CHECKING:
+ from openai.types.chat import ChatCompletionMessageParam
from typing_extensions import override
from pipelex import log
@@ -21,16 +23,14 @@ from pipelex.tools.typing.pydantic_utils import BaseModelTypeVar
class GoogleLLMWorkerError(Exception):
"""Base exception for Google LLM Worker errors."""
- pass
-
class GoogleLLMWorker(LLMWorkerInternalAbstract):
def __init__(
self,
sdk_instance: genai.Client,
inference_model: InferenceModelSpec,
- structure_method: Optional[StructureMethod] = None,
- reporting_delegate: Optional[ReportingProtocol] = None,
+ structure_method: StructureMethod | None = None,
+ reporting_delegate: ReportingProtocol | None = None,
):
super().__init__(
inference_model=inference_model,
@@ -75,16 +75,19 @@ class GoogleLLMWorker(LLMWorkerInternalAbstract):
# Extract text from response
if not response.candidates:
- raise LLMCompletionError(f"No candidates returned from model: {self.inference_model.desc}")
+ msg = f"No candidates returned from model: {self.inference_model.desc}"
+ raise LLMCompletionError(msg)
candidate = response.candidates[0]
if not candidate.content or not candidate.content.parts:
- raise LLMCompletionError(f"No content parts in response from model: {self.inference_model.desc}")
+ msg = f"No content parts in response from model: {self.inference_model.desc}"
+ raise LLMCompletionError(msg)
# Extract text from the first part
text_content = candidate.content.parts[0].text
if not text_content:
- raise LLMCompletionError(f"No text content in response from model: {self.inference_model.desc}")
+ msg = f"No text content in response from model: {self.inference_model.desc}"
+ raise LLMCompletionError(msg)
# Track token usage if available
if llm_job.job_report.llm_tokens_usage and response.usage_metadata:
@@ -96,7 +99,7 @@ class GoogleLLMWorker(LLMWorkerInternalAbstract):
async def _gen_object(
self,
llm_job: LLMJob,
- schema: Type[BaseModelTypeVar],
+ schema: type[BaseModelTypeVar],
) -> BaseModelTypeVar:
"""Generate structured output using Google Gemini API with instructor."""
# Prepare contents (text and images)
@@ -111,14 +114,15 @@ class GoogleLLMWorker(LLMWorkerInternalAbstract):
)
result_object, completion = await self.instructor_for_objects.chat.completions.create_with_completion(
- messages=[cast(ChatCompletionMessageParam, contents)],
+ messages=[cast("ChatCompletionMessageParam", contents)],
response_model=schema,
max_retries=llm_job.job_config.max_retries,
model=self.inference_model.model_id,
generation_config=generation_config,
)
if not isinstance(result_object, schema):
- raise GoogleLLMWorkerError(f"Google Gemini API returned an object that is not of type {schema}: {result_object}")
+ msg = f"Google Gemini API returned an object that is not of type {schema}: {result_object}"
+ raise GoogleLLMWorkerError(msg)
# Track token usage if available from completion
if llm_job.job_report.llm_tokens_usage:
diff --git a/pipelex/plugins/mistral/mistral_factory.py b/pipelex/plugins/mistral/mistral_factory.py
index 100934d72..e08322db4 100644
--- a/pipelex/plugins/mistral/mistral_factory.py
+++ b/pipelex/plugins/mistral/mistral_factory.py
@@ -1,5 +1,3 @@
-from typing import Dict, List
-
from mistralai import Mistral, OCRImageObject, OCRResponse
from mistralai.models import (
ContentChunk,
@@ -47,17 +45,14 @@ class MistralFactory:
#########################################################
@classmethod
- def make_simple_messages(cls, llm_job: LLMJob) -> List[Messages]:
- """
- Makes a list of messages with a system message (if provided) and followed by a user message.
- """
- messages: List[Messages] = []
- user_content: List[ContentChunk] = []
+ def make_simple_messages(cls, llm_job: LLMJob) -> list[Messages]:
+ """Makes a list of messages with a system message (if provided) and followed by a user message."""
+ messages: list[Messages] = []
+ user_content: list[ContentChunk] = []
if user_text := llm_job.llm_prompt.user_text:
user_content.append(TextChunk(text=user_text))
if user_images := llm_job.llm_prompt.user_images:
- for user_image in user_images:
- user_content.append(cls.make_mistral_image_url(user_image))
+ user_content.extend(cls.make_mistral_image_url(user_image) for user_image in user_images)
if user_content:
messages.append(UserMessage(content=user_content))
@@ -70,28 +65,26 @@ class MistralFactory:
def make_mistral_image_url(cls, prompt_image: PromptImage) -> ImageURLChunk:
if isinstance(prompt_image, PromptImageUrl):
return ImageURLChunk(image_url=prompt_image.url)
- elif isinstance(prompt_image, PromptImagePath):
+ if isinstance(prompt_image, PromptImagePath):
image_bytes = load_binary_as_base64(prompt_image.file_path).decode("utf-8")
file_type = detect_file_type_from_path(prompt_image.file_path)
return ImageURLChunk(image_url=f"data:{file_type.mime};base64,{image_bytes}")
- elif isinstance(prompt_image, PromptImageBase64):
+ if isinstance(prompt_image, PromptImageBase64):
image_bytes = prompt_image.base_64.decode("utf-8")
file_type = detect_file_type_from_base64(prompt_image.base_64)
return ImageURLChunk(image_url=f"data:{file_type.mime};base64,{image_bytes}")
- else:
- raise PromptImageFormatError(f"prompt_image of type {type(prompt_image)} is not supported")
+ msg = f"prompt_image of type {type(prompt_image)} is not supported"
+ raise PromptImageFormatError(msg)
@classmethod
def make_simple_messages_openai_typed(
cls,
llm_job: LLMJob,
- ) -> List[ChatCompletionMessageParam]:
- """
- Makes a list of messages with a system message (if provided) and followed by a user message.
- """
+ ) -> list[ChatCompletionMessageParam]:
+ """Makes a list of messages with a system message (if provided) and followed by a user message."""
llm_prompt = llm_job.llm_prompt
- messages: List[ChatCompletionMessageParam] = []
- user_contents: List[ChatCompletionContentPartParam] = []
+ messages: list[ChatCompletionMessageParam] = []
+ user_contents: list[ChatCompletionContentPartParam] = []
if system_content := llm_prompt.system_text:
messages.append(ChatCompletionSystemMessageParam(role="system", content=system_content))
# TODO: confirm that we can prompt without user_contents, for instance if we have only images,
@@ -124,7 +117,7 @@ class MistralFactory:
should_include_images: bool = False,
# export_dir: Optional[str] = None,
) -> OcrOutput:
- pages: Dict[int, Page] = {}
+ pages: dict[int, Page] = {}
for ocr_response_page in mistral_ocr_response.pages:
page = Page(
text=ocr_response_page.markdown,
@@ -145,7 +138,7 @@ class MistralFactory:
cls,
mistral_ocr_image_obj: OCRImageObject,
) -> ExtractedImageFromPage:
- extracted_image = ExtractedImageFromPage(
+ return ExtractedImageFromPage(
image_id=mistral_ocr_image_obj.id,
top_left_x=mistral_ocr_image_obj.top_left_x,
top_left_y=mistral_ocr_image_obj.top_left_y,
@@ -153,4 +146,3 @@ class MistralFactory:
bottom_right_y=mistral_ocr_image_obj.bottom_right_y,
base_64=mistral_ocr_image_obj.image_base64 if mistral_ocr_image_obj.image_base64 else None,
)
- return extracted_image
diff --git a/pipelex/plugins/mistral/mistral_llm_worker.py b/pipelex/plugins/mistral/mistral_llm_worker.py
index f0de7ad25..ef4160c3f 100644
--- a/pipelex/plugins/mistral/mistral_llm_worker.py
+++ b/pipelex/plugins/mistral/mistral_llm_worker.py
@@ -1,8 +1,10 @@
-from typing import Any, Optional, Type
+from typing import TYPE_CHECKING, Any
import instructor
from mistralai import Mistral
-from mistralai.models import ChatCompletionResponse
+
+if TYPE_CHECKING:
+ from mistralai.models import ChatCompletionResponse
from typing_extensions import override
from pipelex import log
@@ -22,8 +24,8 @@ class MistralLLMWorker(LLMWorkerInternalAbstract):
self,
sdk_instance: Any,
inference_model: InferenceModelSpec,
- structure_method: Optional[StructureMethod] = None,
- reporting_delegate: Optional[ReportingProtocol] = None,
+ structure_method: StructureMethod | None = None,
+ reporting_delegate: ReportingProtocol | None = None,
):
LLMWorkerInternalAbstract.__init__(
self,
@@ -33,14 +35,14 @@ class MistralLLMWorker(LLMWorkerInternalAbstract):
)
if not isinstance(sdk_instance, Mistral):
- raise SdkTypeError(f"Provided LLM sdk_instance for {self.__class__.__name__} is not of type Mistral: it's a '{type(sdk_instance)}'")
+ msg = f"Provided LLM sdk_instance for {self.__class__.__name__} is not of type Mistral: it's a '{type(sdk_instance)}'"
+ raise SdkTypeError(msg)
if default_max_tokens := inference_model.max_tokens:
self.default_max_tokens = default_max_tokens
else:
- raise MistralWorkerConfigurationError(
- f"No max_tokens provided for llm model '{self.inference_model.desc}', but it is required for Mistral"
- )
+ msg = f"No max_tokens provided for llm model '{self.inference_model.desc}', but it is required for Mistral"
+ raise MistralWorkerConfigurationError(msg)
self.mistral_client_for_text: Mistral = sdk_instance
if structure_method:
@@ -56,19 +58,22 @@ class MistralLLMWorker(LLMWorkerInternalAbstract):
llm_job: LLMJob,
) -> str:
messages = MistralFactory.make_simple_messages(llm_job=llm_job)
- response: Optional[ChatCompletionResponse] = await self.mistral_client_for_text.chat.complete_async(
+ response: ChatCompletionResponse | None = await self.mistral_client_for_text.chat.complete_async(
messages=messages,
model=self.inference_model.model_id,
temperature=llm_job.job_params.temperature,
max_tokens=llm_job.job_params.max_tokens or self.default_max_tokens,
)
if not response:
- raise LLMCompletionError("Mistral response is None")
+ msg = "Mistral response is None"
+ raise LLMCompletionError(msg)
if not response.choices:
- raise LLMCompletionError("Mistral response.choices is None")
+ msg = "Mistral response.choices is None"
+ raise LLMCompletionError(msg)
mistral_response_content = response.choices[0].message.content
if not isinstance(mistral_response_content, str):
- raise LLMCompletionError("Mistral response.choices[0].message.content is not a string")
+ msg = "Mistral response.choices[0].message.content is not a string"
+ raise LLMCompletionError(msg)
if (llm_tokens_usage := llm_job.job_report.llm_tokens_usage) and (usage := response.usage):
llm_tokens_usage.nb_tokens_by_category = MistralFactory.make_nb_tokens_by_category(usage=usage)
@@ -79,7 +84,7 @@ class MistralLLMWorker(LLMWorkerInternalAbstract):
async def _gen_object(
self,
llm_job: LLMJob,
- schema: Type[BaseModelTypeVar],
+ schema: type[BaseModelTypeVar],
) -> BaseModelTypeVar:
result_object, completion = await self.instructor_for_objects.chat.completions.create_with_completion(
response_model=schema,
diff --git a/pipelex/plugins/mistral/mistral_llms.py b/pipelex/plugins/mistral/mistral_llms.py
index c9a22d23f..6179927f3 100644
--- a/pipelex/plugins/mistral/mistral_llms.py
+++ b/pipelex/plugins/mistral/mistral_llms.py
@@ -1,5 +1,3 @@
-from typing import List
-
from mistralai.models import Data
from pipelex.hub import get_models_manager
@@ -7,13 +5,15 @@ from pipelex.plugins.mistral.mistral_exceptions import MistralModelListingError
from pipelex.plugins.mistral.mistral_factory import MistralFactory
-def list_mistral_models() -> List[Data]:
+def list_mistral_models() -> list[Data]:
backend = get_models_manager().get_required_inference_backend("mistral")
mistral_client = MistralFactory.make_mistral_client(backend=backend)
models_list_response = mistral_client.models.list()
if not models_list_response:
- raise MistralModelListingError("No models found")
+ msg = "No models found"
+ raise MistralModelListingError(msg)
models_list = models_list_response.data
if not models_list:
- raise MistralModelListingError("No models found")
+ msg = "No models found"
+ raise MistralModelListingError(msg)
return sorted(models_list, key=lambda model: model.id)
diff --git a/pipelex/plugins/mistral/mistral_ocr_worker.py b/pipelex/plugins/mistral/mistral_ocr_worker.py
index c63004b01..6a1c43b0c 100644
--- a/pipelex/plugins/mistral/mistral_ocr_worker.py
+++ b/pipelex/plugins/mistral/mistral_ocr_worker.py
@@ -1,4 +1,4 @@
-from typing import Any, Dict, Optional
+from typing import Any
from mistralai import Mistral
from typing_extensions import override
@@ -22,9 +22,9 @@ class MistralOcrWorker(OcrWorkerAbstract):
def __init__(
self,
sdk_instance: Any,
- extra_config: Dict[str, Any],
+ extra_config: dict[str, Any],
inference_model: InferenceModelSpec,
- reporting_delegate: Optional[ReportingProtocol] = None,
+ reporting_delegate: ReportingProtocol | None = None,
):
super().__init__(
extra_config=extra_config,
@@ -33,7 +33,8 @@ class MistralOcrWorker(OcrWorkerAbstract):
)
if not isinstance(sdk_instance, Mistral):
- raise SdkTypeError(f"Provided OCR sdk_instance for {self.__class__.__name__} is not of type Mistral: it's a '{type(sdk_instance)}'")
+ msg = f"Provided OCR sdk_instance for {self.__class__.__name__} is not of type Mistral: it's a '{type(sdk_instance)}'"
+ raise SdkTypeError(msg)
self.mistral_client: Mistral = sdk_instance
@@ -57,7 +58,8 @@ class MistralOcrWorker(OcrWorkerAbstract):
should_include_page_views=ocr_job.job_params.should_include_page_views,
)
else:
- raise OcrInputError("No image nor PDF URI provided in OcrJob")
+ msg = "No image nor PDF URI provided in OcrJob"
+ raise OcrInputError(msg)
return ocr_output
async def _make_ocr_output_from_image(
@@ -66,17 +68,17 @@ class MistralOcrWorker(OcrWorkerAbstract):
should_caption_image: bool = False,
) -> OcrOutput:
if should_caption_image:
- raise NotImplementedError("Captioning is not implemented for Mistral OCR.")
+ msg = "Captioning is not implemented for Mistral OCR."
+ raise NotImplementedError(msg)
image_path, image_url = clarify_path_or_url(path_or_uri=image_uri)
if image_url:
return await self.extract_from_image_url(
image_url=image_url,
)
- else:
- assert image_path is not None # Type narrowing for mypy
- return await self.extract_from_image_file(
- image_path=image_path,
- )
+ assert image_path is not None
+ return await self.extract_from_image_file(
+ image_path=image_path,
+ )
async def _make_ocr_output_from_pdf(
self,
@@ -86,14 +88,15 @@ class MistralOcrWorker(OcrWorkerAbstract):
should_include_page_views: bool,
) -> OcrOutput:
if should_caption_images:
- raise OcrCapabilityError("Captioning is not implemented for Mistral OCR.")
+ msg = "Captioning is not implemented for Mistral OCR."
+ raise OcrCapabilityError(msg)
if should_include_page_views:
log.debug("Page views are not implemented for Mistral OCR.")
# TODO: use a model capability flag to check possibility before asking for it
# it it's asked and not available, raise
# the caller will be responsible to get the page views using other solution if needed
# raise OcrCapabilityError("Page views are not implemented for Mistral OCR.")
- pdf_path, pdf_url = clarify_path_or_url(path_or_uri=pdf_uri) # pyright: ignore
+ pdf_path, pdf_url = clarify_path_or_url(path_or_uri=pdf_uri)
ocr_output: OcrOutput
if pdf_url:
ocr_output = await self.extract_from_pdf_url(
@@ -101,7 +104,7 @@ class MistralOcrWorker(OcrWorkerAbstract):
should_include_images=should_include_images,
)
else: # pdf_path must be provided based on validation
- assert pdf_path is not None # Type narrowing for mypy
+ assert pdf_path is not None
ocr_output = await self.extract_from_pdf_file(
pdf_path=pdf_path,
should_include_images=should_include_images,
@@ -119,10 +122,9 @@ class MistralOcrWorker(OcrWorkerAbstract):
"image_url": image_url,
},
)
- ocr_output = await MistralFactory.make_ocr_output_from_mistral_response(
+ return await MistralFactory.make_ocr_output_from_mistral_response(
mistral_ocr_response=ocr_response,
)
- return ocr_output
async def extract_from_image_file(
self,
@@ -137,10 +139,9 @@ class MistralOcrWorker(OcrWorkerAbstract):
model=self.inference_model.model_id,
document={"type": "image_url", "image_url": f"data:{mime_type};base64,{b64.decode('utf-8')}"},
)
- ocr_output = await MistralFactory.make_ocr_output_from_mistral_response(
+ return await MistralFactory.make_ocr_output_from_mistral_response(
mistral_ocr_response=ocr_response,
)
- return ocr_output
async def extract_from_pdf_url(
self,
@@ -156,11 +157,10 @@ class MistralOcrWorker(OcrWorkerAbstract):
include_image_base64=should_include_images,
)
- ocr_output = await MistralFactory.make_ocr_output_from_mistral_response(
+ return await MistralFactory.make_ocr_output_from_mistral_response(
mistral_ocr_response=ocr_response,
should_include_images=should_include_images,
)
- return ocr_output
async def extract_from_pdf_file(
self,
diff --git a/pipelex/plugins/mistral/mistral_utils.py b/pipelex/plugins/mistral/mistral_utils.py
index fbbb43ffe..d09e37c01 100644
--- a/pipelex/plugins/mistral/mistral_utils.py
+++ b/pipelex/plugins/mistral/mistral_utils.py
@@ -8,16 +8,17 @@ async def upload_file_for_ocr(
mistral_client: Mistral,
file_path: str,
) -> str:
- """
- Upload a local file to Mistral.
+ """Upload a local file to Mistral.
Args:
- pdf_path: Path to the local file to upload
+ file_path: Path to the local file to upload
+ mistral_client: Mistral client
Returns:
ID of the uploaded file
+
"""
- async with aiofiles.open(file_path, "rb") as file: # type: ignore[reportUnknownMemberType]
+ async with aiofiles.open(file_path, "rb") as file: # pyright: ignore[reportUnknownMemberType]
file_content = await file.read()
uploaded_file = await mistral_client.files.upload_async(
diff --git a/pipelex/plugins/openai/openai_factory.py b/pipelex/plugins/openai/openai_factory.py
index 9bf8427a2..fe28215e8 100644
--- a/pipelex/plugins/openai/openai_factory.py
+++ b/pipelex/plugins/openai/openai_factory.py
@@ -1,5 +1,3 @@
-from typing import Dict, List
-
import openai
from openai.types.chat import (
ChatCompletionContentPartImageParam,
@@ -45,15 +43,17 @@ class OpenAIFactory:
) -> openai.AsyncClient:
try:
sdk_variant = OpenAISdkVariant(plugin.sdk)
- except ValueError:
- raise OpenAIFactoryError(f"Plugin '{plugin}' is not supported by OpenAIFactory")
+ except ValueError as exc:
+ msg = f"Plugin '{plugin}' is not supported by OpenAIFactory"
+ raise OpenAIFactoryError(msg) from exc
the_client: openai.AsyncOpenAI
match sdk_variant:
case OpenAISdkVariant.AZURE_OPENAI:
log.debug(f"Making AsyncOpenAI client with endpoint: {backend.endpoint}")
if backend.endpoint is None:
- raise OpenAIFactoryError("Azure OpenAI endpoint is not set")
+ msg = "Azure OpenAI endpoint is not set"
+ raise OpenAIFactoryError(msg)
the_client = openai.AsyncAzureOpenAI(
azure_endpoint=backend.endpoint,
api_key=backend.api_key,
@@ -73,13 +73,11 @@ class OpenAIFactory:
def make_simple_messages(
cls,
llm_job: LLMJob,
- ) -> List[ChatCompletionMessageParam]:
- """
- Makes a list of messages with a system message (if provided) and followed by a user message.
- """
+ ) -> list[ChatCompletionMessageParam]:
+ """Makes a list of messages with a system message (if provided) and followed by a user message."""
llm_prompt = llm_job.llm_prompt
- messages: List[ChatCompletionMessageParam] = []
- user_contents: List[ChatCompletionContentPartParam] = []
+ messages: list[ChatCompletionMessageParam] = []
+ user_contents: list[ChatCompletionContentPartParam] = []
if system_content := llm_prompt.system_text:
messages.append(ChatCompletionSystemMessageParam(role="system", content=system_content))
# TODO: confirm that we can prompt without user_contents, for instance if we have only images,
@@ -109,12 +107,13 @@ class OpenAIFactory:
image_bytes = load_binary_as_base64(path=prompt_image.file_path)
return cls.make_openai_image_url(PromptImageBase64(base_64=image_bytes))
else:
- raise LLMPromptParameterError(f"prompt_image of type {type(prompt_image)} is not supported")
+ msg = f"prompt_image of type {type(prompt_image)} is not supported"
+ raise LLMPromptParameterError(msg)
return openai_image_url
@staticmethod
def make_openai_error_info(exception: Exception) -> str:
- error_mapping: Dict[type, str] = {
+ error_mapping: dict[type, str] = {
openai.BadRequestError: "OpenAI API request was invalid.",
openai.InternalServerError: "OpenAI is having trouble. Please try again later.",
openai.RateLimitError: "OpenAI API request exceeded rate limit.",
@@ -125,8 +124,7 @@ class OpenAIFactory:
openai.APIConnectionError: "OpenAI API request failed to connect.",
openai.APIError: "OpenAI API returned an API Error.",
}
- error_info = error_mapping.get(type(exception), "An unexpected error occurred with the OpenAI API.")
- return error_info
+ return error_mapping.get(type(exception), "An unexpected error occurred with the OpenAI API.")
# reference:
# https://help.openai.com/en/articles/5247780-using-logit-bias-to-define-token-probability
diff --git a/pipelex/plugins/openai/openai_func.py b/pipelex/plugins/openai/openai_func.py
index 598dc0212..71aeb42cc 100644
--- a/pipelex/plugins/openai/openai_func.py
+++ b/pipelex/plugins/openai/openai_func.py
@@ -1,42 +1,51 @@
import inspect
-from typing import Any, Callable, Dict, List, Tuple, Type
+from collections.abc import Callable
+from typing import Any
from instructor import OpenAISchema, openai_schema
from openai.types.chat import ChatCompletionMessage
from pydantic import BaseModel, create_model
-def create_pydantic_model_from_function(func: Callable[..., Any]) -> Type[BaseModel]:
- """
- Creates a Pydantic BaseModel from a function's signature.
+def create_pydantic_model_from_function(func: Callable[..., Any]) -> type[BaseModel]:
+ """Create a Pydantic BaseModel from a function's signature.
- Parameters:
- func (Callable): The function from which to derive the BaseModel.
+ Args:
+ func: The function to inspect. Its parameters become model fields.
Returns:
- Type[BaseModel]: A dynamically created Pydantic BaseModel class.
+ type[BaseModel]: A dynamically created Pydantic model class.
+
+ Raises:
+ ValueError: If the function has unsupported parameter kinds.
+
"""
sig = inspect.signature(func)
- fields: Dict[str, Tuple[Type[Any], Any]] = {
+ fields: dict[str, tuple[type[Any], Any]] = {
name: (param.annotation, param.default if param.default is not inspect.Parameter.empty else ...) for name, param in sig.parameters.items()
}
model_name = func.__name__
- return create_model(__model_name=model_name, **fields) # type: ignore
+ return create_model(__model_name=model_name, **fields) # type: ignore[name-defined,call-overload,no-any-return] # pyright: ignore[reportCallIssue, reportUnknownVariableType]
-def create_openai_schema_from_function(func: Callable[..., Any]) -> Dict[str, Any]:
- """
- Creates an OpenAI schema from a function.
+def create_openai_schema_from_function(func: Callable[..., Any]) -> dict[str, Any]:
+ """Creates an OpenAI schema from a function.
- Parameters:
- func (Callable): The function from which to derive the OpenAI schema.
+ Parameters
+ ----------
+ func : Callable[..., Any]
+ The Python function to introspect and convert into an OpenAI tool schema.
Returns:
- Dict[str, Any]: The OpenAI schema as a dictionary.
+ -------
+ dict[str, Any]
+ The OpenAI-compatible schema describing the function's name, description,
+ and parameters.
+
"""
- model: Type[BaseModel] = create_pydantic_model_from_function(func)
+ model: type[BaseModel] = create_pydantic_model_from_function(func)
the_openai_schema: OpenAISchema = openai_schema(model)
- as_openai_schema: Dict[str, Any] = the_openai_schema.openai_schema
+ as_openai_schema: dict[str, Any] = the_openai_schema.openai_schema
# replace description using first non-empty line of func.__doc__ description
if func.__doc__:
@@ -47,10 +56,10 @@ def create_openai_schema_from_function(func: Callable[..., Any]) -> Dict[str, An
return as_openai_schema
-def list_openai_tools(openai_message: ChatCompletionMessage) -> List[str]:
+def list_openai_tools(openai_message: ChatCompletionMessage) -> list[str]:
if not openai_message.tool_calls:
return []
- tools: List[str] = []
+ tools: list[str] = []
for tool_call in openai_message.tool_calls:
if tool_call.type == "function":
tools.append(tool_call.function.name)
diff --git a/pipelex/plugins/openai/openai_img_gen_factory.py b/pipelex/plugins/openai/openai_img_gen_factory.py
index 4d99ca48d..00279f979 100644
--- a/pipelex/plugins/openai/openai_img_gen_factory.py
+++ b/pipelex/plugins/openai/openai_img_gen_factory.py
@@ -28,7 +28,8 @@ class OpenAIImgGenFactory:
| AspectRatio.PORTRAIT_9_16
| AspectRatio.PORTRAIT_9_21
):
- raise ImgGenParameterError(f"Aspect ratio '{aspect_ratio}' is not supported by GPT Image 1 model")
+ msg = f"Aspect ratio '{aspect_ratio}' is not supported by GPT Image 1 model"
+ raise ImgGenParameterError(msg)
@classmethod
def output_format_for_gpt_image_1(cls, output_format: OutputFormat) -> GptImage1OutputFormatType:
diff --git a/pipelex/plugins/openai/openai_img_gen_worker.py b/pipelex/plugins/openai/openai_img_gen_worker.py
index 4b8720153..c87d949e6 100644
--- a/pipelex/plugins/openai/openai_img_gen_worker.py
+++ b/pipelex/plugins/openai/openai_img_gen_worker.py
@@ -1,4 +1,4 @@
-from typing import Any, List, Optional
+from typing import Any
import openai
import shortuuid
@@ -24,12 +24,13 @@ class OpenAIImgGenWorker(ImgGenWorkerAbstract):
self,
sdk_instance: Any,
inference_model: InferenceModelSpec,
- reporting_delegate: Optional[ReportingProtocol] = None,
+ reporting_delegate: ReportingProtocol | None = None,
):
super().__init__(inference_model=inference_model, reporting_delegate=reporting_delegate)
if not isinstance(sdk_instance, openai.AsyncOpenAI):
- raise SdkTypeError(f"Provided Imgg sdk_instance is not of type openai.AsyncOpenAI: it's a '{type(sdk_instance)}'")
+ msg = f"Provided ImgGen sdk_instance is not of type openai.AsyncOpenAI: it's a '{type(sdk_instance)}'"
+ raise SdkTypeError(msg)
self.openai_client = sdk_instance
@@ -39,15 +40,14 @@ class OpenAIImgGenWorker(ImgGenWorkerAbstract):
img_gen_job: ImgGenJob,
) -> GeneratedImage:
one_image_list = await self.gen_image_list(img_gen_job=img_gen_job, nb_images=1)
- generated_image = one_image_list[0]
- return generated_image
+ return one_image_list[0]
@override
async def _gen_image_list(
self,
img_gen_job: ImgGenJob,
nb_images: int,
- ) -> List[GeneratedImage]:
+ ) -> list[GeneratedImage]:
image_size = OpenAIImgGenFactory.image_size_for_gpt_image_1(aspect_ratio=img_gen_job.job_params.aspect_ratio)
output_format = OpenAIImgGenFactory.output_format_for_gpt_image_1(output_format=img_gen_job.job_params.output_format)
moderation = OpenAIImgGenFactory.moderation_for_gpt_image_1(is_moderated=img_gen_job.job_params.is_moderated)
@@ -66,14 +66,16 @@ class OpenAIImgGenWorker(ImgGenWorkerAbstract):
n=nb_images,
)
if not result.data:
- raise ImgGenGenerationError("No result from OpenAI")
+ msg = "No result from OpenAI"
+ raise ImgGenGenerationError(msg)
- generated_image_list: List[GeneratedImage] = []
+ generated_image_list: list[GeneratedImage] = []
image_id = shortuuid.uuid()[:4]
for image_index, image_data in enumerate(result.data):
image_base64 = image_data.b64_json
if not image_base64:
- raise ImgGenGenerationError("No base64 image data received from OpenAI")
+ msg = "No base64 image data received from OpenAI"
+ raise ImgGenGenerationError(msg)
folder_path = TEMP_OUTPUTS_DIR
ensure_path(folder_path)
@@ -85,6 +87,6 @@ class OpenAIImgGenWorker(ImgGenWorkerAbstract):
url=img_path,
width=1024,
height=1024,
- )
+ ),
)
return generated_image_list
diff --git a/pipelex/plugins/openai/openai_llm_worker.py b/pipelex/plugins/openai/openai_llm_worker.py
index cd4380647..f3e6cd9bb 100644
--- a/pipelex/plugins/openai/openai_llm_worker.py
+++ b/pipelex/plugins/openai/openai_llm_worker.py
@@ -1,10 +1,12 @@
-from typing import Any, Optional, Type
+from typing import TYPE_CHECKING, Any
import instructor
import openai
from instructor.exceptions import InstructorRetryException
from openai import NOT_GIVEN, APIConnectionError, BadRequestError, NotFoundError
-from openai.types.chat import ChatCompletionMessage
+
+if TYPE_CHECKING:
+ from openai.types.chat import ChatCompletionMessage
from typing_extensions import override
from pipelex import log
@@ -24,8 +26,8 @@ class OpenAILLMWorker(LLMWorkerInternalAbstract):
self,
sdk_instance: Any,
inference_model: InferenceModelSpec,
- structure_method: Optional[StructureMethod],
- reporting_delegate: Optional[ReportingProtocol] = None,
+ structure_method: StructureMethod | None,
+ reporting_delegate: ReportingProtocol | None = None,
):
LLMWorkerInternalAbstract.__init__(
self,
@@ -35,9 +37,8 @@ class OpenAILLMWorker(LLMWorkerInternalAbstract):
)
if not isinstance(sdk_instance, openai.AsyncOpenAI):
- raise SdkTypeError(
- f"Provided LLM sdk_instance for {self.__class__.__name__} is not of type openai.AsyncOpenAI: it's a '{type(sdk_instance)}'"
- )
+ msg = f"Provided LLM sdk_instance for {self.__class__.__name__} is not of type openai.AsyncOpenAI: it's a '{type(sdk_instance)}'"
+ raise SdkTypeError(msg)
self.openai_client_for_text: openai.AsyncOpenAI = sdk_instance
if structure_method:
@@ -48,6 +49,13 @@ class OpenAILLMWorker(LLMWorkerInternalAbstract):
self.instructor_for_objects = instructor.from_openai(client=sdk_instance)
#########################################################
+ @override
+ def setup(self):
+ pass
+
+ @override
+ def teardown(self):
+ pass
@override
async def _gen_text(
@@ -72,18 +80,20 @@ class OpenAILLMWorker(LLMWorkerInternalAbstract):
)
except NotFoundError as not_found_error:
# TODO: record llm config so it can be displayed here
- raise LLMModelNotFoundError(
- f"OpenAI model or deployment not found:\n{self.inference_model.desc}\nmodel: {self.inference_model.desc}\n{not_found_error}"
- ) from not_found_error
+ msg = f"OpenAI model or deployment not found:\n{self.inference_model.desc}\nmodel: {self.inference_model.desc}\n{not_found_error}"
+ raise LLMModelNotFoundError(msg) from not_found_error
except APIConnectionError as api_connection_error:
- raise LLMCompletionError(f"OpenAI API connection error: {api_connection_error}") from api_connection_error
+ msg = f"OpenAI API connection error: {api_connection_error}"
+ raise LLMCompletionError(msg) from api_connection_error
except BadRequestError as bad_request_error:
- raise LLMCompletionError(f"OpenAI bad request error with model: {self.inference_model.desc}:\n{bad_request_error}") from bad_request_error
+ msg = f"OpenAI bad request error with model: {self.inference_model.desc}:\n{bad_request_error}"
+ raise LLMCompletionError(msg) from bad_request_error
openai_message: ChatCompletionMessage = response.choices[0].message
response_text = openai_message.content
if response_text is None:
- raise LLMCompletionError(f"OpenAI response message content is None: {response}\nmodel: {self.inference_model.desc}")
+ msg = f"OpenAI response message content is None: {response}\nmodel: {self.inference_model.desc}"
+ raise LLMCompletionError(msg)
if (llm_tokens_usage := llm_job.job_report.llm_tokens_usage) and (usage := response.usage):
llm_tokens_usage.nb_tokens_by_category = OpenAIFactory.make_nb_tokens_by_category(usage=usage)
@@ -93,7 +103,7 @@ class OpenAILLMWorker(LLMWorkerInternalAbstract):
async def _gen_object(
self,
llm_job: LLMJob,
- schema: Type[BaseModelTypeVar],
+ schema: type[BaseModelTypeVar],
) -> BaseModelTypeVar:
messages = OpenAIFactory.make_simple_messages(llm_job=llm_job)
try:
@@ -114,13 +124,14 @@ class OpenAILLMWorker(LLMWorkerInternalAbstract):
max_retries=llm_job.job_config.max_retries,
)
except InstructorRetryException as exc:
- raise LLMCompletionError(
- f"OpenAI instructor failed with model: {self.inference_model.desc} trying to generate schema: {schema} with error: {exc}"
- ) from exc
+ msg = f"OpenAI instructor failed with model: {self.inference_model.desc} trying to generate schema: {schema} with error: {exc}"
+ raise LLMCompletionError(msg) from exc
except NotFoundError as exc:
- raise LLMCompletionError(f"OpenAI model or deployment '{self.inference_model.model_id}' not found: {exc}") from exc
+ msg = f"OpenAI model or deployment '{self.inference_model.model_id}' not found: {exc}"
+ raise LLMCompletionError(msg) from exc
except BadRequestError as bad_request_error:
- raise LLMCompletionError(f"OpenAI bad request error with model: {self.inference_model.desc}:\n{bad_request_error}") from bad_request_error
+ msg = f"OpenAI bad request error with model: {self.inference_model.desc}:\n{bad_request_error}"
+ raise LLMCompletionError(msg) from bad_request_error
if (llm_tokens_usage := llm_job.job_report.llm_tokens_usage) and (usage := completion.usage):
llm_tokens_usage.nb_tokens_by_category = OpenAIFactory.make_nb_tokens_by_category(usage=usage)
diff --git a/pipelex/plugins/openai/openai_llms.py b/pipelex/plugins/openai/openai_llms.py
index 61f5a13ca..d790e3fe3 100644
--- a/pipelex/plugins/openai/openai_llms.py
+++ b/pipelex/plugins/openai/openai_llms.py
@@ -1,5 +1,3 @@
-from typing import List
-
from openai.types import Model
from pipelex.cogt.model_backends.backend import InferenceBackend
@@ -10,7 +8,7 @@ from pipelex.plugins.plugin_sdk_registry import Plugin
async def openai_list_available_models(
plugin: Plugin,
backend: InferenceBackend,
-) -> List[Model]:
+) -> list[Model]:
openai_client_async = OpenAIFactory.make_openai_client(
plugin=plugin,
backend=backend,
@@ -18,5 +16,4 @@ async def openai_list_available_models(
models = await openai_client_async.models.list()
data = models.data
- sorted_data = sorted(data, key=lambda model: model.id)
- return sorted_data
+ return sorted(data, key=lambda model: model.id)
diff --git a/pipelex/plugins/openai/vertexai_factory.py b/pipelex/plugins/openai/vertexai_factory.py
index e8b5a71e0..3adb49e5a 100644
--- a/pipelex/plugins/openai/vertexai_factory.py
+++ b/pipelex/plugins/openai/vertexai_factory.py
@@ -1,4 +1,4 @@
-from typing import Any, Dict, Tuple
+from typing import Any
from pipelex.cogt.exceptions import CogtError, MissingDependencyError
from pipelex.tools.config.config_model import ConfigModel
@@ -26,22 +26,25 @@ class VertexAIFactory(ConfigModel):
@classmethod
def make_endpoint_and_api_key(
cls,
- extra_config: Dict[str, Any],
- ) -> Tuple[str, str]:
+ extra_config: dict[str, Any],
+ ) -> tuple[str, str]:
"""Configure and return endpoint, and API key."""
gcp_project_id = extra_config.get(VertexAIExtraField.GCP_PROJECT_ID)
if not gcp_project_id:
- raise VertexAIConfigError("GCP project ID is not set in VertexAI config")
+ msg = "GCP project ID is not set in VertexAI config"
+ raise VertexAIConfigError(msg)
gcp_location = extra_config.get(VertexAIExtraField.GCP_LOCATION)
if not gcp_location:
- raise VertexAIConfigError("GCP location is not set in VertexAI config")
+ msg = "GCP location is not set in VertexAI config"
+ raise VertexAIConfigError(msg)
endpoint = cls._make_endpoint(gcp_project_id=gcp_project_id, gcp_location=gcp_location)
gcp_credentials_file_path = extra_config.get(VertexAIExtraField.GCP_CREDENTIALS_FILE_PATH)
if not gcp_credentials_file_path or value_is_placeholder(gcp_credentials_file_path):
- raise VertexAIConfigError("GCP credentials file path is not properly set for VertexAI.")
+ msg = "GCP credentials file path is not properly set for VertexAI."
+ raise VertexAIConfigError(msg)
api_key = cls._make_api_key(gcp_credentials_file_path=gcp_credentials_file_path)
@@ -55,31 +58,31 @@ class VertexAIFactory(ConfigModel):
def _make_api_key(cls, gcp_credentials_file_path: str) -> str:
"""Get GCP credentials and return access token."""
try:
- from google.auth.transport.requests import Request
- from google.oauth2.service_account import Credentials
+ from google.auth.transport.requests import Request # noqa: PLC0415
+ from google.oauth2.service_account import Credentials # noqa: PLC0415
except ImportError as exc:
- raise MissingDependencyError(
- "google-auth-oauthlib",
- "google",
- (
- "The google-auth-oauthlib SDK is required to use Google connection. "
- "You can install it with 'pip install pipelex[google]', or use this model via another provider "
- "(such as Azure OpenAI, OpenAI, anthropic or bedrock)."
- ),
- ) from exc
+ lib_name = "google-auth-oauthlib"
+ lib_extra_name = "google"
+ msg = (
+ "The google-auth-oauthlib SDK is required to use Google connection. "
+ "You can install it with 'pip install pipelex[google]', or use this model via another provider "
+ "(such as Azure OpenAI, OpenAI, anthropic or bedrock)."
+ )
+ raise MissingDependencyError(lib_name, lib_extra_name, msg) from exc
try:
- credentials_dict: Dict[str, Any] = load_json_dict_from_path(path=gcp_credentials_file_path)
+ credentials_dict: dict[str, Any] = load_json_dict_from_path(path=gcp_credentials_file_path)
except FileNotFoundError as exc:
- raise VertexAICredentialsError(
- f"Could not get VertexAI credentials from GCP credentials file: File not found: {gcp_credentials_file_path}"
- ) from exc
+ msg = f"Could not get VertexAI credentials from GCP credentials file: File not found: {gcp_credentials_file_path}"
+ raise VertexAICredentialsError(msg) from exc
credentials = Credentials.from_service_account_info( # pyright: ignore[reportUnknownMemberType]
- credentials_dict, scopes=["https://www.googleapis.com/auth/cloud-platform"]
+ credentials_dict,
+ scopes=["https://www.googleapis.com/auth/cloud-platform"],
)
auth_req = Request()
credentials.refresh(auth_req) # pyright: ignore[reportUnknownMemberType]
if not isinstance(credentials.token, str): # pyright: ignore[reportUnknownMemberType]
- raise VertexAICredentialsError("Token is not a string")
+ msg = "Token is not a string"
+ raise VertexAICredentialsError(msg)
return credentials.token
diff --git a/pipelex/plugins/plugin_sdk_registry.py b/pipelex/plugins/plugin_sdk_registry.py
index 3fd7f8168..0cb5f84de 100644
--- a/pipelex/plugins/plugin_sdk_registry.py
+++ b/pipelex/plugins/plugin_sdk_registry.py
@@ -1,4 +1,4 @@
-from typing import Any, Dict, Optional
+from typing import Any
from pydantic import BaseModel, Field, RootModel
@@ -21,7 +21,7 @@ class Plugin(BaseModel):
)
-PluginSdkRegistryRoot = Dict[str, Any]
+PluginSdkRegistryRoot = dict[str, Any]
class PluginSdkRegistry(RootModel[PluginSdkRegistryRoot]):
@@ -33,7 +33,7 @@ class PluginSdkRegistry(RootModel[PluginSdkRegistryRoot]):
sdk_instance.teardown()
self.root = {}
- def get_sdk_instance(self, plugin: Plugin) -> Optional[Any]:
+ def get_sdk_instance(self, plugin: Plugin) -> Any | None:
return self.root.get(plugin.sdk_handle)
def set_sdk_instance(self, plugin: Plugin, sdk_instance: Any) -> Any:
diff --git a/pipelex/plugins/pypdfium2/__init__.py b/pipelex/plugins/pypdfium2/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/pipelex/plugins/pypdfium2/pypdfium2_worker.py b/pipelex/plugins/pypdfium2/pypdfium2_worker.py
index 399cd55c8..388178efa 100644
--- a/pipelex/plugins/pypdfium2/pypdfium2_worker.py
+++ b/pipelex/plugins/pypdfium2/pypdfium2_worker.py
@@ -1,4 +1,4 @@
-from typing import Any, Dict, Optional
+from typing import Any
from typing_extensions import override
@@ -15,10 +15,9 @@ from pipelex.tools.pdf.pypdfium2_renderer import pypdfium2_renderer
class Pypdfium2Worker(OcrWorkerAbstract):
def __init__(
self,
- sdk_instance: Any,
- extra_config: Dict[str, Any],
+ extra_config: dict[str, Any],
inference_model: InferenceModelSpec,
- reporting_delegate: Optional[ReportingProtocol] = None,
+ reporting_delegate: ReportingProtocol | None = None,
):
super().__init__(extra_config=extra_config, inference_model=inference_model, reporting_delegate=reporting_delegate)
@@ -28,10 +27,11 @@ class Pypdfium2Worker(OcrWorkerAbstract):
ocr_job: OcrJob,
) -> OcrOutput:
if ocr_job.ocr_input.image_uri:
- raise NotImplementedError("Pypdfium2 only extracts text from PDFs, not images")
+ msg = "Pypdfium2 only extracts text from PDFs, not images"
+ raise NotImplementedError(msg)
- elif pdf_uri := ocr_job.ocr_input.pdf_uri:
- pdf_path, pdf_url = clarify_path_or_url(path_or_uri=pdf_uri) # pyright: ignore
+ if pdf_uri := ocr_job.ocr_input.pdf_uri:
+ pdf_path, pdf_url = clarify_path_or_url(path_or_uri=pdf_uri)
ocr_output: OcrOutput
if pdf_url:
ocr_output = await self.extract_from_pdf_url(
@@ -43,7 +43,8 @@ class Pypdfium2Worker(OcrWorkerAbstract):
pdf_path=pdf_path,
)
else:
- raise OcrInputError("No PDF URI provided in OcrJob")
+ msg = "No PDF URI provided in OcrJob"
+ raise OcrInputError(msg)
return ocr_output
async def extract_from_pdf_url(
@@ -51,27 +52,25 @@ class Pypdfium2Worker(OcrWorkerAbstract):
pdf_url: str,
) -> OcrOutput:
page_texts = await pypdfium2_renderer.get_text_from_pdf_pages_from_uri(pdf_uri=pdf_url)
- pages: Dict[int, Page] = {}
+ pages: dict[int, Page] = {}
for page_index, page_text in enumerate(page_texts):
pages[page_index + 1] = Page(
text=page_text,
)
- ocr_output = OcrOutput(
+ return OcrOutput(
pages=pages,
)
- return ocr_output
async def extract_from_pdf_file(
self,
pdf_path: str,
) -> OcrOutput:
page_texts = await pypdfium2_renderer.get_text_from_pdf_pages(pdf_input=pdf_path)
- pages: Dict[int, Page] = {}
+ pages: dict[int, Page] = {}
for page_index, page_text in enumerate(page_texts):
pages[page_index + 1] = Page(
text=page_text,
)
- ocr_output = OcrOutput(
+ return OcrOutput(
pages=pages,
)
- return ocr_output
diff --git a/pipelex/reporting/__init__.py b/pipelex/reporting/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/pipelex/reporting/reporting_manager.py b/pipelex/reporting/reporting_manager.py
index 85c063824..8169a7865 100644
--- a/pipelex/reporting/reporting_manager.py
+++ b/pipelex/reporting/reporting_manager.py
@@ -1,5 +1,3 @@
-from typing import Dict, List, Optional
-
from pydantic import Field, RootModel
from typing_extensions import override
@@ -13,12 +11,13 @@ from pipelex.config import ReportingConfig
from pipelex.pipeline.pipeline_models import SpecialPipelineId
from pipelex.reporting.reporting_protocol import ReportingProtocol
from pipelex.tools.misc.file_utils import ensure_path, get_incremental_file_path
+from pipelex.tools.typing.pydantic_utils import empty_list_factory_of
-LLMUsageRegistryRoot = List[LLMTokensUsage]
+LLMUsageRegistryRoot = list[LLMTokensUsage]
class UsageRegistry(RootModel[LLMUsageRegistryRoot]):
- root: LLMUsageRegistryRoot = Field(default_factory=list)
+ root: LLMUsageRegistryRoot = Field(default_factory=empty_list_factory_of(LLMTokensUsage))
def get_current_tokens_usage(self) -> LLMUsageRegistryRoot:
return self.root
@@ -29,7 +28,7 @@ class UsageRegistry(RootModel[LLMUsageRegistryRoot]):
class ReportingManager(ReportingProtocol):
def __init__(self, reporting_config: ReportingConfig):
- self._usage_registries: Dict[str, UsageRegistry] = {}
+ self._usage_registries: dict[str, UsageRegistry] = {}
self._reporting_config = reporting_config
############################################################
@@ -51,7 +50,8 @@ class ReportingManager(ReportingProtocol):
def _get_registry(self, pipeline_run_id: str) -> UsageRegistry:
if pipeline_run_id not in self._usage_registries:
- raise ReportingManagerError(f"Registry for pipeline '{pipeline_run_id}' does not exist")
+ msg = f"Registry for pipeline '{pipeline_run_id}' does not exist"
+ raise ReportingManagerError(msg)
return self._usage_registries[pipeline_run_id]
def _report_llm_job(self, llm_job: LLMJob):
@@ -61,7 +61,7 @@ class ReportingManager(ReportingProtocol):
log.warning("LLM job has no llm_tokens_usage")
return
- llm_token_cost_report: Optional[LLMTokenCostReport] = None
+ llm_token_cost_report: LLMTokenCostReport | None = None
if self._reporting_config.is_log_costs_to_console:
llm_token_cost_report = CostRegistry.complete_cost_report(llm_tokens_usage=llm_tokens_usage)
@@ -79,7 +79,8 @@ class ReportingManager(ReportingProtocol):
@override
def open_registry(self, pipeline_run_id: str):
if pipeline_run_id in self._usage_registries:
- raise ReportingManagerError(f"Registry for pipeline '{pipeline_run_id}' already exists")
+ msg = f"Registry for pipeline '{pipeline_run_id}' already exists"
+ raise ReportingManagerError(msg)
self._usage_registries[pipeline_run_id] = UsageRegistry()
@override
@@ -93,8 +94,8 @@ class ReportingManager(ReportingProtocol):
self._report_llm_job(llm_job=llm_job)
@override
- def generate_report(self, pipeline_run_id: Optional[str] = None):
- cost_report_file_path: Optional[str] = None
+ def generate_report(self, pipeline_run_id: str | None = None):
+ cost_report_file_path: str | None = None
if self._reporting_config.is_generate_cost_report_file_enabled:
ensure_path(self._reporting_config.cost_report_dir_path)
cost_report_file_path = get_incremental_file_path(
@@ -103,7 +104,7 @@ class ReportingManager(ReportingProtocol):
extension=self._reporting_config.cost_report_extension,
)
- registries_to_process: Dict[str, UsageRegistry] = {}
+ registries_to_process: dict[str, UsageRegistry] = {}
if pipeline_run_id:
registries_to_process = {pipeline_run_id: self._get_registry(pipeline_run_id)}
else:
diff --git a/pipelex/reporting/reporting_protocol.py b/pipelex/reporting/reporting_protocol.py
index 2c63e4d9a..9d8664a5e 100644
--- a/pipelex/reporting/reporting_protocol.py
+++ b/pipelex/reporting/reporting_protocol.py
@@ -1,4 +1,4 @@
-from typing import Optional, Protocol
+from typing import Protocol
from typing_extensions import override
@@ -10,7 +10,7 @@ class ReportingProtocol(Protocol):
def report_inference_job(self, inference_job: InferenceJobAbstract): ...
- def generate_report(self, pipeline_run_id: Optional[str] = None): ...
+ def generate_report(self, pipeline_run_id: str | None = None): ...
def close_registry(self, pipeline_run_id: str): ...
@@ -29,7 +29,7 @@ class ReportingNoOp(ReportingProtocol):
pass
@override
- def generate_report(self, pipeline_run_id: Optional[str] = None):
+ def generate_report(self, pipeline_run_id: str | None = None):
pass
@override
diff --git a/pipelex/test_extras/registry_test_models.py b/pipelex/test_extras/registry_test_models.py
index d453f729c..33a7bf088 100644
--- a/pipelex/test_extras/registry_test_models.py
+++ b/pipelex/test_extras/registry_test_models.py
@@ -1,4 +1,4 @@
-from typing import ClassVar, List
+from typing import ClassVar
from pipelex.core.stuffs.stuff_content import StructuredContent
from pipelex.tools.registry_models import ModelType, RegistryModels
@@ -12,4 +12,4 @@ class FictionCharacter(StructuredContent):
class PipelexTestModels(RegistryModels):
- TEST_MODELS: ClassVar[List[ModelType]] = [FictionCharacter]
+ TEST_MODELS: ClassVar[list[ModelType]] = [FictionCharacter]
diff --git a/pipelex/test_extras/shared_pytest_plugins.py b/pipelex/test_extras/shared_pytest_plugins.py
index 6e5dd3880..6c9ed48f8 100644
--- a/pipelex/test_extras/shared_pytest_plugins.py
+++ b/pipelex/test_extras/shared_pytest_plugins.py
@@ -1,9 +1,8 @@
import os
-from typing import List
import pytest
from pytest import FixtureRequest, Parser
-from rich import print
+from rich import print # noqa: A004
from pipelex.core.pipes.pipe_run_params import PipeRunMode
from pipelex.tools.environment import is_env_var_set, set_env
@@ -61,7 +60,7 @@ def pipe_run_mode(request: FixtureRequest) -> PipeRunMode:
return PipeRunMode(mode_str)
-def _setup_env_var_placeholders(env_var_keys: List[str]) -> None:
+def _setup_env_var_placeholders(env_var_keys: list[str]) -> None:
"""Set placeholder environment variables when running in CI to prevent import failures.
These placeholders allow the code to import successfully, while actual inference tests
@@ -69,6 +68,7 @@ def _setup_env_var_placeholders(env_var_keys: List[str]) -> None:
Args:
env_var_keys: List of environment variable keys that need placeholders
+
"""
# Set placeholders for env vars who's presence is required for the code to run properly
# even if their value is not used in the test
@@ -83,7 +83,7 @@ def _setup_env_var_placeholders(env_var_keys: List[str]) -> None:
print(f"[yellow]Set {substitutions_counter} placeholder environment variables[/yellow]")
-def _cleanup_placeholder_env_vars(env_var_keys: List[str]) -> None:
+def _cleanup_placeholder_env_vars(env_var_keys: list[str]) -> None:
"""Remove placeholder environment variables that were set during CI testing.
This function identifies and removes any environment variables that contain
@@ -91,6 +91,7 @@ def _cleanup_placeholder_env_vars(env_var_keys: List[str]) -> None:
Args:
env_var_keys: List of environment variable keys to check for placeholders
+
"""
removed_counter = 0
diff --git a/pipelex/tools/aws/aws_config.py b/pipelex/tools/aws/aws_config.py
index b3aa13c63..cace6d869 100644
--- a/pipelex/tools/aws/aws_config.py
+++ b/pipelex/tools/aws/aws_config.py
@@ -1,5 +1,3 @@
-from typing import Tuple
-
from pydantic import Field
from pipelex import log
@@ -28,10 +26,10 @@ AWS_REGION_VAR_NAME = "AWS_REGION"
class AwsConfig(ConfigModel):
api_key_method: AwsKeyMethod = Field(strict=False)
- def get_aws_access_keys(self) -> Tuple[str, str, str]:
+ def get_aws_access_keys(self) -> tuple[str, str, str]:
return self.get_aws_access_keys_with_method(api_key_method=self.api_key_method)
- def get_aws_access_keys_with_method(self, api_key_method: AwsKeyMethod) -> Tuple[str, str, str]:
+ def get_aws_access_keys_with_method(self, api_key_method: AwsKeyMethod) -> tuple[str, str, str]:
match api_key_method:
case AwsKeyMethod.ENV:
log.debug("Getting AWS access keys from environment (key id and secret access key).")
@@ -40,7 +38,8 @@ class AwsConfig(ConfigModel):
aws_secret_access_key = get_required_env(AWS_SECRET_ACCESS_KEY_VAR_NAME)
aws_region = get_required_env(AWS_REGION_VAR_NAME)
except EnvVarNotFoundError as exc:
- raise AwsCredentialsError(f"Error getting AWS access keys from environment: {exc}") from exc
+ msg = f"Error getting AWS access keys from environment: {exc}"
+ raise AwsCredentialsError(msg) from exc
log.debug("Getting AWS region from environment (priority override) or from aws_config.")
case AwsKeyMethod.SECRET_PROVIDER:
@@ -50,7 +49,8 @@ class AwsConfig(ConfigModel):
aws_secret_access_key = get_secret(AWS_SECRET_ACCESS_KEY_VAR_NAME)
aws_region = get_secret(AWS_REGION_VAR_NAME)
except SecretNotFoundError as exc:
- raise AwsCredentialsError("Error getting AWS access keys from secrets provider.") from exc
+ msg = "Error getting AWS access keys from secrets provider."
+ raise AwsCredentialsError(msg) from exc
log.debug("Getting AWS region from environment (priority override) or from aws_config.")
return aws_access_key_id, aws_secret_access_key, aws_region
diff --git a/pipelex/tools/class_registry_utils.py b/pipelex/tools/class_registry_utils.py
index f44b33592..18e7384b8 100644
--- a/pipelex/tools/class_registry_utils.py
+++ b/pipelex/tools/class_registry_utils.py
@@ -1,18 +1,24 @@
+import types
from pathlib import Path
-from typing import Any, Dict, List, Optional, Type, Union, get_args, get_origin
+from typing import TYPE_CHECKING, Annotated, Any, Union, get_args, get_origin
from kajson.kajson_manager import KajsonManager
-from pydantic.fields import FieldInfo
+
+if TYPE_CHECKING:
+ from pydantic.fields import FieldInfo
from pipelex.tools.typing.module_inspector import find_classes_in_module, import_module_from_file
+_NoneType = type(None)
+_UnionType = getattr(types, "UnionType", None) # Py3.10+: types.UnionType
+
class ClassRegistryUtils:
@classmethod
def register_classes_in_file(
cls,
file_path: str,
- base_class: Optional[Type[Any]],
+ base_class: type[Any] | None,
is_include_imported: bool,
) -> None:
"""Processes a Python file to find and register classes."""
@@ -31,23 +37,21 @@ class ClassRegistryUtils:
def register_classes_in_folder(
cls,
folder_path: str,
- base_class: Optional[Type[Any]] = None,
+ base_class: type[Any] | None = None,
is_recursive: bool = True,
is_include_imported: bool = False,
) -> None:
- """
- Registers all classes in Python files within folders that are subclasses of base_class.
+ """Registers all classes in Python files within folders that are subclasses of base_class.
If base_class is None, registers all classes.
Args:
- folder_paths: List of paths to folders containing Python files
+ folder_path: Path to folder containing Python files
base_class: Optional base class to filter registerable classes
- recursive: Whether to search recursively in subdirectories
- exclude_files: List of filenames to exclude
- exclude_dirs: List of directory names to exclude
+ is_recursive: Whether to search recursively in subdirectories
include_imported: Whether to include classes imported from other modules
- """
+ is_include_imported: Whether to include classes imported from other modules
+ """
python_files = cls.find_files_in_dir(
dir_path=folder_path,
pattern="*.py",
@@ -62,46 +66,45 @@ class ClassRegistryUtils:
)
@classmethod
- def find_files_in_dir(cls, dir_path: str, pattern: str, is_recursive: bool) -> List[Path]:
- """
- Find files matching a pattern in a directory.
+ def find_files_in_dir(cls, dir_path: str, pattern: str, is_recursive: bool) -> list[Path]:
+ """Find files matching a pattern in a directory.
Args:
dir_path: Directory path to search in
pattern: File pattern to match (e.g. "*.py")
- recursive: Whether to search recursively in subdirectories
+ is_recursive: Whether to search recursively in subdirectories
Returns:
List of matching Path objects
+
"""
path = Path(dir_path)
if is_recursive:
return list(path.rglob(pattern))
- else:
- return list(path.glob(pattern))
+ return list(path.glob(pattern))
@staticmethod
- def are_classes_equivalent(class_1: Type[Any], class_2: Type[Any]) -> bool:
+ def are_classes_equivalent(class_1: type[Any], class_2: type[Any]) -> bool:
"""Check if two Pydantic classes are equivalent (same fields, types, descriptions)."""
if not (hasattr(class_1, "model_fields") and hasattr(class_2, "model_fields")):
return class_1 == class_2
# Compare model schemas using Pydantic's built-in capabilities
try:
- schema_1: Dict[str, Any] = class_1.model_json_schema() # type: ignore[attr-defined]
- schema_2: Dict[str, Any] = class_2.model_json_schema() # type: ignore[attr-defined]
+ schema_1: dict[str, Any] = class_1.model_json_schema()
+ schema_2: dict[str, Any] = class_2.model_json_schema()
return schema_1 == schema_2
except Exception:
# Fallback to manual field comparison if schema comparison fails
- fields_1: Dict[str, FieldInfo] = class_1.model_fields # type: ignore[attr-defined]
- fields_2: Dict[str, FieldInfo] = class_2.model_fields # type: ignore[attr-defined]
+ fields_1: dict[str, FieldInfo] = class_1.model_fields
+ fields_2: dict[str, FieldInfo] = class_2.model_fields
if set(fields_1.keys()) != set(fields_2.keys()):
return False
- for field_name in fields_1.keys():
- field_1: FieldInfo = fields_1[field_name]
- field_2: FieldInfo = fields_2[field_name]
+ for field_1_name, field_1_info in fields_1.items():
+ field_1: FieldInfo = field_1_info
+ field_2: FieldInfo = fields_2[field_1_name]
# Compare field types
if field_1.annotation != field_2.annotation:
@@ -118,32 +121,34 @@ class ClassRegistryUtils:
return True
@staticmethod
- def has_compatible_field(class_1: Type[Any], class_2: Type[Any]) -> bool:
- """Check if class_1 has a field that is compatible with class_2."""
+ def has_compatible_field(class_1: type[Any], class_2: type[Any]) -> bool:
+ """Check if class_1 has a field whose (possibly wrapped) type matches/subclasses class_2."""
if not hasattr(class_1, "model_fields"):
return False
- fields: Dict[str, FieldInfo] = class_1.model_fields # type: ignore[attr-defined]
- for _field_name, field_info in fields.items():
- field_type = field_info.annotation
+ fields: dict[str, FieldInfo] = class_1.model_fields # type: ignore[attr-defined]
- # Handle Optional types by extracting the inner type
- origin = get_origin(field_type)
- if origin is Union:
- args = get_args(field_type)
- # Check if this is Optional[T] (Union[T, None])
- if len(args) == 2 and type(None) in args:
- field_type = args[0] if args[1] is type(None) else args[1]
+ def _is_compatible(t: Any) -> bool:
+ # Unwrap Annotated[T, ...]
+ if get_origin(t) is Annotated:
+ t = get_args(t)[0]
- # Check if the field type matches class_2
- if field_type == class_2:
- return True
+ origin = get_origin(t)
- # Check if field_type is a subclass of class_2
+ # Handle unions, including PEP 604 (T | None)
+ if origin in (Union, _UnionType):
+ for arg in get_args(t):
+ if arg is _NoneType:
+ continue
+ if _is_compatible(arg):
+ return True
+ return False
+
+ # Base case: direct match / subclass
try:
- if isinstance(field_type, type) and issubclass(field_type, class_2):
- return True
+ return t is class_2 or (isinstance(t, type) and issubclass(t, class_2))
except TypeError:
- pass
+ # Not a class type (e.g., typing constructs you don't care about)
+ return False
- return False
+ return any(_is_compatible(field.annotation) for field in fields.values())
diff --git a/pipelex/tools/config/config_model.py b/pipelex/tools/config/config_model.py
index 64a3f7e0d..f92ae20b9 100644
--- a/pipelex/tools/config/config_model.py
+++ b/pipelex/tools/config/config_model.py
@@ -1,4 +1,4 @@
-from typing import Dict, List, Optional, Type, TypeVar, Union
+from typing import TypeVar
from pydantic import BaseModel, ConfigDict
@@ -13,12 +13,11 @@ class ConfigModel(BaseModel):
@staticmethod
def transform_dict_str_to_enum(
- input_dict: Dict[str, str],
- key_enum_cls: Optional[Type[StrEnumType]] = None,
- value_enum_cls: Optional[Type[StrEnumType]] = None,
- ) -> Union[Dict[str, StrEnumType], Dict[StrEnumType, str], Dict[StrEnumType, StrEnumType]]:
- """
- Transforms a dictionary with str values into a dictionary with enum values.
+ input_dict: dict[str, str],
+ key_enum_cls: type[StrEnumType] | None = None,
+ value_enum_cls: type[StrEnumType] | None = None,
+ ) -> dict[str, StrEnumType] | dict[StrEnumType, str] | dict[StrEnumType, StrEnumType]:
+ """Transforms a dictionary with str values into a dictionary with enum values.
Args:
input_dict: Dictionary with string values to be transformed.
@@ -27,24 +26,24 @@ class ConfigModel(BaseModel):
Returns:
A dictionary where the values are converted to the given StrEnum type.
+
"""
# return {key: value_enum_cls(value) for key, value in input_dict.items()}
if key_enum_cls and value_enum_cls:
return {key_enum_cls(key): value_enum_cls(value) for key, value in input_dict.items()}
- elif key_enum_cls:
+ if key_enum_cls:
return {key_enum_cls(key): value for key, value in input_dict.items()}
- elif value_enum_cls:
+ if value_enum_cls:
return {key: value_enum_cls(value) for key, value in input_dict.items()}
- else:
- raise ConfigModelError("Either key_enum_cls or value_enum_cls must be provided.")
+ msg = "Either key_enum_cls or value_enum_cls must be provided."
+ raise ConfigModelError(msg)
@staticmethod
def transform_dict_of_floats_str_to_enum(
- input_dict: Dict[str, float],
- key_enum_cls: Type[StrEnumType],
- ) -> Dict[StrEnumType, float]:
- """
- Transforms a dictionary with str keys and float values into a dictionary with enum keys and float values.
+ input_dict: dict[str, float],
+ key_enum_cls: type[StrEnumType],
+ ) -> dict[StrEnumType, float]:
+ """Transforms a dictionary with str keys and float values into a dictionary with enum keys and float values.
Args:
input_dict: Dictionary with string values to be transformed.
@@ -52,12 +51,13 @@ class ConfigModel(BaseModel):
Returns:
A dictionary where the keys are converted to the given StrEnum type.
+
"""
return {key_enum_cls(key): value for key, value in input_dict.items()}
@staticmethod
def transform_list_of_str_to_enum(
- input_list: List[str],
- enum_cls: Type[StrEnumType],
- ) -> List[StrEnumType]:
+ input_list: list[str],
+ enum_cls: type[StrEnumType],
+ ) -> list[StrEnumType]:
return [enum_cls(item) for item in input_list]
diff --git a/pipelex/tools/config/config_root.py b/pipelex/tools/config/config_root.py
index fcf42c180..65096c4cf 100644
--- a/pipelex/tools/config/config_root.py
+++ b/pipelex/tools/config/config_root.py
@@ -1,4 +1,4 @@
-from typing import Any, Optional
+from typing import Any
from pydantic import ValidationError
@@ -18,26 +18,25 @@ class SecretMethod(StrEnum):
class ConfigRoot(ConfigModel):
- """
- Main configuration class for the project.
+ """Main configuration class for the project.
Attributes:
project_name (str): Name of the current project.
+
"""
- project_name: Optional[str] = None
+ project_name: str | None = None
def __init__(self, **kwargs: Any):
- """
- Initialize the Config instance.
+ """Initialize the Config instance.
Args:
**kwargs: Keyword arguments for configuration.
Raises:
ConfigValidationError: If the provided data is invalid.
- """
+ """
try:
super().__init__(**kwargs)
except ValidationError as exc:
diff --git a/pipelex/tools/config/manager.py b/pipelex/tools/config/manager.py
index 53f1079a1..15376ec22 100644
--- a/pipelex/tools/config/manager.py
+++ b/pipelex/tools/config/manager.py
@@ -1,7 +1,7 @@
import importlib
import os
from configparser import ConfigParser
-from typing import Any, Dict, Optional
+from typing import Any
import toml
@@ -19,7 +19,7 @@ CONFIG_TEMPLATE_SUBPATH = "config_template"
INFERENCE_CONFIG_SUBPATH = "inference"
-class ConfigException(Exception):
+class ConfigError(Exception):
pass
@@ -45,7 +45,8 @@ class ConfigManager:
@property
def local_root_dir(self) -> str:
"""Get the root directory of the project using pipelex.
- This is the directory from where the command is being run."""
+ This is the directory from where the command is being run.
+ """
return os.path.abspath(os.getcwd())
@property
@@ -56,29 +57,28 @@ class ConfigManager:
def pipelex_specific_config_file_path(self) -> str:
return os.path.join(self.pipelex_config_dir, CONFIG_NAME)
- def get_pipelex_config(self) -> Dict[str, Any]:
+ def get_pipelex_config(self) -> dict[str, Any]:
"""Get the pipelex configuration from pipelex.toml.
Returns:
Dict[str, Any]: The configuration dictionary from pipelex.toml
+
"""
config_path = self.pipelex_root_config_path
- config = load_toml_from_path(config_path)
- return config
+ return load_toml_from_path(config_path)
- def get_local_config(self) -> Dict[str, Any]:
+ def get_local_config(self) -> dict[str, Any]:
"""Get the local pipelex configuration from pipelex.toml in the project root.
Returns:
Dict[str, Any]: The configuration dictionary from the local pipelex.toml
+
"""
config_path = os.path.join(self.local_root_dir, CONFIG_NAME)
- config = failable_load_toml_from_path(config_path)
- return config or {}
+ return failable_load_toml_from_path(config_path) or {}
- def load_inheritance_config(self, the_pipelex_config: Dict[str, Any]):
- """
- Load the config by inheritance in a pyproject.toml file.
+ def load_inheritance_config(self, the_pipelex_config: dict[str, Any]):
+ """Load the config by inheritance in a pyproject.toml file.
This will be removed in the future.
Requires to have a pyproject.toml file in the project root.
"""
@@ -87,7 +87,7 @@ class ConfigManager:
print(f"pyproject.toml not found in {self.local_root_dir}")
return
- def _find_package_path(package_name: str) -> Optional[str]:
+ def _find_package_path(package_name: str) -> str | None:
"""Find package path by importing it"""
try:
module = importlib.import_module(package_name)
@@ -109,7 +109,7 @@ class ConfigManager:
if config:
deep_update(the_pipelex_config, config)
- def load_config(self, specific_config_path: Optional[str] = None) -> Dict[str, Any]:
+ def load_config(self, specific_config_path: str | None = None) -> dict[str, Any]:
"""Load and merge configurations from pipelex and local config files.
The configuration is loaded and merged in the following order:
@@ -122,8 +122,8 @@ class ConfigManager:
Returns:
Dict[str, Any]: The merged configuration dictionary
- """
+ """
#################### 1. Load pipelex config ####################
pipelex_config = self.get_pipelex_config()
@@ -138,9 +138,12 @@ class ConfigManager:
deep_update(pipelex_config, local_config)
#################### 4. Load overrides for the current project ####################
- list_of_overrides = (
- CONFIG_BASE_OVERRIDES_BEFORE_ENV + [runtime_manager.environment] + [runtime_manager.run_mode] + CONFIG_BASE_OVERRIDES_AFTER_ENV
- )
+ list_of_overrides = [
+ *CONFIG_BASE_OVERRIDES_BEFORE_ENV,
+ runtime_manager.environment,
+ runtime_manager.run_mode,
+ *CONFIG_BASE_OVERRIDES_AFTER_ENV,
+ ]
for override in list_of_overrides:
if override:
if override == runtime_manager.run_mode.UNIT_TEST:
@@ -156,11 +159,12 @@ class ConfigManager:
if config:
deep_update(pipelex_config, config)
else:
- raise ConfigException(f"Failed to load specific config from {specific_config_path}")
+ msg = f"Failed to load specific config from {specific_config_path}"
+ raise ConfigError(msg)
return pipelex_config
- def get_project_name(self) -> Optional[str]:
+ def get_project_name(self) -> str | None:
"""Get the project name from configuration files.
Checks the following files in order:
@@ -170,15 +174,15 @@ class ConfigManager:
4. setup.py
Returns:
- Optional[str]: The project name or None if not found
+ str | None: The project name or None if not found
+
"""
# First check pipelex's pyproject.toml
pipelex_pyproject_path = os.path.join(os.path.dirname(self.local_root_dir), "pyproject.toml")
try:
pyproject = toml.load(pipelex_pyproject_path)
- if project_name := pyproject.get("project", {}).get("name"):
- if isinstance(project_name, str):
- return project_name
+ if (project_name := pyproject.get("project", {}).get("name")) and isinstance(project_name, str):
+ return str(project_name)
except FileNotFoundError:
pass
except toml.TomlDecodeError as exc:
@@ -190,9 +194,9 @@ class ConfigManager:
pyproject_path = os.path.join(self.local_root_dir, "pyproject.toml")
try:
pyproject = toml.load(pyproject_path)
- if project_name := pyproject.get("project", {}).get("name") or pyproject.get("tool", {}).get("poetry", {}).get("name"):
- if isinstance(project_name, str):
- return project_name
+ name_obj: object = pyproject.get("project", {}).get("name") or pyproject.get("tool", {}).get("poetry", {}).get("name")
+ if isinstance(name_obj, str):
+ return name_obj
except FileNotFoundError as exc:
print(f"Local pyproject.toml not found at {pyproject_path}: {exc}")
except toml.TomlDecodeError as exc:
@@ -205,9 +209,8 @@ class ConfigManager:
try:
config = ConfigParser()
config.read(setup_cfg_path)
- if config.has_section("metadata"):
- if cfg_name := config.get("metadata", "name", fallback=None):
- return cfg_name
+ if (cfg_name := config.get("metadata", "name", fallback=None)) and config.has_section("metadata"):
+ return cfg_name
except FileNotFoundError as exc:
print(f"setup.cfg not found at {setup_cfg_path}: {exc}")
except (ValueError, OSError) as exc:
@@ -230,7 +233,7 @@ class ConfigManager:
return line[start + 1 : end]
except FileNotFoundError as exc:
print(f"setup.py not found at {setup_py_path}: {exc}")
- except (IOError, UnicodeDecodeError) as exc:
+ except (OSError, UnicodeDecodeError) as exc:
print(f"Failed to read setup.py at {setup_py_path}: {exc}")
print("Could not find project name in any of the configuration files")
diff --git a/pipelex/tools/environment.py b/pipelex/tools/environment.py
index 97f14fc59..ec2cef78f 100644
--- a/pipelex/tools/environment.py
+++ b/pipelex/tools/environment.py
@@ -1,5 +1,4 @@
import os
-from typing import List, Optional
from dotenv import load_dotenv
@@ -16,27 +15,24 @@ class EnvVarNotFoundError(ToolException):
def get_required_env(key: str) -> str:
value = os.getenv(key)
if not value:
- raise EnvVarNotFoundError(f"Environment variable '{key}' is required but not set")
+ msg = f"Environment variable '{key}' is required but not set"
+ raise EnvVarNotFoundError(msg)
return value
-def get_optional_env(key: str) -> Optional[str]:
- value = os.getenv(key)
- return value
+def get_optional_env(key: str) -> str | None:
+ return os.getenv(key)
def is_env_var_set(key: str) -> bool:
return os.getenv(key) is not None
-def all_env_vars_are_set(keys: List[str]) -> bool:
- for each_key in keys:
- if not is_env_var_set(each_key):
- return False
- return True
+def all_env_vars_are_set(keys: list[str]) -> bool:
+ return all(is_env_var_set(each_key) for each_key in keys)
-def any_env_var_is_placeholder(keys: List[str]) -> bool:
+def any_env_var_is_placeholder(keys: list[str]) -> bool:
for each_key in keys:
env_value = os.getenv(each_key)
if value_is_placeholder(env_value):
@@ -46,4 +42,3 @@ def any_env_var_is_placeholder(keys: List[str]) -> bool:
def set_env(key: str, value: str) -> None:
os.environ[key] = value
- return None
diff --git a/pipelex/tools/func_registry.py b/pipelex/tools/func_registry.py
index 223e19cb9..34b58e97e 100644
--- a/pipelex/tools/func_registry.py
+++ b/pipelex/tools/func_registry.py
@@ -1,6 +1,7 @@
import inspect
import logging
-from typing import Any, Callable, Dict, List, Optional, TypeVar, get_type_hints
+from collections.abc import Callable
+from typing import Any, TypeVar, get_type_hints
from pydantic import Field, PrivateAttr, RootModel
@@ -10,7 +11,7 @@ FUNC_REGISTRY_LOGGER_CHANNEL_NAME = "func_registry"
# Type variable for generic function types
T = TypeVar("T")
-FuncRegistryDict = Dict[str, Callable[..., Any]]
+FuncRegistryDict = dict[str, Callable[..., Any]]
class FuncRegistryError(ToolException):
@@ -34,7 +35,7 @@ class FuncRegistry(RootModel[FuncRegistryDict]):
def register_function(
self,
func: Callable[..., Any],
- name: Optional[str] = None,
+ name: str | None = None,
should_warn_if_already_registered: bool = True,
) -> None:
"""Registers a function in the registry with a name if it meets eligibility criteria."""
@@ -46,7 +47,8 @@ class FuncRegistry(RootModel[FuncRegistryDict]):
if should_warn_if_already_registered:
self.log(f"Function '{key}' already exists in registry")
else:
- raise FuncRegistryError(f"Function '{key}' already exists in registry")
+ msg = f"Function '{key}' already exists in registry"
+ raise FuncRegistryError(msg)
else:
self.log(f"Registered new single function '{key}' in registry")
self.root[key] = func
@@ -55,52 +57,56 @@ class FuncRegistry(RootModel[FuncRegistryDict]):
"""Unregisters a function from the registry."""
key = func.__name__
if key not in self.root:
- raise FuncRegistryError(f"Function '{key}' not found in registry")
+ msg = f"Function '{key}' not found in registry"
+ raise FuncRegistryError(msg)
del self.root[key]
self.log(f"Unregistered single function '{key}' from registry")
def unregister_function_by_name(self, name: str) -> None:
"""Unregisters a function from the registry by its name."""
if name not in self.root:
- raise FuncRegistryError(f"Function '{name}' not found in registry")
+ msg = f"Function '{name}' not found in registry"
+ raise FuncRegistryError(msg)
del self.root[name]
- def register_functions_dict(self, functions: Dict[str, Callable[..., Any]]) -> None:
+ def register_functions_dict(self, functions: dict[str, Callable[..., Any]]) -> None:
"""Registers multiple functions in the registry with names if they meet eligibility criteria."""
for name, func in functions.items():
self.register_function(func=func, name=name, should_warn_if_already_registered=False)
- def register_functions(self, functions: List[Callable[..., Any]]) -> None:
+ def register_functions(self, functions: list[Callable[..., Any]]) -> None:
"""Registers multiple functions in the registry with names if they meet eligibility criteria."""
for func in functions:
self.register_function(func=func, should_warn_if_already_registered=False)
- def get_function(self, name: str) -> Optional[Callable[..., Any]]:
+ def get_function(self, name: str) -> Callable[..., Any] | None:
"""Retrieves a function from the registry by its name. Returns None if not found."""
return self.root.get(name)
def get_required_function(self, name: str) -> Callable[..., Any]:
"""Retrieves a function from the registry by its name. Raises an error if not found."""
if name not in self.root:
- raise FuncRegistryError(
- f"Function '{name}' not found in registry: \
- See how to register a function here: https://docs.pipelex.com/pages/build-reliable-ai-workflows-with-pipelex/pipe-operators/PipeFunc"
+ msg = (
+ f"Function '{name}' not found in registry:"
+ "See how to register a function here: https://docs.pipelex.com/pages/build-reliable-ai-workflows-with-pipelex/pipe-operators/PipeFunc"
)
+ raise FuncRegistryError(msg)
return self.root[name]
- def get_required_function_with_signature(self, name: str, expected_signature: Callable[..., T]) -> Callable[..., T]:
- """
- Retrieves a function from the registry by its name and verifies it matches the expected signature.
+ def get_required_function_with_signature(self, name: str) -> Callable[..., object]:
+ """Retrieves a function from the registry by its name and verifies it matches the expected signature.
Raises an error if not found or if signature doesn't match.
"""
if name not in self.root:
- raise FuncRegistryError(f"Function '{name}' not found in registry")
+ msg = f"Function '{name}' not found in registry"
+ raise FuncRegistryError(msg)
func = self.root[name]
# Note: This is a basic signature check. For more thorough type checking,
# you might want to use typing.get_type_hints() or a more sophisticated type checker
if not callable(func):
- raise FuncRegistryError(f"'{name}' is not a callable function")
+ msg = f"'{name}' is not a callable function"
+ raise FuncRegistryError(msg)
return func
def has_function(self, name: str) -> bool:
@@ -108,15 +114,14 @@ class FuncRegistry(RootModel[FuncRegistryDict]):
return name in self.root
def is_eligible_function(self, func: Callable[..., Any]) -> bool:
- """
- Checks if a function matches the criteria for PipeFunc registration:
+ """Checks if a function matches the criteria for PipeFunc registration:
- Exactly 1 parameter named "working_memory" with type WorkingMemory
- Return type that is a subclass of StuffContent
"""
try:
# Import here to avoid circular imports
- from pipelex.core.memory.working_memory import WorkingMemory
- from pipelex.core.stuffs.stuff_content import StuffContent
+ from pipelex.core.memory.working_memory import WorkingMemory # noqa: PLC0415
+ from pipelex.core.stuffs.stuff_content import StuffContent # noqa: PLC0415
# Get function signature
sig = inspect.signature(func)
@@ -153,7 +158,7 @@ class FuncRegistry(RootModel[FuncRegistryDict]):
return True
# Handle generic types like ListContent[SomeType]
if hasattr(return_type, "__origin__"):
- origin = getattr(return_type, "__origin__")
+ origin = return_type.__origin__
if inspect.isclass(origin) and issubclass(origin, StuffContent):
return True
except TypeError:
diff --git a/pipelex/tools/func_registry_utils.py b/pipelex/tools/func_registry_utils.py
index c16bd06f9..469f8646d 100644
--- a/pipelex/tools/func_registry_utils.py
+++ b/pipelex/tools/func_registry_utils.py
@@ -1,6 +1,7 @@
import inspect
+from collections.abc import Callable
from pathlib import Path
-from typing import Any, Callable, List
+from typing import Any
from pipelex.tools.func_registry import func_registry
from pipelex.tools.typing.module_inspector import import_module_from_file
@@ -13,8 +14,7 @@ class FuncRegistryUtils:
folder_path: str,
is_recursive: bool = True,
) -> None:
- """
- Discovers and attempts to register all functions in Python files within a folder.
+ """Discovers and attempts to register all functions in Python files within a folder.
Only functions that meet the eligibility criteria will be registered:
- Must be an async function
- Exactly 1 parameter named "working_memory" with type WorkingMemory
@@ -25,6 +25,7 @@ class FuncRegistryUtils:
Args:
folder_path: Path to folder containing Python files
is_recursive: Whether to search recursively in subdirectories
+
"""
python_files = cls._find_files_in_dir(
dir_path=folder_path,
@@ -55,11 +56,9 @@ class FuncRegistryUtils:
print(f"Error processing file {file_path}: {e}")
@classmethod
- def _find_functions_in_module(cls, module: Any) -> List[Callable[..., Any]]:
- """
- Finds all functions in a module (eligibility will be checked during registration).
- """
- functions: List[Callable[..., Any]] = []
+ def _find_functions_in_module(cls, module: Any) -> list[Callable[..., Any]]:
+ """Finds all functions in a module (eligibility will be checked during registration)."""
+ functions: list[Callable[..., Any]] = []
module_name = module.__name__
# Find all functions in the module (not imported ones)
@@ -74,9 +73,8 @@ class FuncRegistryUtils:
return functions
@classmethod
- def _find_files_in_dir(cls, dir_path: str, pattern: str, is_recursive: bool) -> List[Path]:
- """
- Find files matching a pattern in a directory.
+ def _find_files_in_dir(cls, dir_path: str, pattern: str, is_recursive: bool) -> list[Path]:
+ """Find files matching a pattern in a directory.
Args:
dir_path: Directory path to search in
@@ -85,9 +83,9 @@ class FuncRegistryUtils:
Returns:
List of matching Path objects
+
"""
path = Path(dir_path)
if is_recursive:
return list(path.rglob(pattern))
- else:
- return list(path.glob(pattern))
+ return list(path.glob(pattern))
diff --git a/pipelex/tools/log/log.py b/pipelex/tools/log/log.py
index 8e418d9be..32da57727 100644
--- a/pipelex/tools/log/log.py
+++ b/pipelex/tools/log/log.py
@@ -1,5 +1,5 @@
import logging
-from typing import Any, Dict, Optional, Union
+from typing import Any
from pipelex.tools.log.log_config import LogConfig, LogMode
from pipelex.tools.log.log_dispatch import LogDispatch
@@ -8,22 +8,18 @@ from pipelex.tools.log.log_levels import LOGGING_LEVEL_DEV, LOGGING_LEVEL_OFF, L
class Log:
- """
- A class for managing logging configurations and operations.
- """
+ """A class for managing logging configurations and operations."""
########################################################
# Init and Configure
########################################################
def __init__(self):
- """
- Initialize the Log class with default attributes.
- """
- self.project_name: Optional[str] = None
- self._log_config_instance: Optional[LogConfig] = None
- self.rich_handler: Optional[logging.Handler] = None
- self.poor_handler: Optional[logging.Handler] = None
+ """Initialize the Log class with default attributes."""
+ self.project_name: str | None = None
+ self._log_config_instance: LogConfig | None = None
+ self.rich_handler: logging.Handler | None = None
+ self.poor_handler: logging.Handler | None = None
self.log_dispatch: LogDispatch = LogDispatch()
def set_log_mode(self, mode: LogMode):
@@ -31,23 +27,22 @@ class Log:
@property
def _log_config(self) -> LogConfig:
- """
- Get the log configuration, raising an error if it's not set.
+ """Get the log configuration, raising an error if it's not set.
Returns:
LogConfig: The current log configuration.
Raises:
RuntimeError: If the log configuration is not set.
+
"""
if self._log_config_instance is None:
- raise RuntimeError("LogConfig is not set. You must initialize Pipelex first.")
+ msg = "LogConfig is not set. You must initialize Pipelex first."
+ raise RuntimeError(msg)
return self._log_config_instance
def reset(self):
- """
- Reset the logging system.
- """
+ """Reset the logging system."""
# Remove all handlers from the root logger
root_logger = logging.getLogger()
for handler in root_logger.handlers[:]:
@@ -56,7 +51,10 @@ class Log:
# Remove handlers from poor loggers
if self._log_config_instance:
- poor_loggers = set(self._log_config_instance.poor_loggers + [self._log_config_instance.generic_poor_logger])
+ poor_loggers = {
+ *self._log_config_instance.poor_loggers,
+ self._log_config_instance.generic_poor_logger,
+ }
for logger_name in poor_loggers:
logger = logging.getLogger(logger_name)
for handler in logger.handlers[:]:
@@ -75,8 +73,7 @@ class Log:
project_name: str,
log_config: LogConfig,
):
- """
- Configure the logging system with the given project name and log configuration.
+ """Configure the logging system with the given project name and log configuration.
Args:
project_name (str): The name of the project.
@@ -84,9 +81,11 @@ class Log:
Raises:
RuntimeError: If the log configuration is already set.
+
"""
if self._log_config_instance is not None:
- raise RuntimeError("LogConfig is already set. You can only call log.configure() once.")
+ msg = "LogConfig is already set. You can only call log.configure() once."
+ raise RuntimeError(msg)
self.log_dispatch.configure(
project_name=project_name,
@@ -106,7 +105,7 @@ class Log:
self.poor_handler = logging.StreamHandler()
self.poor_handler.setFormatter(LevelAndEmojiLogFormatter())
- poor_loggers = set(log_config.poor_loggers + [log_config.generic_poor_logger])
+ poor_loggers = {*log_config.poor_loggers, log_config.generic_poor_logger}
for logger_name in poor_loggers:
logger = logging.getLogger(logger_name)
logger.setLevel(log_config.default_log_level.int_logging_level)
@@ -119,50 +118,48 @@ class Log:
self.verbose(f"Config set for {project_name}")
def set_poor_log_formatter(self, formatter: logging.Formatter):
- """
- Set the formatter for the poor log handler.
+ """Set the formatter for the poor log handler.
Args:
formatter (logging.Formatter): The formatter to use for poor logging.
+
"""
if self.poor_handler is None:
- raise RuntimeError("Poor log handler is not set.")
+ msg = "Poor log handler is not set."
+ raise RuntimeError(msg)
self.poor_handler.setFormatter(formatter)
- def _should_ignore(self, problem_id: Optional[str] = None) -> bool:
- """
- Check if a log message should be ignored based on the problem ID.
+ def _should_ignore(self, problem_id: str | None = None) -> bool:
+ """Check if a log message should be ignored based on the problem ID.
Args:
- problem_id (Optional[str]): The problem ID to check.
+ problem_id (str | None): The problem ID to check.
Returns:
bool: True if the message should be ignored, False otherwise.
+
"""
- if problem_id and problem_id in self._log_config.silenced_problem_ids:
- return True
- else:
- return False
+ return bool(problem_id) and problem_id in self._log_config.silenced_problem_ids
########################################################
# Public methods
########################################################
def set_level_by_int(self, level_int: int):
- """
- Set the log level using an integer value.
+ """Set the log level using an integer value.
Args:
level_int (int): The integer representation of the log level.
+
"""
logging.getLogger().setLevel(level_int)
def set_level_by_name(self, level_name: str):
- """
- Set the log level using a string name.
+ """Set the log level using a string name.
Args:
level_name (str): The name of the log level.
+
"""
if level_name.upper() == LogLevel.DEV:
level = LOGGING_LEVEL_DEV
@@ -173,122 +170,123 @@ class Log:
self.set_level_by_int(level)
def set_level(self, level: LogLevel):
- """
- Set the default log level for all loggers.
+ """Set the default log level for all loggers.
+
Args:
level (LogLevel): The log level to set.
+
"""
self.set_level_by_int(level_int=level.int_logging_level)
def set_level_for_package(self, package_name: str, level: LogLevel):
- """
- Set the log level for a specific package.
+ """Set the log level for a specific package.
Args:
package_name (str): The name of the package.
level (LogLevel): The log level to set for the package.
+
"""
logger_name = package_name.replace("-", ".")
logging.getLogger(logger_name).setLevel(level.int_logging_level)
- def set_levels_for_packages(self, package_log_levels: Dict[str, LogLevel]):
- """
- Set log levels for multiple packages.
+ def set_levels_for_packages(self, package_log_levels: dict[str, LogLevel]):
+ """Set log levels for multiple packages.
Args:
package_log_levels (Dict[str, LogLevel]): A dictionary mapping package names to log levels.
+
"""
for package_name, level in package_log_levels.items():
self.set_level_for_package(package_name=package_name, level=level)
def verbose(
self,
- content: Union[str, Any],
- title: Optional[str] = None,
- inline: Optional[str] = None,
+ content: str | Any,
+ title: str | None = None,
+ inline: str | None = None,
):
- """
- Log a verbose message.
+ """Log a verbose message.
Args:
content (Union[str, Any]): The content to log.
- title (Optional[str], optional): The title of the log message. Defaults to None.
- inline (Optional[str], optional): Inline title for the log message. Defaults to None.
+ title (str | None, optional): The title of the log message. Defaults to None.
+ inline (str | None, optional): Inline title for the log message. Defaults to None.
Used to display the title inline, only if the title arg is None.
+
"""
severity = LOGGING_LEVEL_VERBOSE
self.log_dispatch.dispatch(content=content, severity=severity, title=title, inline=inline)
def debug(
self,
- content: Union[str, Any],
- title: Optional[str] = None,
- inline: Optional[str] = None,
+ content: str | Any,
+ title: str | None = None,
+ inline: str | None = None,
):
- """
- Log a debug message.
+ """Log a debug message.
Args:
content (Union[str, Any]): The content to log.
- title (Optional[str], optional): The title of the log message. Defaults to None.
- inline (Optional[str], optional): Inline title for the log message. Defaults to None.
+ title (str | None, optional): The title of the log message. Defaults to None.
+ inline (str | None, optional): Inline title for the log message. Defaults to None.
Used to display the title inline, only if the title arg is None.
+
"""
severity = logging.DEBUG
self.log_dispatch.dispatch(content=content, severity=severity, title=title, inline=inline)
def dev(
self,
- content: Union[str, Any],
- title: Optional[str] = None,
- inline: Optional[str] = None,
+ content: str | Any,
+ title: str | None = None,
+ inline: str | None = None,
):
- """
- Log a development message.
+ """Log a development message.
Args:
content (Union[str, Any]): The content to log.
- title (Optional[str], optional): The title of the log message. Defaults to None.
- inline (Optional[str], optional): Inline title for the log message. Defaults to None.
+ title (str | None, optional): The title of the log message. Defaults to None.
+ inline (str | None, optional): Inline title for the log message. Defaults to None.
Used to display the title inline, only if the title arg is None.
+
"""
severity = LOGGING_LEVEL_DEV
self.log_dispatch.dispatch(content=content, severity=severity, title=title, inline=inline)
def info(
self,
- content: Union[str, Any],
- title: Optional[str] = None,
- inline: Optional[str] = None,
+ content: str | Any,
+ title: str | None = None,
+ inline: str | None = None,
):
- """
- Log an info message.
+ """Log an info message.
Args:
content (Union[str, Any]): The content to log.
- title (Optional[str], optional): The title of the log message. Defaults to None.
- inline (Optional[str], optional): Inline title for the log message. Defaults to None.
+ title (str | None, optional): The title of the log message. Defaults to None.
+ inline (str | None, optional): Inline title for the log message. Defaults to None.
Used to display the title inline, only if the title arg is None.
+
"""
severity = logging.INFO
self.log_dispatch.dispatch(content=content, severity=severity, title=title, inline=inline)
def warning(
self,
- content: Union[str, Any],
- title: Optional[str] = None,
- inline: Optional[str] = None,
- problem_id: Optional[str] = None,
+ content: str | Any,
+ title: str | None = None,
+ inline: str | None = None,
+ problem_id: str | None = None,
):
- """
- Log a warning message.
+ """Log a warning message.
Args:
content (Union[str, Any]): The content to log.
- title (Optional[str], optional): The title of the log message. Defaults to None.
- inline (Optional[str], optional): Inline title for the log message. Defaults to None.
+ title (str | None, optional): The title of the log message. Defaults to None.
+ inline (str | None, optional): Inline title for the log message. Defaults to None.
Used to display the title inline, only if the title arg is None.
- problem_id (Optional[str], optional): A problem ID to associate with the warning. Defaults to None.
+ problem_id (str | None, optional): A problem ID to associate with the warning. Defaults to None.
+
"""
if self._should_ignore(problem_id=problem_id):
return
@@ -297,22 +295,22 @@ class Log:
def error(
self,
- content: Union[str, Any],
- title: Optional[str] = None,
- inline: Optional[str] = None,
+ content: str | Any,
+ title: str | None = None,
+ inline: str | None = None,
include_exception: bool = False,
- problem_id: Optional[str] = None,
+ problem_id: str | None = None,
):
- """
- Log an error message.
+ """Log an error message.
Args:
content (Union[str, Any]): The content to log.
- title (Optional[str], optional): The title of the log message. Defaults to None.
- inline (Optional[str], optional): Inline title for the log message. Defaults to None.
+ title (str | None, optional): The title of the log message. Defaults to None.
+ inline (str | None, optional): Inline title for the log message. Defaults to None.
Used to display the title inline, only if the title arg is None.
include_exception (bool, optional): Whether to include exception information. Defaults to False.
- problem_id (Optional[str], optional): A problem ID to associate with the error. Defaults to None.
+ problem_id (str | None, optional): A problem ID to associate with the error. Defaults to None.
+
"""
if self._should_ignore(problem_id=problem_id):
return
@@ -327,22 +325,22 @@ class Log:
def critical(
self,
- content: Union[str, Any],
- title: Optional[str] = None,
- inline: Optional[str] = None,
+ content: str | Any,
+ title: str | None = None,
+ inline: str | None = None,
include_exception: bool = False,
- problem_id: Optional[str] = None,
+ problem_id: str | None = None,
):
- """
- Log a critical message.
+ """Log a critical message.
Args:
content (Union[str, Any]): The content to log.
- title (Optional[str], optional): The title of the log message. Defaults to None.
- inline (Optional[str], optional): Inline title for the log message. Defaults to None.
+ title (str | None, optional): The title of the log message. Defaults to None.
+ inline (str | None, optional): Inline title for the log message. Defaults to None.
Used to display the title inline, only if the title arg is None.
include_exception (bool, optional): Whether to include exception information. Defaults to False.
- problem_id (Optional[str], optional): A problem ID to associate with the critical message. Defaults to None.
+ problem_id (str | None, optional): A problem ID to associate with the critical message. Defaults to None.
+
"""
if self._should_ignore(problem_id=problem_id):
return
diff --git a/pipelex/tools/log/log_config.py b/pipelex/tools/log/log_config.py
index 1a49e6295..c2ae2e5ac 100644
--- a/pipelex/tools/log/log_config.py
+++ b/pipelex/tools/log/log_config.py
@@ -1,5 +1,5 @@
from enum import Enum
-from typing import Dict, List, cast
+from typing import cast
from pydantic import Field, field_validator
from rich.highlighter import Highlighter, JSONHighlighter, ReprHighlighter
@@ -20,7 +20,7 @@ class HighlighterName(StrEnum):
REPR = "repr"
-class PROBLEM_IDS(StrEnum):
+class ProblemIds(StrEnum):
AZURE_OPENAI_NO_STREAM_OPTIONS = "Azure OpenAI no stream_options"
@@ -35,7 +35,7 @@ class CallerInfoTemplate(Enum):
@classmethod
def for_template_key(cls, key: "CallerInfoTemplate") -> str:
- templates: Dict[CallerInfoTemplate, str] = {
+ templates: dict[CallerInfoTemplate, str] = {
cls.FILE_LINE: "{file}:{line}",
cls.FILE_LINE_FUNC: "{file}:{line} {func}",
cls.FUNC: "{func}",
@@ -56,8 +56,8 @@ class RichLogConfig(ConfigModel):
is_rich_tracebacks: bool
is_tracebacks_word_wrap: bool
is_tracebacks_show_locals: bool
- tracebacks_suppress: List[str]
- keywords_to_hilight: List[str]
+ tracebacks_suppress: list[str]
+ keywords_to_hilight: list[str]
def make_rich_handler(self) -> RichHandler:
highlighter: Highlighter
@@ -83,7 +83,7 @@ class RichLogConfig(ConfigModel):
class LogConfig(ConfigModel):
default_log_level: LogLevel = Field(strict=False)
- package_log_levels: Dict[str, LogLevel]
+ package_log_levels: dict[str, LogLevel]
log_mode: LogMode = Field(strict=False)
is_console_logging_enabled: bool
@@ -93,21 +93,18 @@ class LogConfig(ConfigModel):
is_caller_info_enabled: bool
caller_info_template: CallerInfoTemplate = Field(strict=False)
- silenced_problem_ids: List[str]
+ silenced_problem_ids: list[str]
rich_log_config: RichLogConfig
# logger name to use for safe logging without fancy features like code filepath and stuff
generic_poor_logger: str
- poor_loggers: List[str]
+ poor_loggers: list[str]
@field_validator("package_log_levels", mode="before")
- def validate_package_log_levels(cls, value: Dict[str, str]) -> Dict[str, LogLevel]:
- """
- Validate the package log levels dictionary.
- """
- the_dict = cast(
- Dict[str, LogLevel],
+ @staticmethod
+ def validate_package_log_levels(value: dict[str, str]) -> dict[str, LogLevel]:
+ return cast(
+ "dict[str, LogLevel]",
ConfigModel.transform_dict_str_to_enum(input_dict=value, value_enum_cls=LogLevel),
)
- return the_dict
diff --git a/pipelex/tools/log/log_dispatch.py b/pipelex/tools/log/log_dispatch.py
index c9a7f2e5b..a902afbf6 100644
--- a/pipelex/tools/log/log_dispatch.py
+++ b/pipelex/tools/log/log_dispatch.py
@@ -2,49 +2,47 @@ import inspect
import logging
import os
import traceback
-from typing import Any, List, Optional, Union
+from pathlib import Path
+from typing import Any, cast
from pipelex.tools.log.log_config import CallerInfoTemplate, LogConfig, LogMode
from pipelex.tools.misc.json_utils import purify_json, purify_json_dict, purify_json_list
class LogDispatch:
- """
- A class for handling log dispatching to both console and Google Cloud.
- """
+ """A class for handling log dispatching to both console and Google Cloud."""
########################################################
# Init and Configure
########################################################
# TODO: more elegant init for log_dispatch / log
def __init__(self):
- self.project_name: Optional[str] = None
- self._log_config_instance: Optional[LogConfig] = None
+ self.project_name: str | None = None
+ self._log_config_instance: LogConfig | None = None
self.log_mode: LogMode = LogMode.RICH
def set_log_mode(self, mode: LogMode):
self.log_mode = mode
def reset(self):
- """
- Reset the log dispatch.
- """
+ """Reset the log dispatch."""
self.project_name = None
self._log_config_instance = None
@property
def _log_config(self) -> LogConfig:
- """
- Retrieves the log configuration.
+ """Retrieves the log configuration.
Raises:
RuntimeError: If LogConfig is not set.
Returns:
LogConfig: The current log configuration.
+
"""
if self._log_config_instance is None:
- raise RuntimeError("LogConfig is not set. You must call pipelex_hub.set_config().")
+ msg = "LogConfig is not set. You must call pipelex_hub.set_config()."
+ raise RuntimeError(msg)
return self._log_config_instance
def configure(
@@ -52,8 +50,7 @@ class LogDispatch:
project_name: str,
log_config: LogConfig,
):
- """
- Configures the LogDispatch with project name and log configuration.
+ """Configures the LogDispatch with project name and log configuration.
Args:
project_name (str): The name of the project.
@@ -61,9 +58,11 @@ class LogDispatch:
Raises:
RuntimeError: If LogConfig is already set.
+
"""
if self._log_config_instance is not None:
- raise RuntimeError("LogConfig is already set. You can only call log.configure() once.")
+ msg = "LogConfig is already set. You can only call log.configure() once."
+ raise RuntimeError(msg)
self._log_config_instance = log_config
self.project_name = project_name
self.log_mode = log_config.log_mode
@@ -74,24 +73,24 @@ class LogDispatch:
def dispatch(
self,
- content: Union[str, Any],
+ content: str | Any,
severity: int,
- title: Optional[str] = None,
- inline: Optional[str] = None,
+ title: str | None = None,
+ inline: str | None = None,
include_exception: bool = False,
):
- """
- Dispatches a log message to appropriate logging methods based on content type.
+ """Dispatches a log message to appropriate logging methods based on content type.
Args:
content (Union[str, Any]): The content to be logged.
severity (int): The severity level of the log message.
- title (Optional[str], optional): The title of the log message. Defaults to None.
- inline (Optional[str], optional): Inline title for the log message. Defaults to None.
+ title (str | None, optional): The title of the log message. Defaults to None.
+ inline (str | None, optional): Inline title for the log message. Defaults to None.
Used to display the title inline, only if the title arg is None.
include_exception (bool, optional): Whether to include exception traceback. Defaults to False.
+
"""
- caller_info_str: Optional[str] = None
+ caller_info_str: str | None = None
if (
(self._log_config.is_caller_info_enabled)
and (frame0 := inspect.currentframe())
@@ -100,10 +99,9 @@ class LogDispatch:
and (caller_frame := frame1.f_back)
):
caller_info = inspect.getframeinfo(caller_frame)
- caller_file = caller_info.filename
- cwd = os.getcwd()
+ caller_path = Path(caller_info.filename)
try:
- caller_file = os.path.relpath(caller_file, cwd)
+ caller_path = caller_path.relative_to(Path.cwd())
except ValueError:
# This can happen if the file is on a different drive (on Windows)
# In this case, we'll keep the absolute path
@@ -111,7 +109,7 @@ class LogDispatch:
caller_line = caller_info.lineno
caller_func = caller_info.function
template_str = CallerInfoTemplate.for_template_key(key=self._log_config.caller_info_template)
- caller_info_str = template_str.format(file=caller_file, line=caller_line, func=caller_func)
+ caller_info_str = template_str.format(file=str(caller_path), line=caller_line, func=caller_func)
if isinstance(content, str):
self._log_message(
@@ -135,22 +133,22 @@ class LogDispatch:
self,
message: str,
severity: int,
- caller_info_str: Optional[str],
- title: Optional[str] = None,
- inline: Optional[str] = None,
+ caller_info_str: str | None,
+ title: str | None = None,
+ inline: str | None = None,
include_exception: bool = False,
):
- """
- Logs a message to both console and Google Cloud.
+ """Logs a message to both console and Google Cloud.
Args:
message (str): The message to be logged.
severity (int): The severity level of the log message.
- caller_info_str (Optional[str]): Information about the caller.
- title (Optional[str], optional): The title of the log message. Defaults to None.
- inline (Optional[str], optional): Inline title for the log message. Defaults to None.
+ caller_info_str (str | None): Information about the caller.
+ title (str | None, optional): The title of the log message. Defaults to None.
+ inline (str | None, optional): Inline title for the log message. Defaults to None.
Used to display the title inline, only if the title arg is None.
include_exception (bool, optional): Whether to include exception traceback. Defaults to False.
+
"""
if title is not None:
message = f"{title}:\n{message}"
@@ -169,19 +167,19 @@ class LogDispatch:
self,
data: Any,
severity: int,
- caller_info_str: Optional[str],
- title: Optional[str] = None,
+ caller_info_str: str | None,
+ title: str | None = None,
include_exception: bool = False,
):
- """
- Logs potentially structured data (maybe it's a dict or a list) to both console and Google Cloud.
+ """Logs potentially structured data (maybe it's a dict or a list) to both console and Google Cloud.
Args:
data (Any): The data to be logged.
severity (int): The severity level of the log message.
- caller_info_str (Optional[str]): Information about the caller.
- title (Optional[str], optional): The title of the log message. Defaults to None.
+ caller_info_str (str | None): Information about the caller.
+ title (str | None, optional): The title of the log message. Defaults to None.
include_exception (bool, optional): Whether to include exception traceback. Defaults to False.
+
"""
if data is None:
message = "None"
@@ -208,7 +206,7 @@ class LogDispatch:
message += f"\n{traceback.format_exc()}"
self._log_to_console(message=message, severity=severity)
elif isinstance(data, list):
- list_data: List[Any] = data
+ list_data = cast("list[Any]", data)
_, list_string = purify_json_list(
data=list_data,
indent=self._log_config.json_logs_indent,
@@ -239,12 +237,12 @@ class LogDispatch:
self._log_to_console(message=message, severity=severity)
def _log_to_console(self, message: str, severity: int):
- """
- Logs a message to the console.
+ """Logs a message to the console.
Args:
message (str): The message to be logged.
severity (int): The severity level of the log message.
+
"""
if not self._log_config.is_console_logging_enabled:
return
@@ -258,7 +256,7 @@ class LogDispatch:
stack = inspect.stack()
try:
- logging_module_path = os.path.abspath(__file__)
+ logging_module_path = Path(__file__).absolute()
log_origin_name = "unknown"
for frame_info in stack[1:]:
@@ -276,14 +274,14 @@ class LogDispatch:
if module_file == logging_module_path or module_file.endswith("/log.py"):
continue
+ if module.__name__ == "__main__":
+ if self.project_name is None:
+ msg = "Project name is not set. You must call initialize Pipelex first."
+ raise RuntimeError(msg)
+ log_origin_name = self.project_name
else:
- if module.__name__ == "__main__":
- if self.project_name is None:
- raise RuntimeError("Project name is not set. You must call initialize Pipelex first.")
- log_origin_name = self.project_name
- else:
- log_origin_name = module.__name__.split(sep=".", maxsplit=1)[0]
- break
+ log_origin_name = module.__name__.split(sep=".", maxsplit=1)[0]
+ break
finally:
del frame_info
logger = logging.getLogger(log_origin_name)
diff --git a/pipelex/tools/log/log_formatter.py b/pipelex/tools/log/log_formatter.py
index ac4de0704..fea7a3191 100644
--- a/pipelex/tools/log/log_formatter.py
+++ b/pipelex/tools/log/log_formatter.py
@@ -1,5 +1,4 @@
import logging
-from typing import Dict, Optional
from typing_extensions import override
@@ -8,14 +7,14 @@ from pipelex.tools.misc.terminal_utils import BOLD_FONT, RESET_FONT, TerminalCol
# TODO: move these to the config
-def emoji_for_channel(channel_name: str) -> Optional[str]:
- channel_emojis: Dict[str, str] = {
+def emoji_for_channel(channel_name: str) -> str | None:
+ channel_emojis: dict[str, str] = {
"root": "",
"werkzeug": "📡",
"urllib3.connectionpool": "⚡️",
}
- emoji = channel_emojis.get(channel_name, None)
+ emoji = channel_emojis.get(channel_name)
if emoji == "":
# blank emoji is OK
return emoji
@@ -53,7 +52,7 @@ class EmojiLogFormatter(logging.Formatter):
return formatter.format(record)
-log_level_color: Dict[int, TerminalColor] = {
+log_level_color: dict[int, TerminalColor] = {
LOGGING_LEVEL_VERBOSE: TerminalColor.WHITE,
logging.DEBUG: TerminalColor.GREEN,
LOGGING_LEVEL_DEV: TerminalColor.CYAN,
@@ -64,7 +63,7 @@ log_level_color: Dict[int, TerminalColor] = {
}
# added spaces and truncated to 5 characters to make it look better
-log_level_tag: Dict[int, str] = {
+log_level_tag: dict[int, str] = {
LOGGING_LEVEL_VERBOSE: "VERBO",
logging.DEBUG: "DEBUG",
LOGGING_LEVEL_DEV: "DEV ",
diff --git a/pipelex/tools/misc/attribute_utils.py b/pipelex/tools/misc/attribute_utils.py
index 8cfec1008..11a64369f 100644
--- a/pipelex/tools/misc/attribute_utils.py
+++ b/pipelex/tools/misc/attribute_utils.py
@@ -1,4 +1,4 @@
-from typing import Any, ClassVar, Union
+from typing import Any, ClassVar
class AttributePolisher:
@@ -25,19 +25,17 @@ class AttributePolisher:
if not isinstance(value, (str, bytes)):
return False
- if name == "base_64" and len(value) > cls.base_64_truncate_length:
- return True
- elif name == "url" and isinstance(value, str) and value.startswith("data:image/") and len(value) > cls.url_truncate_length:
- return True
- return False
+ return (name == "base_64" and len(value) > cls.base_64_truncate_length) or (
+ name == "url" and isinstance(value, str) and value.startswith("data:image/") and len(value) > cls.url_truncate_length
+ )
@classmethod
- def get_truncated_value(cls, name: str, value: Union[str, bytes]) -> Union[str, bytes]:
+ def get_truncated_value(cls, name: str, value: str | bytes) -> str | bytes:
"""Get the truncated value based on the field name and value type."""
if isinstance(value, bytes):
return cls._truncate_bytes(value, cls.base_64_truncate_length)
- elif name == "base_64":
+ if name == "base_64":
return cls._truncate_string(value, cls.base_64_truncate_length)
- elif name == "url" and value.startswith("data:image/"):
+ if name == "url" and value.startswith("data:image/"):
return cls._truncate_string(value, cls.url_truncate_length)
return value
diff --git a/pipelex/tools/misc/base_64_utils.py b/pipelex/tools/misc/base_64_utils.py
index a95d5ebaa..14cda8867 100644
--- a/pipelex/tools/misc/base_64_utils.py
+++ b/pipelex/tools/misc/base_64_utils.py
@@ -12,26 +12,23 @@ def load_binary_as_base64(path: str) -> bytes:
async def load_binary_as_base64_async(path: str) -> bytes:
- async with aiofiles.open(path, "rb") as fp: # type: ignore[reportUnknownMemberType]
+ async with aiofiles.open(path, "rb") as fp: # pyright: ignore[reportUnknownMemberType]
data_bytes = await fp.read()
return base64.b64encode(data_bytes)
async def load_binary_async(path: str) -> bytes:
- async with aiofiles.open(path, "rb") as fp: # type: ignore[reportUnknownMemberType]
- data_bytes = await fp.read()
- return data_bytes
+ async with aiofiles.open(path, "rb") as fp: # pyright: ignore[reportUnknownMemberType]
+ return await fp.read()
def encode_to_base64(data_bytes: bytes) -> bytes:
- b64 = base64.b64encode(data_bytes)
- return b64
+ return base64.b64encode(data_bytes)
async def encode_to_base64_async(data_bytes: bytes) -> bytes:
# Use asyncio.to_thread to run the CPU-bound task in a separate thread
- b64 = await asyncio.to_thread(base64.b64encode, data_bytes)
- return b64
+ return await asyncio.to_thread(base64.b64encode, data_bytes)
def save_base64_to_binary_file(
diff --git a/pipelex/tools/misc/context_provider_abstract.py b/pipelex/tools/misc/context_provider_abstract.py
index 8bfd24567..1cff3ac83 100644
--- a/pipelex/tools/misc/context_provider_abstract.py
+++ b/pipelex/tools/misc/context_provider_abstract.py
@@ -1,5 +1,5 @@
from abc import ABC, abstractmethod
-from typing import Any, Dict, Optional, Type
+from typing import Any
from pipelex.tools.exceptions import ToolException
@@ -11,15 +11,14 @@ class ContextProviderException(ToolException):
class ContextProviderAbstract(ABC):
- """
- A ContextProvider provides context to templating engine. This interface is implemented by WorkingMemory.
+ """A ContextProvider provides context to templating engine. This interface is implemented by WorkingMemory.
It exists to make these features available to lower level classes.
"""
@abstractmethod
- def get_typed_object_or_attribute(self, name: str, wanted_type: Optional[Type[Any]] = None) -> Any:
+ def get_typed_object_or_attribute(self, name: str, wanted_type: type[Any] | None = None) -> Any:
pass
@abstractmethod
- def generate_context(self) -> Dict[str, Any]:
+ def generate_context(self) -> dict[str, Any]:
pass
diff --git a/pipelex/tools/misc/dict_utils.py b/pipelex/tools/misc/dict_utils.py
index 1d54df1e7..fe8c88a83 100644
--- a/pipelex/tools/misc/dict_utils.py
+++ b/pipelex/tools/misc/dict_utils.py
@@ -1,14 +1,14 @@
"""Dictionary utility functions for manipulating dictionary order and structure."""
-from typing import Any, Callable, Dict, List, TypeVar, cast
+from collections.abc import Callable
+from typing import Any, TypeVar, cast
K = TypeVar("K")
V = TypeVar("V")
-def insert_before(dictionary: Dict[K, V], target_key: K, new_key: K, new_value: V) -> Dict[K, V]:
- """
- Insert a new key-value pair before a target key in a dictionary.
+def insert_before(dictionary: dict[K, V], target_key: K, new_key: K, new_value: V) -> dict[K, V]:
+ """Insert a new key-value pair before a target key in a dictionary.
Creates a new dictionary with the new item positioned before the target key.
If the target key doesn't exist, the new item is added at the end.
@@ -26,8 +26,9 @@ def insert_before(dictionary: Dict[K, V], target_key: K, new_key: K, new_value:
>>> d = {'a': 1, 'c': 3}
>>> insert_before(d, 'c', 'b', 2)
{'a': 1, 'b': 2, 'c': 3}
+
"""
- result: Dict[K, V] = {}
+ result: dict[K, V] = {}
inserted = False
for key, value in dictionary.items():
@@ -43,9 +44,8 @@ def insert_before(dictionary: Dict[K, V], target_key: K, new_key: K, new_value:
return result
-def apply_to_strings_recursive(data: Any, transform_func: Callable[[str], str]) -> Dict[str, Any]:
- """
- Recursively traverse a data structure and apply a transformation function to all string values.
+def apply_to_strings_recursive(data: Any, transform_func: Callable[[str], str]) -> dict[str, Any]:
+ """Recursively traverse a data structure and apply a transformation function to all string values.
This function walks through dictionaries, lists, and other nested structures,
applying the provided transformation function only to string values while
@@ -62,13 +62,14 @@ def apply_to_strings_recursive(data: Any, transform_func: Callable[[str], str])
>>> data = {'a': 'hello ${USER}', 'b': [1, 'world ${HOME}'], 'c': {'d': 'test ${PATH}'}}
>>> result = apply_to_strings_recursive(data, lambda s: s.replace('${USER}', 'john'))
>>> # Returns: {'a': 'hello john', 'b': [1, 'world ${HOME}'], 'c': {'d': 'test ${PATH}'}}
+
"""
- result: Dict[str, Any] = {}
+ result: dict[str, Any] = {}
for key, value in data.items():
if isinstance(value, dict):
result[key] = apply_to_strings_recursive(value, transform_func)
elif isinstance(value, list):
- list_value: List[Any] = cast(List[Any], value)
+ list_value: list[Any] = cast("list[Any]", value)
result[key] = apply_to_strings_in_list(list_value, transform_func)
elif isinstance(value, str):
result[key] = transform_func(value)
@@ -78,14 +79,14 @@ def apply_to_strings_recursive(data: Any, transform_func: Callable[[str], str])
return result
-def apply_to_strings_in_list(data: List[Any], transform_func: Callable[[str], str]) -> List[Any]:
+def apply_to_strings_in_list(data: list[Any], transform_func: Callable[[str], str]) -> list[Any]:
"""Helper function to apply string transformation to items in a list."""
- result: List[Any] = []
+ result: list[Any] = []
for item in data:
if isinstance(item, dict):
result.append(apply_to_strings_recursive(item, transform_func))
elif isinstance(item, list):
- list_item: List[Any] = cast(List[Any], item)
+ list_item: list[Any] = cast("list[Any]", item)
result.append(apply_to_strings_in_list(list_item, transform_func))
elif isinstance(item, str):
result.append(transform_func(item))
diff --git a/pipelex/tools/misc/file_fetch_utils.py b/pipelex/tools/misc/file_fetch_utils.py
index 2d9b73683..cb295dbe2 100644
--- a/pipelex/tools/misc/file_fetch_utils.py
+++ b/pipelex/tools/misc/file_fetch_utils.py
@@ -1,33 +1,30 @@
-from typing import Optional
-
import httpx
from httpx import Response
async def fetch_file_from_url_httpx_async(
url: str,
- timeout: Optional[int] = None,
+ request_timeout: int | None = None,
) -> bytes:
async with httpx.AsyncClient() as client:
response: Response = await client.get(
url,
- timeout=timeout,
+ timeout=request_timeout,
follow_redirects=True,
)
response.raise_for_status() # Raise exception for 4XX/5XX status codes
- bytes_content: bytes = response.content
- return bytes_content
+ return response.content
def fetch_file_from_url_httpx(
url: str,
- timeout: Optional[int] = None,
+ request_timeout: int | None = None,
) -> bytes:
with httpx.Client() as client:
response: Response = client.get(
url,
- timeout=timeout,
+ timeout=request_timeout,
follow_redirects=True,
)
response.raise_for_status() # Raise exception for 4XX/5XX status codes
diff --git a/pipelex/tools/misc/file_utils.py b/pipelex/tools/misc/file_utils.py
index e87e5404d..2aa7e5a09 100644
--- a/pipelex/tools/misc/file_utils.py
+++ b/pipelex/tools/misc/file_utils.py
@@ -2,7 +2,6 @@ import importlib.resources
import os
import shutil
from pathlib import Path
-from typing import List, Optional
########################################################
# Save & Load
@@ -10,15 +9,17 @@ from typing import List, Optional
def save_bytes_to_binary_file(file_path: str, byte_data: bytes, create_directory: bool = False) -> str:
- """
- Write binary data to a file.
+ """Write binary data to a file.
Args:
file_path (str): Path where the binary data will be saved
byte_data (bytes): Binary data to be written
+ create_directory (bool, optional): Whether to create the directory if it doesn't exist.
+ Defaults to False.
Returns:
str: Path to the saved file
+
"""
# Ensure the directory exists
if create_directory:
@@ -30,8 +31,7 @@ def save_bytes_to_binary_file(file_path: str, byte_data: bytes, create_directory
def save_text_to_path(text: str, path: str, create_directory: bool = False):
- """
- Writes text content to a file at the specified path.
+ """Writes text content to a file at the specified path.
This function opens a file in write mode and writes the provided text to it.
If the file already exists, it will be overwritten.
@@ -44,6 +44,7 @@ def save_text_to_path(text: str, path: str, create_directory: bool = False):
Raises:
IOError: If there are issues writing to the file (e.g., permission denied).
+
"""
if create_directory:
directory = os.path.dirname(path)
@@ -55,8 +56,7 @@ def save_text_to_path(text: str, path: str, create_directory: bool = False):
def load_text_from_path(path: str) -> str:
- """
- Reads and returns the entire contents of a text file.
+ """Reads and returns the entire contents of a text file.
This function opens a file in text mode using UTF-8 encoding and reads
its entire contents into a string.
@@ -69,14 +69,14 @@ def load_text_from_path(path: str) -> str:
Raises:
FileNotFoundError: If the file does not exist.
+
"""
with open(path, encoding="utf-8") as file:
return file.read()
-def failable_load_text_from_path(path: str) -> Optional[str]:
- """
- Attempts to read a text file, returning None if the file doesn't exist.
+def failable_load_text_from_path(path: str) -> str | None:
+ """Attempts to read a text file, returning None if the file doesn't exist.
This function is a safer version of load_text_from_path that handles missing files
gracefully by returning None instead of raising an error.
@@ -86,6 +86,7 @@ def failable_load_text_from_path(path: str) -> Optional[str]:
Returns:
Optional[str]: The complete contents of the file as a string, or None if the file doesn't exist.
+
"""
if not path_exists(path):
return None
@@ -98,8 +99,7 @@ def failable_load_text_from_path(path: str) -> Optional[str]:
def copy_file(source_path: str, target_path: str, overwrite: bool = True) -> None:
- """
- Copies a file from the source path to the target path.
+ """Copies a file from the source path to the target path.
Creates any necessary parent directories for the target path if they don't exist.
@@ -107,6 +107,7 @@ def copy_file(source_path: str, target_path: str, overwrite: bool = True) -> Non
source_path (str): The path to the source file.
target_path (str): The path to the target file.
overwrite (bool, optional): Whether to overwrite existing files. Defaults to True.
+
"""
# Ensure the target directory exists
target_dir = os.path.dirname(target_path)
@@ -123,9 +124,7 @@ def copy_file_from_package(
target_path: str,
overwrite: bool = True,
) -> None:
- """
- Copies a file from a package to a target directory.
- """
+ """Copies a file from a package to a target directory."""
file_path = str(importlib.resources.files(package_name).joinpath(file_path_in_package))
copy_file(
source_path=file_path,
@@ -139,10 +138,9 @@ def copy_folder_from_package(
folder_path_in_package: str,
target_dir: str,
overwrite: bool = True,
- non_overwrite_files: Optional[List[str]] = None,
+ non_overwrite_files: list[str] | None = None,
) -> None:
- """
- Copies a folder from a package to a target directory.
+ """Copies a folder from a package to a target directory.
This function walks through the specified folder in the package and copies
all files and directories to the target directory, preserving the directory
@@ -153,6 +151,8 @@ def copy_folder_from_package(
folder_path_in_package (str): The path to the folder in the package to copy.
target_dir (str): The target directory to copy the folder to.
overwrite (bool, optional): Whether to overwrite existing files. Defaults to True.
+ non_overwrite_files (Optional[List[str]], optional): List of files to not overwrite. Defaults to None.
+
"""
os.makedirs(target_dir, exist_ok=True)
@@ -177,7 +177,7 @@ def copy_folder_from_package(
dest_file = os.path.join(target_subdir, file)
# Check if the file exists and respect the overwrite parameter
- if not os.path.exists(dest_file) or (overwrite and file not in non_overwrite_files):
+ if not Path(dest_file).exists() or (overwrite and file not in non_overwrite_files):
copy_file(
source_path=src_file,
target_path=dest_file,
@@ -187,8 +187,7 @@ def copy_folder_from_package(
def remove_file(file_path: str):
- """
- Removes a file if it exists at the specified path.
+ """Removes a file if it exists at the specified path.
This function checks if a file exists before attempting to remove it,
preventing errors from trying to remove non-existent files.
@@ -198,22 +197,23 @@ def remove_file(file_path: str):
Note:
This function silently succeeds if the file doesn't exist.
+
"""
if path_exists(file_path):
- os.remove(file_path)
+ Path(file_path).unlink()
def remove_folder(folder_path: str) -> None:
- """
- Removes a folder if it exists at the specified path.
+ """Removes a folder if it exists at the specified path.
This function checks if a folder exists before attempting to remove it,
preventing errors from trying to remove non-existent folders.
Args:
folder_path (str): The path to the folder to be removed.
+
"""
- if os.path.exists(folder_path):
+ if Path(folder_path).exists():
shutil.rmtree(folder_path)
@@ -223,18 +223,17 @@ def remove_folder(folder_path: str) -> None:
def ensure_directory_exists(directory_path: str) -> None:
- """
- Creates a directory and any necessary parent directories if they don't exist.
+ """Creates a directory and any necessary parent directories if they don't exist.
Args:
directory_path (str): The path to the directory to create.
+
"""
- os.makedirs(directory_path, exist_ok=True)
+ Path(directory_path).mkdir(parents=True, exist_ok=True)
def ensure_path(path: str) -> bool:
- """
- Ensures a directory exists at the specified path, creating it if necessary.
+ """Ensures a directory exists at the specified path, creating it if necessary.
This function checks if a directory exists at the given path. If it doesn't exist,
it creates the directory and any necessary parent directories.
@@ -244,17 +243,16 @@ def ensure_path(path: str) -> bool:
Returns:
bool: True if the directory was created, False if it already existed.
+
"""
- typed_path = Path(path)
- if typed_path.exists():
+ if Path(path).exists():
return False
- typed_path.mkdir(parents=True, exist_ok=True)
+ Path(path).mkdir(parents=True, exist_ok=True)
return True
def path_exists(path_str: str) -> bool:
- """
- Checks if a file or directory exists at the specified path.
+ """Checks if a file or directory exists at the specified path.
This function converts the input string path to a Path object and checks
if anything exists at that location in the filesystem.
@@ -264,14 +262,13 @@ def path_exists(path_str: str) -> bool:
Returns:
bool: True if a file or directory exists at the path, False otherwise.
+
"""
- path = Path(path_str)
- return path.exists()
+ return Path(path_str).exists()
def get_incremental_directory_path(base_path: str, base_name: str, start_at: int = 1) -> str:
- """
- Generates a unique directory path by incrementing a counter until an unused path is found.
+ """Generates a unique directory path by incrementing a counter until an unused path is found.
This function creates a directory path in the format 'base_path/base_name_XX' where XX
is a two-digit number that starts at start_at and increments until an unused path is found.
@@ -284,6 +281,7 @@ def get_incremental_directory_path(base_path: str, base_name: str, start_at: int
Returns:
str: The path to the newly created directory.
+
"""
counter = start_at
while True:
@@ -302,8 +300,7 @@ def get_incremental_file_path(
start_at: int = 1,
avoid_suffix_if_possible: bool = False,
) -> str:
- """
- Generates a unique file path by incrementing a counter until an unused path is found.
+ """Generates a unique file path by incrementing a counter until an unused path is found.
This function creates a file path in the format 'base_path/base_name_XX.extension' where XX
is a two-digit number that starts at start_at and increments until an unused path is found.
@@ -314,9 +311,11 @@ def get_incremental_file_path(
base_name (str): The base name for the file (will be appended with _XX).
extension (str): The file extension (without the dot).
start_at (int, optional): The number to start counting from. Defaults to 1.
+ avoid_suffix_if_possible (bool, optional): If True, avoids adding a suffix if possible. Defaults to False.
Returns:
str: A unique file path that does not exist in the filesystem.
+
"""
if avoid_suffix_if_possible:
# try without adding the suffix
@@ -339,20 +338,19 @@ def get_incremental_file_path(
########################################################
-def find_files_in_dir(dir_path: str, pattern: str, is_recursive: bool) -> List[Path]:
- """
- Find files matching a pattern in a directory.
+def find_files_in_dir(dir_path: str, pattern: str, is_recursive: bool) -> list[Path]:
+ """Find files matching a pattern in a directory.
Args:
dir_path: Directory path to search in
pattern: File pattern to match (e.g. "*.py")
- recursive: Whether to search recursively in subdirectories
+ is_recursive: Whether to search recursively in subdirectories
Returns:
List of matching Path objects
+
"""
path = Path(dir_path)
if is_recursive:
return list(path.rglob(pattern))
- else:
- return list(path.glob(pattern))
+ return list(path.glob(pattern))
diff --git a/pipelex/tools/misc/filetype_utils.py b/pipelex/tools/misc/filetype_utils.py
index 1ee664d8f..af274d770 100644
--- a/pipelex/tools/misc/filetype_utils.py
+++ b/pipelex/tools/misc/filetype_utils.py
@@ -1,7 +1,6 @@
import base64
import binascii
from pathlib import Path
-from typing import Union
import filetype
from pydantic import BaseModel
@@ -18,9 +17,8 @@ class FileType(BaseModel):
mime: str
-def detect_file_type_from_path(path: Union[str, Path]) -> FileType:
- """
- Detect the file type of a file at a given path.
+def detect_file_type_from_path(path: str | Path) -> FileType:
+ """Detect the file type of a file at a given path.
Args:
path: The path to the file to detect the type of.
@@ -30,18 +28,19 @@ def detect_file_type_from_path(path: Union[str, Path]) -> FileType:
Raises:
FileTypeException: If the file type cannot be identified.
+
"""
kind = filetype.guess(path) # pyright: ignore[reportUnknownMemberType]
if kind is None:
- raise FileTypeException(f"Could not identify file type of '{path!s}'")
+ msg = f"Could not identify file type of '{path!s}'"
+ raise FileTypeException(msg)
extension = f"{kind.extension}"
mime = f"{kind.mime}"
return FileType(extension=extension, mime=mime)
def detect_file_type_from_bytes(buf: bytes) -> FileType:
- """
- Detect the file type of a given bytes object.
+ """Detect the file type of a given bytes object.
Args:
buf: The bytes object to detect the type of.
@@ -51,18 +50,19 @@ def detect_file_type_from_bytes(buf: bytes) -> FileType:
Raises:
FileTypeException: If the file type cannot be identified.
+
"""
kind = filetype.guess(buf) # pyright: ignore[reportUnknownMemberType]
if kind is None:
- raise FileTypeException(f"Could not identify file type of given bytes: {buf[:300]!r}")
+ msg = f"Could not identify file type of given bytes: {buf[:300]!r}"
+ raise FileTypeException(msg)
extension = f"{kind.extension}"
mime = f"{kind.mime}"
return FileType(extension=extension, mime=mime)
-def detect_file_type_from_base64(b64: Union[str, bytes]) -> FileType:
- """
- Detect the file type of a given Base-64-encoded string.
+def detect_file_type_from_base64(b64: str | bytes) -> FileType:
+ """Detect the file type of a given Base-64-encoded string.
Args:
b64: The Base-64-encoded bytes or string to detect the type of.
@@ -72,6 +72,7 @@ def detect_file_type_from_base64(b64: Union[str, bytes]) -> FileType:
Raises:
FileTypeException: If the file type cannot be identified.
+
"""
# Normalise to bytes holding only the Base-64 alphabet
if isinstance(b64, bytes):
@@ -84,6 +85,7 @@ def detect_file_type_from_base64(b64: Union[str, bytes]) -> FileType:
try:
raw = base64.b64decode(b64_bytes, validate=True)
except binascii.Error as exc: # malformed Base-64
- raise FileTypeException(f"Could not identify file type of given bytes because input is not valid Base-64: {exc}") from exc
+ msg = f"Could not identify file type of given bytes because input is not valid Base-64: {exc}"
+ raise FileTypeException(msg) from exc
return detect_file_type_from_bytes(buf=raw)
diff --git a/pipelex/tools/misc/json_utils.py b/pipelex/tools/misc/json_utils.py
index a1c528128..8a230f122 100644
--- a/pipelex/tools/misc/json_utils.py
+++ b/pipelex/tools/misc/json_utils.py
@@ -1,5 +1,6 @@
import json
-from typing import Any, Dict, List, Mapping, Optional, Tuple, Union, cast
+from collections.abc import Mapping
+from typing import Any, Union, cast
from kajson import kajson
from pydantic import BaseModel
@@ -8,7 +9,7 @@ from pipelex.tools.exceptions import ToolException
from pipelex.tools.misc.file_utils import save_text_to_path
from pipelex.tools.typing.pydantic_utils import CustomBaseModel
-JsonContent = Union[Dict[str, Any], List[Any]]
+JsonContent = Union[dict[str, Any], list[Any]]
class ArgumentTypeError(ToolException):
@@ -19,9 +20,8 @@ class JsonTypeError(ToolException):
pass
-def json_str(some_object: Any, title: Optional[str] = None, is_spaced: bool = False) -> str:
- """
- Creates a formatted JSON string representation of any Python object with optional title and spacing.
+def json_str(some_object: Any, title: str | None = None, is_spaced: bool = False) -> str:
+ """Creates a formatted JSON string representation of any Python object with optional title and spacing.
This function is a higher-level wrapper around purify_json that provides additional formatting
options. It always uses 4-space indentation and disables warning wrapping for non-serializable
@@ -30,7 +30,7 @@ def json_str(some_object: Any, title: Optional[str] = None, is_spaced: bool = Fa
Args:
some_object (Any): The object to convert to a JSON string. Can be any type supported
by purify_json.
- title (Optional[str], optional): A title to prepend to the JSON string. If provided,
+ title (str | None, optional): A title to prepend to the JSON string. If provided,
the output will be in the format "title: {json_string}". Defaults to None.
is_spaced (bool, optional): If True, adds newlines before and after the JSON string
for better readability. Defaults to False.
@@ -59,11 +59,10 @@ def json_str(some_object: Any, title: Optional[str] = None, is_spaced: bool = Fa
def save_as_json_to_path(
object_to_save: Any,
path: str,
- indent: Optional[int] = 4,
+ indent: int | None = 4,
is_warning_enabled: bool = True,
):
- """
- Saves a Python object as a JSON file at the specified path.
+ """Saves a Python object as a JSON file at the specified path.
This function converts a Python object to a JSON string and saves it to a file. The object
is first purified to ensure JSON compatibility before saving.
@@ -71,19 +70,19 @@ def save_as_json_to_path(
Args:
object_to_save (Any): The Python object to be saved as JSON. Can be any JSON-serializable object.
path (str): The file path where the JSON file will be saved.
- indent (Optional[int], optional): Number of spaces for JSON formatting indentation. Defaults to 4.
+ indent (int | None, optional): Number of spaces for JSON formatting indentation. Defaults to 4.
is_warning_enabled (bool, optional): Whether to show warnings during JSON purification. Defaults to True.
Returns:
None
+
"""
_, json_string = purify_json(object_to_save, indent=indent, is_warning_enabled=is_warning_enabled)
save_text_to_path(json_string, path)
def load_json_from_path(path: str) -> JsonContent:
- """
- Loads and parses a JSON file from the specified path.
+ """Loads and parses a JSON file from the specified path.
This function reads a JSON file and returns its contents as a Python object.
The file is read using UTF-8 encoding.
@@ -97,15 +96,15 @@ def load_json_from_path(path: str) -> JsonContent:
Raises:
FileNotFoundError: If the file does not exist.
json.JSONDecodeError: If the file contains invalid JSON.
+
"""
with open(path, encoding="utf-8") as file:
json_content: JsonContent = json.load(file)
return json_content
-def load_json_dict_from_path(path: str) -> Dict[Any, Any]:
- """
- Loads a JSON file and ensures it contains a dictionary.
+def load_json_dict_from_path(path: str) -> dict[Any, Any]:
+ """Loads a JSON file and ensures it contains a dictionary.
This function reads a JSON file and verifies that its content is a dictionary.
It uses load_json_from_path internally and adds type checking.
@@ -120,17 +119,17 @@ def load_json_dict_from_path(path: str) -> Dict[Any, Any]:
JsonTypeError: If the JSON content is not a dictionary.
FileNotFoundError: If the file does not exist.
json.JSONDecodeError: If the file contains invalid JSON.
+
"""
json_content: JsonContent = load_json_from_path(path)
if isinstance(json_content, dict):
return json_content
- else:
- raise JsonTypeError(f"{path} is not a dict")
+ msg = f"{path} is not a dict"
+ raise JsonTypeError(msg)
-def load_json_list_from_path(path: str) -> List[Any]:
- """
- Loads a JSON file and ensures it contains a list.
+def load_json_list_from_path(path: str) -> list[Any]:
+ """Loads a JSON file and ensures it contains a list.
This function reads a JSON file and verifies that its content is a list.
It uses load_json_from_path internally and adds type checking.
@@ -145,17 +144,17 @@ def load_json_list_from_path(path: str) -> List[Any]:
JsonTypeError: If the JSON content is not a list.
FileNotFoundError: If the file does not exist.
json.JSONDecodeError: If the file contains invalid JSON.
+
"""
json_content: JsonContent = load_json_from_path(path)
if isinstance(json_content, list):
return json_content
- else:
- raise JsonTypeError(f"{path} is not a list")
+ msg = f"{path} is not a list"
+ raise JsonTypeError(msg)
-def deep_update(target_dict: Dict[str, Any], updates: Dict[str, Any]):
- """
- Recursively updates a dictionary with values from another dictionary.
+def deep_update(target_dict: dict[str, Any], updates: dict[str, Any]):
+ """Recursively updates a dictionary with values from another dictionary.
This function performs a deep merge of two dictionaries, handling nested
dictionaries and lists. For dictionaries, it recursively updates values,
@@ -172,19 +171,19 @@ def deep_update(target_dict: Dict[str, Any], updates: Dict[str, Any]):
>>> deep_update(base, updates)
>>> print(base)
{'a': 1, 'b': {'x': 2, 'y': 4, 'z': 5}, 'c': [1, 2, 3, 4]}
+
"""
for key, value in updates.items():
if isinstance(value, dict) and key in target_dict and isinstance(target_dict[key], dict):
- deep_update(target_dict[key], value) # type: ignore
+ deep_update(target_dict[key], value) # pyright: ignore[reportUnknownArgumentType]
elif isinstance(value, list) and key in target_dict and isinstance(target_dict[key], list):
- target_dict[key] = list(target_dict[key] + value) # type: ignore
+ target_dict[key] = list(target_dict[key] + value)
else:
target_dict[key] = value
def remove_none_values(json_content: JsonContent | Any) -> JsonContent | Any:
- """
- Recursively removes all None values from a JSON-compatible data structure.
+ """Recursively removes all None values from a JSON-compatible data structure.
This function traverses dictionaries and lists, removing any keys with None values
from dictionaries and processing nested structures. It preserves the structure of
@@ -212,37 +211,37 @@ def remove_none_values(json_content: JsonContent | Any) -> JsonContent | Any:
"nested": {"key": "exists"},
"list": [1, {}]
}
+
"""
if isinstance(json_content, dict):
- json_content = cast(Dict[str, Any], json_content) # pyright: ignore[reportUnnecessaryCast]
- cleaned_dict: Dict[str, Any] = {}
+ json_content = cast("dict[str, Any]", json_content) # pyright: ignore[reportUnnecessaryCast]
+ cleaned_dict: dict[str, Any] = {}
for key, value in json_content.items():
if value is not None:
cleaned_dict[key] = remove_none_values(json_content=value)
return cleaned_dict
elif isinstance(json_content, list):
- json_content = cast(List[Any], json_content) # pyright: ignore[reportUnnecessaryCast]
+ json_content = cast("list[Any]", json_content) # pyright: ignore[reportUnnecessaryCast]
return [remove_none_values(item) for item in json_content]
else:
return json_content
-def remove_none_values_from_dict(data: Mapping[str, Any]) -> Dict[str, Any]:
+def remove_none_values_from_dict(data: Mapping[str, Any]) -> dict[str, Any]:
processed = remove_none_values(json_content=data)
if not isinstance(processed, dict):
- raise JsonTypeError("Removing None values from a dict, we expected a dict in return")
- processed = cast(Dict[str, Any], processed) # pyright: ignore[reportUnnecessaryCast]
- return processed
+ msg = "Removing None values from a dict, we expected a dict in return"
+ raise JsonTypeError(msg)
+ return cast("dict[str, Any]", processed) # pyright: ignore[reportUnnecessaryCast]
def purify_json(
data: Any,
- indent: Optional[int] = None,
+ indent: int | None = None,
is_truncate_bytes_enabled: bool = False,
is_warning_enabled: bool = True,
-) -> Tuple[Union[Dict[Any, Any], List[Any]], str]:
- """
- Converts any Python object into a JSON-serializable format and its string representation.
+) -> tuple[dict[Any, Any] | list[Any], str]:
+ """Converts any Python object into a JSON-serializable format and its string representation.
This function handles various types of input data:
- Pydantic BaseModel instances are converted using their model_dump method
@@ -252,7 +251,8 @@ def purify_json(
Args:
data (Any): The data to convert. Can be a Pydantic model, list, dict, or any other type.
- indent (Optional[int], optional): Number of spaces for JSON formatting indentation. Defaults to None.
+ indent (int | None, optional): Number of spaces for JSON formatting indentation. Defaults to None.
+ is_truncate_bytes_enabled (bool, optional): If True, truncates bytes values to a string representation. Defaults to False.
is_warning_enabled (bool, optional): If True, wraps non-serializable data in a warning object.
Defaults to True.
@@ -266,6 +266,7 @@ def purify_json(
>>> data, json_str = purify_json(model)
>>> print(json_str)
'{"name": "test"}'
+
"""
dict_string: str
if isinstance(data, CustomBaseModel) and is_truncate_bytes_enabled:
@@ -275,7 +276,7 @@ def purify_json(
is_truncate_bytes_enabled=is_truncate_bytes_enabled,
is_warning_enabled=is_warning_enabled,
)
- elif isinstance(data, BaseModel):
+ if isinstance(data, BaseModel):
return purify_json(
data.model_dump(serialize_as_any=True),
indent=indent,
@@ -284,28 +285,29 @@ def purify_json(
)
if isinstance(data, list):
- the_list = data # type: ignore
+ the_list = data # pyright: ignore[reportUnknownVariableType]
if not the_list:
return [], "[]"
if isinstance(the_list[0], CustomBaseModel) and is_truncate_bytes_enabled:
- the_list_of_custom_base_models: List[CustomBaseModel] = the_list
+ the_list_of_custom_base_models = cast("list[CustomBaseModel]", the_list)
pure_list = [item.model_dump_truncated(serialize_as_any=True) for item in the_list_of_custom_base_models]
dict_string = json.dumps(pure_list, indent=indent, default=str)
return pure_list, dict_string
- elif isinstance(the_list[0], BaseModel):
- the_list_of_base_models: List[BaseModel] = the_list
+ if isinstance(the_list[0], BaseModel):
+ the_list_of_base_models = cast("list[BaseModel]", the_list)
pure_list = [item.model_dump(serialize_as_any=True) for item in the_list_of_base_models]
dict_string = json.dumps(pure_list, indent=indent, default=str)
return pure_list, dict_string
try:
dict_string = json.dumps(data, indent=indent)
- pure_dict: Union[Dict[Any, Any], List[Any]] = data
+ pure_dict = cast("dict[Any, Any] | list[Any]", data)
except TypeError:
try:
dict_string = kajson.dumps(data, indent=indent) # pyright: ignore[reportUnknownMemberType]
except Exception:
if is_warning_enabled:
+ data = cast("dict[Any, Any] | list[Any]", data)
data = {"!": data}
dict_string = json.dumps(data, indent=indent, default=str)
pure_dict = json.loads(dict_string)
@@ -313,12 +315,11 @@ def purify_json(
def purify_json_list(
- data: List[Any],
- indent: Optional[int] = None,
+ data: list[Any],
+ indent: int | None = None,
is_truncate_bytes_enabled: bool = False,
-) -> Tuple[List[Any], str]:
- """
- Converts a list of Python objects into a JSON-serializable list and its string representation.
+) -> tuple[list[Any], str]:
+ """Converts a list of Python objects into a JSON-serializable list and its string representation.
This function specifically handles lists and provides specialized processing for lists of
Pydantic BaseModel instances. It attempts multiple serialization methods to ensure successful
@@ -326,8 +327,9 @@ def purify_json_list(
Args:
data (List[Any]): The list to convert. Can contain Pydantic models or other types.
- indent (Optional[int], optional): Number of spaces for JSON formatting indentation.
+ indent (int | None, optional): Number of spaces for JSON formatting indentation.
Defaults to None.
+ is_truncate_bytes_enabled (bool, optional): If True, truncates bytes values to a string representation. Defaults to False.
Returns:
Tuple[List[Any], str]: A tuple containing:
@@ -339,19 +341,20 @@ def purify_json_list(
>>> data, json_str = purify_json_list(models)
>>> print(json_str)
'[{"name": "test1"}, {"name": "test2"}]'
+
"""
list_string: str
- pure_list: List[Any]
+ pure_list: list[Any]
if not data:
return [], "[]"
if isinstance(data[0], CustomBaseModel) and is_truncate_bytes_enabled:
- the_list_of_custom_base_models: List[CustomBaseModel] = data
+ the_list_of_custom_base_models = cast("list[CustomBaseModel]", data)
pure_list = [item.model_dump_truncated(serialize_as_any=True) for item in the_list_of_custom_base_models]
list_string = json.dumps(pure_list, indent=indent, default=str)
return pure_list, list_string
- elif isinstance(data[0], BaseModel):
- the_list_of_base_models: List[BaseModel] = data
+ if isinstance(data[0], BaseModel):
+ the_list_of_base_models: list[BaseModel] = data
pure_list = [item.model_dump(serialize_as_any=True) for item in the_list_of_base_models]
list_string = json.dumps(pure_list, indent=indent, default=str)
return pure_list, list_string
@@ -368,9 +371,8 @@ def purify_json_list(
return pure_list, list_string
-def purify_json_dict(data: Any, indent: Optional[int] = None, is_warning_enabled: bool = True) -> Tuple[Dict[str, Any], str]:
- """
- Converts any Python object into a JSON-serializable dictionary and its string representation.
+def purify_json_dict(data: Any, indent: int | None = None, is_warning_enabled: bool = True) -> tuple[dict[str, Any], str]:
+ """Converts any Python object into a JSON-serializable dictionary and its string representation.
This function specifically handles dictionary-like objects and Pydantic BaseModel instances,
converting them to pure dictionaries that can be JSON serialized. It includes multiple
@@ -378,7 +380,7 @@ def purify_json_dict(data: Any, indent: Optional[int] = None, is_warning_enabled
Args:
data (Any): The data to convert. Can be a Pydantic model or dictionary-like object.
- indent (Optional[int], optional): Number of spaces for JSON formatting indentation.
+ indent (int | None, optional): Number of spaces for JSON formatting indentation.
Defaults to None.
is_warning_enabled (bool, optional): If True, wraps non-serializable data in a warning
object with a "!" key. Defaults to True.
@@ -396,6 +398,7 @@ def purify_json_dict(data: Any, indent: Optional[int] = None, is_warning_enabled
>>> data, json_str = purify_json_dict(model)
>>> print(json_str)
'{"name": "test", "value": 123}'
+
"""
dict_string: str
if isinstance(data, BaseModel):
@@ -406,11 +409,12 @@ def purify_json_dict(data: Any, indent: Optional[int] = None, is_warning_enabled
)
if isinstance(data, list):
- raise ArgumentTypeError("The data is a list, not a dict")
+ msg = "The data is a list, not a dict"
+ raise ArgumentTypeError(msg)
try:
dict_string = json.dumps(data, indent=indent)
- pure_dict: Dict[str, Any] = data
+ pure_dict: dict[str, Any] = data
except TypeError:
try:
dict_string = kajson.dumps(data, indent=indent) # pyright: ignore[reportUnknownMemberType]
@@ -422,9 +426,8 @@ def purify_json_dict(data: Any, indent: Optional[int] = None, is_warning_enabled
return pure_dict, dict_string
-def pure_json_str(data: Any, indent: Optional[int] = None, is_warning_enabled: bool = True) -> str:
- """
- Converts any Python object directly to its JSON string representation.
+def pure_json_str(data: Any, indent: int | None = None, is_warning_enabled: bool = True) -> str:
+ """Converts any Python object directly to its JSON string representation.
This is a convenience wrapper around purify_json that returns only the string
representation, discarding the intermediate data structure. It inherits all the
@@ -433,7 +436,7 @@ def pure_json_str(data: Any, indent: Optional[int] = None, is_warning_enabled: b
Args:
data (Any): The data to convert to a JSON string. Can be any type supported
by purify_json.
- indent (Optional[int], optional): Number of spaces for JSON formatting indentation.
+ indent (int | None, optional): Number of spaces for JSON formatting indentation.
Defaults to None.
is_warning_enabled (bool, optional): If True, wraps non-serializable data in a
warning object. Defaults to True.
@@ -448,6 +451,7 @@ def pure_json_str(data: Any, indent: Optional[int] = None, is_warning_enabled: b
'{
"name": "test"
}'
+
"""
_, json_string = purify_json(data, indent=indent, is_warning_enabled=is_warning_enabled)
return json_string
diff --git a/pipelex/tools/misc/markdown_utils.py b/pipelex/tools/misc/markdown_utils.py
index c0695ed8e..d0f69ee9d 100644
--- a/pipelex/tools/misc/markdown_utils.py
+++ b/pipelex/tools/misc/markdown_utils.py
@@ -1,29 +1,25 @@
-from typing import Any, Dict, List, Optional
+from typing import Any, cast
from pipelex.tools.misc.attribute_utils import AttributePolisher
from pipelex.tools.misc.json_utils import purify_json_dict
from pipelex.tools.misc.string_utils import snake_to_capitalize_first_letter
-def convert_to_markdown(data: Any, level: int = 1, is_pretty: bool = False, key: Optional[str] = None) -> str:
- """
- Convert arbitrary JSON-compatible Python data to a Markdown string
+def convert_to_markdown(data: Any, level: int = 1, is_pretty: bool = False, key: str | None = None) -> str:
+ """Convert arbitrary JSON-compatible Python data to a Markdown string
without needing to specify the markdown type explicitly.
"""
if isinstance(data, dict):
- the_dict: Dict[str, Any] = data
+ the_dict = cast("dict[str, Any]", data)
# Treat keys as headings and values as their content
- dict_result_lines: List[str] = []
- for key, value in the_dict.items():
+ dict_result_lines: list[str] = []
+ for _key, _value in the_dict.items():
heading_prefix = "#" * min(level, 6) # Limit heading levels to 6
# Use the key as a heading
- if is_pretty:
- converted_line = f"{heading_prefix} {snake_to_capitalize_first_letter(key)}"
- else:
- converted_line = f"{heading_prefix} {key}"
+ converted_line = f"{heading_prefix} {snake_to_capitalize_first_letter(_key)}" if is_pretty else f"{heading_prefix} {_key}"
# Convert the value recursively, increasing the heading level
# dict_result_lines.append(convert_to_markdown(data=value, level=level + 1))
- converted_value = convert_to_markdown(data=value, level=level + 1, key=key)
+ converted_value = convert_to_markdown(data=_value, level=level + 1, key=_key)
converted_value_nb_lines = len(converted_value.split("\n"))
if converted_value_nb_lines > 1:
dict_result_lines.append(converted_line)
@@ -38,8 +34,8 @@ def convert_to_markdown(data: Any, level: int = 1, is_pretty: bool = False, key:
# they become list items.
if not data:
return ""
- the_list: List[Any] = data
- list_result_lines: List[str] = []
+ the_list = cast("list[Any]", data)
+ list_result_lines: list[str] = []
for item in the_list:
# Convert the item first
item_md = convert_to_markdown(item, level=level)
diff --git a/pipelex/tools/misc/mermaid_utils.py b/pipelex/tools/misc/mermaid_utils.py
index b88f9e43a..994a6a158 100644
--- a/pipelex/tools/misc/mermaid_utils.py
+++ b/pipelex/tools/misc/mermaid_utils.py
@@ -24,13 +24,11 @@ def make_mermaid_url(mermaid_code: str) -> str:
},
}
encoded = encode_pako_from_string(json.dumps(as_dict))
- url = f"https://mermaid.ink/img/{encoded}"
- return url
+ return f"https://mermaid.ink/img/{encoded}"
def clean_str_for_mermaid_node_title(text: str) -> str:
- """
- Cleans a string to be safely used as a Mermaid node title by replacing quotes
+ """Cleans a string to be safely used as a Mermaid node title by replacing quotes
with similar Unicode characters that won't interfere with Mermaid syntax.
Args:
@@ -38,11 +36,11 @@ def clean_str_for_mermaid_node_title(text: str) -> str:
Returns:
The cleaned string with quotes replaced
+
"""
# Replace single and double quotes with similar Unicode characters
text = text.replace('"', "″") # Replace with prime symbol
- text = text.replace("'", "′") # Replace with curly quote
- return text
+ return text.replace("'", "′") # Replace with curly quote
def print_mermaid_url(url: str, title: str):
diff --git a/pipelex/tools/misc/path_utils.py b/pipelex/tools/misc/path_utils.py
index 052cd3002..fbf894d29 100644
--- a/pipelex/tools/misc/path_utils.py
+++ b/pipelex/tools/misc/path_utils.py
@@ -1,6 +1,5 @@
import os
import urllib.parse
-from typing import Optional, Tuple
from pipelex.types import StrEnum
@@ -28,8 +27,7 @@ class InterpretedPathOrUrl(StrEnum):
def interpret_path_or_url(path_or_uri: str) -> InterpretedPathOrUrl:
- """
- Determines whether a string represents a file URI, URL, or file path.
+ """Determines whether a string represents a file URI, URL, or file path.
This function analyzes the input string to categorize it as one of three types:
- File URI (starts with "file://")
@@ -55,6 +53,7 @@ def interpret_path_or_url(path_or_uri: str) -> InterpretedPathOrUrl:
InterpretedPathOrUrl.URL
>>> interpret_path_or_url("/home/user/file.txt")
InterpretedPathOrUrl.FILE_PATH
+
"""
if path_or_uri.startswith("file://"):
return InterpretedPathOrUrl.FILE_URI
@@ -66,9 +65,8 @@ def interpret_path_or_url(path_or_uri: str) -> InterpretedPathOrUrl:
return InterpretedPathOrUrl.FILE_NAME
-def clarify_path_or_url(path_or_uri: str) -> Tuple[Optional[str], Optional[str]]:
- """
- Separates a path_or_uri string into either a file path or online URL component.
+def clarify_path_or_url(path_or_uri: str) -> tuple[str | None, str | None]:
+ """Separates a path_or_uri string into either a file path or online URL component.
This function processes the input string to determine its type and returns
the appropriate components. For file URIs, it converts them to regular file paths.
@@ -90,9 +88,10 @@ def clarify_path_or_url(path_or_uri: str) -> Tuple[Optional[str], Optional[str]]
(None, 'https://example.com')
>>> clarify_path_or_url("/home/user/file.txt")
('/home/user/file.txt', None)
+
"""
- file_path: Optional[str]
- url: Optional[str]
+ file_path: str | None
+ url: str | None
match interpret_path_or_url(path_or_uri):
case InterpretedPathOrUrl.FILE_URI:
parsed_uri = urllib.parse.urlparse(path_or_uri)
@@ -109,5 +108,6 @@ def clarify_path_or_url(path_or_uri: str) -> Tuple[Optional[str], Optional[str]]
file_path = path_or_uri
url = None
case InterpretedPathOrUrl.BASE_64:
- raise NotImplementedError("Base 64 is not supported yet by clarify_path_or_url")
+ msg = "Base 64 is not supported yet by clarify_path_or_url"
+ raise NotImplementedError(msg)
return file_path, url
diff --git a/pipelex/tools/misc/placeholder.py b/pipelex/tools/misc/placeholder.py
index a8275ebfe..db84ab0af 100644
--- a/pipelex/tools/misc/placeholder.py
+++ b/pipelex/tools/misc/placeholder.py
@@ -1,5 +1,3 @@
-from typing import Optional
-
PLACEHOLDER_PREFIX = "placeholder"
@@ -11,11 +9,12 @@ def make_placeholder_value(key: str) -> str:
Returns:
A placeholder value in the format "placeholder-for-{key}"
+
"""
return f"{PLACEHOLDER_PREFIX}-for-{key}"
-def value_is_placeholder(value: Optional[str]) -> bool:
+def value_is_placeholder(value: str | None) -> bool:
"""Check if a value is a placeholder based on the prefix.
Args:
@@ -23,5 +22,6 @@ def value_is_placeholder(value: Optional[str]) -> bool:
Returns:
True if the value starts with the placeholder prefix, False otherwise
+
"""
return value is not None and value.startswith(PLACEHOLDER_PREFIX)
diff --git a/pipelex/tools/misc/pretty.py b/pipelex/tools/misc/pretty.py
index 49d030275..42a9e4d19 100644
--- a/pipelex/tools/misc/pretty.py
+++ b/pipelex/tools/misc/pretty.py
@@ -1,5 +1,5 @@
import shutil
-from typing import Any, ClassVar, List, Optional, Union
+from typing import Any, ClassVar
from rich import print as rich_print
from rich.markdown import Markdown
@@ -18,19 +18,19 @@ BORDER_COLOR = TerminalColor.YELLOW
def pretty_print(
- content: Union[str, Any],
- title: Optional[TextType] = None,
- subtitle: Optional[TextType] = None,
- border_style: Optional[StyleType] = None,
+ content: str | Any,
+ title: TextType | None = None,
+ subtitle: TextType | None = None,
+ border_style: StyleType | None = None,
):
PrettyPrinter.pretty_print(content=content, title=title, subtitle=subtitle, border_style=border_style)
def pretty_print_md(
content: str,
- title: Optional[TextType] = None,
- subtitle: Optional[TextType] = None,
- border_style: Optional[StyleType] = None,
+ title: TextType | None = None,
+ subtitle: TextType | None = None,
+ border_style: StyleType | None = None,
):
md_content = Markdown(content)
PrettyPrinter.pretty_print(content=md_content, title=title, subtitle=subtitle, border_style=border_style)
@@ -47,10 +47,10 @@ class PrettyPrinter:
@classmethod
def pretty_print(
cls,
- content: Union[str, Any],
- title: Optional[TextType] = None,
- subtitle: Optional[TextType] = None,
- border_style: Optional[StyleType] = None,
+ content: str | Any,
+ title: TextType | None = None,
+ subtitle: TextType | None = None,
+ border_style: StyleType | None = None,
):
match cls.mode:
case PrettyPrintMode.RICH:
@@ -61,10 +61,10 @@ class PrettyPrinter:
@classmethod
def pretty_print_using_rich(
cls,
- content: Union[str, Any],
- title: Optional[TextType] = None,
- subtitle: Optional[TextType] = None,
- border_style: Optional[StyleType] = None,
+ content: str | Any,
+ title: TextType | None = None,
+ subtitle: TextType | None = None,
+ border_style: StyleType | None = None,
):
if isinstance(content, str):
if content.startswith(("http://", "https://")):
@@ -99,9 +99,9 @@ class PrettyPrinter:
@classmethod
def pretty_print_without_rich(
cls,
- content: Union[str, Any],
- title: Optional[TextType] = None,
- subtitle: Optional[TextType] = None,
+ content: str | Any,
+ title: TextType | None = None,
+ subtitle: TextType | None = None,
):
if isinstance(content, str) and content.startswith(("http://", "https://")):
cls.pretty_print_url_without_rich(content=content, title=title, subtitle=subtitle)
@@ -112,7 +112,7 @@ class PrettyPrinter:
terminal_width = shutil.get_terminal_size().columns
content_str = f"{content}"
max_content_width = terminal_width - len(title) - 8 # Accounting for frame and padding
- wrapped_lines: List[str] = []
+ wrapped_lines: list[str] = []
for line in content_str.splitlines():
while len(line) > max_content_width:
wrapped_lines.append(line[:max_content_width])
@@ -137,9 +137,9 @@ class PrettyPrinter:
@classmethod
def pretty_print_url_without_rich(
cls,
- content: Union[str, Any],
- title: Optional[TextType] = None,
- subtitle: Optional[TextType] = None,
+ content: str | Any,
+ title: TextType | None = None,
+ subtitle: TextType | None = None,
):
title = title or ""
if subtitle:
diff --git a/pipelex/tools/misc/string_utils.py b/pipelex/tools/misc/string_utils.py
index 6237f1dfe..48cddf388 100644
--- a/pipelex/tools/misc/string_utils.py
+++ b/pipelex/tools/misc/string_utils.py
@@ -1,10 +1,9 @@
import re
-from typing import Any, List, Optional
+from typing import Any
def has_text(text: str) -> bool:
- """
- Checks if a string contains any alphanumeric characters.
+ """Checks if a string contains any alphanumeric characters.
This function uses a regex pattern to check for the presence of any alphanumeric
character, including non-ASCII characters (e.g., é, ñ, etc.).
@@ -14,41 +13,42 @@ def has_text(text: str) -> bool:
Returns:
bool: True if the string contains at least one alphanumeric character, False otherwise.
+
"""
return bool(re.search(r"\w", text))
-def is_none_or_has_text(text: Optional[str]) -> bool:
- """
- Checks if a string is None or contains alphanumeric characters.
+def is_none_or_has_text(text: str | None) -> bool:
+ """Checks if a string is None or contains alphanumeric characters.
This function combines a None check with the has_text function to validate that
a string is either None or contains meaningful content.
Args:
- text (Optional[str]): The string to check, which can be None.
+ text (str | None): The string to check, which can be None.
Returns:
bool: True if the string is None or contains at least one alphanumeric character,
False if it's an empty string or contains only non-alphanumeric characters.
+
"""
return text is None or has_text(text)
-def can_inject_text(value: Optional[Any]) -> bool:
- """
- Checks if a value can be safely converted to a string containing alphanumeric characters.
+def can_inject_text(value: Any | None) -> bool:
+ """Checks if a value can be safely converted to a string containing alphanumeric characters.
This function attempts to convert any value to a string and checks if it contains
alphanumeric characters. It handles None values and exceptions gracefully, making it
safe for type-uncertain inputs.
Args:
- value (Optional[Any]): Any value that might be convertible to a string, including None.
+ value (Any | None): Any value that might be convertible to a string, including None.
Returns:
bool: True if the value can be converted to a string containing alphanumeric characters,
False if the value is None, empty, or cannot be safely converted to a string.
+
"""
if not value:
return False
@@ -58,26 +58,25 @@ def can_inject_text(value: Optional[Any]) -> bool:
return False
-def is_not_none_and_has_text(text: Optional[str]) -> bool:
- """
- Checks if a string is not None and contains alphanumeric characters.
+def is_not_none_and_has_text(text: str | None) -> bool:
+ """Checks if a string is not None and contains alphanumeric characters.
This function combines a None check with the has_text function to validate that
a string exists and contains meaningful content. It's the opposite of is_none_or_has_text.
Args:
- text (Optional[str]): The string to check, which can be None.
+ text (str | None): The string to check, which can be None.
Returns:
bool: True if the string is not None and contains at least one alphanumeric character,
False if it's None, empty, or contains only non-alphanumeric characters.
+
"""
return text is not None and has_text(text)
def camel_to_snake_case(name: str) -> str:
- """
- Converts a camelCase string to snake_case format.
+ """Converts a camelCase string to snake_case format.
This function identifies word boundaries in camelCase by looking for transitions
between lowercase and uppercase letters, then joins the words with underscores.
@@ -91,14 +90,14 @@ def camel_to_snake_case(name: str) -> str:
Example:
>>> camel_to_snake_case("thisIsATest")
'this_is_a_test'
+
"""
s1 = re.sub("(.)([A-Z][a-z]+)", r"\1_\2", name)
return re.sub("([a-z0-9])([A-Z])", r"\1_\2", s1).lower()
def pascal_case_to_snake_case(name: str) -> str:
- """
- Converts a PascalCase string to snake_case format.
+ """Converts a PascalCase string to snake_case format.
This function is a wrapper around camel_to_snake_case, as the same conversion
rules apply to both PascalCase and camelCase strings.
@@ -112,13 +111,13 @@ def pascal_case_to_snake_case(name: str) -> str:
Example:
>>> pascal_case_to_snake_case("ThisIsATest")
'this_is_a_test'
+
"""
return camel_to_snake_case(name=name)
def pascal_case_to_sentence(name: str) -> str:
- """
- Converts a PascalCase string to a capitalized sentence.
+ """Converts a PascalCase string to a capitalized sentence.
This function splits a PascalCase string at word boundaries and spaces.
- Preserves fully uppercase words (e.g., "BOB", "JSON")
@@ -137,10 +136,11 @@ def pascal_case_to_sentence(name: str) -> str:
'BOB low key'
>>> pascal_case_to_sentence("ParseJSONData")
'Parse JSON data'
+
"""
# First split by spaces
- space_parts: List[str] = name.split()
- result_parts: List[str] = []
+ space_parts: list[str] = name.split()
+ result_parts: list[str] = []
for part in space_parts:
if part.isupper():
@@ -148,16 +148,15 @@ def pascal_case_to_sentence(name: str) -> str:
result_parts.append(part)
else:
# Handle PascalCase parts
- words: List[str] = re.findall(r"([A-Z][a-z]+|[A-Z]{2,}(?=[A-Z][a-z]|\d|\W|$)|[A-Z]{2,}|[0-9]+)", part)
- processed_words: List[str] = [word if word.isupper() else word.lower() for word in words]
+ words: list[str] = re.findall(r"([A-Z][a-z]+|[A-Z]{2,}(?=[A-Z][a-z]|\d|\W|$)|[A-Z]{2,}|[0-9]+)", part)
+ processed_words: list[str] = [word if word.isupper() else word.lower() for word in words]
result_parts.append(" ".join(processed_words))
return " ".join(result_parts).capitalize()
def snake_to_pascal_case(snake_str: str) -> str:
- """
- Converts a snake_case string to PascalCase format.
+ """Converts a snake_case string to PascalCase format.
This function splits the string at underscores, capitalizes the first letter
of each component, and joins them together without separators.
@@ -171,14 +170,14 @@ def snake_to_pascal_case(snake_str: str) -> str:
Example:
>>> snake_to_pascal_case("hello_world")
'HelloWorld'
+
"""
components = snake_str.split("_")
return "".join(component.title() for component in components)
def snake_to_capitalize_first_letter(snake_str: str) -> str:
- """
- Converts a snake_case string to a space-separated string with the first letter capitalized.
+ """Converts a snake_case string to a space-separated string with the first letter capitalized.
This function splits the string at underscores, joins the components with spaces,
and capitalizes only the first letter of the entire string.
@@ -192,6 +191,7 @@ def snake_to_capitalize_first_letter(snake_str: str) -> str:
Example:
>>> snake_to_capitalize_first_letter("this_is_a_test")
'This is a test'
+
"""
components = snake_str.split("_")
phrase = " ".join(components)
@@ -230,11 +230,11 @@ def matches_wildcard_pattern(text: str, pattern: str) -> bool:
True
>>> matches_wildcard_pattern("exact-match", "exact-match")
True
+
"""
if pattern == "*":
return True
-
- if pattern.startswith("*") and pattern.endswith("*"):
+ elif pattern.startswith("*") and pattern.endswith("*"):
# Pattern like "*sonnet*"
middle = pattern[1:-1]
return middle in text
diff --git a/pipelex/tools/misc/toml_utils.py b/pipelex/tools/misc/toml_utils.py
index c14e5c286..d20b5bec8 100644
--- a/pipelex/tools/misc/toml_utils.py
+++ b/pipelex/tools/misc/toml_utils.py
@@ -1,7 +1,8 @@
from __future__ import annotations
import re
-from typing import Any, Dict, List, Mapping, Optional, cast
+from collections.abc import Mapping
+from typing import Any, cast
import toml
import tomlkit
@@ -16,13 +17,11 @@ from pipelex.tools.misc.json_utils import remove_none_values_from_dict
class TOMLValidationError(Exception):
"""Raised when TOML file has formatting issues that could cause problems."""
- pass
-
-def validate_toml_content(content: str, file_path: Optional[str] = None) -> None:
+def validate_toml_content(content: str, file_path: str | None = None) -> None:
"""Validate TOML content for common formatting issues."""
lines = content.splitlines()
- issues: List[str] = []
+ issues: list[str] = []
for line_num, line in enumerate(lines, 1):
# Check for trailing whitespace
@@ -55,8 +54,9 @@ def validate_toml_file(path: str) -> None:
Raises:
TOMLValidationError: If formatting issues are detected
+
"""
- with open(path, "r", encoding="utf-8") as file:
+ with open(path, encoding="utf-8") as file:
content = file.read()
validate_toml_content(content, path)
@@ -74,6 +74,7 @@ def clean_trailing_whitespace(content: str) -> str:
Returns:
The cleaned TOML content with trailing whitespace removed and an empty line at EOF
+
"""
# Split into lines and clean each line
lines = [line.rstrip() for line in content.splitlines()]
@@ -90,7 +91,7 @@ def clean_trailing_whitespace(content: str) -> str:
return "\n".join(lines) + "\n\n"
-def load_toml_from_path(path: str) -> Dict[str, Any]:
+def load_toml_from_path(path: str) -> dict[str, Any]:
"""Load TOML from path.
Args:
@@ -101,9 +102,10 @@ def load_toml_from_path(path: str) -> Dict[str, Any]:
Raises:
toml.TomlDecodeError: If TOML parsing fails, with file path included
+
"""
try:
- with open(path, "r", encoding="utf-8") as file:
+ with open(path, encoding="utf-8") as file:
content = file.read()
cleaned_content = clean_trailing_whitespace(content)
@@ -114,14 +116,13 @@ def load_toml_from_path(path: str) -> Dict[str, Any]:
file.write(cleaned_content)
# Parse TOML first
- dict_from_toml = toml.loads(cleaned_content)
-
- return dict_from_toml
+ return toml.loads(cleaned_content)
except toml.TomlDecodeError as exc:
- raise toml.TomlDecodeError(f"TOML parsing error in file '{path}': {exc}", exc.doc, exc.pos) from exc
+ msg = f"TOML parsing error in file '{path}': {exc}"
+ raise toml.TomlDecodeError(msg, exc.doc, exc.pos) from exc
-def failable_load_toml_from_path(path: str) -> Optional[Dict[str, Any]]:
+def failable_load_toml_from_path(path: str) -> dict[str, Any] | None:
"""Load TOML from path with failure handling."""
if not path_exists(path):
return None
@@ -139,8 +140,7 @@ def make_toml_string(
ensure_trailing_newline: bool = True,
ensure_leading_blank_line_in_value: bool = False,
):
- """
- Build a tomlkit string node.
+ r"""Build a tomlkit string node.
- If `force_multiline` or the text contains '\\n', we emit a triple-quoted multiline string.
- When multiline, `ensure_trailing_newline` puts the closing quotes on their own line.
- When multiline, `ensure_leading_blank_line_in_value` inserts a real blank line at the start of the value.
@@ -163,15 +163,10 @@ def _convert_to_inline(value: Any) -> Any:
# Handle Pydantic models by converting them to dict first
if isinstance(value, BaseModel):
# For RootModel, use the root attribute; for regular models, use model_dump()
- if hasattr(value, "root"):
- # This is a RootModel, use its root value
- value = getattr(value, "root")
- else:
- # This is a regular BaseModel, convert to dict
- value = value.model_dump()
+ value = value.root if hasattr(value, "root") else value.model_dump() # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType, reportUnknownVariableType]
if isinstance(value, Mapping):
- value = cast(Mapping[str, Any], value)
+ value = cast("Mapping[str, Any]", value)
inline_table_obj = inline_table()
for key, value_item in value.items():
inline_table_obj[key] = _convert_to_inline(value_item)
@@ -179,12 +174,12 @@ def _convert_to_inline(value: Any) -> Any:
return inline_table_obj
if isinstance(value, list):
- value = cast(List[Any], value)
+ value = cast("list[Any]", value)
array_obj = array()
array_obj.multiline(True) # set to False for single-line arrays
for element in value:
if isinstance(element, Mapping):
- element = cast(Mapping[str, Any], element)
+ element = cast("Mapping[str, Any]", element)
inline_element = inline_table()
for inner_key, inner_value in element.items():
inline_element[inner_key] = _convert_to_inline(inner_value)
@@ -199,32 +194,30 @@ def _convert_to_inline(value: Any) -> Any:
# This looks like a prompt template - escape newlines for single-line format
# Return the string directly and let tomlkit handle the escaping properly
return value
- else:
- # Triple quotes if needed (or forced); closing quotes on their own line.
- return make_toml_string(
- value,
- prefer_literal=False, # flip to True for '''...'''
- force_multiline=False, # flip to True to force """...""" even without \n
- ensure_trailing_newline=True, # keep closing """ on its own line
- ensure_leading_blank_line_in_value=False, # flip to True to keep a blank first line
- )
- return value
+ # Triple quotes if needed (or forced); closing quotes on their own line.
+ return make_toml_string(
+ value,
+ prefer_literal=False, # flip to True for '''...'''
+ force_multiline=False, # flip to True to force """...""" even without \n
+ ensure_trailing_newline=True, # keep closing """ on its own line
+ ensure_leading_blank_line_in_value=False, # flip to True to keep a blank first line
+ )
+ return value # pyright: ignore[reportUnknownVariableType]
def _filter_empty_values(value: Any) -> Any:
"""Filter out empty lists and None values from data structures."""
if isinstance(value, dict):
- filtered: Dict[str, Any] = {}
- for k, v in cast(Dict[str, Any], value).items():
+ filtered: dict[str, Any] = {}
+ for k, v in cast("dict[str, Any]", value).items():
filtered_v = _filter_empty_values(v)
# Keep empty dicts but skip empty lists and None values
if filtered_v is not None and (not isinstance(filtered_v, list) or filtered_v):
filtered[k] = filtered_v
return filtered
- elif isinstance(value, list):
- return [_filter_empty_values(item) for item in cast(List[Any], value) if _filter_empty_values(item) is not None]
- else:
- return value
+ if isinstance(value, list):
+ return [_filter_empty_values(item) for item in cast("list[Any]", value) if _filter_empty_values(item) is not None]
+ return value
def _create_ordered_inline_table(data: Mapping[str, Any]) -> Any:
@@ -268,7 +261,7 @@ def dict_to_toml(data: Mapping[str, Any]) -> str:
# Handle sections (concepts, pipes)
for section_key, section_value in data.items():
if isinstance(section_value, Mapping):
- section_value = cast(Mapping[str, Any], section_value)
+ section_value = cast("Mapping[str, Any]", section_value)
# Skip empty sections
if not section_value:
@@ -284,7 +277,7 @@ def dict_to_toml(data: Mapping[str, Any]) -> str:
section_table.add(item_key, _convert_to_inline(item_value))
elif isinstance(item_value, Mapping):
# Complex object that needs its own table
- item_value = cast(Mapping[str, Any], item_value)
+ item_value = cast("Mapping[str, Any]", item_value)
item_table = table()
# Handle the structure field specially
@@ -293,9 +286,9 @@ def dict_to_toml(data: Mapping[str, Any]) -> str:
if field_key == "structure" and isinstance(field_value, Mapping):
# Structure should be its own table [section.item.structure]
structure_table = table()
- for struct_key, struct_value in cast(Mapping[str, Any], field_value).items():
+ for struct_key, struct_value in cast("Mapping[str, Any]", field_value).items():
if isinstance(struct_value, Mapping):
- structure_table.add(struct_key, _create_ordered_inline_table(cast(Mapping[str, Any], struct_value)))
+ structure_table.add(struct_key, _create_ordered_inline_table(cast("Mapping[str, Any]", struct_value)))
else:
structure_table.add(struct_key, _convert_to_inline(struct_value))
item_table.add(field_key, structure_table)
@@ -377,17 +370,16 @@ def dict_to_toml(data: Mapping[str, Any]) -> str:
return "\n".join(processed_lines)
- dumped_content = fix_inline_table_spacing(dumped_content)
+ return fix_inline_table_spacing(dumped_content)
- return dumped_content
-
-def save_toml_to_path(data: Dict[str, Any], path: str) -> None:
+def save_toml_to_path(data: dict[str, Any], path: str) -> None:
"""Save dictionary as TOML to file path.
Args:
data: Dictionary to save as TOML
path: Path where to save the TOML file
+
"""
data_cleaned = data
with open(path, "w", encoding="utf-8") as file:
diff --git a/pipelex/tools/pdf/pypdfium2_renderer.py b/pipelex/tools/pdf/pypdfium2_renderer.py
index 4df3eb70e..35a996a97 100644
--- a/pipelex/tools/pdf/pypdfium2_renderer.py
+++ b/pipelex/tools/pdf/pypdfium2_renderer.py
@@ -2,16 +2,17 @@ from __future__ import annotations
import asyncio
import pathlib
-from typing import List
+from typing import TYPE_CHECKING
import pypdfium2 as pdfium
-from PIL import Image
from pypdfium2.raw import FPDFBitmap_BGRA
from pipelex.tools.exceptions import ToolException
from pipelex.tools.misc.file_fetch_utils import fetch_file_from_url_httpx_async
from pipelex.tools.misc.path_utils import clarify_path_or_url
+if TYPE_CHECKING:
+ from PIL import Image
PDFIUM2_REFERENCE_DPI = 72
@@ -23,8 +24,7 @@ PdfInput = str | pathlib.Path | bytes
class PyPdfium2Renderer:
- """
- Thread-safe PDF page renderer built on pypdfium2.
+ """Thread-safe PDF page renderer built on pypdfium2.
• All entry into the native PDFium library is protected by a single
asyncio.Lock, so the enclosing *process* is safe even if other
@@ -39,9 +39,9 @@ class PyPdfium2Renderer:
# ---- internal blocking helper ------------------------------------
# TODO: Needs UT
@staticmethod
- def _render_pdf_pages_sync(pdf_input: PdfInput, scale: float) -> List[Image.Image]:
+ def _render_pdf_pages_sync(pdf_input: PdfInput, scale: float) -> list[Image.Image]:
pdf_doc = pdfium.PdfDocument(pdf_input)
- images: List[Image.Image] = []
+ images: list[Image.Image] = []
for index in range(len(pdf_doc)):
# page = doc.get_page(index)
page = pdf_doc[index]
@@ -59,9 +59,9 @@ class PyPdfium2Renderer:
# TODO: Needs UT
@staticmethod
- def _get_text_from_pdf_pages_sync(pdf_input: PdfInput) -> List[str]:
+ def _get_text_from_pdf_pages_sync(pdf_input: PdfInput) -> list[str]:
pdf_doc = pdfium.PdfDocument(pdf_input)
- texts: List[str] = []
+ texts: list[str] = []
for index in range(len(pdf_doc)):
page = pdf_doc[index]
text = page.get_textpage().get_text_bounded() # pyright: ignore[reportUnknownMemberType]
@@ -71,37 +71,37 @@ class PyPdfium2Renderer:
return texts
# ---- public async façade -----------------------------------------
- async def render_pdf_pages(self, pdf_input: PdfInput, dpi: int) -> List[Image.Image]:
+ async def render_pdf_pages(self, pdf_input: PdfInput, dpi: int) -> list[Image.Image]:
scale = dpi / PDFIUM2_REFERENCE_DPI
"""Render *one* page and return PNG bytes."""
async with self._pdfium_lock:
return await asyncio.to_thread(self._render_pdf_pages_sync, pdf_input, scale)
- async def get_text_from_pdf_pages(self, pdf_input: PdfInput) -> List[str]:
+ async def get_text_from_pdf_pages(self, pdf_input: PdfInput) -> list[str]:
"""Extract text from all pages of a PDF."""
async with self._pdfium_lock:
return await asyncio.to_thread(self._get_text_from_pdf_pages_sync, pdf_input)
- async def render_pdf_pages_from_uri(self, pdf_uri: str, dpi: int) -> List[Image.Image]:
- pdf_path, pdf_url = clarify_path_or_url(path_or_uri=pdf_uri) # pyright: ignore
+ async def render_pdf_pages_from_uri(self, pdf_uri: str, dpi: int) -> list[Image.Image]:
+ pdf_path, pdf_url = clarify_path_or_url(path_or_uri=pdf_uri)
if pdf_url:
pdf_bytes = await fetch_file_from_url_httpx_async(url=pdf_url)
return await self.render_pdf_pages(pdf_input=pdf_bytes, dpi=dpi)
- elif pdf_path:
+ if pdf_path:
return await self.render_pdf_pages(pdf_input=pdf_path, dpi=dpi)
- else:
- raise PyPdfium2RendererError(f"Invalid PDF URI: {pdf_uri}")
+ msg = f"Invalid PDF URI: {pdf_uri}"
+ raise PyPdfium2RendererError(msg)
- async def get_text_from_pdf_pages_from_uri(self, pdf_uri: str) -> List[str]:
+ async def get_text_from_pdf_pages_from_uri(self, pdf_uri: str) -> list[str]:
"""Extract text from all pages of a PDF from URI."""
- pdf_path, pdf_url = clarify_path_or_url(path_or_uri=pdf_uri) # pyright: ignore
+ pdf_path, pdf_url = clarify_path_or_url(path_or_uri=pdf_uri)
if pdf_url:
pdf_bytes = await fetch_file_from_url_httpx_async(url=pdf_url)
return await self.get_text_from_pdf_pages(pdf_input=pdf_bytes)
- elif pdf_path:
+ if pdf_path:
return await self.get_text_from_pdf_pages(pdf_input=pdf_path)
- else:
- raise PyPdfium2RendererError(f"Invalid PDF URI: {pdf_uri}")
+ msg = f"Invalid PDF URI: {pdf_uri}"
+ raise PyPdfium2RendererError(msg)
pypdfium2_renderer = PyPdfium2Renderer()
diff --git a/pipelex/tools/registry_models.py b/pipelex/tools/registry_models.py
index d4f175d5f..76d087a0a 100644
--- a/pipelex/tools/registry_models.py
+++ b/pipelex/tools/registry_models.py
@@ -1,15 +1,16 @@
-from typing import Any, Callable, List, Set, Type
+from collections.abc import Callable
+from typing import Any
from pydantic import BaseModel
-ModelType = Type[BaseModel]
+ModelType = type[BaseModel]
class RegistryModels:
@classmethod
- def get_all_models(cls) -> List[ModelType]:
- model_lists: List[List[ModelType]] = [getattr(cls, attr) for attr in dir(cls) if isinstance(getattr(cls, attr), list)]
- all_models: Set[ModelType] = set()
+ def get_all_models(cls) -> list[ModelType]:
+ model_lists: list[list[ModelType]] = [getattr(cls, attr) for attr in dir(cls) if isinstance(getattr(cls, attr), list)]
+ all_models: set[ModelType] = set()
for model_list in model_lists:
all_models.update(model_list)
@@ -18,8 +19,8 @@ class RegistryModels:
class RegistryFuncs:
@classmethod
- def get_all_functions(cls) -> List[Callable[..., Any]]:
- functions: List[Callable] = [] # pyright: ignore[reportMissingTypeArgument, reportUnknownVariableType]
+ def get_all_functions(cls) -> list[Callable[..., Any]]:
+ functions: list[Callable] = [] # pyright: ignore[reportMissingTypeArgument, reportUnknownVariableType]
for attr in dir(cls):
attr_value = getattr(cls, attr)
if callable(attr_value):
diff --git a/pipelex/tools/runtime_manager.py b/pipelex/tools/runtime_manager.py
index 2d0874e29..6a3fb9f02 100644
--- a/pipelex/tools/runtime_manager.py
+++ b/pipelex/tools/runtime_manager.py
@@ -1,5 +1,3 @@
-from typing import Optional
-
from pydantic import BaseModel
from pipelex.tools.environment import get_optional_env
@@ -17,14 +15,11 @@ class RunMode(StrEnum):
def get_from_env_var(cls) -> "RunMode":
if mode_str := get_optional_env(RUN_MODE_ENV_VAR_KEY):
return RunMode(mode_str)
- else:
- return RunMode.NORMAL
+ return RunMode.NORMAL
class WorkerMode(StrEnum):
- """
- Used for external worker, note that it can be run "for unit tests" even if it's not a unit test.
- """
+ """Used for external worker, note that it can be run "for unit tests" even if it's not a unit test."""
NORMAL = "normal"
UNIT_TEST = "unit_test"
@@ -56,7 +51,7 @@ class ProblemReactions(BaseModel):
class RuntimeManager(BaseModel):
_environment: RunEnvironment = RunEnvironment.get_from_env_var()
_run_mode: RunMode = RunMode.get_from_env_var()
- _worker_mode: Optional[WorkerMode] = None
+ _worker_mode: WorkerMode | None = None
problem_reactions: ProblemReactions = ProblemReactions(
template_inputs=ProblemReaction.LOG,
@@ -79,7 +74,7 @@ class RuntimeManager(BaseModel):
return self._run_mode
@property
- def worker_mode(self) -> Optional[WorkerMode]:
+ def worker_mode(self) -> WorkerMode | None:
return self._worker_mode
@property
diff --git a/pipelex/tools/secrets/env_secrets_provider.py b/pipelex/tools/secrets/env_secrets_provider.py
index dc12a9a8a..14ccf218e 100644
--- a/pipelex/tools/secrets/env_secrets_provider.py
+++ b/pipelex/tools/secrets/env_secrets_provider.py
@@ -1,5 +1,3 @@
-from typing import Optional
-
from typing_extensions import override
from pipelex.tools.environment import EnvVarNotFoundError, get_optional_env, get_required_env
@@ -13,19 +11,22 @@ class EnvSecretsProvider(SecretsProviderAbstract):
try:
return get_required_env(key=secret_id)
except EnvVarNotFoundError as exc:
- raise SecretNotFoundError(f"Secret '{secret_id}' not found") from exc
+ msg = f"Secret '{secret_id}' not found"
+ raise SecretNotFoundError(msg) from exc
@override
- def get_optional_secret(self, secret_id: str) -> Optional[str]:
+ def get_optional_secret(self, secret_id: str) -> str | None:
return get_optional_env(key=secret_id)
@override
def get_required_secret_specific_version(self, secret_id: str, version_id: str) -> str:
- raise NotImplementedError("EnvSecretsProvider does not support specific versions")
+ msg = "EnvSecretsProvider does not support specific versions"
+ raise NotImplementedError(msg)
@override
- def get_optional_secret_specific_version(self, secret_id: str, version_id: str) -> Optional[str]:
- raise NotImplementedError("EnvSecretsProvider does not support specific versions")
+ def get_optional_secret_specific_version(self, secret_id: str, version_id: str) -> str | None:
+ msg = "EnvSecretsProvider does not support specific versions"
+ raise NotImplementedError(msg)
@override
def set_secret_as_env_var(self, secret_id: str, version_id: str = "latest"):
diff --git a/pipelex/tools/secrets/secrets_provider_abstract.py b/pipelex/tools/secrets/secrets_provider_abstract.py
index a5ce8ca94..dd83a6733 100644
--- a/pipelex/tools/secrets/secrets_provider_abstract.py
+++ b/pipelex/tools/secrets/secrets_provider_abstract.py
@@ -1,5 +1,4 @@
from abc import ABC, abstractmethod
-from typing import Optional
LATEST_SECRET_VERSION_NAME = "latest"
@@ -9,13 +8,13 @@ class SecretsProviderAbstract(ABC):
def get_required_secret(self, secret_id: str) -> str: ...
@abstractmethod
- def get_optional_secret(self, secret_id: str) -> Optional[str]: ...
+ def get_optional_secret(self, secret_id: str) -> str | None: ...
@abstractmethod
def get_required_secret_specific_version(self, secret_id: str, version_id: str) -> str: ...
@abstractmethod
- def get_optional_secret_specific_version(self, secret_id: str, version_id: str) -> Optional[str]: ...
+ def get_optional_secret_specific_version(self, secret_id: str, version_id: str) -> str | None: ...
@abstractmethod
def set_secret_as_env_var(self, secret_id: str, version_id: str = LATEST_SECRET_VERSION_NAME): ...
diff --git a/pipelex/tools/secrets/secrets_utils.py b/pipelex/tools/secrets/secrets_utils.py
index 50d2c400c..30d952d41 100644
--- a/pipelex/tools/secrets/secrets_utils.py
+++ b/pipelex/tools/secrets/secrets_utils.py
@@ -49,6 +49,7 @@ def substitute_vars(content: str) -> str:
Raises:
VarNotFoundError: If required variable is missing from all specified sources
+
"""
def replace_var(match: re.Match[str]) -> str:
@@ -65,11 +66,12 @@ def substitute_vars(content: str) -> str:
try:
prefix = VarPrefix(prefix_str)
- except ValueError:
+ except ValueError as exc:
+ msg = f"Unknown variable prefix: '{prefix_str}'"
raise UnknownVarPrefixError(
var_name=var_name,
- message=f"Unknown variable prefix: '{prefix_str}'",
- )
+ message=msg,
+ ) from exc
match prefix:
case VarPrefix.ENV:
@@ -97,11 +99,12 @@ def _handle_fallback_pattern(var_spec: str) -> str:
try:
prefix = VarPrefix(prefix_str)
- except ValueError:
+ except ValueError as exc:
+ msg = f"Unknown variable prefix: '{prefix_str}'"
raise UnknownVarPrefixError(
var_name=var_name,
- message=f"Unknown variable prefix: '{prefix_str}'",
- )
+ message=msg,
+ ) from exc
match prefix:
case VarPrefix.ENV:
@@ -119,10 +122,8 @@ def _handle_fallback_pattern(var_spec: str) -> str:
return get_secrets_provider().get_secret(secret_id=part)
except SecretNotFoundError:
continue # Try next option
-
- raise VarFallbackPatternError(
- message=f"Could not get variable from fallback pattern: {var_spec}",
- )
+ msg = f"Could not get variable from fallback pattern: {var_spec}"
+ raise VarFallbackPatternError(message=msg)
def _get_env_var(var_name: str) -> str:
@@ -130,7 +131,8 @@ def _get_env_var(var_name: str) -> str:
try:
return get_required_env(var_name)
except EnvVarNotFoundError as exc:
- raise VarNotFoundError(message=f"Could not get variable '{var_name}': {str(exc)}", var_name=var_name) from exc
+ msg = f"Could not get variable '{var_name}': {exc!s}"
+ raise VarNotFoundError(message=msg, var_name=var_name) from exc
def _get_secret(secret_name: str) -> str:
@@ -138,4 +140,5 @@ def _get_secret(secret_name: str) -> str:
try:
return get_secrets_provider().get_secret(secret_id=secret_name)
except SecretNotFoundError as exc:
- raise VarNotFoundError(message=f"Could not get variable '{secret_name}': {str(exc)}", var_name=secret_name) from exc
+ msg = f"Could not get variable '{secret_name}': {exc!s}"
+ raise VarNotFoundError(message=msg, var_name=secret_name) from exc
diff --git a/pipelex/tools/templating/jinja2_blueprint.py b/pipelex/tools/templating/jinja2_blueprint.py
index ac136d9d9..8210ba811 100644
--- a/pipelex/tools/templating/jinja2_blueprint.py
+++ b/pipelex/tools/templating/jinja2_blueprint.py
@@ -1,4 +1,4 @@
-from typing import Any, Dict, Optional
+from typing import Any
from pydantic import BaseModel, Field
@@ -7,11 +7,11 @@ from pipelex.tools.templating.templating_models import PromptingStyle
class Jinja2Blueprint(BaseModel):
- jinja2_name: Optional[str] = Field(default=None, description="Name of the Jinja2 template to use")
- jinja2: Optional[str] = Field(default=None, description="Raw Jinja2 template string")
- prompting_style: Optional[PromptingStyle] = Field(default=None, description="Style of prompting to use (typically for different LLMs)")
+ jinja2_name: str | None = Field(default=None, description="Name of the Jinja2 template to use")
+ jinja2: str | None = Field(default=None, description="Raw Jinja2 template string")
+ prompting_style: PromptingStyle | None = Field(default=None, description="Style of prompting to use (typically for different LLMs)")
template_category: Jinja2TemplateCategory = Field(
default=Jinja2TemplateCategory.LLM_PROMPT,
description="Category of the template (could also be HTML, MARKDOWN, MERMAID, etc.), influences Jinja2 rendering environment config",
)
- extra_context: Optional[Dict[str, Any]] = Field(default=None, description="Additional context variables for template rendering")
+ extra_context: dict[str, Any] | None = Field(default=None, description="Additional context variables for template rendering")
diff --git a/pipelex/tools/templating/jinja2_environment.py b/pipelex/tools/templating/jinja2_environment.py
index 6d513780a..ba8dd6262 100644
--- a/pipelex/tools/templating/jinja2_environment.py
+++ b/pipelex/tools/templating/jinja2_environment.py
@@ -1,5 +1,3 @@
-from typing import Tuple
-
from jinja2 import BaseLoader, Environment, PackageLoader
from pipelex.tools.templating.jinja2_template_category import Jinja2TemplateCategory
@@ -51,7 +49,7 @@ def make_jinja2_env_from_package(
template_category: Jinja2TemplateCategory,
package_name: str,
package_path: str,
-) -> Tuple[Environment, BaseLoader]:
+) -> tuple[Environment, BaseLoader]:
full_package_path = f"{package_path}/jinja2_{template_category}"
loader = PackageLoader(
package_name=package_name,
@@ -65,18 +63,17 @@ def make_jinja2_env_without_loader(
template_category: Jinja2TemplateCategory,
) -> Environment:
loader = BaseLoader()
- jinja2_env = make_jinja2_env_from_loader(template_category=template_category, loader=loader)
- return jinja2_env
+ return make_jinja2_env_from_loader(template_category=template_category, loader=loader)
def make_jinja2_env_from_template_provider(
template_category: Jinja2TemplateCategory,
template_provider: TemplateProviderAbstract,
-) -> Tuple[Environment, BaseLoader]:
+) -> tuple[Environment, BaseLoader]:
loader = Jinja2TemplateLoader(template_provider=template_provider)
jinja2_env = make_jinja2_env_from_loader(template_category=template_category, loader=loader)
filters = template_category.filters
for filter_name, filter_function in filters.items():
- jinja2_env.filters[filter_name] = filter_function # type: ignore
+ jinja2_env.filters[filter_name] = filter_function # pyright: ignore[reportArgumentType]
return jinja2_env, loader
diff --git a/pipelex/tools/templating/jinja2_errors.py b/pipelex/tools/templating/jinja2_errors.py
index 79bf0195f..97ac0c3cb 100644
--- a/pipelex/tools/templating/jinja2_errors.py
+++ b/pipelex/tools/templating/jinja2_errors.py
@@ -1,5 +1,3 @@
-from typing import Optional
-
from pipelex.tools.exceptions import ToolException
@@ -23,7 +21,7 @@ class Jinja2DetectVariablesError(ToolException):
pass
-def make_jinja2_error_explanation(jinja2_name: Optional[str], template_text: Optional[str]) -> str:
+def make_jinja2_error_explanation(jinja2_name: str | None, template_text: str | None) -> str:
explanation = ""
if jinja2_name:
explanation += f"\nJinja2 name: '{jinja2_name}'\n"
diff --git a/pipelex/tools/templating/jinja2_filters.py b/pipelex/tools/templating/jinja2_filters.py
index 63f3a391e..c798c5293 100644
--- a/pipelex/tools/templating/jinja2_filters.py
+++ b/pipelex/tools/templating/jinja2_filters.py
@@ -1,4 +1,4 @@
-from typing import Any, Optional
+from typing import Any
from jinja2 import pass_context
from jinja2.runtime import Context, Undefined
@@ -17,40 +17,39 @@ ALLOWED_FILTERS = ["tag", "format"]
# Filter to format some Stuff or any object with the appropriate text formatting methods
@pass_context
-def text_format(context: Context, value: Any, text_format: Optional[TextFormat] = None) -> Any:
+def text_format(context: Context, value: Any, text_format: TextFormat | None = None) -> Any:
if text_format:
if isinstance(text_format, str): # pyright: ignore[reportUnnecessaryIsInstance]
applied_text_format = TextFormat(text_format)
elif isinstance(text_format, TextFormat): # pyright: ignore[reportUnnecessaryIsInstance]
applied_text_format = text_format
else:
- raise Jinja2ContextError(f"Invalid text format: '{text_format}'")
+ msg = f"Invalid text format: '{text_format}'"
+ raise Jinja2ContextError(msg)
else:
applied_text_format = TextFormat(context.get(Jinja2ContextKey.TEXT_FORMAT, default=TextFormat.PLAIN))
if hasattr(value, "rendered_str"):
- rendered_str = value.rendered_str(text_format=applied_text_format)
- return rendered_str
- elif hasattr(value, applied_text_format.render_method_name):
+ return value.rendered_str(text_format=applied_text_format)
+ if hasattr(value, applied_text_format.render_method_name):
render_method = getattr(value, applied_text_format.render_method_name)
- rendered_str = render_method()
- return rendered_str
- elif isinstance(value, StrEnum):
+ return render_method()
+ if isinstance(value, StrEnum):
return value.value
- else:
- return value
+ return value
# TODO: better separate tag and render
# Filter to tag the variable with a tag style and a provided name, appropriate for tagging in a prompt
@pass_context
-def tag(context: Context, value: Any, tag_name: Optional[str] = None) -> Any:
+def tag(context: Context, value: Any, tag_name: str | None = None) -> Any:
if isinstance(value, Undefined):
# maybe we don't need this check
if tag_name:
- raise Jinja2ContextError(f"Jinja2 undefined value with tag_name '{tag_name}'")
- else:
- raise Jinja2ContextError("Jinja2 undefined value.")
+ msg = f"Jinja2 undefined value with tag_name '{tag_name}'"
+ raise Jinja2ContextError(msg)
+ msg = "Jinja2 undefined value."
+ raise Jinja2ContextError(msg)
if isinstance(value, Jinja2TaggableAbstract):
value, tag_name = value.render_tagged_for_jinja2(context=context, tag_name=tag_name)
@@ -58,7 +57,7 @@ def tag(context: Context, value: Any, tag_name: Optional[str] = None) -> Any:
return render_any_tagged_for_jinja2(context=context, value=value, tag_name=tag_name)
-def render_any_tagged_for_jinja2(context: Context, value: Any, tag_name: Optional[str] = None) -> Any:
+def render_any_tagged_for_jinja2(context: Context, value: Any, tag_name: str | None = None) -> Any:
tag_style_str = context.get(Jinja2ContextKey.TAG_STYLE)
tag_style: TagStyle
if tag_style_str:
diff --git a/pipelex/tools/templating/jinja2_models.py b/pipelex/tools/templating/jinja2_models.py
index 09a129413..29aa180f4 100644
--- a/pipelex/tools/templating/jinja2_models.py
+++ b/pipelex/tools/templating/jinja2_models.py
@@ -1,5 +1,5 @@
from abc import ABC, abstractmethod
-from typing import Any, Optional, Tuple
+from typing import Any
from jinja2.runtime import Context
@@ -18,5 +18,5 @@ class Jinja2ContextKey(StrEnum):
class Jinja2TaggableAbstract(ABC):
@abstractmethod
- def render_tagged_for_jinja2(self, context: Context, tag_name: Optional[str] = None) -> Tuple[Any, Optional[str]]:
+ def render_tagged_for_jinja2(self, context: Context, tag_name: str | None = None) -> tuple[Any, str | None]:
pass
diff --git a/pipelex/tools/templating/jinja2_parsing.py b/pipelex/tools/templating/jinja2_parsing.py
index 901b426fc..eb6a773b7 100644
--- a/pipelex/tools/templating/jinja2_parsing.py
+++ b/pipelex/tools/templating/jinja2_parsing.py
@@ -16,4 +16,5 @@ def check_jinja2_parsing(
try:
jinja2_env.parse(jinja2_template_source)
except TemplateSyntaxError as exc:
- raise Jinja2ParsingError(f"Could not parse Jinja2 template because of: {exc}. The template is:\n{jinja2_template_source}") from exc
+ msg = f"Could not parse Jinja2 template because of: {exc}. The template is:\n{jinja2_template_source}"
+ raise Jinja2ParsingError(msg) from exc
diff --git a/pipelex/tools/templating/jinja2_rendering.py b/pipelex/tools/templating/jinja2_rendering.py
index 2b1f42e54..fd2cc23cf 100644
--- a/pipelex/tools/templating/jinja2_rendering.py
+++ b/pipelex/tools/templating/jinja2_rendering.py
@@ -1,4 +1,4 @@
-from typing import Any, Dict, Optional
+from typing import Any
from jinja2 import Template, meta
from jinja2.exceptions import (
@@ -27,19 +27,20 @@ from pipelex.tools.templating.templating_models import PromptingStyle
########################################################################################
-def _add_to_templating_context(temlating_context: Dict[str, Any], jinja2_context_key: Jinja2ContextKey, value: Any) -> None:
+def _add_to_templating_context(temlating_context: dict[str, Any], jinja2_context_key: Jinja2ContextKey, value: Any) -> None:
if jinja2_context_key in temlating_context:
- raise Jinja2StuffError(f"Jinja2 context key '{jinja2_context_key}' already in temlating_context")
+ msg = f"Jinja2 context key '{jinja2_context_key}' already in temlating_context"
+ raise Jinja2StuffError(msg)
temlating_context[jinja2_context_key] = value
async def render_jinja2(
template_category: Jinja2TemplateCategory,
template_provider: TemplateProviderAbstract,
- temlating_context: Dict[str, Any],
- jinja2_name: Optional[str] = None,
- jinja2: Optional[str] = None,
- prompting_style: Optional[PromptingStyle] = None,
+ temlating_context: dict[str, Any],
+ jinja2_name: str | None = None,
+ jinja2: str | None = None,
+ prompting_style: PromptingStyle | None = None,
) -> str:
jinja2_env, loader = make_jinja2_env_from_template_provider(
template_category=template_category,
@@ -56,10 +57,12 @@ async def render_jinja2(
template = jinja2_env.get_template(jinja2_name)
template_source = loader.get_source(jinja2_env, jinja2_name)[0]
else:
- raise Jinja2StuffError("No jinja2 or jinja2_name in Jinja2Assignment")
+ msg = "No jinja2 or jinja2_name in Jinja2Assignment"
+ raise Jinja2StuffError(msg)
except TemplateAssertionError as exc:
explanation = make_jinja2_error_explanation(jinja2_name=jinja2_name, template_text=jinja2)
- raise Jinja2RenderError(f"Jinja2 render error: '{exc}' {explanation}") from exc
+ msg = f"Jinja2 render error: '{exc}' {explanation}"
+ raise Jinja2RenderError(msg) from exc
template_source = preprocess_template(template_source)
check_jinja2_parsing(
@@ -97,14 +100,18 @@ async def render_jinja2(
generated_text: str = await template.render_async(**temlating_context)
except Jinja2StuffError as stuff_error:
explanation = make_jinja2_error_explanation(jinja2_name=jinja2_name, template_text=template_source)
- raise Jinja2RenderError(f"Jinja2 render — stuff error: '{stuff_error}' {explanation}") from stuff_error
+ msg = f"Jinja2 render — stuff error: '{stuff_error}' {explanation}"
+ raise Jinja2RenderError(msg) from stuff_error
except TemplateSyntaxError as syntax_error:
explanation = make_jinja2_error_explanation(jinja2_name=jinja2_name, template_text=template_source)
- raise Jinja2RenderError(f"Jinja2 render — syntax error: '{syntax_error}' {explanation}") from syntax_error
+ msg = f"Jinja2 render — syntax error: '{syntax_error}' {explanation}"
+ raise Jinja2RenderError(msg) from syntax_error
except UndefinedError as undef_error:
explanation = make_jinja2_error_explanation(jinja2_name=jinja2_name, template_text=template_source)
- raise Jinja2RenderError(f"Jinja2 render — undefined error: '{undef_error}' {explanation}") from undef_error
+ msg = f"Jinja2 render — undefined error: '{undef_error}' {explanation}"
+ raise Jinja2RenderError(msg) from undef_error
except Jinja2ContextError as context_error:
explanation = make_jinja2_error_explanation(jinja2_name=jinja2_name, template_text=template_source)
- raise Jinja2RenderError(f"Jinja2 render — context error: '{context_error}' {explanation}") from context_error
+ msg = f"Jinja2 render — context error: '{context_error}' {explanation}"
+ raise Jinja2RenderError(msg) from context_error
return generated_text
diff --git a/pipelex/tools/templating/jinja2_required_variables.py b/pipelex/tools/templating/jinja2_required_variables.py
index 346d57bd9..e901a79d6 100644
--- a/pipelex/tools/templating/jinja2_required_variables.py
+++ b/pipelex/tools/templating/jinja2_required_variables.py
@@ -1,5 +1,3 @@
-from typing import Optional, Set
-
from jinja2 import meta
from jinja2.exceptions import (
TemplateSyntaxError,
@@ -15,15 +13,14 @@ from pipelex.tools.templating.template_provider_abstract import TemplateProvider
def detect_jinja2_required_variables(
template_category: Jinja2TemplateCategory,
template_provider: TemplateProviderAbstract,
- jinja2_name: Optional[str] = None,
- jinja2: Optional[str] = None,
-) -> Set[str]:
- """
- Returns a list of variables required by the Jinja2 template.
+ jinja2_name: str | None = None,
+ jinja2: str | None = None,
+) -> set[str]:
+ """Returns a list of variables required by the Jinja2 template.
Args:
template_category: Category of the template (HTML, MARKDOWN, etc.), used to set the appropriate jinja2 environment settings
- template_library: Library containing templates
+ template_provider: Library containing templates
jinja2_name: Name of template in library (optional)
jinja2: Direct Jinja2 template string (optional)
@@ -32,6 +29,7 @@ def detect_jinja2_required_variables(
Raises:
Jinja2StuffError: If neither jinja2 nor jinja2_name is provided
+
"""
jinja2_env, loader = make_jinja2_env_from_template_provider(
template_category=template_category,
@@ -44,19 +42,23 @@ def detect_jinja2_required_variables(
elif jinja2_name:
template_source = loader.get_source(jinja2_env, jinja2_name)[0]
else:
- raise Jinja2StuffError("No jinja2 or jinja2_name provided")
+ msg = "No jinja2 or jinja2_name provided"
+ raise Jinja2StuffError(msg)
try:
parsed_ast = jinja2_env.parse(template_source)
undeclared_variables = meta.find_undeclared_variables(parsed_ast)
except Jinja2StuffError as stuff_error:
explanation = make_jinja2_error_explanation(jinja2_name=jinja2_name, template_text=template_source)
- raise Jinja2DetectVariablesError(f"Jinja2 detect variables — stuff error: '{stuff_error}' {explanation}") from stuff_error
+ msg = f"Jinja2 detect variables — stuff error: '{stuff_error}' {explanation}"
+ raise Jinja2DetectVariablesError(msg) from stuff_error
except TemplateSyntaxError as syntax_error:
explanation = make_jinja2_error_explanation(jinja2_name=jinja2_name, template_text=template_source)
- raise Jinja2DetectVariablesError(f"Jinja2 detect variables — syntax error: '{syntax_error}' {explanation}") from syntax_error
+ msg = f"Jinja2 detect variables — syntax error: '{syntax_error}' {explanation}"
+ raise Jinja2DetectVariablesError(msg) from syntax_error
except UndefinedError as undef_error:
explanation = make_jinja2_error_explanation(jinja2_name=jinja2_name, template_text=template_source)
- raise Jinja2DetectVariablesError(f"Jinja2 detect variables — undefined error: '{undef_error}' {explanation}") from undef_error
+ msg = f"Jinja2 detect variables — undefined error: '{undef_error}' {explanation}"
+ raise Jinja2DetectVariablesError(msg) from undef_error
return undeclared_variables
diff --git a/pipelex/tools/templating/jinja2_template_category.py b/pipelex/tools/templating/jinja2_template_category.py
index dd609d6fd..cae802315 100644
--- a/pipelex/tools/templating/jinja2_template_category.py
+++ b/pipelex/tools/templating/jinja2_template_category.py
@@ -1,4 +1,5 @@
-from typing import Any, Callable, Dict, Optional
+from collections.abc import Callable
+from typing import Any
from jinja2.runtime import Context
@@ -15,7 +16,7 @@ class Jinja2TemplateCategory(StrEnum):
LLM_PROMPT = "llm_prompt"
@property
- def filters(self) -> Dict[Jinja2FilterName, Callable[[Context, Any, Optional[TextFormat]], Any]]:
+ def filters(self) -> dict[Jinja2FilterName, Callable[[Context, Any, TextFormat | None], Any]]:
match self:
case Jinja2TemplateCategory.MERMAID:
return {}
diff --git a/pipelex/tools/templating/jinja2_template_loader.py b/pipelex/tools/templating/jinja2_template_loader.py
index fcdbf7361..417d36254 100644
--- a/pipelex/tools/templating/jinja2_template_loader.py
+++ b/pipelex/tools/templating/jinja2_template_loader.py
@@ -1,4 +1,4 @@
-from typing import Callable, Optional, Tuple
+from collections.abc import Callable
from jinja2 import BaseLoader, Environment
from typing_extensions import override
@@ -13,7 +13,7 @@ class Jinja2TemplateLoader(BaseLoader):
self.template_provider = template_provider
@override
- def get_source(self, environment: Environment, template: str) -> Tuple[str, Optional[str], Optional[Callable[[], bool]]]:
+ def get_source(self, environment: Environment, template: str) -> tuple[str, str | None, Callable[[], bool] | None]:
the_template = self.template_provider.get_template(template_name=template)
log.debug(f"TemplateLoader.get_source: template='{template}'")
return the_template, None, None
diff --git a/pipelex/tools/templating/template_library.py b/pipelex/tools/templating/template_library.py
index 70e7a8624..82132d484 100644
--- a/pipelex/tools/templating/template_library.py
+++ b/pipelex/tools/templating/template_library.py
@@ -1,4 +1,4 @@
-from typing import Any, ClassVar, Dict
+from typing import Any, ClassVar, cast
from jinja2 import TemplateSyntaxError
from pydantic import Field, RootModel, ValidationError
@@ -14,7 +14,7 @@ from pipelex.tools.templating.template_preprocessor import preprocess_template
from pipelex.tools.templating.template_provider_abstract import TemplateNotFoundError, TemplateProviderAbstract
from pipelex.tools.typing.pydantic_utils import format_pydantic_validation_error
-TemplateLibraryRoot = Dict[str, str]
+TemplateLibraryRoot = dict[str, str]
class TemplateLibraryError(ToolException):
@@ -46,7 +46,8 @@ class TemplateLibrary(TemplateProviderAbstract, RootModel[TemplateLibraryRoot]):
try:
return self.root[template_name]
except KeyError as exc:
- raise TemplateNotFoundError(f"Template '{template_name}' not found in template library") from exc
+ msg = f"Template '{template_name}' not found in template library"
+ raise TemplateNotFoundError(msg) from exc
def _set_template(self, template: str, name: str):
preprocessed_template = preprocess_template(template)
@@ -54,7 +55,8 @@ class TemplateLibrary(TemplateProviderAbstract, RootModel[TemplateLibraryRoot]):
def _add_new_template(self, template: str, name: str):
if name in self.root:
- raise TemplateLibraryError(f"Template '{name}' already exists in the library")
+ msg = f"Template '{name}' already exists in the library"
+ raise TemplateLibraryError(msg)
self._set_template(template=template, name=name)
def _load_from_toml(self, toml_path: str):
@@ -65,7 +67,7 @@ class TemplateLibrary(TemplateProviderAbstract, RootModel[TemplateLibraryRoot]):
toml_name = toml_path.split("/")[-1]
log.debug(f"Loaded {len(self.root) - nb_concepts_before} templates from '{toml_name}'")
- def _load_from_recursive_dict(self, domain: str, recursive_dict: Dict[str, Any]):
+ def _load_from_recursive_dict(self, domain: str, recursive_dict: dict[str, Any]):
for name, obj in recursive_dict.items():
try:
if isinstance(obj, str):
@@ -74,14 +76,16 @@ class TemplateLibrary(TemplateProviderAbstract, RootModel[TemplateLibraryRoot]):
self._add_new_template(template=template, name=name)
elif isinstance(obj, dict):
# this is not a templae but a subdomain
- sub_recursive_dict: Dict[str, str] = obj
+ sub_recursive_dict = cast("dict[str, str]", obj)
domain = f"{domain}/{name}"
self._load_from_recursive_dict(domain=domain, recursive_dict=sub_recursive_dict)
else:
- raise TemplateLibraryError(f"Unexpected type for key '{name}' in recursive_dict: {type(obj)}")
+ msg = f"Unexpected type for key '{name}' in recursive_dict: {type(obj)}"
+ raise TemplateLibraryError(msg)
except ValidationError as exc:
error_msg = format_pydantic_validation_error(exc)
- raise TemplateLibraryError(f"Error loading concept '{name}' of domain '{domain}' because of: {error_msg}") from exc
+ msg = f"Error loading concept '{name}' of domain '{domain}' because of: {error_msg}"
+ raise TemplateLibraryError(msg) from exc
def validate_templates(self, template_category: Jinja2TemplateCategory):
for template_name, template in self.root.items():
@@ -96,4 +100,5 @@ class TemplateLibrary(TemplateProviderAbstract, RootModel[TemplateLibraryRoot]):
error_msg += f"\nThe template is:\n{template}"
else:
error_msg += "The template is empty."
- raise TemplateLibraryError(error_msg) from exc
+ msg = error_msg
+ raise TemplateLibraryError(msg) from exc
diff --git a/pipelex/tools/templating/template_preprocessor.py b/pipelex/tools/templating/template_preprocessor.py
index 115e97859..a7b82174b 100644
--- a/pipelex/tools/templating/template_preprocessor.py
+++ b/pipelex/tools/templating/template_preprocessor.py
@@ -1,6 +1,5 @@
import re
from re import Match
-from typing import List
from pipelex.tools.templating.jinja2_errors import Jinja2TemplateError
from pipelex.tools.templating.jinja2_filters import ALLOWED_FILTERS
@@ -14,6 +13,7 @@ def _detect_non_existent_filters(template_str: str) -> None:
Raises:
Jinja2TemplateError: If any non-allowed filters are found
+
"""
# Pattern to match Jinja2 filter syntax: {{ variable|filter() }} or {{ variable|filter(param) }}
# This handles:
@@ -24,14 +24,15 @@ def _detect_non_existent_filters(template_str: str) -> None:
filter_pattern = r"\{\{\s*[^|}]+\|\s*([a-zA-Z0-9_]+)(?:\([^)]*\))?\s*\}\}"
matches = re.finditer(filter_pattern, template_str)
- invalid_filters: List[str] = []
+ invalid_filters: list[str] = []
for match in matches:
filter_name = match.group(1)
if filter_name not in ALLOWED_FILTERS:
invalid_filters.append(filter_name)
if invalid_filters:
- raise Jinja2TemplateError(f"Invalid Jinja2 filters found: {invalid_filters}. Only the following filters are allowed: {ALLOWED_FILTERS}")
+ msg = f"Invalid Jinja2 filters found: {invalid_filters}. Only the following filters are allowed: {ALLOWED_FILTERS}"
+ raise Jinja2TemplateError(msg)
# Handle @variable patterns
@@ -57,8 +58,7 @@ def replace_dollar_variable(match: Match[str]) -> str:
def preprocess_template(template: str) -> str:
- """
- Preprocess a template string to interpret our syntax patterns and convert them to Jinja2 syntax.
+ """Preprocess a template string to interpret our syntax patterns and convert them to Jinja2 syntax.
Also, detect the use of non-existent filters.
"""
_detect_non_existent_filters(template_str=template)
diff --git a/pipelex/tools/templating/templating_models.py b/pipelex/tools/templating/templating_models.py
index cd95b0439..098128a61 100644
--- a/pipelex/tools/templating/templating_models.py
+++ b/pipelex/tools/templating/templating_models.py
@@ -1,7 +1,7 @@
from pydantic import BaseModel, Field
-from typing_extensions import Self, override
+from typing_extensions import override
-from pipelex.types import StrEnum
+from pipelex.types import Self, StrEnum
class TextFormat(StrEnum):
diff --git a/pipelex/tools/typing/__init__.py b/pipelex/tools/typing/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/pipelex/tools/typing/module_inspector.py b/pipelex/tools/typing/module_inspector.py
index bbe6dac70..e86a143bc 100644
--- a/pipelex/tools/typing/module_inspector.py
+++ b/pipelex/tools/typing/module_inspector.py
@@ -2,14 +2,12 @@ import importlib.util
import inspect
import os
import sys
-from typing import Any, List, Optional, Type
+from typing import Any
class ModuleFileError(Exception):
"""Exception raised for errors related to module file operations."""
- pass
-
def import_module_from_file(file_path: str) -> Any:
"""Imports a module from a file path.
@@ -22,10 +20,12 @@ def import_module_from_file(file_path: str) -> Any:
Raises:
ModuleFileError: If the file is not a Python file or cannot be loaded
+
"""
# Validate that the file is a Python file
if not file_path.endswith(".py"):
- raise ModuleFileError(f"File {file_path} is not a Python file (must end with .py)")
+ msg = f"File {file_path} is not a Python file (must end with .py)"
+ raise ModuleFileError(msg)
# Convert file path to module-style path to use as the actual module name
module_name = _convert_file_path_to_module_path(file_path)
@@ -37,7 +37,8 @@ def import_module_from_file(file_path: str) -> Any:
# Use importlib.util to load the module from file path
spec = importlib.util.spec_from_file_location(module_name, file_path)
if spec is None or spec.loader is None:
- raise ModuleFileError(f"Could not load module from {file_path}")
+ msg = f"Could not load module from {file_path}"
+ raise ModuleFileError(msg)
module = importlib.util.module_from_spec(spec)
@@ -53,25 +54,21 @@ def import_module_from_file(file_path: str) -> Any:
def _convert_file_path_to_module_path(file_path: str) -> str:
"""Convert a file path to a module-style path."""
# Remove .py extension
- module_path = file_path[:-3] if file_path.endswith(".py") else file_path
+ module_path = file_path.removesuffix(".py")
# Replace path separators with dots
module_path = module_path.replace(os.sep, ".")
# Handle __init__.py files by removing the __init__ part
- if module_path.endswith(".__init__"):
- module_path = module_path[:-9]
-
- return module_path
+ return module_path.removesuffix(".__init__")
def find_classes_in_module(
module: Any,
- base_class: Optional[Type[Any]],
+ base_class: type[Any] | None,
include_imported: bool,
-) -> List[Type[Any]]:
- """
- Finds all classes in a module that match the criteria.
+) -> list[type[Any]]:
+ """Finds all classes in a module that match the criteria.
Args:
module: The module to search for classes
@@ -80,8 +77,9 @@ def find_classes_in_module(
Returns:
List of class types that match the criteria
+
"""
- classes: List[Type[Any]] = []
+ classes: list[type[Any]] = []
module_name = module.__name__
# Find all classes in the module
diff --git a/pipelex/tools/typing/pydantic_utils.py b/pipelex/tools/typing/pydantic_utils.py
index d4115b02d..8190dc636 100644
--- a/pipelex/tools/typing/pydantic_utils.py
+++ b/pipelex/tools/typing/pydantic_utils.py
@@ -1,4 +1,5 @@
-from typing import Any, Dict, List, Optional, Sequence, Set, TypeVar, Union
+from collections.abc import Callable, Sequence
+from typing import Any, TypeVar, cast
from pydantic import BaseModel, ValidationError
from rich.repr import Result as RichReprResult
@@ -9,6 +10,15 @@ from pipelex.types import StrEnum
BaseModelTypeVar = TypeVar("BaseModelTypeVar", bound=BaseModel)
+T = TypeVar("T")
+
+
+def empty_list_factory_of(_: type[T]) -> Callable[[], list[T]]:
+ def _factory() -> list[T]:
+ return []
+
+ return _factory
+
def format_pydantic_validation_error(exc: ValidationError) -> str:
"""Format a Pydantic ValidationError into a readable string with detailed error information.
@@ -18,6 +28,7 @@ def format_pydantic_validation_error(exc: ValidationError) -> str:
Returns:
A formatted string containing categorized validation errors
+
"""
error_msg = "Validation error(s):"
@@ -29,7 +40,7 @@ def format_pydantic_validation_error(exc: ValidationError) -> str:
enum_errors = [
f"{'.'.join(map(str, err['loc']))}: invalid enum value '{err.get('input', 'unknown')}'" for err in exc.errors() if err["type"] == "enum"
]
- model_type_errors: List[str] = []
+ model_type_errors: list[str] = []
for err in exc.errors():
if err["type"] == "model_type":
field_path = ".".join(map(str, err["loc"]))
@@ -63,18 +74,19 @@ def format_pydantic_validation_error(exc: ValidationError) -> str:
def convert_strenum_to_str(
- obj: Dict[str, Any] | List[Any] | StrEnum | Any,
-) -> Dict[str, Any] | List[Any] | str | Any:
+ obj: dict[str, Any] | list[Any] | StrEnum | Any,
+) -> dict[str, Any] | list[Any] | str | Any:
if isinstance(obj, dict):
- obj_dict: Dict[str, Any] = obj
+ obj_dict = cast("dict[str, Any]", obj)
return {str(key): convert_strenum_to_str(value) for key, value in obj_dict.items()}
elif isinstance(obj, list):
- obj_list: List[Any] = obj
+ obj_list = cast("list[Any]", obj)
return [convert_strenum_to_str(item) for item in obj_list]
elif isinstance(obj, StrEnum):
if hasattr(obj, "display_name"):
- return obj.display_name() # type: ignore
- return str(obj)
+ return obj.display_name() # pyright: ignore[reportUnknownVariableType, reportUnknownMemberType, reportAttributeAccessIssue]
+ else:
+ return str(obj)
else:
return obj
@@ -89,25 +101,24 @@ class FieldVisibility(StrEnum):
ONLY_HIDDEN_FIELDS = "only_hidden_fields"
-def clean_model_to_dict(obj: BaseModel) -> Dict[str, Any]:
+def clean_model_to_dict(obj: BaseModel) -> dict[str, Any]:
dict_dump = serialize_model(
obj=obj,
field_visibility=FieldVisibility.NO_HIDDEN_FIELDS,
is_stringify_enums=True,
)
if not isinstance(dict_dump, dict):
- raise TypeError(f"Expected dict, got {type(dict_dump)}")
- result_dict: Dict[str, Any] = dict_dump
- return result_dict
+ msg = f"Expected dict, got {type(dict_dump)}"
+ raise TypeError(msg)
+ return cast("dict[str, Any]", dict_dump)
def serialize_model(
obj: Any,
field_visibility: FieldVisibility = FieldVisibility.NO_HIDDEN_FIELDS,
is_stringify_enums: bool = True,
-) -> Union[Dict[str, Any], List[Any], Any]:
- """
- Recursively serialize a Pydantic BaseModel (and its nested BaseModels)
+) -> dict[str, Any] | list[Any] | Any:
+ """Recursively serialize a Pydantic BaseModel (and its nested BaseModels)
into a dictionary, omitting any fields marked with
'json_schema_extra={ExtraFieldAttribute.IS_HIDDEN: True}'.
@@ -120,11 +131,16 @@ def serialize_model(
return obj
# Identify which fields should be excluded
- fields_to_exclude: Set[str] = set()
+ fields_to_exclude: set[str] = set()
for field_name, field_info in obj.__class__.model_fields.items():
json_schema_extra = field_info.json_schema_extra
- is_hidden = json_schema_extra and isinstance(json_schema_extra, dict) and json_schema_extra.get(ExtraFieldAttribute.IS_HIDDEN) is True
+ is_hidden: bool
+ if json_schema_extra and isinstance(json_schema_extra, dict):
+ typed_json_schema_extra = cast("dict[str, Any]", json_schema_extra)
+ is_hidden = typed_json_schema_extra.get(ExtraFieldAttribute.IS_HIDDEN) is True
+ else:
+ is_hidden = False
match field_visibility:
case FieldVisibility.ALL_FIELDS:
pass
@@ -136,8 +152,8 @@ def serialize_model(
fields_to_exclude.add(field_name)
# Build a dict, omitting hidden fields. Recursively handle nested models.
- data: Dict[str, Any] = {}
- for field_name, _ in obj.__class__.model_fields.items():
+ data: dict[str, Any] = {}
+ for field_name in obj.__class__.model_fields:
if field_name in fields_to_exclude:
continue # Skip hidden fields
@@ -153,7 +169,7 @@ def serialize_model(
# If it's a list, we recurse for each item
elif isinstance(value, list):
- value_list: List[Any] = value
+ value_list = cast("list[Any]", value)
data[field_name] = [
serialize_model(
obj=item,
@@ -165,7 +181,7 @@ def serialize_model(
# If it's a dict, we can similarly recurse for any nested BaseModels inside the dict
elif isinstance(value, dict):
- value_dict: Dict[str, Any] = value
+ value_dict = cast("dict[str, Any]", value)
data[field_name] = {
key: serialize_model(
obj=value,
@@ -177,7 +193,7 @@ def serialize_model(
elif is_stringify_enums and isinstance(value, StrEnum):
if hasattr(value, "display_name"):
- data[field_name] = value.display_name() # type: ignore
+ data[field_name] = value.display_name() # pyright: ignore[reportUnknownMemberType, reportAttributeAccessIssue]
else:
data[field_name] = str(value)
@@ -190,10 +206,10 @@ def serialize_model(
class CustomBaseModel(BaseModel):
@override
- def __rich_repr__(self) -> RichReprResult: # type: ignore
- for item in super().__rich_repr__(): # type: ignore
+ def __rich_repr__(self) -> RichReprResult:
+ for item in super().__rich_repr__(): # type: ignore[misc]
if isinstance(item, tuple):
- tuple_item: tuple[Any, ...] = item
+ tuple_item = cast("tuple[Any, ...]", item)
if len(tuple_item) >= 2:
name = tuple_item[0]
value = tuple_item[1]
@@ -209,8 +225,8 @@ class CustomBaseModel(BaseModel):
yield item
@override
- def __repr_args__(self) -> Sequence[tuple[Optional[str], Any]]:
- processed_args: list[tuple[Optional[str], Any]] = []
+ def __repr_args__(self) -> Sequence[tuple[str | None, Any]]:
+ processed_args: list[tuple[str | None, Any]] = []
for name, value in super().__repr_args__():
if name and AttributePolisher.should_truncate(name=name, value=value):
truncated_value = AttributePolisher.get_truncated_value(name, value)
@@ -220,8 +236,8 @@ class CustomBaseModel(BaseModel):
return processed_args
def model_dump_truncated(self, **kwargs: Any) -> Any:
- """
- Dump the model to a dictionary with serialize_as_any=True and apply
+ """Dump the model to a dictionary with serialize_as_any=True and apply
+
AttributePolisher truncation to fields that should be truncated.
Handles nested attributes recursively.
@@ -230,6 +246,7 @@ class CustomBaseModel(BaseModel):
Returns:
Dictionary with truncated values where appropriate
+
"""
# Get the model dump with serialize_as_any=True
dumped_data = self.model_dump(**kwargs)
@@ -237,9 +254,8 @@ class CustomBaseModel(BaseModel):
# Apply truncation logic recursively
return self._apply_truncation_recursive(dumped_data)
- def _apply_truncation_recursive(self, obj: Any, name: Optional[str] = None) -> Any:
- """
- Recursively apply AttributePolisher truncation logic to a data structure.
+ def _apply_truncation_recursive(self, obj: Any, name: str | None = None) -> Any:
+ """Recursively apply AttributePolisher truncation logic to a data structure.
Args:
obj: The object to process
@@ -247,6 +263,7 @@ class CustomBaseModel(BaseModel):
Returns:
The processed object with truncation applied where appropriate
+
"""
# First check if this specific object should be truncated
if name and AttributePolisher.should_truncate(name=name, value=obj):
@@ -254,21 +271,20 @@ class CustomBaseModel(BaseModel):
# If it's a dictionary, recurse into its values
if isinstance(obj, dict):
- obj_dict: Dict[str, Any] = obj
- truncated_dict: Dict[str, Any] = {}
+ obj_dict = cast("dict[str, Any]", obj)
+ truncated_dict: dict[str, Any] = {}
for key, value in obj_dict.items():
truncated_dict[key] = self._apply_truncation_recursive(value, name=key)
return truncated_dict
# If it's a list, recurse into its items
- elif isinstance(obj, list):
- obj_list: List[Any] = obj
+ if isinstance(obj, list):
+ obj_list = cast("list[Any]", obj)
return [self._apply_truncation_recursive(item, name=name) for item in obj_list]
# If it's a tuple, recurse into its items and return as tuple
- elif isinstance(obj, tuple):
- return tuple(self._apply_truncation_recursive(item, name=name) for item in obj) # type: ignore
+ if isinstance(obj, tuple):
+ return tuple(self._apply_truncation_recursive(item, name=name) for item in obj) # pyright: ignore[reportUnknownVariableType]
# For all other types, return as-is
- else:
- return obj
+ return obj
diff --git a/pipelex/tools/typing/structure_printer.py b/pipelex/tools/typing/structure_printer.py
index 982064336..7cc8c4a0e 100644
--- a/pipelex/tools/typing/structure_printer.py
+++ b/pipelex/tools/typing/structure_printer.py
@@ -1,5 +1,6 @@
+import types
from enum import Enum
-from typing import Any, Dict, List, Optional, Set, Type, Union, get_args, get_origin, get_type_hints
+from typing import Any, Union, get_args, get_origin, get_type_hints
from pydantic import BaseModel
@@ -9,24 +10,27 @@ from pipelex.types import StrEnum
class StructurePrinter:
def pretty_type(self, tp: object) -> str:
"""Pretty print a type, with special handling for containers, literals and enums."""
- origin = getattr(tp, "__origin__", None)
args = getattr(tp, "__args__", None)
+ origin = get_origin(tp)
if origin is None:
if isinstance(tp, type):
return tp.__name__
return str(tp)
- if origin is Union and args:
+ args = get_args(tp)
+
+ # was: if origin is Union and args:
+ if origin in (Union, types.UnionType) and args:
non_none = [a for a in args if a is not type(None)]
- if len(non_none) == 1 and len(args) == 2:
- return f"Optional[{self.pretty_type(non_none[0])}]"
- return f"Union[{', '.join(self.pretty_type(a) for a in args)}]"
+ if len(args) == 2 and len(non_none) == 1:
+ return f"{self.pretty_type(non_none[0])} | None"
+ return " | ".join(self.pretty_type(a) for a in args)
if str(origin).endswith("Literal") and args: # Handle both typing.Literal and typing_extensions.Literal
# For enum values, just get their values
- values: List[str] = []
+ values: list[str] = []
for arg in args:
- if isinstance(arg, Enum) or isinstance(arg, StrEnum):
+ if isinstance(arg, (Enum, StrEnum)):
values.append(f'"{arg.value}"')
else:
values.append(repr(arg))
@@ -35,22 +39,21 @@ class StructurePrinter:
return "Literal[\n " + ",\n ".join(values) + ",\n ]"
return f"Literal[{', '.join(values)}]"
- if (origin is list or origin is List) and args:
+ if (origin is list or origin is list) and args:
return f"List[{self.pretty_type(args[0])}]"
- if (origin is dict or origin is Dict) and args:
+ if (origin is dict or origin is dict) and args:
return f"Dict[{self.pretty_type(args[0])}, {self.pretty_type(args[1])}]"
return str(tp)
def get_type_structure(
self,
- tp: Type[Any],
- seen_types: Optional[Set[str]] = None,
- collected_types: Optional[Dict[str, Type[Any]]] = None,
- collected_enums: Optional[Dict[str, Type[Enum]]] = None,
- base_class: Type[Any] = BaseModel,
- ) -> List[str]:
- """
- Get the structure of a type, listing referenced subclasses of base_class and enums.
+ tp: type[Any],
+ seen_types: set[str] | None = None,
+ collected_types: dict[str, type[Any]] | None = None,
+ collected_enums: dict[str, type[Enum]] | None = None,
+ base_class: type[Any] = BaseModel,
+ ) -> list[str]:
+ """Get the structure of a type, listing referenced subclasses of base_class and enums.
Args:
tp: The type to analyze
@@ -58,6 +61,7 @@ class StructurePrinter:
collected_types: Dictionary of collected types to analyze
collected_enums: Dictionary of collected enums
base_class: The base class to check for inheritance (defaults to BaseModel)
+
"""
if seen_types is None:
seen_types = set()
@@ -75,18 +79,19 @@ class StructurePrinter:
return str(tp)
args = get_args(tp)
- if origin is Union:
+ # was: if origin is Union:
+ if origin in (Union, types.UnionType):
non_none = [a for a in args if a is not type(None)]
- if len(non_none) == 1:
- return f"Optional[{format_type(non_none[0])}]"
- return f"Union[{', '.join(format_type(a) for a in non_none)}]"
+ if len(args) == 2 and len(non_none) == 1:
+ return f"{format_type(non_none[0])} | None"
+ return " | ".join(format_type(a) for a in args)
if str(origin).endswith("Literal") and args: # Handle both typing.Literal and typing_extensions.Literal
# For enum values, just get their values
- values: List[str] = []
+ values: list[str] = []
enum_type = None
for arg in args:
- if isinstance(arg, Enum) or isinstance(arg, StrEnum):
+ if isinstance(arg, (Enum, StrEnum)):
values.append(arg.value)
if enum_type is None:
enum_type = type(arg)
@@ -97,41 +102,38 @@ class StructurePrinter:
collected_enums[enum_type.__name__] = enum_type
# Return multi-line format for Literal fields
if len(values) > 1:
- lines: List[str] = []
- for value in values:
- lines.append(f'"{value}"')
+ lines = [f'"{value}"' for value in values]
return "Literal[\n " + ",\n ".join(lines) + ",\n ]"
return f"Literal[{', '.join(values)}]"
- if origin in (list, List):
+ if origin in (list, list):
return f"List[{format_type(args[0])}]"
- if origin in (dict, Dict):
+ if origin in (dict, dict):
return f"Dict[{format_type(args[0])}, {format_type(args[1])}]"
return str(tp)
- def collect_types(tp: Type[Any]) -> None:
+ def collect_types(tp: type[Any]) -> None:
"""Recursively collect types and enums"""
+ args = get_args(tp)
+
origin = get_origin(tp)
args = get_args(tp)
if origin:
- if origin is Union:
- non_none = [a for a in args if a is not type(None)]
- for arg in non_none:
- if isinstance(arg, type):
- collect_types(arg)
- elif hasattr(arg, "__origin__"): # Handle nested generics
- collect_types(arg)
- elif origin in (list, List):
- if isinstance(args[0], type):
- collect_types(args[0])
- elif hasattr(args[0], "__origin__"): # Handle nested generics
- collect_types(args[0])
- elif origin in (dict, Dict):
+ # was: if origin is Union:
+ if origin in (Union, types.UnionType):
for arg in args:
- if isinstance(arg, type):
+ if arg is type(None):
+ continue
+ if isinstance(arg, type) or get_origin(arg) is not None:
collect_types(arg)
- elif hasattr(arg, "__origin__"): # Handle nested generics
+ elif origin is list:
+ inner = args[0]
+ if isinstance(inner, type) or get_origin(inner) is not None:
+ collect_types(inner)
+ elif origin is dict:
+ for arg in args:
+ if isinstance(arg, type) or get_origin(arg) is not None:
collect_types(arg)
return
@@ -150,8 +152,7 @@ class StructurePrinter:
if (
issubclass(base, BaseModel)
and base is not BaseModel
- and base.__module__ != "pydantic.main"
- and base.__module__ != "abc"
+ and base.__module__ not in ("pydantic.main", "abc")
and not base.__module__.startswith("pipelex.core")
):
collect_types(base)
@@ -161,11 +162,11 @@ class StructurePrinter:
model_fields = getattr(tp, "model_fields", {})
if model_fields:
- for fname, _ in model_fields.items():
+ for fname in model_fields:
ftype = type_hints[fname]
collect_types(ftype)
elif hasattr(tp, "__annotations__"):
- for fname, ftype in type_hints.items():
+ for ftype in type_hints.values():
collect_types(ftype)
except (TypeError, AttributeError):
# Handle cases where type hints cannot be retrieved
@@ -175,7 +176,7 @@ class StructurePrinter:
collect_types(tp)
# Generate output
- output: List[str] = []
+ output: list[str] = []
# First output the main class and its dependencies
for class_name, class_type in collected_types.items():
@@ -236,7 +237,7 @@ class StructurePrinter:
fields = [(type_k, type_hints[type_k]) for type_k in sorted(type_hints.keys())]
# Check if all fields are inherited
- parent_fields: Set[str] = set()
+ parent_fields: set[str] = set()
for base in class_type.__bases__:
try:
parent_fields.update(get_type_hints(base).keys())
@@ -257,10 +258,11 @@ class StructurePrinter:
else:
field_type = type_hints[fname]
ftype_str = format_type(field_type)
- # Check if field is Optional
+
field_origin = get_origin(field_type)
field_args = get_args(field_type)
- is_optional = field_origin is Union and type(None) in field_args
+
+ is_optional = field_origin in (Union, types.UnionType) and type(None) in field_args
# Handle default values
field_default = None
@@ -281,9 +283,9 @@ class StructurePrinter:
field_line += " = None"
elif field_default is not None:
if isinstance(field_default, bool):
- field_line += f" = {str(field_default)}"
+ field_line += f" = {field_default!s}"
elif not isinstance(field_default, (BaseModel, list, dict)):
- field_line += f" = {repr(field_default)}"
+ field_line += f" = {field_default!r}"
# Add description as a comment if available
# First check if there's a direct field description from model_fields
@@ -291,8 +293,8 @@ class StructurePrinter:
field_line += f" # {field_description}"
# Then check if the field type itself has model_fields and a description
# This handles nested content types that have field descriptions
- elif hasattr(ftype, "model_fields") and fname in ftype.model_fields and hasattr(ftype.model_fields[fname], "description"): # type: ignore
- field_line += f" # {ftype.model_fields[fname].description}" # type: ignore
+ elif hasattr(ftype, "model_fields") and fname in ftype.model_fields and hasattr(ftype.model_fields[fname], "description"): # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType]
+ field_line += f" # {ftype.model_fields[fname].description}" # pyright: ignore[reportUnknownMemberType]
# Split multi-line field lines
if "\n" in field_line:
@@ -319,7 +321,6 @@ class StructurePrinter:
if enum_type.__doc__ and enum_type.__doc__.strip() != "An enumeration.":
doc = enum_type.__doc__.strip()
output.append(f' """{doc}"""')
- for member in enum_type:
- output.append(f' {member.name} = "{member.value}"')
+ output += [f' {member.name} = "{member.value}"' for member in enum_type]
return output
diff --git a/pipelex/tools/typing/validation_utils.py b/pipelex/tools/typing/validation_utils.py
index e99aaf2fa..447df6d62 100644
--- a/pipelex/tools/typing/validation_utils.py
+++ b/pipelex/tools/typing/validation_utils.py
@@ -1,9 +1,8 @@
-from typing import Any, List, Optional
+from typing import Any
-def has_exactly_one_among_attributes_from_list(obj: Any, attributes_list: List[str]) -> bool:
- """
- Checks if exactly one attribute from a list of attributes is non-None in an object.
+def has_exactly_one_among_attributes_from_list(obj: Any, attributes_list: list[str]) -> bool:
+ """Checks if exactly one attribute from a list of attributes is non-None in an object.
This function is useful for validating mutually exclusive attributes in data models
or configurations, ensuring that exactly one of the specified attributes has a value.
@@ -15,14 +14,14 @@ def has_exactly_one_among_attributes_from_list(obj: Any, attributes_list: List[s
Returns:
bool: True if exactly one attribute from the list is non-None,
False otherwise.
+
"""
provided_attributes = [attribute for attribute in attributes_list if getattr(obj, attribute) is not None]
return len(provided_attributes) == 1
-def has_more_than_one_among_attributes_from_list(obj: Any, attributes_list: List[str]) -> bool:
- """
- Checks if more than one attribute from a list of attributes is non-None in an object.
+def has_more_than_one_among_attributes_from_list(obj: Any, attributes_list: list[str]) -> bool:
+ """Checks if more than one attribute from a list of attributes is non-None in an object.
This function is useful for detecting conflicts in configurations or data models
where attributes are supposed to be mutually exclusive.
@@ -34,14 +33,14 @@ def has_more_than_one_among_attributes_from_list(obj: Any, attributes_list: List
Returns:
bool: True if more than one attribute from the list is non-None,
False otherwise.
+
"""
provided_attributes = [attribute for attribute in attributes_list if getattr(obj, attribute) is not None]
return len(provided_attributes) > 1
-def has_more_than_one_among_attributes_from_lists(obj: Any, attributes_lists: List[List[str]]) -> Optional[List[str]]:
- """
- Checks if more than one attribute from lists (plural) of attributes is non-None in an object.
+def has_more_than_one_among_attributes_from_lists(obj: Any, attributes_lists: list[list[str]]) -> list[str] | None:
+ """Checks if more than one attribute from lists (plural) of attributes is non-None in an object.
This function is useful for detecting conflicts in configurations or data models
where attributes are supposed to be mutually exclusive.
@@ -51,7 +50,8 @@ def has_more_than_one_among_attributes_from_lists(obj: Any, attributes_lists: Li
attributes_lists (List[List[str]]): List of lists of attribute names to check.
Returns:
- Optional[List[str]]: The list of attributes that are non-None if more than one, None otherwise.
+ List[str] | None: The list of attributes that are non-None if more than one, None otherwise.
+
"""
for attributes_list in attributes_lists:
if has_more_than_one_among_attributes_from_list(obj, attributes_list):
diff --git a/pipelex/types.py b/pipelex/types.py
index c4dacc782..1f1c2116c 100644
--- a/pipelex/types.py
+++ b/pipelex/types.py
@@ -1,8 +1,13 @@
-import sys
+# ruff: noqa: UP
-if sys.version_info >= (3, 11): # real std-lib
- from enum import StrEnum
-else: # back-port
- from backports.strenum import StrEnum
+try:
+ from typing import Self # Python 3.11+
+except ImportError: # Python 3.10
+ from typing_extensions import Self # type: ignore[assignment]
-__all__ = ["StrEnum"]
+try:
+ from enum import StrEnum # Python 3.11+
+except ImportError: # Python 3.10
+ from backports.strenum import StrEnum # type: ignore[assignment, import-not-found, no-redef]
+
+__all__ = ["Self", "StrEnum"]
diff --git a/pyproject.toml b/pyproject.toml
index eebc2b9c8..0cad96831 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -1,6 +1,6 @@
[project]
name = "pipelex"
-version = "0.10.3"
+version = "0.10.2"
description = "Pipelex is an open-source dev tool based on a simple declarative language that lets you define replicable, structured, composable LLM pipelines."
authors = [{ name = "Evotis S.A.S.", email = "[email protected]" }]
maintainers = [{ name = "Pipelex staff", email = "[email protected]" }]
@@ -70,8 +70,9 @@ dev = [
"boto3-stubs>=1.35.24",
"cocode==0.1.2",
"mypy>=1.11.2",
- "pyright==1.1.398",
+ "pyright>=1.1.405",
"pandas-stubs>=2.2.3.241126",
+ "pylint>=3.3.8",
"pytest>=8.3.3",
"pytest-asyncio>=0.24.0",
"pytest-cov>=6.1.1",
@@ -112,15 +113,15 @@ ignore_missing_imports = true
module = ["filetype", "json2html", "pypdfium2", "pypdfium2.raw"]
[tool.pyright]
+pythonVersion = "3.11"
+include = ["pipelex", "tests"]
+exclude = ["**/__pycache__", ".venv", ".git", "build", "dist", "tests"]
analyzeUnannotatedFunctions = true
deprecateTypingAliases = false
disableBytesTypePromotions = true
enableExperimentalFeatures = false
enableTypeIgnoreComments = true
-exclude = ["**/node_modules", "**/__pycache__"]
extraPaths = ["./tests"]
-include = ["pipelex", "tests"]
-pythonVersion = "3.11"
reportAbstractUsage = "error"
reportArgumentType = "error"
reportAssertAlwaysTrue = "error"
@@ -201,6 +202,7 @@ reportUnusedFunction = "error"
reportUnusedImport = "none"
reportUnusedVariable = "error"
reportWildcardImportFromLibrary = "error"
+reportUnusedParameter = "error"
strictDictionaryInference = true
strictListInference = true
strictParameterNoneValue = true
@@ -240,6 +242,7 @@ exclude_lines = [
[tool.ruff]
exclude = [
+ "tests", # TO REMOVE
".cursor",
".git",
".github",
@@ -255,23 +258,104 @@ target-version = "py311"
[tool.ruff.format]
[tool.ruff.lint]
-ignore = ["F401"]
-external = ["F401"]
-select = [
- "E4",
- "E7",
- "E9",
- "F",
- "A001",
- "A002",
- "A003",
- "RUF008",
- "RUF009",
- "RUF012",
- "RUF013",
- "RUF100",
- "E501",
- "I",
+select = ["ALL"]
+ignore = [
+ "ANN201", # Missing return type annotation for public function `my_func`
+ "ANN202", # Missing return type annotation for private function `my_func`
+ "ANN204", # Missing return type annotation for special method `my_func`
+ "ANN206", # Missing return type annotation for classmethod `my_func`
+ "ANN401", # Dynamically typed expressions (typing.Any) are disallowed in `...`
+ "ASYNC230", # Async functions should not open files with blocking methods like `open`
+
+ "C901", # Is to complex
+ "COM812", # Checks for the absence of trailing commas.
+
+ "D100", # Missing docstring in public module
+ "D101", # Missing docstring in public class
+ "D102", # Missing docstring in public method
+ "D103", # Missing docstring in public function
+ "D104", # Missing docstring in public package
+ "D105", # Missing docstring in magic method
+ "D107", # Missing docstring in __init__
+ "D205", # 1 blank line required between summary line and description
+ "D400", # First line should end with a period
+ "D401", # First line of docstring should be in imperative mood: "My docstring...."
+ "D404", # First word of the docstring should not be "This"
+ "D415", # First line should end with a period, question mark, or exclamation point
+
+ "DTZ005", # `datetime.datetime.now()` called without a `tz` argument
+
+ "ERA001", # Found commented-out code
+
+ "FBT001", # Boolean-typed positional argument in function definition
+ "FBT002", # Boolean default positional argument in function definition
+ "FBT003", #Boolean positional value in function call
+
+ "FIX002", # Line contains TODO, consider resolving the issue
+
+ "PLC0105", # `TypeVar` name "SomethingType" does not reflect its covariance; consider renaming it to "SomethingType_co"
+
+ "PLR0911", # Too many return statements (/6)
+ "PLR0912", # Too many branches (/12)
+ "PLR0913", # Too many arguments in function definition (/5)
+ "PLR0915", # Too many statements (/50)
+ "PLR2004", # Magic value used in comparison, consider replacing `2` with a constant variable
+
+ "PT013", # Incorrect import of `pytest`; use `import pytest` instead
+
+ "PTH100", # `os.path.abspath()` should be replaced by `Path.resolve()`
+ "PTH103", # `os.makedirs()` should be replaced by `Path.mkdir(parents=True)`
+ "PTH109", # `os.getcwd()` should be replaced by `Path.cwd()`
+ "PTH118", # `os.path.join()` should be replaced by `Path` with `/` operator
+ "PTH120", # `os.path.dirname()` should be replaced by `Path.parent`
+ "PTH110", # `os.path.exists()` should be replaced by `Path.exists()`
+ "PTH112", # `os.path.isdir()` should be replaced by `Path.is_dir()`
+ "PTH119", # `os.path.basename()` should be replaced by `Path.name`
+ "PTH123", # `open()` should be replaced by `Path.open()`
+ "PTH208", # Use `pathlib.Path.iterdir()` instead.
+
+ "PYI051", # `Literal["auto"]` is redundant in a union with `str`
+
+ "RET505", # superfluous-else-return
+
+ "RUF001", # String contains ambiguous `′` (PRIME). Did you mean ``` (GRAVE ACCENT)?
+ "RUF003", # Comment contains ambiguous `’` (RIGHT SINGLE QUOTATION MARK). Did you mean ``` (GRAVE ACCENT)?
+
+ "SIM105", # Use `contextlib.suppress(ValueError)` instead of `try`-`except`-`pass`
+ "SIM108", # Use ternary operator `description = func.__doc__.strip().split("\n")[0] if func.__doc__ else func.__name__` instead of `if`-`else`-block
+
+ "S101", # Use of `assert` detected
+ "S102", # Use of `exec` detected
+ "S106", # Possible hardcoded password assigned to argument: "secret"
+ "S105", # Possible hardcoded password assigned to: "child_secret"
+
+ "TD002", # Missing author in TODO; try: `# TODO(<author_name>): ...` or `# TODO @<author_name>: ...`
+ "TD003", # Missing issue link for this TODO
+
+ "T201", # `print` found
+
+ # TODO: stop ignoring these rules
+ "BLE001", # Do not catch blind exception: `Exception`
+ "B027", # Checks for empty methods in abstract base classes without an abstract decorator.
+ "UP007", # Use `X | Y` for type annotations
+ "UP036", # Version block is outdated for minimum Python version
+ "SIM102", # Use a single `if` statement instead of nested `if` statements
+ "S701", # Using jinja2 templates with `autoescape=False` is dangerous and can lead to XSS. Ensure `autoescape=True` or use the `select_autoescape` function.
+ "N818", # Exception name `RootException` should be named with an Error suffix
+ "TRY301", # Abstract `raise` to an inner function
+ "PERF401", # Use a list comprehension to create a transformed list
+ "PLW2901", # `for` loop variable `line` overwritten by assignment target
+ "TRY300", # Consider moving this statement to an `else` block
+ "UP035", # `typing.List` is deprecated, use `list` instead
+ "RET503", # Missing explicit `return` at the end of function able to return non-`None` value
+]
+
+[tool.ruff.lint.pydocstyle]
+convention = "google"
+
+[tool.ruff.lint.per-file-ignores]
+"tests/**/*.py" = [
+ "T201", # Allow print statements in test files
]
[tool.uv]
@@ -291,3 +375,14 @@ packages = ["pipelex"]
[tool.hatch.metadata]
allow-direct-references = true
+
+[tool.pylint.main]
+py-version = "3.11"
+reports = false
+
+[tool.pylint.messages_control]
+disable = ["all"]
+enable = [
+ "W0101", # Unreachable code: Used when there is some code behind a "return" or "raise" statement, which will never be accessed.
+]
+ignore = [".venv", "__pycache__", "build", "dist", ".git"]
diff --git a/tests/cases/__init__.py b/tests/cases/__init__.py
index 2621add48..135dbd61f 100644
--- a/tests/cases/__init__.py
+++ b/tests/cases/__init__.py
@@ -11,11 +11,11 @@ from .templates import JINJA2TestCases
from .urls import TestURLs
__all__ = [
- "PDFTestCases",
- "ImageTestCases",
- "TestURLs",
"ClassRegistryTestCases",
"FileHelperTestCases",
"Fruit",
+ "ImageTestCases",
"JINJA2TestCases",
+ "PDFTestCases",
+ "TestURLs",
]
diff --git a/tests/cases/documents.py b/tests/cases/documents.py
index 841a1942f..27a18dcb9 100644
--- a/tests/cases/documents.py
+++ b/tests/cases/documents.py
@@ -1,6 +1,6 @@
"""Document constants for testing."""
-from typing import ClassVar, List
+from typing import ClassVar
class PDFTestCases:
@@ -12,10 +12,10 @@ class PDFTestCases:
# Local file paths
PDF_FILE_PATH_1 = f"{TEST_DOCUMENT_DIRECTORY}/solar_system.pdf"
PDF_FILE_PATH_2 = f"{TEST_DOCUMENT_DIRECTORY}/illustrated_train_article.pdf"
- DOCUMENT_FILE_PATHS: ClassVar[List[str]] = [
+ DOCUMENT_FILE_PATHS: ClassVar[list[str]] = [
PDF_FILE_PATH_1,
PDF_FILE_PATH_2,
]
# Remote URLs
- DOCUMENT_URLS: ClassVar[List[str]] = ["https://www.w3.org/WAI/ER/tests/xhtml/testfiles/resources/pdf/dummy.pdf"]
+ DOCUMENT_URLS: ClassVar[list[str]] = ["https://www.w3.org/WAI/ER/tests/xhtml/testfiles/resources/pdf/dummy.pdf"]
diff --git a/tests/cases/images.py b/tests/cases/images.py
index b4b6c4795..8bb42a77c 100644
--- a/tests/cases/images.py
+++ b/tests/cases/images.py
@@ -1,6 +1,6 @@
"""Image constants for testing."""
-from typing import ClassVar, List
+from typing import ClassVar
class ImageTestCases:
@@ -16,13 +16,13 @@ class ImageTestCases:
IMAGE_URL_PNG = "https://www.python.org/static/community_logos/python-logo-master-v3-TM.png"
# File path collections
- IMAGE_FILE_PATHS: ClassVar[List[str]] = [
+ IMAGE_FILE_PATHS: ClassVar[list[str]] = [
f"{TEST_IMAGE_DIRECTORY}/ai_lympics.png",
f"{TEST_IMAGE_DIRECTORY}/animal_lympics.jpg",
]
# URL collections
- IMAGE_URLS: ClassVar[List[str]] = [
+ IMAGE_URLS: ClassVar[list[str]] = [
"https://www.w3.org/People/mimasa/test/imgformat/img/w3c_home.png",
"https://www.google.com/images/branding/googlelogo/1x/googlelogo_color_272x92dp.png",
]
diff --git a/tests/cases/registry.py b/tests/cases/registry.py
index 2b90d6852..ecde1e5e5 100644
--- a/tests/cases/registry.py
+++ b/tests/cases/registry.py
@@ -1,6 +1,6 @@
"""Registry and model constants for testing."""
-from typing import ClassVar, List
+from typing import ClassVar
from pydantic import BaseModel
from typing_extensions import override
@@ -11,14 +11,14 @@ class ClassRegistryTestCases:
MODEL_FOLDER_PATH = "tests/data/tools_data/mock_folder_with_classes"
- CLASSES_TO_REGISTER: ClassVar[List[str]] = [
+ CLASSES_TO_REGISTER: ClassVar[list[str]] = [
"Class1",
"Class2",
"Class3",
"Class4",
]
- CLASSES_NOT_TO_REGISTER: ClassVar[List[str]] = [
+ CLASSES_NOT_TO_REGISTER: ClassVar[list[str]] = [
"ClassA",
"ClassB",
]
diff --git a/tests/cases/source_code.py b/tests/cases/source_code.py
index 4253aa1b9..75c390e3e 100644
--- a/tests/cases/source_code.py
+++ b/tests/cases/source_code.py
@@ -1,27 +1,26 @@
"""Source code manipulation functions for testing."""
-from typing import List
from pipelex.core.memory.working_memory import WorkingMemory
from pipelex.core.stuffs.stuff_content import TextContent
def wrap_lines(working_memory: WorkingMemory) -> TextContent:
- """
- Wraps each line of source code text with HTML span tags.
+ """Wraps each line of source code text with HTML span tags.
Args:
working_memory: The working memory containing the source text
Returns:
TextContent with each line wrapped in span tags with class="line"
+
"""
# Get the source text from working memory as string
source_text = working_memory.get_stuff_as_str("source_text")
# Split into lines and wrap each line
lines = source_text.split("\n")
- wrapped_lines: List[str] = []
+ wrapped_lines: list[str] = []
for line in lines:
wrapped_line = f'<span class="line">{line}</span>'
diff --git a/tests/cases/templates.py b/tests/cases/templates.py
index de891ea42..9f1d3e719 100644
--- a/tests/cases/templates.py
+++ b/tests/cases/templates.py
@@ -1,6 +1,6 @@
"""Template constants for testing."""
-from typing import Any, ClassVar, List, Tuple
+from typing import Any, ClassVar
from pipelex.tools.templating.templating_models import PromptingStyle, TagStyle, TextFormat
@@ -11,12 +11,12 @@ class JINJA2TestCases:
"""Jinja2 template test constants."""
# Template names
- JINJA2_NAME: ClassVar[List[str]] = [
+ JINJA2_NAME: ClassVar[list[str]] = [
"jinja2_test_template",
]
# Basic templates
- JINJA2_FOR_ANY: ClassVar[List[str]] = [
+ JINJA2_FOR_ANY: ClassVar[list[str]] = [
"I want a {{ place_holder }} cocktail.",
]
@@ -78,7 +78,7 @@ Format filter html:
"""
# Template collections
- JINJA2_FOR_STUFF: ClassVar[List[str]] = [
+ JINJA2_FOR_STUFF: ClassVar[list[str]] = [
JINJA2_FILTER_TAG,
JINJA2_FILTER_FORMAT,
JINJA2_FILTER_FORMAT_PLAIN,
@@ -90,7 +90,7 @@ Format filter html:
]
# Style configurations
- STYLE: ClassVar[List[PromptingStyle]] = [
+ STYLE: ClassVar[list[PromptingStyle]] = [
PromptingStyle(
tag_style=TagStyle.NO_TAG,
text_format=TextFormat.PLAIN,
@@ -110,20 +110,20 @@ Format filter html:
]
# Test data
- COLOR: ClassVar[List[str]] = [
+ COLOR: ClassVar[list[str]] = [
"red",
"blue",
"green",
]
- FRUIT: ClassVar[List[Fruit]] = [
+ FRUIT: ClassVar[list[Fruit]] = [
Fruit(color="red", name="cherry"),
Fruit(color="blue", name="blueberry"),
Fruit(color="green", name="grape"),
]
# Mixed object types for comprehensive testing
- ANY_OBJECT: ClassVar[List[Tuple[str, Any]]] = [ # topic, any_object
+ ANY_OBJECT: ClassVar[list[tuple[str, Any]]] = [ # topic, any_object
("string", "test_string"),
("integer", 42),
("float", 3.14),
diff --git a/tests/cases/urls.py b/tests/cases/urls.py
index 173768f0b..cb707aa64 100644
--- a/tests/cases/urls.py
+++ b/tests/cases/urls.py
@@ -1,6 +1,6 @@
"""URL constants for testing."""
-from typing import ClassVar, List
+from typing import ClassVar
class TestURLs:
@@ -9,7 +9,7 @@ class TestURLs:
GCP_PUBLIC = "https://storage.googleapis.com/public_test_files_7fa6_4277_9ab/diagrams/gantt_tree_house.png"
WIKIPEDIA = "https://upload.wikimedia.org/wikipedia/commons/thumb/e/ec/Olympic_rings_on_the_Eiffel_Tower_2024_%2819%29.jpg/440px-Olympic_rings_on_the_Eiffel_Tower_2024_%2819%29.jpg"
- PUBLIC_URLS: ClassVar[List[str]] = [
+ PUBLIC_URLS: ClassVar[list[str]] = [
GCP_PUBLIC,
WIKIPEDIA,
]
diff --git a/tests/conftest.py b/tests/conftest.py
index e280c2d64..4e5954112 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -40,7 +40,7 @@ def reset_pipelex_config_fixture():
@pytest.fixture(scope="function", autouse=True)
def pretty():
# Code to run before each test
- yield
+ return
# Code to run after each test
diff --git a/tests/e2e/pipelex/migration/test_migrate_v0_1_0_to_v0_2_0_e2e.py b/tests/e2e/pipelex/migration/test_migrate_v0_1_0_to_v0_2_0_e2e.py
index 635c55de3..a6f726191 100644
--- a/tests/e2e/pipelex/migration/test_migrate_v0_1_0_to_v0_2_0_e2e.py
+++ b/tests/e2e/pipelex/migration/test_migrate_v0_1_0_to_v0_2_0_e2e.py
@@ -1,7 +1,7 @@
"""End-to-end tests for the migration functionality."""
from pathlib import Path
-from typing import Any, List, Tuple
+from typing import Any
import pytest
@@ -13,7 +13,6 @@ class TestMigrationE2E:
def test_real_world_migration_scenario(self, tmp_path: Path) -> None:
"""Test a realistic migration scenario with multiple files and complex structures."""
-
# Create a realistic pipelex libraries structure
pipelines_dir = tmp_path / "pipelines"
pipelines_dir.mkdir()
@@ -177,7 +176,6 @@ prompt_template = "Process: @input_text"
def test_migration_with_complex_formatting(self, tmp_path: Path) -> None:
"""Test migration preserves complex formatting and edge cases."""
-
test_file = tmp_path / "complex_formatting.toml"
original_content = """# Complex formatting test file
domain = "formatting_test"
@@ -272,7 +270,6 @@ Process this: @input_text
def test_dry_run_does_not_modify_files(self, tmp_path: Path) -> None:
"""Test that dry run mode doesn't modify any files."""
-
test_file = tmp_path / "dry_run_test.toml"
original_content = """domain = "dry_run_test"
@@ -310,7 +307,6 @@ refines = "Document"
def test_error_handling_with_permission_issues(self, tmp_path: Path, mocker: Any) -> None:
"""Test proper error handling when file operations fail."""
-
test_file = tmp_path / "permission_test.toml"
test_file.write_text("""[concept.Test]
Concept = "Test concept"
@@ -375,7 +371,7 @@ Concept = "Test concept"
# Verify the challenge case is still there and unchanged
challenge_line_found = False
- concept_lines_outside_multiline: List[Tuple[int, str]] = []
+ concept_lines_outside_multiline: list[tuple[int, str]] = []
lines = migrated_content.split("\n")
for i, line in enumerate(lines, 1):
diff --git a/tests/e2e/pipelex/pipes/pipe_controller/__init__.py b/tests/e2e/pipelex/pipes/pipe_controller/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/tests/e2e/pipelex/pipes/pipe_controller/pipe_condition/__init__.py b/tests/e2e/pipelex/pipes/pipe_controller/pipe_condition/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/tests/e2e/pipelex/pipes/pipe_controller/pipe_condition/test_pipe_condition_simple.py b/tests/e2e/pipelex/pipes/pipe_controller/pipe_condition/test_pipe_condition_simple.py
index 9c6fca945..41fd7d788 100644
--- a/tests/e2e/pipelex/pipes/pipe_controller/pipe_condition/test_pipe_condition_simple.py
+++ b/tests/e2e/pipelex/pipes/pipe_controller/pipe_condition/test_pipe_condition_simple.py
@@ -86,8 +86,8 @@ class TestPipeConditionSimple:
structure_class_name="CategoryInput",
),
structure_class=CategoryInput,
- )
- ]
+ ),
+ ],
)
try:
diff --git a/tests/e2e/pipelex/pipes/pipe_controller/pipe_parallel/__init__.py b/tests/e2e/pipelex/pipes/pipe_controller/pipe_parallel/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/tests/e2e/pipelex/pipes/pipe_controller/pipe_parallel/test_pipe_parallel_1.py b/tests/e2e/pipelex/pipes/pipe_controller/pipe_parallel/test_pipe_parallel_1.py
index 581fd4ed4..4f2fb755a 100644
--- a/tests/e2e/pipelex/pipes/pipe_controller/pipe_parallel/test_pipe_parallel_1.py
+++ b/tests/e2e/pipelex/pipes/pipe_controller/pipe_parallel/test_pipe_parallel_1.py
@@ -80,8 +80,8 @@ class TestPipeParallelDocumentAnalysis:
assert hasattr(combined_analysis, "content_result")
# Verify that both analyses contain meaningful content
- length_analysis = cast(LengthAnalysis, length_result_stuff.content)
- content_analysis = cast(ContentAnalysis, content_result_stuff.content)
+ length_analysis = cast("LengthAnalysis", length_result_stuff.content)
+ content_analysis = cast("ContentAnalysis", content_result_stuff.content)
assert isinstance(length_analysis.analysis, str)
assert isinstance(content_analysis.analysis, str)
diff --git a/tests/e2e/pipelex/pipes/pipe_controller/pipe_sequence/__init__.py b/tests/e2e/pipelex/pipes/pipe_controller/pipe_sequence/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/tests/e2e/pipelex/pipes/pipe_controller/pipe_sequence/test_pipe_sequence_batching.py b/tests/e2e/pipelex/pipes/pipe_controller/pipe_sequence/test_pipe_sequence_batching.py
index 6b5ea1fd1..14dc0cf0c 100644
--- a/tests/e2e/pipelex/pipes/pipe_controller/pipe_sequence/test_pipe_sequence_batching.py
+++ b/tests/e2e/pipelex/pipes/pipe_controller/pipe_sequence/test_pipe_sequence_batching.py
@@ -36,7 +36,7 @@ async def test_review_analysis_sequence_with_batching(pipe_run_mode: PipeRunMode
),
structure_class=Document,
),
- ]
+ ],
)
pipe = get_required_pipe(pipe_code="analyze_reviews_sequence")
pipe_output = await pipe.run_pipe(
@@ -82,7 +82,7 @@ async def test_review_analysis_sequence_with_batching(pipe_run_mode: PipeRunMode
# Verify final product rating
stuff = pipe_output.working_memory.get_stuff("product_rating")
# Use cast to tell the type system what we know about the object
- product_rating_stuff = cast(ProductRating, stuff.content)
+ product_rating_stuff = cast("ProductRating", stuff.content)
assert product_rating_stuff is not None
# Check that the ProductRating has meaningful values
diff --git a/tests/helpers/paths.py b/tests/helpers/paths.py
index 45011de38..9fc99a60b 100644
--- a/tests/helpers/paths.py
+++ b/tests/helpers/paths.py
@@ -16,6 +16,7 @@ def data_path(relative: str) -> Path:
Example:
>>> img = data_path("images/white_square.png")
>>> doc = data_path("documents/solar_system.pdf")
+
"""
data_dir = resources.files("tests.data")
return Path(str(data_dir)) / relative
diff --git a/tests/integration/pipelex/client/__init__.py b/tests/integration/pipelex/client/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/tests/integration/pipelex/client/test_client.py b/tests/integration/pipelex/client/test_client.py
index 62777912c..f91ca2ac0 100644
--- a/tests/integration/pipelex/client/test_client.py
+++ b/tests/integration/pipelex/client/test_client.py
@@ -1,4 +1,3 @@
-from typing import List
import pytest
from pydantic import BaseModel
@@ -17,16 +16,15 @@ from pipelex.core.stuffs.stuff_factory import StuffFactory
class Example(BaseModel):
pipe_code: str
- memory: List[Stuff]
+ memory: list[Stuff]
@pytest.mark.pipelex_api
@pytest.mark.asyncio(loop_scope="class")
class TestPipelexApiClient:
@pytest.fixture
- def examples(self) -> List[Example]:
- """
- Fixture providing test example for API client tests.
+ def examples(self) -> list[Example]:
+ """Fixture providing test example for API client tests.
"""
return [
Example(
@@ -74,12 +72,12 @@ class TestPipelexApiClient:
distinctive bionic design inspired by peregrine falcons, the train uses an innovative
hybrid propulsion system that combines traditional electric motors with
compressed air boosters for acceleration phases.
- """
+ """,
),
),
StuffFactory.make_stuff(
concept=ConceptFactory.make(
- concept_code="Question", domain="answer", definition="answer.Question", structure_class_name="Question"
+ concept_code="Question", domain="answer", definition="answer.Question", structure_class_name="Question",
),
name="question",
content=TextContent(text="Aerodynamic features?"),
@@ -90,13 +88,13 @@ class TestPipelexApiClient:
async def test_client_execute_pipeline(
self,
- examples: List[Example],
+ examples: list[Example],
):
- """
- Test the execute_pipe method with the example.
+ """Test the execute_pipe method with the example.
Args:
examples: List of test examples from the fixture
+
"""
for example in examples:
# Create working memory from example data
diff --git a/tests/integration/pipelex/cogt/__init__.py b/tests/integration/pipelex/cogt/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/tests/integration/pipelex/cogt/test_content_generator.py b/tests/integration/pipelex/cogt/test_content_generator.py
index 658fb5235..e5781b45c 100644
--- a/tests/integration/pipelex/cogt/test_content_generator.py
+++ b/tests/integration/pipelex/cogt/test_content_generator.py
@@ -1,4 +1,3 @@
-from typing import List
import pytest
from pytest import FixtureRequest
@@ -90,7 +89,7 @@ class TestContentGenerator:
async def test_make_object_list_direct(self, request: FixtureRequest):
llm_setting_for_object = get_model_deck().get_llm_setting(llm_choice="llm_for_testing_gen_object")
- person_list_direct: List[Employee] = await get_content_generator().make_object_list_direct(
+ person_list_direct: list[Employee] = await get_content_generator().make_object_list_direct(
job_metadata=JobMetadata(job_name=request.node.originalname), # type: ignore
object_class=Employee,
llm_prompt_for_object_list=LLMPrompt(user_text=USER_TEXTS_FOR_PEOPLE_STR),
diff --git a/tests/integration/pipelex/cogt/test_data.py b/tests/integration/pipelex/cogt/test_data.py
index 279051250..72c75a44b 100644
--- a/tests/integration/pipelex/cogt/test_data.py
+++ b/tests/integration/pipelex/cogt/test_data.py
@@ -1,4 +1,4 @@
-from typing import Any, ClassVar, Dict, List, Tuple
+from typing import Any, ClassVar
from pydantic import BaseModel, Field, field_validator
@@ -59,17 +59,17 @@ class LLMVisionTestCases:
PATH_IMG_GANTT_1 = f"{TEST_IMAGE_DIRECTORY}/gantt_tree_house.png"
- IMAGE_PATHS: ClassVar[List[Tuple[str, str]]] = [ # topic, image_path
+ IMAGE_PATHS: ClassVar[list[tuple[str, str]]] = [ # topic, image_path
("AI Lympics PNG", PATH_IMG_PNG_1),
("AI Lympics JPEG", PATH_IMG_JPEG_1),
("Gantt Chart", PATH_IMG_GANTT_1),
("Eiffel Tower", PATH_IMG_JPEG_3),
]
- IMAGE_PATH_PAIRS: ClassVar[List[Tuple[str, Tuple[str, str]]]] = [ # topic, image_pair
+ IMAGE_PATH_PAIRS: ClassVar[list[tuple[str, tuple[str, str]]]] = [ # topic, image_pair
("AI Lympics PNG", (PATH_IMG_PNG_1, PATH_IMG_PNG_2)),
]
- IMAGE_URLS: ClassVar[List[Tuple[str, str]]] = [ # topic, image_uri
+ IMAGE_URLS: ClassVar[list[tuple[str, str]]] = [ # topic, image_uri
(
"Alan Turing",
URL_WIKIPEDIA_ALAN_TURING,
@@ -85,7 +85,7 @@ class LLMTestConstants:
USER_TEXT_SHORT = "In one sentence, who is Bill Gates?"
# USER_TEXT_SHORT = "What's the biggest football match tonight in Europe?"
PROMPT_TEMPLATE_TEXT = "Can you give one example of flower which is {color} in color ?"
- PROMPT_COLOR_EXAMPLES: ClassVar[List[str]] = [
+ PROMPT_COLOR_EXAMPLES: ClassVar[list[str]] = [
"red",
"blue",
"green",
@@ -104,17 +104,17 @@ class LLMTestCases:
When my son was 7 he was 3ft tall. When he was 8 he was 4ft tall. When he was 9 he was 5ft tall.
How tall do you think he was when he was 12? and at 15?
"""
- SINGLE_TEXT: ClassVar[List[Tuple[str, str]]] = [ # topic, prompt_text
+ SINGLE_TEXT: ClassVar[list[tuple[str, str]]] = [ # topic, prompt_text
("Haiku", USER_TEXT_HAIKU),
("Tricky", USER_TEXT_TRICKY),
]
- SINGLE_OBJECT: ClassVar[List[Tuple[str, BaseModel]]] = [
+ SINGLE_OBJECT: ClassVar[list[tuple[str, BaseModel]]] = [
("name: John, age: 30", Person(name="John", age=30)),
("Betty Draper, 51", Person(name="Betty Draper", age=51)),
("Whiskers, the cat", Pet(species=PetSpecies.CAT, name="Whiskers")),
("Whiskers, the dog", Pet(species=PetSpecies.DOG, name="Whiskers")),
]
- MULTIPLE_OBJECTS: ClassVar[List[List[Tuple[str, BaseModel]]]] = [
+ MULTIPLE_OBJECTS: ClassVar[list[list[tuple[str, BaseModel]]]] = [
[
("name: John, age: 30", Person(name="John", age=30)),
# ("Betty Draper, 51", Person(name="Betty Draper", age=51)),
@@ -155,7 +155,7 @@ class SerDeTestLLMCases:
)
# Dictionary representation example ------------------------------------
- DICT_1: ClassVar[Dict[str, Any]] = {
+ DICT_1: ClassVar[dict[str, Any]] = {
"proto_prompt": LLMPrompt(
system_text=None,
user_text="Some user text in the template",
@@ -176,13 +176,13 @@ class SerDeTestLLMCases:
)
# Group constants for parametrization ----------------------------------
- PYDANTIC_EXAMPLES: ClassVar[List[BaseModel]] = [
+ PYDANTIC_EXAMPLES: ClassVar[list[BaseModel]] = [
MY_PROMPT_TEMPLATE_MODEL_1,
MY_PROMPT_TEMPLATE_MODEL_2,
]
- PYDANTIC_EXAMPLES_USING_SUBCLASS: ClassVar[List[BaseModel]] = [
+ PYDANTIC_EXAMPLES_USING_SUBCLASS: ClassVar[list[BaseModel]] = [
PROMPT_WITH_IMAGE_PATH,
]
- PYDANTIC_EXAMPLES_DICT: ClassVar[List[Dict[str, Any]]] = [
+ PYDANTIC_EXAMPLES_DICT: ClassVar[list[dict[str, Any]]] = [
DICT_1,
]
diff --git a/tests/integration/pipelex/cogt/test_llm_gen_object.py b/tests/integration/pipelex/cogt/test_llm_gen_object.py
index 3a609494b..321bd7782 100644
--- a/tests/integration/pipelex/cogt/test_llm_gen_object.py
+++ b/tests/integration/pipelex/cogt/test_llm_gen_object.py
@@ -1,5 +1,4 @@
import asyncio
-from typing import List, Tuple
import pytest
from pydantic import BaseModel
@@ -56,12 +55,12 @@ class TestLLMGenObject:
@pytest.mark.parametrize("case_tuples", LLMTestCases.MULTIPLE_OBJECTS)
async def test_gen_object_async_multiple_using_handle(
- self, llm_job_params: LLMJobParams, llm_handle: str, case_tuples: List[Tuple[str, BaseModel]]
+ self, llm_job_params: LLMJobParams, llm_handle: str, case_tuples: list[tuple[str, BaseModel]],
):
llm_worker = get_llm_worker(llm_handle=llm_handle)
if not llm_worker.is_gen_object_supported:
pytest.skip(f"'{llm_worker.desc}' does not support object generation")
- tasks: List[asyncio.Task[BaseModel]] = []
+ tasks: list[asyncio.Task[BaseModel]] = []
for case_tuple in case_tuples:
user_text, expected_instance = case_tuple
expected_class = expected_instance.__class__
@@ -81,11 +80,11 @@ class TestLLMGenObject:
assert output_instance.model_dump(serialize_as_any=True) == expected_instance.model_dump(serialize_as_any=True)
@pytest.mark.parametrize("case_tuples", LLMTestCases.MULTIPLE_OBJECTS)
- async def test_gen_object_async_multiple_using_llm_preset(self, llm_preset_id: str, case_tuples: List[Tuple[str, BaseModel]]):
+ async def test_gen_object_async_multiple_using_llm_preset(self, llm_preset_id: str, case_tuples: list[tuple[str, BaseModel]]):
llm_worker, llm_job = get_async_worker_and_job(llm_preset_id=llm_preset_id, user_text=case_tuples[0][0])
if not llm_worker.is_gen_object_supported:
pytest.skip(f"'{llm_worker.desc}' does not support object generation")
- tasks: List[asyncio.Task[BaseModel]] = []
+ tasks: list[asyncio.Task[BaseModel]] = []
for case_tuple in case_tuples:
user_text, expected_instance = case_tuple
expected_class = expected_instance.__class__
diff --git a/tests/integration/pipelex/cogt/test_llm_gen_text.py b/tests/integration/pipelex/cogt/test_llm_gen_text.py
index b340b04b8..d0b5e90b2 100644
--- a/tests/integration/pipelex/cogt/test_llm_gen_text.py
+++ b/tests/integration/pipelex/cogt/test_llm_gen_text.py
@@ -1,5 +1,4 @@
import asyncio
-from typing import List, Tuple
import pytest
@@ -14,7 +13,7 @@ from pipelex.hub import get_llm_worker, get_model_deck
from tests.integration.pipelex.cogt.test_data import LLMTestCases
-def get_worker_and_job(llm_preset_id: str, user_text: str) -> Tuple[LLMWorkerAbstract, LLMJob]:
+def get_worker_and_job(llm_preset_id: str, user_text: str) -> tuple[LLMWorkerAbstract, LLMJob]:
llm_setting = get_model_deck().get_llm_setting(llm_choice=llm_preset_id)
pretty_print(llm_setting, title=llm_preset_id)
pretty_print(user_text)
@@ -56,7 +55,7 @@ class TestLLMGenText:
job_params_base = llm_job.job_params
max_tokens = 30
temperature = 0.1
- tasks: List[asyncio.Task[str]] = []
+ tasks: list[asyncio.Task[str]] = []
for _ in range(4):
max_tokens += 50
temperature += 0.2
diff --git a/tests/integration/pipelex/cogt/test_llm_report.py b/tests/integration/pipelex/cogt/test_llm_report.py
index 96807b22c..2ac786f5b 100644
--- a/tests/integration/pipelex/cogt/test_llm_report.py
+++ b/tests/integration/pipelex/cogt/test_llm_report.py
@@ -1,5 +1,4 @@
import asyncio
-from typing import List
import pytest
@@ -44,7 +43,7 @@ class TestLLMReport:
"llm_for_testing_gen_text",
"llm_for_testing_gen_object",
]
- tasks: List[asyncio.Task[str]] = []
+ tasks: list[asyncio.Task[str]] = []
for llm_preset_id in llm_preset_ids:
llm_worker, llm_job = self._get_async_worker_and_job(llm_preset_id=llm_preset_id, prompt_text=prompt_text)
job_params_base = llm_job.job_params
diff --git a/tests/integration/pipelex/cogt/test_llm_vision.py b/tests/integration/pipelex/cogt/test_llm_vision.py
index da74826aa..99b63a216 100644
--- a/tests/integration/pipelex/cogt/test_llm_vision.py
+++ b/tests/integration/pipelex/cogt/test_llm_vision.py
@@ -1,4 +1,3 @@
-from typing import Tuple
import pytest
@@ -74,7 +73,7 @@ class TestLLMVision:
pytest.skip(f"Prompt Image format not supported for this LLM: {llm_handle_for_vision} because {exc}")
@pytest.mark.parametrize("topic, image_pair", LLMVisionTestCases.IMAGE_PATH_PAIRS)
- async def test_gen_text_from_vision_2_images(self, llm_handle_for_vision: str, topic: str, image_pair: Tuple[str, str]):
+ async def test_gen_text_from_vision_2_images(self, llm_handle_for_vision: str, topic: str, image_pair: tuple[str, str]):
prompt_image1 = PromptImagePath(file_path=image_pair[0])
prompt_image2 = PromptImagePath(file_path=image_pair[1])
llm_worker = get_llm_worker(llm_handle=llm_handle_for_vision)
diff --git a/tests/integration/pipelex/cogt/test_serde_llm_classes.py b/tests/integration/pipelex/cogt/test_serde_llm_classes.py
index 532830be3..caca27a2c 100644
--- a/tests/integration/pipelex/cogt/test_serde_llm_classes.py
+++ b/tests/integration/pipelex/cogt/test_serde_llm_classes.py
@@ -1,4 +1,4 @@
-from typing import Any, Dict
+from typing import Any
import pytest
from kajson import kajson
@@ -42,7 +42,7 @@ class TestSerDeLLMClasses:
@pytest.mark.parametrize("test_dict", SerDeTestLLMCases.PYDANTIC_EXAMPLES_DICT)
def test_serde_model_validate_from_dict(
self,
- test_dict: Dict[str, Any],
+ test_dict: dict[str, Any],
):
pretty_print(test_dict, title="Original dict")
# Deserialize the dictionary to a LLMPromptTemplate model
@@ -63,7 +63,7 @@ class TestSerDeLLMClasses:
@pytest.mark.parametrize("test_dict", SerDeTestLLMCases.PYDANTIC_EXAMPLES_DICT)
def test_serde_instantiate_from_kwargs(
self,
- test_dict: Dict[str, Any],
+ test_dict: dict[str, Any],
):
pretty_print(test_dict, title="Original dict")
# Deserialize the dictionary to a LLMPromptTemplate model
diff --git a/tests/integration/pipelex/conftest.py b/tests/integration/pipelex/conftest.py
index f284d92d7..898cb0107 100644
--- a/tests/integration/pipelex/conftest.py
+++ b/tests/integration/pipelex/conftest.py
@@ -1,4 +1,3 @@
-from typing import Optional
import pytest
@@ -11,7 +10,7 @@ from pipelex.plugins.plugin_sdk_registry import Plugin
# "llm_for_testing_gen_text",
# "llm_for_testing_gen_object",
"llm_for_creative_writing",
- ]
+ ],
)
def llm_preset_id(request: pytest.FixtureRequest) -> str:
assert isinstance(request.param, str)
@@ -63,7 +62,7 @@ def llm_preset_id(request: pytest.FixtureRequest) -> str:
"gemini-2.5-flash-lite",
# "gemini-2.5-flash",
# "gemini-2.5-pro",
- ]
+ ],
)
def llm_handle(request: pytest.FixtureRequest) -> str:
assert isinstance(request.param, str)
@@ -95,7 +94,7 @@ def llm_handle(request: pytest.FixtureRequest) -> str:
# "gemini-2.5-flash",
# "gemini-2.5-pro",
# "qwen3:8b",
- ]
+ ],
)
def llm_handle_for_vision(request: pytest.FixtureRequest) -> str:
assert isinstance(request.param, str)
@@ -106,7 +105,7 @@ def llm_handle_for_vision(request: pytest.FixtureRequest) -> str:
params=[
Plugin(sdk="openai", backend="openai"),
Plugin(sdk="azure_openai", backend="azure_openai"),
- ]
+ ],
)
def plugin_for_openai(request: pytest.FixtureRequest) -> Plugin:
assert isinstance(request.param, Plugin)
@@ -117,7 +116,7 @@ def plugin_for_openai(request: pytest.FixtureRequest) -> Plugin:
params=[
Plugin(sdk="anthropic", backend="anthropic"),
Plugin(sdk="bedrock_anthropic", backend="bedrock_anthropic"),
- ]
+ ],
)
def plugin_for_anthropic(request: pytest.FixtureRequest) -> Plugin:
assert isinstance(request.param, Plugin)
@@ -128,9 +127,9 @@ def plugin_for_anthropic(request: pytest.FixtureRequest) -> Plugin:
params=[
# None,
"https://inference.pipelex.com/v1",
- ]
+ ],
)
-def openai_endpoint(request: pytest.FixtureRequest) -> Optional[str]:
+def openai_endpoint(request: pytest.FixtureRequest) -> str | None:
assert isinstance(request.param, str) or request.param is None
return request.param
@@ -176,7 +175,7 @@ def openai_endpoint(request: pytest.FixtureRequest) -> Optional[str]:
# "pipelex/gpt-4o-mini",
# "pipelex/claude-3.7-sonnet",
# "vertex_ai/gemini-2.0-flash",
- ]
+ ],
)
def llm_id(request: pytest.FixtureRequest) -> str:
assert isinstance(request.param, str)
@@ -190,7 +189,7 @@ def llm_id(request: pytest.FixtureRequest) -> str:
max_tokens=None,
seed=None,
),
- ]
+ ],
)
def llm_job_params(request: pytest.FixtureRequest) -> LLMJobParams:
assert isinstance(request.param, LLMJobParams)
@@ -204,7 +203,7 @@ def llm_job_params(request: pytest.FixtureRequest) -> LLMJobParams:
# "flux-pro/v1.1-ultra",
"fast-lightning-sdxl",
"gpt-image-1",
- ]
+ ],
)
def img_gen_handle(request: pytest.FixtureRequest) -> str:
assert isinstance(request.param, str)
@@ -215,7 +214,7 @@ def img_gen_handle(request: pytest.FixtureRequest) -> str:
params=[
"pypdfium2-extract-text",
"mistral-ocr",
- ]
+ ],
)
def ocr_handle(request: pytest.FixtureRequest) -> str:
assert isinstance(request.param, str)
@@ -225,7 +224,7 @@ def ocr_handle(request: pytest.FixtureRequest) -> str:
@pytest.fixture(
params=[
"mistral-ocr",
- ]
+ ],
)
def ocr_handle_from_image(request: pytest.FixtureRequest) -> str:
assert isinstance(request.param, str)
@@ -235,7 +234,7 @@ def ocr_handle_from_image(request: pytest.FixtureRequest) -> str:
@pytest.fixture(
params=[
"base_ocr_mistral",
- ]
+ ],
)
def ocr_choice_for_pdf(request: pytest.FixtureRequest) -> str:
assert isinstance(request.param, str)
@@ -245,7 +244,7 @@ def ocr_choice_for_pdf(request: pytest.FixtureRequest) -> str:
@pytest.fixture(
params=[
"base_ocr_mistral",
- ]
+ ],
)
def ocr_choice_for_image(request: pytest.FixtureRequest) -> str:
assert isinstance(request.param, str)
diff --git a/tests/integration/pipelex/core/memory/__init__.py b/tests/integration/pipelex/core/memory/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/tests/integration/pipelex/core/memory/test_working_memory_factory.py b/tests/integration/pipelex/core/memory/test_working_memory_factory.py
index 95ccdff34..0f39216ca 100644
--- a/tests/integration/pipelex/core/memory/test_working_memory_factory.py
+++ b/tests/integration/pipelex/core/memory/test_working_memory_factory.py
@@ -11,7 +11,7 @@ class TestWorkingMemoryFactory:
"text_item": {
"concept_code": NativeConceptEnum.TEXT,
"content": "Hello, world!",
- }
+ },
}
working_memory = WorkingMemoryFactory.make_from_compact_memory(compact_memory)
@@ -33,7 +33,7 @@ class TestWorkingMemoryFactory:
"text_and_images": {
"text": {
"text": "This is a complex document page with multiple images and rich text content. "
- "It demonstrates nested structured content handling."
+ "It demonstrates nested structured content handling.",
},
"images": [
{
@@ -51,7 +51,7 @@ class TestWorkingMemoryFactory:
},
"page_view": {"url": "mock_url", "caption": "Full page screenshot"},
},
- }
+ },
}
working_memory = WorkingMemoryFactory.make_from_compact_memory(compact_memory)
diff --git a/tests/integration/pipelex/pipes/__init__.py b/tests/integration/pipelex/pipes/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/tests/integration/pipelex/pipes/conftest.py b/tests/integration/pipelex/pipes/conftest.py
index 33ff3509f..c635d5faf 100644
--- a/tests/integration/pipelex/pipes/conftest.py
+++ b/tests/integration/pipelex/pipes/conftest.py
@@ -1,5 +1,5 @@
import os
-from typing import AsyncIterator, Awaitable, Callable
+from collections.abc import AsyncIterator, Awaitable, Callable
import pytest_asyncio
from pytest import FixtureRequest
@@ -14,8 +14,7 @@ from tests.conftest import TEST_OUTPUTS_DIR
@pytest_asyncio.fixture # pyright: ignore[reportUntypedFunctionDecorator, reportUnknownMemberType]
async def pipe_result_handler(request: FixtureRequest) -> AsyncIterator[tuple[str, ActivityHandlerForResultFiles]]:
- """
- This fixture is used to handle the result of a pipe run in unit tests.
+ """This fixture is used to handle the result of a pipe run in unit tests.
It creates and registers an activity handler to save the activities of the pipe into a specific directory.
It returns a tuple with the result directory path and the activity handler, which enables the
calling test to save whatever it wants into that result directory.
diff --git a/tests/integration/pipelex/pipes/controller/pipe_batch/__init__.py b/tests/integration/pipelex/pipes/controller/pipe_batch/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/tests/integration/pipelex/pipes/controller/pipe_batch/test_pipe_batch_simple.py b/tests/integration/pipelex/pipes/controller/pipe_batch/test_pipe_batch_simple.py
index bd5c25b78..b34893d67 100644
--- a/tests/integration/pipelex/pipes/controller/pipe_batch/test_pipe_batch_simple.py
+++ b/tests/integration/pipelex/pipes/controller/pipe_batch/test_pipe_batch_simple.py
@@ -1,6 +1,6 @@
"""Simple integration test for PipeBatch controller."""
-from typing import List, cast
+from typing import cast
import pytest
from pytest import FixtureRequest
@@ -73,7 +73,7 @@ class TestPipeBatchSimple:
text_list_stuff = StuffFactory.make_stuff(
concept=ConceptFactory.make_native_concept(native_concept_data=NATIVE_CONCEPTS_DATA[NativeConceptEnum.TEXT]),
- content=ListContent[StuffContent](items=cast(List[StuffContent], text_items)),
+ content=ListContent[StuffContent](items=cast("list[StuffContent]", text_items)),
name="text_list",
)
@@ -95,7 +95,7 @@ class TestPipeBatchSimple:
assert isinstance(text_list.content, ListContent)
# Cast the content to the proper type for type checking
- list_content = cast(ListContent[TextContent], text_list.content) # type: ignore
+ list_content = cast("ListContent[TextContent]", text_list.content) # type: ignore
assert len(list_content.items) == 3
# Verify each item in the list
@@ -109,7 +109,7 @@ class TestPipeBatchSimple:
# Actually run the PipeBatch pipe
pipe_output = await pipe_batch._run_controller_pipe( # pyright: ignore[reportPrivateUsage]
- job_metadata=JobMetadata(job_name=cast(str, request.node.originalname)), # type: ignore
+ job_metadata=JobMetadata(job_name=cast("str", request.node.originalname)), # type: ignore
working_memory=working_memory,
pipe_run_params=PipeRunParamsFactory.make_run_params(pipe_run_mode=pipe_run_mode),
output_name="batch_result",
@@ -141,7 +141,7 @@ class TestPipeBatchSimple:
original_list = final_working_memory.get_stuff("text_list")
assert original_list is not None
assert isinstance(original_list.content, ListContent)
- original_items = cast(ListContent[TextContent], original_list.content) # type: ignore
+ original_items = cast("ListContent[TextContent]", original_list.content) # type: ignore
assert len(original_items.items) == 3
assert original_items.items[0].text == "hello"
assert original_items.items[1].text == "world"
@@ -155,7 +155,7 @@ class TestPipeBatchSimple:
# Verify the batch result content matches exactly
assert isinstance(batch_result.content, ListContent)
- result_list = cast(ListContent[TextContent], batch_result.content) # type: ignore
+ result_list = cast("ListContent[TextContent]", batch_result.content) # type: ignore
assert len(result_list.items) == 3
if pipe_run_mode != PipeRunMode.DRY:
assert result_list.items[0].text == "UPPER: HELLO"
diff --git a/tests/integration/pipelex/pipes/controller/pipe_condition/__init__.py b/tests/integration/pipelex/pipes/controller/pipe_condition/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/tests/integration/pipelex/pipes/controller/pipe_condition/test_pipe_condition_simple.py b/tests/integration/pipelex/pipes/controller/pipe_condition/test_pipe_condition_simple.py
index 3b2bb5dd8..9107b8acd 100644
--- a/tests/integration/pipelex/pipes/controller/pipe_condition/test_pipe_condition_simple.py
+++ b/tests/integration/pipelex/pipes/controller/pipe_condition/test_pipe_condition_simple.py
@@ -66,7 +66,7 @@ class TestPipeConditionSimple:
assert input_text.content.text == "hello world"
pipe_output = await pipe_condition.run_pipe(
- job_metadata=JobMetadata(job_name=cast(str, request.node.originalname)), # type: ignore
+ job_metadata=JobMetadata(job_name=cast("str", request.node.originalname)), # type: ignore
working_memory=working_memory,
pipe_run_params=PipeRunParamsFactory.make_run_params(pipe_run_mode=pipe_run_mode),
)
@@ -135,7 +135,7 @@ class TestPipeConditionSimple:
# Actually run the PipeCondition pipe
pipe_output = await pipe_condition.run_pipe(
- job_metadata=JobMetadata(job_name=cast(str, request.node.originalname)), # type: ignore
+ job_metadata=JobMetadata(job_name=cast("str", request.node.originalname)), # type: ignore
working_memory=working_memory,
pipe_run_params=PipeRunParamsFactory.make_run_params(pipe_run_mode=pipe_run_mode),
)
diff --git a/tests/integration/pipelex/pipes/controller/pipe_condition/test_pipe_condition_validation.py b/tests/integration/pipelex/pipes/controller/pipe_condition/test_pipe_condition_validation.py
index cd6e2d004..bbe1c6ed4 100644
--- a/tests/integration/pipelex/pipes/controller/pipe_condition/test_pipe_condition_validation.py
+++ b/tests/integration/pipelex/pipes/controller/pipe_condition/test_pipe_condition_validation.py
@@ -18,7 +18,7 @@ class TestPipeConditionValidation:
concept_codes_from_the_same_domain=["TestConcept"],
)
concept_2 = ConceptFactory.make_from_blueprint(
- concept_code="Result", domain=domain, blueprint=ConceptBlueprint(definition="Lorem Ipsum"), concept_codes_from_the_same_domain=["Result"]
+ concept_code="Result", domain=domain, blueprint=ConceptBlueprint(definition="Lorem Ipsum"), concept_codes_from_the_same_domain=["Result"],
)
concept_library = get_concept_provider()
concept_library.add_concepts([concept_1, concept_2])
@@ -58,7 +58,7 @@ class TestPipeConditionValidation:
concept_codes_from_the_same_domain=["TestConcept"],
)
concept_2 = ConceptFactory.make_from_blueprint(
- concept_code="Result", domain=domain, blueprint=ConceptBlueprint(definition="Lorem Ipsum"), concept_codes_from_the_same_domain=["Result"]
+ concept_code="Result", domain=domain, blueprint=ConceptBlueprint(definition="Lorem Ipsum"), concept_codes_from_the_same_domain=["Result"],
)
concept_library.add_concepts([concept_1, concept_2])
diff --git a/tests/integration/pipelex/pipes/controller/pipe_parallel/__init__.py b/tests/integration/pipelex/pipes/controller/pipe_parallel/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/tests/integration/pipelex/pipes/controller/pipe_parallel/test_pipe_parallel_simple.py b/tests/integration/pipelex/pipes/controller/pipe_parallel/test_pipe_parallel_simple.py
index 4529ab07b..572850e0c 100644
--- a/tests/integration/pipelex/pipes/controller/pipe_parallel/test_pipe_parallel_simple.py
+++ b/tests/integration/pipelex/pipes/controller/pipe_parallel/test_pipe_parallel_simple.py
@@ -78,7 +78,7 @@ class TestPipeParallelSimple:
# Actually run the PipeParallel pipe
pipe_output = await pipe_parallel.run_pipe(
- job_metadata=JobMetadata(job_name=cast(str, request.node.originalname)), # type: ignore
+ job_metadata=JobMetadata(job_name=cast("str", request.node.originalname)), # type: ignore
working_memory=working_memory,
output_name="parallel_results",
pipe_run_params=PipeRunParamsFactory.make_run_params(pipe_run_mode=pipe_run_mode),
@@ -172,7 +172,7 @@ class TestPipeParallelSimple:
# Actually run the PipeParallel pipe
pipe_output = await pipe_parallel.run_pipe(
- job_metadata=JobMetadata(job_name=cast(str, request.node.originalname)), # type: ignore
+ job_metadata=JobMetadata(job_name=cast("str", request.node.originalname)), # type: ignore
working_memory=working_memory,
output_name="parallel_results",
pipe_run_params=PipeRunParamsFactory.make_run_params(pipe_run_mode=pipe_run_mode),
diff --git a/tests/integration/pipelex/pipes/controller/pipe_parallel/test_pipe_parallel_validation.py b/tests/integration/pipelex/pipes/controller/pipe_parallel/test_pipe_parallel_validation.py
index ed89f5364..1c521b07d 100644
--- a/tests/integration/pipelex/pipes/controller/pipe_parallel/test_pipe_parallel_validation.py
+++ b/tests/integration/pipelex/pipes/controller/pipe_parallel/test_pipe_parallel_validation.py
@@ -20,13 +20,13 @@ class TestPipeParallelValidation:
concept_library = get_concept_provider()
concept_blueprint = ConceptBlueprint(definition="A test document")
concept_1 = ConceptFactory.make_from_blueprint(
- domain=domain, concept_code="Document", blueprint=concept_blueprint, concept_codes_from_the_same_domain=["Document"]
+ domain=domain, concept_code="Document", blueprint=concept_blueprint, concept_codes_from_the_same_domain=["Document"],
)
concept_2 = ConceptFactory.make_from_blueprint(
- domain=domain, concept_code="Context", blueprint=concept_blueprint, concept_codes_from_the_same_domain=["Context"]
+ domain=domain, concept_code="Context", blueprint=concept_blueprint, concept_codes_from_the_same_domain=["Context"],
)
concept_3 = ConceptFactory.make_from_blueprint(
- domain=domain, concept_code="Analysis", blueprint=concept_blueprint, concept_codes_from_the_same_domain=["Analysis"]
+ domain=domain, concept_code="Analysis", blueprint=concept_blueprint, concept_codes_from_the_same_domain=["Analysis"],
)
concept_library.add_concepts(concepts=[concept_1, concept_2, concept_3])
@@ -92,13 +92,13 @@ class TestPipeParallelValidation:
concept_library = get_concept_provider()
concept_blueprint = ConceptBlueprint(definition="Lorem Ipsum")
concept_1 = ConceptFactory.make_from_blueprint(
- domain=domain, concept_code="Document", blueprint=concept_blueprint, concept_codes_from_the_same_domain=["Document"]
+ domain=domain, concept_code="Document", blueprint=concept_blueprint, concept_codes_from_the_same_domain=["Document"],
)
concept_2 = ConceptFactory.make_from_blueprint(
- domain=domain, concept_code="Context", blueprint=concept_blueprint, concept_codes_from_the_same_domain=["Context"]
+ domain=domain, concept_code="Context", blueprint=concept_blueprint, concept_codes_from_the_same_domain=["Context"],
)
concept_3 = ConceptFactory.make_from_blueprint(
- domain=domain, concept_code="ProcessedAnalysis", blueprint=concept_blueprint, concept_codes_from_the_same_domain=["ProcessedAnalysis"]
+ domain=domain, concept_code="ProcessedAnalysis", blueprint=concept_blueprint, concept_codes_from_the_same_domain=["ProcessedAnalysis"],
)
concept_library.add_concepts(concepts=[concept_1, concept_2, concept_3])
@@ -135,13 +135,13 @@ class TestPipeParallelValidation:
concept_library = get_concept_provider()
concept_blueprint = ConceptBlueprint(definition="A test document")
concept_1 = ConceptFactory.make_from_blueprint(
- domain=domain, concept_code="Document", blueprint=concept_blueprint, concept_codes_from_the_same_domain=["Document"]
+ domain=domain, concept_code="Document", blueprint=concept_blueprint, concept_codes_from_the_same_domain=["Document"],
)
concept_2 = ConceptFactory.make_from_blueprint(
- domain=domain, concept_code="Context", blueprint=concept_blueprint, concept_codes_from_the_same_domain=["Context"]
+ domain=domain, concept_code="Context", blueprint=concept_blueprint, concept_codes_from_the_same_domain=["Context"],
)
concept_3 = ConceptFactory.make_from_blueprint(
- domain=domain, concept_code="ProcessedAnalysis", blueprint=concept_blueprint, concept_codes_from_the_same_domain=["ProcessedAnalysis"]
+ domain=domain, concept_code="ProcessedAnalysis", blueprint=concept_blueprint, concept_codes_from_the_same_domain=["ProcessedAnalysis"],
)
concept_library.add_concepts(concepts=[concept_1, concept_2, concept_3])
diff --git a/tests/integration/pipelex/pipes/controller/pipe_sequence/__init__.py b/tests/integration/pipelex/pipes/controller/pipe_sequence/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/tests/integration/pipelex/pipes/controller/pipe_sequence/test_pipe_sequence_dry_run.py b/tests/integration/pipelex/pipes/controller/pipe_sequence/test_pipe_sequence_dry_run.py
index 8f9aa5957..eda074a7d 100644
--- a/tests/integration/pipelex/pipes/controller/pipe_sequence/test_pipe_sequence_dry_run.py
+++ b/tests/integration/pipelex/pipes/controller/pipe_sequence/test_pipe_sequence_dry_run.py
@@ -36,7 +36,7 @@ class TestPipeSequenceDryRun:
# Create structured DiscordChannelUpdate objects
discord_channel_updates = ListContent[DiscordChannelUpdate](
- items=[DiscordChannelUpdate.model_validate(article_data) for article_data in discord_channel_updates_data]
+ items=[DiscordChannelUpdate.model_validate(article_data) for article_data in discord_channel_updates_data],
)
# Create Stuff object for the discord channel updates list
diff --git a/tests/integration/pipelex/pipes/controller/pipe_sequence/test_pipe_sequence_simple.py b/tests/integration/pipelex/pipes/controller/pipe_sequence/test_pipe_sequence_simple.py
index 65c3d8b82..2d60260a1 100644
--- a/tests/integration/pipelex/pipes/controller/pipe_sequence/test_pipe_sequence_simple.py
+++ b/tests/integration/pipelex/pipes/controller/pipe_sequence/test_pipe_sequence_simple.py
@@ -90,7 +90,7 @@ class TestPipeSequenceSimple:
# Actually run the PipeSequence pipe
pipe_output = await pipe_sequence.run_pipe(
- job_metadata=JobMetadata(job_name=cast(str, request.node.originalname)), # type: ignore
+ job_metadata=JobMetadata(job_name=cast("str", request.node.originalname)), # type: ignore
working_memory=working_memory,
output_name="sequence_result",
pipe_run_params=PipeRunParamsFactory.make_run_params(pipe_run_mode=pipe_run_mode),
diff --git a/tests/integration/pipelex/pipes/operator/__init__.py b/tests/integration/pipelex/pipes/operator/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/tests/integration/pipelex/pipes/operator/pipe_imgg/__init__.py b/tests/integration/pipelex/pipes/operator/pipe_imgg/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/tests/integration/pipelex/pipes/operator/pipe_llm/__init__.py b/tests/integration/pipelex/pipes/operator/pipe_llm/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/tests/integration/pipelex/pipes/operator/pipe_llm/test_pipe_llm.py b/tests/integration/pipelex/pipes/operator/pipe_llm/test_pipe_llm.py
index 76c0a9673..c1698885c 100644
--- a/tests/integration/pipelex/pipes/operator/pipe_llm/test_pipe_llm.py
+++ b/tests/integration/pipelex/pipes/operator/pipe_llm/test_pipe_llm.py
@@ -1,4 +1,3 @@
-from typing import List
import pytest
@@ -147,7 +146,7 @@ class TestPipeLLM:
async def test_pipe_llm_attribute_image(
self,
stuff: Stuff,
- attribute_paths: List[str],
+ attribute_paths: list[str],
pipe_run_mode: PipeRunMode,
):
for attribute_path in attribute_paths:
diff --git a/tests/integration/pipelex/pipes/operator/pipe_ocr/__init__.py b/tests/integration/pipelex/pipes/operator/pipe_ocr/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/tests/integration/pipelex/pipes/test_image_inputs.py b/tests/integration/pipelex/pipes/test_image_inputs.py
index df4717e6c..bf6c55e20 100644
--- a/tests/integration/pipelex/pipes/test_image_inputs.py
+++ b/tests/integration/pipelex/pipes/test_image_inputs.py
@@ -52,8 +52,7 @@ class TestImageInputs:
assert pipe_output.main_stuff is not None
async def test_describe_page(self, request: FixtureRequest, pipe_run_mode: PipeRunMode) -> None:
- """
- Test that a pipe can accept a PageContent input, give to the LLM the image via subattributes,
+ """Test that a pipe can accept a PageContent input, give to the LLM the image via subattributes,
But also accepts basic objects
"""
# Create the page content
diff --git a/tests/integration/pipelex/pipes/test_pipe_running_variants.py b/tests/integration/pipelex/pipes/test_pipe_running_variants.py
index 03dea72b8..076b1c201 100644
--- a/tests/integration/pipelex/pipes/test_pipe_running_variants.py
+++ b/tests/integration/pipelex/pipes/test_pipe_running_variants.py
@@ -1,5 +1,5 @@
from pathlib import Path
-from typing import Any, Optional, Tuple, Type
+from typing import Any
import pytest
from pytest import FixtureRequest
@@ -28,7 +28,7 @@ class TestPipeRunningVariants:
self,
pipe_run_mode: PipeRunMode,
request: FixtureRequest,
- pipe_result_handler: Tuple[str, ActivityHandlerForResultFiles],
+ pipe_result_handler: tuple[str, ActivityHandlerForResultFiles],
save_working_memory: Any,
topic: str,
stuff: Stuff,
@@ -54,7 +54,7 @@ class TestPipeRunningVariants:
self,
pipe_run_mode: PipeRunMode,
request: FixtureRequest,
- pipe_result_handler: Tuple[str, ActivityHandlerForResultFiles],
+ pipe_result_handler: tuple[str, ActivityHandlerForResultFiles],
save_working_memory: Any,
topic: str,
pipe_code: str,
@@ -83,11 +83,11 @@ class TestPipeRunningVariants:
self,
pipe_run_mode: PipeRunMode,
request: FixtureRequest,
- pipe_result_handler: Tuple[str, ActivityHandlerForResultFiles],
+ pipe_result_handler: tuple[str, ActivityHandlerForResultFiles],
save_working_memory: Any,
topic: str,
pipe_code: str,
- output_multiplicity: Optional[PipeOutputMultiplicity],
+ output_multiplicity: PipeOutputMultiplicity | None,
):
log.verbose(f"{topic}: just run pipe '{pipe_code}'")
pipe_output = await get_pipe_router().run(
@@ -117,13 +117,13 @@ class TestPipeRunningVariants:
pipe_run_mode: PipeRunMode,
request: FixtureRequest,
pipe_code: str,
- exception: Type[Exception],
+ exception: type[Exception],
expected_error_message: str,
):
failing_pipelines_file_paths = get_config().pipelex.library_config.failing_pipelines_file_paths
library_manager = get_library_manager()
library_manager.load_libraries(
- library_file_paths=[Path(failing_pipeline_file_path) for failing_pipeline_file_path in failing_pipelines_file_paths]
+ library_file_paths=[Path(failing_pipeline_file_path) for failing_pipeline_file_path in failing_pipelines_file_paths],
)
log.verbose(f"This pipe '{pipe_code}' is supposed to cause an error of type: {exception.__name__}")
diff --git a/tests/integration/pipelex/plugins/__init__.py b/tests/integration/pipelex/plugins/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/tests/integration/pipelex/plugins/test_bedrock.py b/tests/integration/pipelex/plugins/test_bedrock.py
index 57c9d4bdf..8c0dad4b3 100644
--- a/tests/integration/pipelex/plugins/test_bedrock.py
+++ b/tests/integration/pipelex/plugins/test_bedrock.py
@@ -1,5 +1,5 @@
import warnings
-from typing import Any, Dict, List
+from typing import Any
import boto3
import pytest
@@ -36,8 +36,8 @@ class TestBedrock:
if any_env_var_is_placeholder(REQUIRED_ENV_VARS):
pytest.skip(f"Some key(s) among {REQUIRED_ENV_VARS} are a placeholder, can't be used to test listing models")
client = boto3.client("bedrock", region_name=bedrock_region_name) # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType]
- response: Dict[str, Any] = client.list_foundation_models(byProvider=bedrock_provider) # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType]
- bedrock_models_list: List[Dict[str, Any]] = response["modelSummaries"] # pyright: ignore[reportUnknownVariableType]
+ response: dict[str, Any] = client.list_foundation_models(byProvider=bedrock_provider) # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType]
+ bedrock_models_list: list[dict[str, Any]] = response["modelSummaries"] # pyright: ignore[reportUnknownVariableType]
if pytestconfig.get_verbosity() >= 2:
# Create and configure the table
console = Console()
@@ -72,8 +72,8 @@ class TestBedrock:
if any_env_var_is_placeholder(REQUIRED_ENV_VARS):
pytest.skip(f"Some key(s) among {REQUIRED_ENV_VARS} are a placeholder, can't be used to test listing models")
client = boto3.client("bedrock", region_name=bedrock_region_name) # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType]
- response: Dict[str, Any] = client.list_inference_profiles() # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType]
- inference_profiles_list: List[Dict[str, Any]] = response["inferenceProfileSummaries"] # pyright: ignore[reportUnknownVariableType]
+ response: dict[str, Any] = client.list_inference_profiles() # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType]
+ inference_profiles_list: list[dict[str, Any]] = response["inferenceProfileSummaries"] # pyright: ignore[reportUnknownVariableType]
if pytestconfig.get_verbosity() >= 2:
# Create and configure the table
console = Console()
diff --git a/tests/integration/pipelex/plugins/test_external_plugin.py b/tests/integration/pipelex/plugins/test_external_plugin.py
index e147437c2..ca309407c 100644
--- a/tests/integration/pipelex/plugins/test_external_plugin.py
+++ b/tests/integration/pipelex/plugins/test_external_plugin.py
@@ -1,4 +1,3 @@
-from typing import Type
import pytest
from polyfactory.factories.pydantic_factory import ModelFactory
@@ -48,7 +47,7 @@ class MockExternalLLMWorker(LLMWorkerAbstract):
async def _gen_object(
self,
llm_job: LLMJob,
- schema: Type[BaseModelTypeVar],
+ schema: type[BaseModelTypeVar],
) -> BaseModelTypeVar:
class ObjectFactory(ModelFactory[schema]): # type: ignore
__model__ = schema
diff --git a/tests/integration/pipelex/plugins/test_openai_image_gen.py b/tests/integration/pipelex/plugins/test_openai_image_gen.py
index b8d5cd718..e66d0542f 100644
--- a/tests/integration/pipelex/plugins/test_openai_image_gen.py
+++ b/tests/integration/pipelex/plugins/test_openai_image_gen.py
@@ -13,7 +13,7 @@ from tests.integration.pipelex.test_data import ImageGenTestCases
@pytest.mark.img_gen
@pytest.mark.inference
@pytest.mark.asyncio(loop_scope="class")
-class TestImggByOpenAIGpt:
+class TestImgGenByOpenAIGpt:
@pytest.mark.parametrize("topic, image_desc", ImageGenTestCases.IMAGE_DESC)
async def test_gpt_image_generation(self, topic: str, image_desc: str):
backend = get_models_manager().get_required_inference_backend("openai")
diff --git a/tests/integration/pipelex/test_data.py b/tests/integration/pipelex/test_data.py
index 63b3bcdbb..a4f22dc0a 100644
--- a/tests/integration/pipelex/test_data.py
+++ b/tests/integration/pipelex/test_data.py
@@ -1,4 +1,4 @@
-from typing import ClassVar, List, Optional, Tuple, Type
+from typing import ClassVar, Optional
from pipelex.core.concepts.concept_factory import ConceptFactory
from pipelex.core.concepts.concept_native import NATIVE_CONCEPTS_DATA, NativeConceptEnum
@@ -23,7 +23,7 @@ class SomeContentWithImageAttribute(StructuredContent):
class SomeContentWithImageSubObjectAttribute(StructuredContent):
image_attribute: ImageContent
- sub_object: Optional["SomeContentWithImageAttribute"] = None
+ sub_object: SomeContentWithImageAttribute | None = None
class PipeTestCases:
@@ -71,7 +71,7 @@ class PipeTestCases:
items=[
TextContent(text="The quick brown fox jumps over the lazy dog"),
ImageContent(url=URL_IMG_GANTT_1),
- ]
+ ],
),
)
@@ -90,7 +90,7 @@ class PipeTestCases:
content=STUFF_CONTENT_WITH_IMAGE_ATTRIBUTE_IN_SUB_OBJECT,
name="stuff_with_image_in_sub_object",
)
- STUFFS_IMAGE_ATTRIBUTES: ClassVar[List[Tuple[Stuff, List[str]]]] = [ # stuff, attribute_paths
+ STUFFS_IMAGE_ATTRIBUTES: ClassVar[list[tuple[Stuff, list[str]]]] = [ # stuff, attribute_paths
(STUFF_WITH_IMAGE_ATTRIBUTE, ["stuff_with_image.image_attribute"]),
(
STUFF_WITH_IMAGE_ATTRIBUTE_IN_SUB_OBJECT,
@@ -113,14 +113,14 @@ class PipeTestCases:
concept_string="answer.Question",
content=USER_TEXT_TRICKY_2,
)
- BLUEPRINT_AND_PIPE: ClassVar[List[Tuple[str, StuffBlueprint, str]]] = [ # topic, blueprint, pipe
+ BLUEPRINT_AND_PIPE: ClassVar[list[tuple[str, StuffBlueprint, str]]] = [ # topic, blueprint, pipe
(
"Tricky question conclude",
TRICKY_QUESTION_BLUEPRINT,
"conclude_tricky_question_by_steps",
),
]
- NO_INPUT: ClassVar[List[Tuple[str, str]]] = [ # topic, pipe
+ NO_INPUT: ClassVar[list[tuple[str, str]]] = [ # topic, pipe
(
"Test with no input",
"test_no_input",
@@ -130,7 +130,7 @@ class PipeTestCases:
"test_no_input_that_could_be_long",
),
]
- NO_INPUT_PARALLEL1: ClassVar[List[Tuple[str, str, Optional[PipeOutputMultiplicity]]]] = [ # topic, pipe, multiplicity
+ NO_INPUT_PARALLEL1: ClassVar[list[tuple[str, str, PipeOutputMultiplicity | None]]] = [ # topic, pipe, multiplicity
(
"Nature colors painting",
"choose_colors",
@@ -148,7 +148,7 @@ class PipeTestCases:
),
]
- STUFF_AND_PIPE: ClassVar[List[Tuple[str, Stuff, str]]] = [ # topic, stuff, pipe_code
+ STUFF_AND_PIPE: ClassVar[list[tuple[str, Stuff, str]]] = [ # topic, stuff, pipe_code
(
"Process Simple Image",
SIMPLE_STUFF_IMAGE,
@@ -160,7 +160,7 @@ class PipeTestCases:
"ocr_page_contents_from_pdf",
),
]
- FAILURE_PIPES: ClassVar[List[Tuple[str, Type[Exception], str]]] = [
+ FAILURE_PIPES: ClassVar[list[tuple[str, type[Exception], str]]] = [
(
"infinite_loop_1",
PipeStackOverflowError,
@@ -170,7 +170,7 @@ class PipeTestCases:
class LibraryTestCases:
- KNOWN_CONCEPTS_AND_PIPES: ClassVar[List[Tuple[str, str]]] = [ # concept, pipe
+ KNOWN_CONCEPTS_AND_PIPES: ClassVar[list[tuple[str, str]]] = [ # concept, pipe
("cars.CarDescription", "generate_car_description"),
("animals.AnimalDescription", "generate_animal_description"),
("flowers.FlowerDescription", "generate_flower_description"),
@@ -178,11 +178,11 @@ class LibraryTestCases:
class PipeOcrTestCases:
- PIPE_OCR_IMAGE_TEST_CASES: ClassVar[List[str]] = [
+ PIPE_OCR_IMAGE_TEST_CASES: ClassVar[list[str]] = [
ImageTestCases.IMAGE_FILE_PATH_PNG,
ImageTestCases.IMAGE_URL_PNG,
]
- PIPE_OCR_PDF_TEST_CASES: ClassVar[List[str]] = PDFTestCases.DOCUMENT_FILE_PATHS + PDFTestCases.DOCUMENT_URLS
+ PIPE_OCR_PDF_TEST_CASES: ClassVar[list[str]] = PDFTestCases.DOCUMENT_FILE_PATHS + PDFTestCases.DOCUMENT_URLS
class ImageGenTestCases:
@@ -192,7 +192,7 @@ class ImageGenTestCases:
Woman typing on a laptop. On the laptop screen you see python code to generate code to write a prompt for an AI model.
"""
- IMAGE_DESC: ClassVar[List[Tuple[str, str]]] = [ # topic, img_gen_prompt_text
+ IMAGE_DESC: ClassVar[list[tuple[str, str]]] = [ # topic, img_gen_prompt_text
# (IMG_GEN_PROMPT_1, IMG_GEN_PROMPT_1),
# (IMG_GEN_PROMPT_2, IMG_GEN_PROMPT_2),
# (IMG_GEN_PROMPT_3, IMG_GEN_PROMPT_3),
@@ -305,7 +305,7 @@ Extract information from the following text:
"""
# Combined test cases for parametrized tests
- STRUCTURE_TEST_CASES: ClassVar[List[Tuple[str, str, str]]] = [ # topic, data, concept
+ STRUCTURE_TEST_CASES: ClassVar[list[tuple[str, str, str]]] = [ # topic, data, concept
("Simple structure basic", SIMPLE_STRUCTURE_1, "ConceptWithSimpleStructure"),
("Simple structure narrative", SIMPLE_STRUCTURE_2, "ConceptWithSimpleStructure"),
("Optionals all present", OPTIONAL_ALL_PRESENT, "ConceptWithOptionals"),
@@ -430,7 +430,7 @@ Extract information from the following text:
"""
# Combined test cases for parametrized tests
- STRUCTURE_TEST_CASES: ClassVar[List[Tuple[str, str, str]]] = [ # topic, data, concept
+ STRUCTURE_TEST_CASES: ClassVar[list[tuple[str, str, str]]] = [ # topic, data, concept
("Dicts with data", DICTS_WITH_DATA, "ConceptWithDicts"),
("Dicts empty", DICTS_EMPTY, "ConceptWithDicts"),
("Unions string variant", UNIONS_STRING_VARIANT, "ConceptWithUnions"),
diff --git a/tests/integration/pipelex/test_inline_structure_concepts.py b/tests/integration/pipelex/test_inline_structure_concepts.py
index c6cbbde14..264aafd54 100644
--- a/tests/integration/pipelex/test_inline_structure_concepts.py
+++ b/tests/integration/pipelex/test_inline_structure_concepts.py
@@ -1,6 +1,5 @@
"""Integration tests for inline structure definitions in concepts."""
-from typing import Dict, Union
import pytest
@@ -21,18 +20,18 @@ class TestInlineStructureConcepts:
def test_inline_structure_definition_creation(self):
"""Test that concepts with inline structure definitions are created correctly."""
# Define inline structure with mixed syntax
- inline_structure: Dict[str, Union[str, ConceptStructureBlueprint]] = {
+ inline_structure: dict[str, str | ConceptStructureBlueprint] = {
"dominant_feature": ConceptStructureBlueprint(
- type=ConceptStructureBlueprintFieldType.TEXT, definition="The most important feature", required=False
+ type=ConceptStructureBlueprintFieldType.TEXT, definition="The most important feature", required=False,
),
"visual_elements": ConceptStructureBlueprint(
- type=ConceptStructureBlueprintFieldType.TEXT, definition="Key visual elements", required=False
+ type=ConceptStructureBlueprintFieldType.TEXT, definition="Key visual elements", required=False,
),
"composition": ConceptStructureBlueprint(
- type=ConceptStructureBlueprintFieldType.TEXT, definition="Analysis of the image composition", required=False
+ type=ConceptStructureBlueprintFieldType.TEXT, definition="Analysis of the image composition", required=False,
),
"color_palette": ConceptStructureBlueprint(
- type=ConceptStructureBlueprintFieldType.TEXT, definition="Description of the main colors", required=False
+ type=ConceptStructureBlueprintFieldType.TEXT, definition="Description of the main colors", required=False,
),
"mood_atmosphere": "The overall mood or atmosphere",
}
@@ -42,7 +41,7 @@ class TestInlineStructureConcepts:
# Create concept from blueprint
concept = ConceptFactory.make_from_blueprint(
- domain="test_domain", concept_code="TestFeatureAnalysis", blueprint=blueprint, concept_codes_from_the_same_domain=["TestFeatureAnalysis"]
+ domain="test_domain", concept_code="TestFeatureAnalysis", blueprint=blueprint, concept_codes_from_the_same_domain=["TestFeatureAnalysis"],
)
# Verify concept properties
@@ -80,7 +79,7 @@ class TestInlineStructureConcepts:
# Create concept from blueprint
concept = ConceptFactory.make_from_blueprint(
- domain="test_domain", concept_code="TestStringRef", blueprint=blueprint, concept_codes_from_the_same_domain=["TestStringRef"]
+ domain="test_domain", concept_code="TestStringRef", blueprint=blueprint, concept_codes_from_the_same_domain=["TestStringRef"],
)
# Verify concept properties
@@ -96,7 +95,7 @@ class TestInlineStructureConcepts:
# Create concept from blueprint
concept = ConceptFactory.make_from_blueprint(
- domain="test_domain", concept_code="TestAutoDetect", blueprint=blueprint, concept_codes_from_the_same_domain=["TestAutoDetect"]
+ domain="test_domain", concept_code="TestAutoDetect", blueprint=blueprint, concept_codes_from_the_same_domain=["TestAutoDetect"],
)
# Should default to TextContent since TestAutoDetect is not a registered class
@@ -104,7 +103,7 @@ class TestInlineStructureConcepts:
def test_inline_structure_with_complex_types(self):
"""Test inline structure with complex field types."""
- inline_structure: Dict[str, Union[str, ConceptStructureBlueprint]] = {
+ inline_structure: dict[str, str | ConceptStructureBlueprint] = {
"title": ConceptStructureBlueprint(type=ConceptStructureBlueprintFieldType.TEXT, definition="Document title"),
"tags": ConceptStructureBlueprint(
type=ConceptStructureBlueprintFieldType.LIST,
@@ -122,14 +121,14 @@ class TestInlineStructureConcepts:
"priority": ConceptStructureBlueprint(choices=["low", "medium", "high"], definition="Priority level", required=False),
"page_count": ConceptStructureBlueprint(type=ConceptStructureBlueprintFieldType.INTEGER, definition="Number of pages", required=False),
"is_active": ConceptStructureBlueprint(
- type=ConceptStructureBlueprintFieldType.BOOLEAN, definition="Whether document is active", required=False
+ type=ConceptStructureBlueprintFieldType.BOOLEAN, definition="Whether document is active", required=False,
),
}
blueprint = ConceptBlueprint(definition="Complex document structure", structure=inline_structure)
concept = ConceptFactory.make_from_blueprint(
- domain="test_domain", concept_code="ComplexDocument", blueprint=blueprint, concept_codes_from_the_same_domain=["ComplexDocument"]
+ domain="test_domain", concept_code="ComplexDocument", blueprint=blueprint, concept_codes_from_the_same_domain=["ComplexDocument"],
)
# Verify concept creation
@@ -174,25 +173,25 @@ not a registered subclass of StuffContent",
def test_multiple_inline_structures_do_not_conflict(self):
"""Test that multiple inline structures with same field names don't conflict."""
# First structure
- structure1: Dict[str, Union[str, ConceptStructureBlueprint]] = {
+ structure1: dict[str, str | ConceptStructureBlueprint] = {
"name": ConceptStructureBlueprint(type=ConceptStructureBlueprintFieldType.TEXT, definition="Person name"),
"age": ConceptStructureBlueprint(type=ConceptStructureBlueprintFieldType.INTEGER, definition="Person age", required=False),
}
blueprint1 = ConceptBlueprint(definition="Person information", structure=structure1)
concept1 = ConceptFactory.make_from_blueprint(
- domain="test_domain", concept_code="Person", blueprint=blueprint1, concept_codes_from_the_same_domain=["Person"]
+ domain="test_domain", concept_code="Person", blueprint=blueprint1, concept_codes_from_the_same_domain=["Person"],
)
# Second structure with same field names but different context
- structure2: Dict[str, Union[str, ConceptStructureBlueprint]] = {
+ structure2: dict[str, str | ConceptStructureBlueprint] = {
"name": "Product name",
"age": ConceptStructureBlueprint(type=ConceptStructureBlueprintFieldType.INTEGER, definition="Product age in days", required=False),
}
blueprint2 = ConceptBlueprint(definition="Product information", structure=structure2)
concept2 = ConceptFactory.make_from_blueprint(
- domain="test_domain", concept_code="Product", blueprint=blueprint2, concept_codes_from_the_same_domain=["Product"]
+ domain="test_domain", concept_code="Product", blueprint=blueprint2, concept_codes_from_the_same_domain=["Product"],
)
# Both should be created successfully
diff --git a/tests/integration/pipelex/test_libraries.py b/tests/integration/pipelex/test_libraries.py
index 0709d76e5..d11302bb6 100644
--- a/tests/integration/pipelex/test_libraries.py
+++ b/tests/integration/pipelex/test_libraries.py
@@ -1,9 +1,7 @@
-"""
-make t TEST=TestLibraries
+"""make t TEST=TestLibraries
"""
from pathlib import Path
-from typing import Optional
import pytest
from rich import box
@@ -20,7 +18,7 @@ from tests.integration.pipelex.test_data import LibraryTestCases
def pretty_print_all_pipes(
pipe_library: PipeLibrary,
- title: Optional[str] = None,
+ title: str | None = None,
):
console = Console()
table = Table(
@@ -54,7 +52,7 @@ def pretty_print_all_pipes(
def pretty_print_all_concepts(
concept_library: ConceptLibrary,
- title: Optional[str] = None,
+ title: str | None = None,
):
console = Console()
table = Table(
@@ -101,7 +99,7 @@ class TestLibraries:
# Test individual concepts and pipes
assert library_manager.concept_library.get_required_concept(concept_string=known_concept) is not None
pretty_print(
- f"Concept: {known_concept} is correctly loaded as {library_manager.concept_library.get_required_concept(concept_string=known_concept)}"
+ f"Concept: {known_concept} is correctly loaded as {library_manager.concept_library.get_required_concept(concept_string=known_concept)}",
)
assert library_manager.pipe_library.get_optional_pipe(known_pipe) is not None
pretty_print(f"Pipe: {known_pipe} is correctly loaded as {library_manager.pipe_library.get_optional_pipe(known_pipe)}")
diff --git a/tests/integration/pipelex/tools/aws/__init__.py b/tests/integration/pipelex/tools/aws/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/tests/integration/pipelex/tools/class_registry/__init__.py b/tests/integration/pipelex/tools/class_registry/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/tests/integration/pipelex/tools/class_registry/test_class_registry_utils.py b/tests/integration/pipelex/tools/class_registry/test_class_registry_utils.py
index 2e4e40af4..0fea03ef5 100644
--- a/tests/integration/pipelex/tools/class_registry/test_class_registry_utils.py
+++ b/tests/integration/pipelex/tools/class_registry/test_class_registry_utils.py
@@ -16,7 +16,7 @@ class TestClassRegistryUtilsIntegration:
"""Integration test for registering StuffContent classes recursively."""
class_registry = get_class_registry()
ClassRegistryUtils.register_classes_in_folder(
- folder_path=ClassRegistryTestCases.MODEL_FOLDER_PATH, base_class=StuffContent, is_recursive=True
+ folder_path=ClassRegistryTestCases.MODEL_FOLDER_PATH, base_class=StuffContent, is_recursive=True,
)
# Should register classes that inherit from StuffContent
@@ -28,7 +28,7 @@ class TestClassRegistryUtilsIntegration:
"""Integration test for registering StuffContent classes non-recursively."""
class_registry = get_class_registry()
ClassRegistryUtils.register_classes_in_folder(
- folder_path=ClassRegistryTestCases.MODEL_FOLDER_PATH, base_class=StuffContent, is_recursive=False
+ folder_path=ClassRegistryTestCases.MODEL_FOLDER_PATH, base_class=StuffContent, is_recursive=False,
)
# Should register only top-level StuffContent classes
diff --git a/tests/integration/pipelex/tools/httpx/__init__.py b/tests/integration/pipelex/tools/httpx/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/tests/integration/pipelex/tools/httpx/test_httpx_fetching.py b/tests/integration/pipelex/tools/httpx/test_httpx_fetching.py
index 8374b460d..18b9b41f9 100644
--- a/tests/integration/pipelex/tools/httpx/test_httpx_fetching.py
+++ b/tests/integration/pipelex/tools/httpx/test_httpx_fetching.py
@@ -15,7 +15,7 @@ class TestHttpxFetching:
assert (
await fetch_file_from_url_httpx_async(
url=url,
- timeout=60,
+ request_timeout=60,
)
is not None
)
@@ -28,7 +28,7 @@ class TestHttpxFetching:
assert (
fetch_file_from_url_httpx(
url=url,
- timeout=60,
+ request_timeout=60,
)
is not None
)
diff --git a/tests/integration/pipelex/tools/pdf/__init__.py b/tests/integration/pipelex/tools/pdf/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/tests/test_pipelines/__init__.py b/tests/test_pipelines/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/tests/test_pipelines/concept_library/__init__.py b/tests/test_pipelines/concept_library/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/tests/test_pipelines/concept_library/is_compatible_concept_code.py b/tests/test_pipelines/concept_library/is_compatible_concept_code.py
index e3ddcd65d..40f1406d3 100644
--- a/tests/test_pipelines/concept_library/is_compatible_concept_code.py
+++ b/tests/test_pipelines/concept_library/is_compatible_concept_code.py
@@ -1,4 +1,3 @@
-from typing import Dict, List, Optional
from pydantic import Field
@@ -6,17 +5,17 @@ from pipelex.core.stuffs.stuff_content import StructuredContent
class FundamentalsDoc(StructuredContent):
- project_overview: Optional[str] = Field(
+ project_overview: str | None = Field(
None,
description="Mission, key features, architecture diagram, demo links",
)
- core_concepts: Optional[Dict[str, str]] = Field(
+ core_concepts: dict[str, str] | None = Field(
None,
description=(
"Names and definitions for project-specific terms, acronyms, data model names, background knowledge, business rules, domain entities"
),
)
- repository_map: Optional[str] = Field(
+ repository_map: str | None = Field(
None,
description="Directory layout explanation and purpose of each folder",
)
@@ -26,20 +25,20 @@ class DocumentationConcept(StructuredContent):
"""A specialized documentation concept that extends FundamentalsDoc."""
title: str = Field(..., description="Title of the documentation")
- sections: List[str] = Field(default_factory=list, description="List of section names")
- last_updated: Optional[str] = Field(None, description="Last update timestamp")
+ sections: list[str] = Field(default_factory=list, description="List of section names")
+ last_updated: str | None = Field(None, description="Last update timestamp")
class MultiMediaConcept(StructuredContent):
"""A concept that combines text and images."""
text_content: str = Field(..., description="The text content")
- image_urls: List[str] = Field(default_factory=list, description="List of image URLs")
- caption: Optional[str] = Field(None, description="Optional caption for the content")
+ image_urls: list[str] = Field(default_factory=list, description="List of image URLs")
+ caption: str | None = Field(None, description="Optional caption for the content")
class IndependentConcept(StructuredContent):
"""An independent concept with its own structure."""
unique_field: str = Field(..., description="A unique field for this concept")
- metadata: Dict[str, str] = Field(default_factory=dict, description="Metadata dictionary")
+ metadata: dict[str, str] = Field(default_factory=dict, description="Metadata dictionary")
diff --git a/tests/test_pipelines/discord_newsletter.py b/tests/test_pipelines/discord_newsletter.py
index 2680b2110..6de3cb373 100644
--- a/tests/test_pipelines/discord_newsletter.py
+++ b/tests/test_pipelines/discord_newsletter.py
@@ -1,4 +1,3 @@
-from typing import List
from pydantic import Field
@@ -25,8 +24,8 @@ class DiscordMessage(StructuredContent):
author: str = Field(..., description="Author of the message")
content: str = Field(..., description="Content of the message")
- attachments: List[Attachment] = Field(default_factory=list, description="List of message attachments")
- embeds: List[Embed] = Field(default_factory=list, description="List of message embeds")
+ attachments: list[Attachment] = Field(default_factory=list, description="List of message attachments")
+ embeds: list[Embed] = Field(default_factory=list, description="List of message embeds")
link: str = Field(..., description="Link to the message")
@@ -35,20 +34,20 @@ class DiscordChannelUpdate(StructuredContent):
name: str = Field(..., description="Name of the Discord channel")
position: int = Field(..., description="Position of the channel")
- messages: List[DiscordMessage] = Field(default_factory=list, description="List of messages in the channel")
+ messages: list[DiscordMessage] = Field(default_factory=list, description="List of messages in the channel")
class ChannelSummary(StructuredContent):
"""Represents a summarized Discord channel for newsletter inclusion"""
channel_name: str = Field(..., description="Name of the Discord channel")
- summary_items: List[str] = Field(..., description="Well-written summaries of the channel's activity")
+ summary_items: list[str] = Field(..., description="Well-written summaries of the channel's activity")
class Newsletter(StructuredContent):
"""Represents the final newsletter content"""
weekly_summary: str = Field(..., description="200 character summary of weekly Share channel content")
- new_members: List[str] = Field(default_factory=list, description="New member introductions in bullet points")
- channel_sections: List[ChannelSummary] = Field(default_factory=list, description="Ordered channel summaries")
- geographic_hubs: List[ChannelSummary] = Field(default_factory=list, description="Geographic hub channels grouped at end")
+ new_members: list[str] = Field(default_factory=list, description="New member introductions in bullet points")
+ channel_sections: list[ChannelSummary] = Field(default_factory=list, description="Ordered channel summaries")
+ geographic_hubs: list[ChannelSummary] = Field(default_factory=list, description="Geographic hub channels grouped at end")
diff --git a/tests/test_pipelines/misc_tests/subfolder_2/subfolder/__init__.py b/tests/test_pipelines/misc_tests/subfolder_2/subfolder/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/tests/test_pipelines/pipe_controllers/pipe_batch/__init__.py b/tests/test_pipelines/pipe_controllers/pipe_batch/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/tests/test_pipelines/pipe_controllers/pipe_condition/__init__.py b/tests/test_pipelines/pipe_controllers/pipe_condition/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/tests/test_pipelines/pipe_controllers/pipe_parallel/__init__.py b/tests/test_pipelines/pipe_controllers/pipe_parallel/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/tests/test_pipelines/pipe_controllers/pipe_sequence/__init__.py b/tests/test_pipelines/pipe_controllers/pipe_sequence/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/tests/test_pipelines/pipe_llm_prompt.py b/tests/test_pipelines/pipe_llm_prompt.py
index 79920a4f1..3464df296 100644
--- a/tests/test_pipelines/pipe_llm_prompt.py
+++ b/tests/test_pipelines/pipe_llm_prompt.py
@@ -1,4 +1,4 @@
-from typing import List, Literal
+from typing import Literal
from pydantic import Field
@@ -60,5 +60,5 @@ class PersonContent(StructuredContent):
class ComplexListContent(StructuredContent):
"""A content class with list fields."""
- items: List[str] = Field(description="List of items")
- tags: List[str] = Field(description="List of tags")
+ items: list[str] = Field(description="List of items")
+ tags: list[str] = Field(description="List of tags")
diff --git a/tests/test_pipelines/test_file_func_registry.py b/tests/test_pipelines/test_file_func_registry.py
index 73a317db4..7730054e9 100644
--- a/tests/test_pipelines/test_file_func_registry.py
+++ b/tests/test_pipelines/test_file_func_registry.py
@@ -1,4 +1,3 @@
-from typing import List
from pydantic import Field
@@ -27,19 +26,19 @@ def read_file_content(working_memory: WorkingMemory) -> ListContent[CodebaseFile
Returns:
ListContent of CodebaseFileContent objects
- """
+ """
file_paths_list = working_memory.get_stuff_as_list("related_file_paths", item_type=FilePath)
- codebase_files: List[CodebaseFileContent] = []
+ codebase_files: list[CodebaseFileContent] = []
for file_path in file_paths_list.items:
try:
- with open(file_path.path, "r", encoding="utf-8") as file:
+ with open(file_path.path, encoding="utf-8") as file:
content = file.read()
codebase_files.append(CodebaseFileContent(file_path=file_path.path, file_content=content))
except Exception as e:
codebase_files.append(
- CodebaseFileContent(file_path=file_path.path, file_content=f"# File not found or unreadable: {file_path.path}\n# Error: {str(e)}")
+ CodebaseFileContent(file_path=file_path.path, file_content=f"# File not found or unreadable: {file_path.path}\n# Error: {e!s}"),
)
return ListContent[CodebaseFileContent](items=codebase_files)
diff --git a/tests/test_pipelines/test_structures_basic.py b/tests/test_pipelines/test_structures_basic.py
index 5460a61f4..cfc6fddfe 100644
--- a/tests/test_pipelines/test_structures_basic.py
+++ b/tests/test_pipelines/test_structures_basic.py
@@ -1,7 +1,6 @@
"""Test structures for basic concepts without union types."""
from datetime import datetime
-from typing import List, Optional
from pydantic import Field
@@ -20,22 +19,22 @@ class ConceptWithOptionals(StructuredContent):
"""A structure with optional fields."""
required_field: str = Field(..., description="A required field")
- optional_string: Optional[str] = Field(None, description="An optional string field")
- optional_number: Optional[int] = Field(None, description="An optional number field")
- optional_date: Optional[datetime] = Field(None, description="An optional date field")
+ optional_string: str | None = Field(None, description="An optional string field")
+ optional_number: int | None = Field(None, description="An optional number field")
+ optional_date: datetime | None = Field(None, description="An optional date field")
class ConceptWithLists(StructuredContent):
"""A structure with list fields."""
- string_list: List[str] = Field(default_factory=list, description="A list of strings")
- number_list: List[int] = Field(default_factory=list, description="A list of numbers")
- optional_list: Optional[List[str]] = Field(None, description="An optional list")
+ string_list: list[str] = Field(default_factory=list, description="A list of strings")
+ number_list: list[int] = Field(default_factory=list, description="A list of numbers")
+ optional_list: list[str] | None = Field(None, description="An optional list")
class ConceptWithNestedStructures(StructuredContent):
"""A structure with nested structures."""
simple_nested: ConceptWithSimpleStructure = Field(..., description="A nested simple structure")
- optional_nested: Optional[ConceptWithOptionals] = Field(None, description="An optional nested structure")
- list_of_nested: List[ConceptWithSimpleStructure] = Field(default_factory=list, description="A list of nested structures")
+ optional_nested: ConceptWithOptionals | None = Field(None, description="An optional nested structure")
+ list_of_nested: list[ConceptWithSimpleStructure] = Field(default_factory=list, description="A list of nested structures")
diff --git a/tests/test_pipelines/test_structures_complex.py b/tests/test_pipelines/test_structures_complex.py
index 06e13705d..7ba682a6d 100644
--- a/tests/test_pipelines/test_structures_complex.py
+++ b/tests/test_pipelines/test_structures_complex.py
@@ -1,6 +1,5 @@
"""Test structures for complex concepts (dicts, unions, etc.)."""
-from typing import Dict, List, Optional, Union
from pydantic import Field
@@ -11,35 +10,35 @@ from tests.test_pipelines.test_structures_basic import ConceptWithOptionals, Con
class ConceptWithDicts(StructuredContent):
"""A structure with dictionary fields."""
- string_dict: Dict[str, str] = Field(default_factory=dict, description="A dictionary of strings")
- number_dict: Dict[str, int] = Field(default_factory=dict, description="A dictionary of numbers")
- optional_dict: Optional[Dict[str, str]] = Field(None, description="An optional dictionary")
+ string_dict: dict[str, str] = Field(default_factory=dict, description="A dictionary of strings")
+ number_dict: dict[str, int] = Field(default_factory=dict, description="A dictionary of numbers")
+ optional_dict: dict[str, str] | None = Field(None, description="An optional dictionary")
class ConceptWithUnions(StructuredContent):
"""A structure with union types."""
- string_or_int: Union[str, int] = Field(..., description="A field that can be string or int")
- optional_union: Optional[Union[str, bool]] = Field(None, description="An optional union field")
- list_of_unions: List[Union[str, int]] = Field(default_factory=list, description="A list of union types")
+ string_or_int: str | int = Field(..., description="A field that can be string or int")
+ optional_union: str | bool | None = Field(None, description="An optional union field")
+ list_of_unions: list[str | int] = Field(default_factory=list, description="A list of union types")
class ConceptWithComplexUnions(StructuredContent):
"""A structure with more complex union combinations."""
- mixed_dict: Dict[str, Union[str, int, bool]] = Field(default_factory=dict, description="A dictionary with mixed value types")
- union_or_list: Union[str, List[int]] = Field(..., description="Either a string or a list of integers")
- optional_complex_union: Optional[Union[Dict[str, str], List[str]]] = Field(None, description="Optional dict or list")
- number_or_bool: Union[int, float, bool] = Field(..., description="Number or boolean value")
+ mixed_dict: dict[str, str | int | bool] = Field(default_factory=dict, description="A dictionary with mixed value types")
+ union_or_list: str | list[int] = Field(..., description="Either a string or a list of integers")
+ optional_complex_union: dict[str, str] | list[str] | None = Field(None, description="Optional dict or list")
+ number_or_bool: int | float | bool = Field(..., description="Number or boolean value")
class ConceptWithNestedUnions(StructuredContent):
"""A structure with nested unions including other structures."""
- simple_or_complex: Union[ConceptWithSimpleStructure, ConceptWithOptionals] = Field(..., description="Simple or complex structure")
- list_of_union_structures: List[Union[ConceptWithSimpleStructure, ConceptWithUnions]] = Field(
- default_factory=list, description="List of different structure types"
+ simple_or_complex: ConceptWithSimpleStructure | ConceptWithOptionals = Field(..., description="Simple or complex structure")
+ list_of_union_structures: list[ConceptWithSimpleStructure | ConceptWithUnions] = Field(
+ default_factory=list, description="List of different structure types",
)
- optional_nested_union: Optional[Union[ConceptWithUnions, ConceptWithComplexUnions]] = Field(
- None, description="Optional union of union structures"
+ optional_nested_union: ConceptWithUnions | ConceptWithComplexUnions | None = Field(
+ None, description="Optional union of union structures",
)
diff --git a/tests/test_pipelines/test_tweet.py b/tests/test_pipelines/test_tweet.py
index c4ae1cf08..b96b842dd 100644
--- a/tests/test_pipelines/test_tweet.py
+++ b/tests/test_pipelines/test_tweet.py
@@ -5,4 +5,3 @@ class OptimizedTweet(TextContent):
"""A tweet optimized for Twitter/X engagement following best practices."""
# TBD
- pass
diff --git a/tests/unit/pipelex/cli/commands/__init__.py b/tests/unit/pipelex/cli/commands/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/tests/unit/pipelex/cli/commands/test_init_cmd.py b/tests/unit/pipelex/cli/commands/test_init_cmd.py
index 69d4755fe..97501af0c 100644
--- a/tests/unit/pipelex/cli/commands/test_init_cmd.py
+++ b/tests/unit/pipelex/cli/commands/test_init_cmd.py
@@ -10,10 +10,7 @@ from pipelex.exceptions import PipelexCLIError
class TestInitCmd:
- """Test cases for the init command functionality."""
-
def test_do_init_config_copies_all_files(self, tmp_path: Path, mocker: MockerFixture) -> None:
- """Test that all files from config template are copied to target directory."""
# Setup directories
template_dir = tmp_path / "config_template"
template_dir.mkdir()
@@ -38,7 +35,6 @@ class TestInitCmd:
mock_echo.assert_any_call(f"✅ Copied 1 files to {target_dir}:")
def test_do_init_config_skips_existing_files(self, tmp_path: Path, mocker: MockerFixture) -> None:
- """Test that existing files are skipped when reset=False."""
# Setup directories with existing file
template_dir = tmp_path / "config_template"
template_dir.mkdir()
@@ -73,7 +69,6 @@ class TestInitCmd:
assert any("ℹ️ Skipped 1 existing files" in call for call in calls)
def test_do_init_config_reset_overwrites_files(self, tmp_path: Path, mocker: MockerFixture) -> None:
- """Test that reset=True overwrites existing files."""
# Setup directories with existing file
template_dir = tmp_path / "config_template"
template_dir.mkdir()
@@ -104,7 +99,6 @@ class TestInitCmd:
assert not any("ℹ️ Skipped" in call for call in calls)
def test_do_init_config_nested_directory_structure(self, tmp_path: Path, mocker: MockerFixture) -> None:
- """Test copying of nested directory structures."""
# Setup complex nested structure
template_dir = tmp_path / "config_template"
template_dir.mkdir()
@@ -144,7 +138,6 @@ class TestInitCmd:
assert any("✅ Copied 3 files to" in call for call in calls)
def test_do_init_config_handles_permission_error(self, tmp_path: Path, mocker: MockerFixture) -> None:
- """Test handling of permission errors during file copying."""
# Setup directories
template_dir = tmp_path / "config_template"
template_dir.mkdir()
@@ -168,7 +161,6 @@ class TestInitCmd:
assert "Failed to initialize configuration" in str(exc_info.value)
def test_do_init_config_creates_target_directory(self, tmp_path: Path, mocker: MockerFixture) -> None:
- """Test that target directory is created if it doesn't exist."""
# Setup template directory only
template_dir = tmp_path / "config_template"
template_dir.mkdir()
diff --git a/tests/unit/pipelex/client/test_api_serialization.py b/tests/unit/pipelex/client/test_api_serialization.py
index ed9b40fee..93f5f1789 100644
--- a/tests/unit/pipelex/client/test_api_serialization.py
+++ b/tests/unit/pipelex/client/test_api_serialization.py
@@ -2,7 +2,7 @@ import json
from datetime import datetime
from decimal import Decimal
from enum import Enum
-from typing import Any, Dict, List, Optional
+from typing import Any
import pytest
from pydantic import BaseModel
@@ -26,8 +26,8 @@ class Priority(Enum):
class TaskStatus(BaseModel):
is_complete: bool
- completion_date: Optional[datetime] = None
- notes: List[str] = []
+ completion_date: datetime | None = None
+ notes: list[str] = []
class ComplexTask(BaseModel):
@@ -35,34 +35,31 @@ class ComplexTask(BaseModel):
title: str
priority: Priority
status: TaskStatus
- due_dates: List[datetime]
- metadata: Dict[str, Any]
- score: Optional[Decimal] = None
+ due_dates: list[datetime]
+ metadata: dict[str, Any]
+ score: Decimal | None = None
class Project(BaseModel):
name: str
created_at: datetime
- tasks: List[ComplexTask]
- settings: Dict[str, Any]
+ tasks: list[ComplexTask]
+ settings: dict[str, Any]
class TestApiSerialization:
- """Test API-specific serialization with kajson, datetime formatting, and cleanup."""
-
@pytest.fixture
def datetime_content_memory(self) -> WorkingMemory:
- """Create WorkingMemory with datetime content."""
datetime_event = DateTimeEvent(
event_name="Project Kickoff Meeting",
- start_time=datetime(2024, 1, 15, 10, 0, 0),
- end_time=datetime(2024, 1, 15, 11, 30, 0),
- created_at=datetime(2024, 1, 1, 9, 0, 0),
+ start_time=datetime(2024, 1, 15, 10, 0, 0, tzinfo=None),
+ end_time=datetime(2024, 1, 15, 11, 30, 0, tzinfo=None),
+ created_at=datetime(2024, 1, 1, 9, 0, 0, tzinfo=None),
)
stuff = StuffFactory.make_stuff(
concept=ConceptFactory.make(
- concept_code="DateTimeEvent", domain="event", definition="event.DateTimeEvent", structure_class_name="DateTimeEvent"
+ concept_code="DateTimeEvent", domain="event", definition="event.DateTimeEvent", structure_class_name="DateTimeEvent",
),
name="project_meeting",
content=datetime_event,
@@ -71,7 +68,6 @@ class TestApiSerialization:
@pytest.fixture
def text_content_memory(self) -> WorkingMemory:
- """Create WorkingMemory with text content."""
stuff = StuffFactory.make_stuff(
concept=ConceptFactory.make_native_concept(native_concept_data=NATIVE_CONCEPTS_DATA[NativeConceptEnum.TEXT]),
name="sample_text",
@@ -81,7 +77,6 @@ class TestApiSerialization:
@pytest.fixture
def number_content_memory(self) -> WorkingMemory:
- """Create WorkingMemory with number content."""
number_content = NumberContent(number=3.14159)
stuff = StuffFactory.make_stuff(
concept=ConceptFactory.make_native_concept(native_concept_data=NATIVE_CONCEPTS_DATA[NativeConceptEnum.NUMBER]),
@@ -91,7 +86,6 @@ class TestApiSerialization:
return WorkingMemoryFactory.make_from_single_stuff(stuff=stuff)
def test_serialize_working_memory_with_datetime(self, datetime_content_memory: WorkingMemory):
- """Test that datetime content is properly serialized to ISO format strings."""
compact_memory = ApiSerializer.serialize_working_memory_for_api(datetime_content_memory)
# Should have one entry for the datetime content
@@ -124,7 +118,6 @@ class TestApiSerialization:
assert "__class__" not in content
def test_api_serialized_memory_is_json_serializable(self, datetime_content_memory: WorkingMemory):
- """Test that API serialized memory is JSON serializable."""
compact_memory = ApiSerializer.serialize_working_memory_for_api(datetime_content_memory)
# This should NOT raise an exception now
@@ -141,7 +134,6 @@ class TestApiSerialization:
assert isinstance(content["created_at"], str)
def test_serialize_text_content(self, text_content_memory: WorkingMemory):
- """Test that text content is handled specially."""
compact_memory = ApiSerializer.serialize_working_memory_for_api(text_content_memory)
assert len(compact_memory) == 1
@@ -153,7 +145,6 @@ class TestApiSerialization:
assert text_blueprint["content"] == "Sample text content"
def test_serialize_number_content(self, number_content_memory: WorkingMemory):
- """Test that number content is properly serialized."""
compact_memory = ApiSerializer.serialize_working_memory_for_api(number_content_memory)
assert len(compact_memory) == 1
diff --git a/tests/unit/pipelex/cogt/models/__init__.py b/tests/unit/pipelex/cogt/models/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/tests/unit/pipelex/cogt/models/test_model_deck.py b/tests/unit/pipelex/cogt/models/test_model_deck.py
index 0620618ce..185f5eaa2 100644
--- a/tests/unit/pipelex/cogt/models/test_model_deck.py
+++ b/tests/unit/pipelex/cogt/models/test_model_deck.py
@@ -1,7 +1,3 @@
-"""Unit tests for ModelDeck class."""
-
-from typing import List, Optional
-
import pytest
from pytest_mock import MockerFixture
@@ -13,10 +9,7 @@ from pipelex.cogt.usage.cost_category import CostCategory
class TestModelDeckGetOptionalInferenceModel:
- """Unit tests for get_optional_inference_model method."""
-
def _create_test_model_spec(self, name: str) -> InferenceModelSpec:
- """Helper to create a test InferenceModelSpec."""
return InferenceModelSpec(
backend_name="test_backend",
name=name,
@@ -30,10 +23,9 @@ class TestModelDeckGetOptionalInferenceModel:
def _create_test_model_deck(
self,
- inference_models: Optional[dict[str, InferenceModelSpec]] = None,
- aliases: Optional[dict[str, str | List[str]]] = None,
+ inference_models: dict[str, InferenceModelSpec] | None = None,
+ aliases: dict[str, str | list[str]] | None = None,
) -> ModelDeck:
- """Helper to create a test ModelDeck."""
return ModelDeck(
inference_models=inference_models or {},
aliases=aliases or {},
@@ -49,7 +41,6 @@ class TestModelDeckGetOptionalInferenceModel:
)
def test_direct_model_lookup_success(self):
- """Test successful direct model lookup."""
# Arrange
model_spec = self._create_test_model_spec("gpt-4")
model_deck = self._create_test_model_deck(inference_models={"gpt-4": model_spec})
@@ -61,7 +52,6 @@ class TestModelDeckGetOptionalInferenceModel:
assert result == model_spec
def test_direct_model_lookup_not_found(self, mocker: MockerFixture):
- """Test direct model lookup when model doesn't exist."""
# Arrange
model_deck = self._create_test_model_deck()
mock_log = mocker.patch("pipelex.cogt.models.model_deck.log")
@@ -74,7 +64,6 @@ class TestModelDeckGetOptionalInferenceModel:
mock_log.warning.assert_called_once_with("Skipping model handle 'nonexistent-model' because it's not found in deck")
def test_simple_string_alias_resolution_success(self, mocker: MockerFixture):
- """Test successful resolution of a simple string alias."""
# Arrange
model_spec = self._create_test_model_spec("gpt-4")
model_deck = self._create_test_model_deck(inference_models={"gpt-4": model_spec}, aliases={"best-gpt": "gpt-4"})
@@ -88,7 +77,6 @@ class TestModelDeckGetOptionalInferenceModel:
mock_log.debug.assert_called_once_with("Redirection for 'best-gpt': gpt-4")
def test_simple_string_alias_resolution_not_found(self, mocker: MockerFixture):
- """Test string alias resolution when target model doesn't exist."""
# Arrange
model_deck = self._create_test_model_deck(aliases={"best-gpt": "nonexistent-model"})
mock_log = mocker.patch("pipelex.cogt.models.model_deck.log")
@@ -103,7 +91,6 @@ class TestModelDeckGetOptionalInferenceModel:
mock_log.warning.assert_called_with("Skipping model handle 'best-gpt' because it's not found in deck")
def test_list_alias_resolution_first_success(self, mocker: MockerFixture):
- """Test list alias resolution where first model in list exists."""
# Arrange
model_spec = self._create_test_model_spec("gpt-4")
model_deck = self._create_test_model_deck(inference_models={"gpt-4": model_spec}, aliases={"best-model": ["gpt-4", "claude-3"]})
@@ -117,11 +104,10 @@ class TestModelDeckGetOptionalInferenceModel:
mock_log.debug.assert_called_once_with("Redirection for 'best-model': ['gpt-4', 'claude-3']")
def test_list_alias_resolution_second_success(self, mocker: MockerFixture):
- """Test list alias resolution where second model in list exists."""
# Arrange
model_spec = self._create_test_model_spec("claude-3")
model_deck = self._create_test_model_deck(
- inference_models={"claude-3": model_spec}, aliases={"best-model": ["nonexistent-model", "claude-3"]}
+ inference_models={"claude-3": model_spec}, aliases={"best-model": ["nonexistent-model", "claude-3"]},
)
mock_log = mocker.patch("pipelex.cogt.models.model_deck.log")
@@ -133,7 +119,6 @@ class TestModelDeckGetOptionalInferenceModel:
mock_log.debug.assert_called_once_with("Redirection for 'best-model': ['nonexistent-model', 'claude-3']")
def test_list_alias_resolution_none_found(self, mocker: MockerFixture):
- """Test list alias resolution where no models in list exist."""
# Arrange
model_deck = self._create_test_model_deck(aliases={"best-model": ["nonexistent-1", "nonexistent-2"]})
mock_log = mocker.patch("pipelex.cogt.models.model_deck.log")
@@ -148,7 +133,6 @@ class TestModelDeckGetOptionalInferenceModel:
assert mock_log.warning.call_count == 3
def test_recursive_alias_resolution_success(self, mocker: MockerFixture):
- """Test recursive alias resolution where alias points to another alias."""
# Arrange
model_spec = self._create_test_model_spec("gpt-4")
model_deck = self._create_test_model_deck(inference_models={"gpt-4": model_spec}, aliases={"best-model": "best-gpt", "best-gpt": "gpt-4"})
@@ -163,11 +147,10 @@ class TestModelDeckGetOptionalInferenceModel:
assert mock_log.debug.call_count == 2
def test_recursive_alias_resolution_with_list(self, mocker: MockerFixture):
- """Test recursive alias resolution where alias points to list containing another alias."""
# Arrange
model_spec = self._create_test_model_spec("gpt-4")
model_deck = self._create_test_model_deck(
- inference_models={"gpt-4": model_spec}, aliases={"best-model": ["nonexistent", "best-gpt"], "best-gpt": "gpt-4"}
+ inference_models={"gpt-4": model_spec}, aliases={"best-model": ["nonexistent", "best-gpt"], "best-gpt": "gpt-4"},
)
mock_log = mocker.patch("pipelex.cogt.models.model_deck.log")
@@ -180,7 +163,6 @@ class TestModelDeckGetOptionalInferenceModel:
assert mock_log.debug.call_count >= 2
def test_empty_alias_list(self, mocker: MockerFixture):
- """Test behavior with empty alias list."""
# Arrange
model_deck = self._create_test_model_deck(aliases={"empty-alias": []})
mock_log = mocker.patch("pipelex.cogt.models.model_deck.log")
@@ -195,7 +177,6 @@ class TestModelDeckGetOptionalInferenceModel:
mock_log.warning.assert_called_once_with("Skipping model handle 'empty-alias' because it's not found in deck")
def test_circular_alias_prevention(self, mocker: MockerFixture):
- """Test that circular aliases don't cause infinite loops."""
# Note: The current implementation doesn't have explicit circular reference detection,
# but Python's recursion limit should prevent infinite loops.
# This test ensures the method handles it gracefully.
@@ -213,7 +194,6 @@ class TestModelDeckGetOptionalInferenceModel:
assert result is None
def test_complex_waterfall_scenario(self, mocker: MockerFixture):
- """Test complex waterfall scenario with multiple levels."""
# Arrange
model_spec = self._create_test_model_spec("claude-3")
model_deck = self._create_test_model_deck(
@@ -231,7 +211,6 @@ class TestModelDeckGetOptionalInferenceModel:
assert mock_log.debug.call_count >= 2
def test_mixed_string_and_list_aliases(self, mocker: MockerFixture):
- """Test scenario with mix of string and list aliases."""
# Arrange
model_spec1 = self._create_test_model_spec("gpt-4")
model_spec2 = self._create_test_model_spec("claude-3")
@@ -263,7 +242,6 @@ class TestModelDeckGetOptionalInferenceModel:
],
)
def test_edge_case_llm_handles(self, llm_handle: str, mocker: MockerFixture):
- """Test edge cases for llm_handle parameter."""
# Arrange
model_deck = self._create_test_model_deck()
mock_log = mocker.patch("pipelex.cogt.models.model_deck.log")
@@ -276,7 +254,6 @@ class TestModelDeckGetOptionalInferenceModel:
mock_log.warning.assert_called_once_with(f"Skipping model handle '{llm_handle}' because it's not found in deck")
def test_logging_behavior(self, mocker: MockerFixture):
- """Test that logging is called with correct messages."""
# Arrange
model_deck = self._create_test_model_deck(aliases={"test-alias": "target-model"})
mock_log = mocker.patch("pipelex.cogt.models.model_deck.log")
diff --git a/tests/unit/pipelex/core/concepts/__init__.py b/tests/unit/pipelex/core/concepts/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/tests/unit/pipelex/core/concepts/concept/__init__.py b/tests/unit/pipelex/core/concepts/concept/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/tests/unit/pipelex/core/concepts/concept/test_concept.py b/tests/unit/pipelex/core/concepts/concept/test_concept.py
index 792702e72..de683c378 100644
--- a/tests/unit/pipelex/core/concepts/concept/test_concept.py
+++ b/tests/unit/pipelex/core/concepts/concept/test_concept.py
@@ -98,7 +98,7 @@ class TestConcept:
valid_domain = "valid_domain"
valid_definition = "Lorem Ipsum"
- for native_concept in [native_concept for native_concept in NativeConceptEnum]:
+ for native_concept in NativeConceptEnum.values_list():
assert Concept.is_native_concept(ConceptFactory.make_native_concept(native_concept_data=NATIVE_CONCEPTS_DATA[native_concept])) is True
assert (
@@ -108,7 +108,7 @@ class TestConcept:
domain=valid_domain,
blueprint=ConceptBlueprint(definition=valid_definition),
concept_codes_from_the_same_domain=["RandomConcept"],
- )
+ ),
)
is True
)
@@ -118,7 +118,7 @@ class TestConcept:
concept_code=NativeConceptEnum.TEXT,
domain=SpecialDomain.NATIVE,
blueprint=ConceptBlueprint(definition=valid_definition),
- )
+ ),
)
is True
)
@@ -129,7 +129,7 @@ class TestConcept:
domain=valid_domain,
blueprint=ConceptBlueprint(definition=valid_definition),
concept_codes_from_the_same_domain=["RandomConcept"],
- )
+ ),
)
is True
)
@@ -140,7 +140,7 @@ class TestConcept:
domain=valid_domain,
blueprint=ConceptBlueprint(definition=valid_definition),
concept_codes_from_the_same_domain=["RandomConcept"],
- )
+ ),
)
is True
)
@@ -151,7 +151,7 @@ class TestConcept:
domain=valid_domain,
blueprint=ConceptBlueprint(definition=valid_definition),
concept_codes_from_the_same_domain=["RandomConcept"],
- )
+ ),
)
is True
)
@@ -162,7 +162,7 @@ class TestConcept:
domain=valid_domain,
blueprint=ConceptBlueprint(definition=valid_definition),
concept_codes_from_the_same_domain=["RandomConcept"],
- )
+ ),
)
is True
)
@@ -173,7 +173,7 @@ class TestConcept:
domain=valid_domain,
blueprint=ConceptBlueprint(definition=valid_definition),
concept_codes_from_the_same_domain=["RandomConcept"],
- )
+ ),
)
is True
)
@@ -184,7 +184,7 @@ class TestConcept:
domain=valid_domain,
blueprint=ConceptBlueprint(definition=valid_definition),
concept_codes_from_the_same_domain=["RandomConcept"],
- )
+ ),
)
is True
)
@@ -195,7 +195,7 @@ class TestConcept:
domain=valid_domain,
blueprint=ConceptBlueprint(definition=valid_definition),
concept_codes_from_the_same_domain=["RandomConcept"],
- )
+ ),
)
is False
)
@@ -260,7 +260,6 @@ class TestConcept:
ConceptBlueprint.validate_concept_string(f"{SpecialDomain.NATIVE}.InvalidNativeConcept")
def test_are_concept_compatible(self):
- """Test are_concept_compatible method."""
concept1 = ConceptFactory.make_from_blueprint(
concept_code="Code1",
domain="domain1",
diff --git a/tests/unit/pipelex/core/concepts/concept_factory.py/data.py b/tests/unit/pipelex/core/concepts/concept_factory.py/data.py
index 85ea77036..5f1c22cc8 100644
--- a/tests/unit/pipelex/core/concepts/concept_factory.py/data.py
+++ b/tests/unit/pipelex/core/concepts/concept_factory.py/data.py
@@ -1,6 +1,4 @@
-"""Test data for ConceptFactory tests."""
-
-from typing import ClassVar, List, Optional, Tuple
+from typing import ClassVar, Optional, List, Tuple
from pipelex.core.concepts.concept import Concept
from pipelex.core.concepts.concept_blueprint import (
@@ -15,7 +13,7 @@ from pipelex.core.domains.domain import SpecialDomain
class TestCases:
# Test cases for make_refines method - only native concepts can be refined
- MAKE_REFINES_TEST_CASES: ClassVar[List[Tuple[str, ConceptBlueprint, str]]] = [
+ MAKE_REFINES_TEST_CASES: ClassVar[list[tuple[str, ConceptBlueprint, str]]] = [
(
"native_concept_string",
ConceptBlueprint(definition="A concept that refines a native text concept", refines=NativeConceptEnum.TEXT),
@@ -32,7 +30,7 @@ class TestCases:
]
# Test cases for make_domain_and_concept_code_from_concept_string_or_code method
- MAKE_DOMAIN_AND_CONCEPT_CODE_TEST_CASES: ClassVar[List[Tuple[str, str, Optional[List[str]], DomainAndConceptCode]]] = [
+ MAKE_DOMAIN_AND_CONCEPT_CODE_TEST_CASES: ClassVar[List[Tuple[str, str, List[str] | None, DomainAndConceptCode]]] = [
# Test case 1: Concept string with dot notation
("my_domain", "other_domain.ConceptName", None, DomainAndConceptCode(domain="other_domain", concept_code="ConceptName")),
# Test case 2: Concept string with dot notation (ignores same domain codes)
@@ -72,7 +70,7 @@ class TestCases:
]
# Test cases for make_from_blueprint method
- MAKE_FROM_BLUEPRINT_TEST_CASES: ClassVar[List[Tuple[str, str, str, ConceptBlueprint, Optional[List[str]], Concept]]] = [
+ MAKE_FROM_BLUEPRINT_TEST_CASES: ClassVar[list[tuple[str, str, str, ConceptBlueprint, list[str] | None, Concept]]] = [
# Test case 1: Simple blueprint with no structure or refines (goes to implicit)
(
"simple_blueprint",
@@ -189,7 +187,7 @@ class TestCases:
"name": "The person's name",
"age": ConceptStructureBlueprint(definition="The person's age", type=ConceptStructureBlueprintFieldType.NUMBER, required=True),
"active": ConceptStructureBlueprint(
- definition="Whether the person is active", type=ConceptStructureBlueprintFieldType.BOOLEAN, required=False, default_value=True
+ definition="Whether the person is active", type=ConceptStructureBlueprintFieldType.BOOLEAN, required=False, default_value=True,
),
},
),
@@ -213,7 +211,7 @@ class TestCases:
"name": ConceptStructureBlueprint(definition="The person's name", type=ConceptStructureBlueprintFieldType.TEXT, required=True),
"age": ConceptStructureBlueprint(definition="The person's age", type=ConceptStructureBlueprintFieldType.NUMBER, required=True),
"active": ConceptStructureBlueprint(
- definition="Whether the person is active", type=ConceptStructureBlueprintFieldType.BOOLEAN, required=False, default_value=True
+ definition="Whether the person is active", type=ConceptStructureBlueprintFieldType.BOOLEAN, required=False, default_value=True,
),
},
),
diff --git a/tests/unit/pipelex/core/concepts/concept_factory.py/test_concept_factory.py b/tests/unit/pipelex/core/concepts/concept_factory.py/test_concept_factory.py
index d07208de3..8ea032084 100644
--- a/tests/unit/pipelex/core/concepts/concept_factory.py/test_concept_factory.py
+++ b/tests/unit/pipelex/core/concepts/concept_factory.py/test_concept_factory.py
@@ -1,6 +1,5 @@
-from typing import Dict, List, Optional, Union
-
import pytest
+from typing import Optional, List
from pipelex.core.concepts.concept import Concept
from pipelex.core.concepts.concept_blueprint import (
@@ -32,37 +31,37 @@ class TestConceptFactory:
def test_normalize_structure_blueprint(self):
"""Test that mixed structure blueprints are properly normalized."""
- mixed_structure_blueprint: Dict[str, Union[str, ConceptStructureBlueprint]] = {
+ mixed_structure_blueprint: dict[str, str | ConceptStructureBlueprint] = {
"name": "The name of the person",
"age": ConceptStructureBlueprint(definition="The age of the person", type=ConceptStructureBlueprintFieldType.NUMBER, required=True),
"active": ConceptStructureBlueprint(
- definition="Whether the person is active", type=ConceptStructureBlueprintFieldType.BOOLEAN, required=False, default_value=True
+ definition="Whether the person is active", type=ConceptStructureBlueprintFieldType.BOOLEAN, required=False, default_value=True,
),
}
- expected_structure: Dict[str, ConceptStructureBlueprint] = {
+ expected_structure: dict[str, ConceptStructureBlueprint] = {
"name": ConceptStructureBlueprint(definition="The name of the person", type=ConceptStructureBlueprintFieldType.TEXT, required=True),
"age": ConceptStructureBlueprint(definition="The age of the person", type=ConceptStructureBlueprintFieldType.NUMBER, required=True),
"active": ConceptStructureBlueprint(
- definition="Whether the person is active", type=ConceptStructureBlueprintFieldType.BOOLEAN, required=False, default_value=True
+ definition="Whether the person is active", type=ConceptStructureBlueprintFieldType.BOOLEAN, required=False, default_value=True,
),
}
assert ConceptFactory.normalize_structure_blueprint(mixed_structure_blueprint) == expected_structure
- mixed_structure_blueprint2: Dict[str, Union[str, ConceptStructureBlueprint]] = {
+ mixed_structure_blueprint2: dict[str, str | ConceptStructureBlueprint] = {
"name": ConceptStructureBlueprint(definition="The name of the person", type=ConceptStructureBlueprintFieldType.TEXT, required=True),
"age": ConceptStructureBlueprint(definition="The age of the person", type=ConceptStructureBlueprintFieldType.NUMBER, required=True),
"active": ConceptStructureBlueprint(
- definition="Whether the person is active", type=ConceptStructureBlueprintFieldType.BOOLEAN, required=False, default_value=True
+ definition="Whether the person is active", type=ConceptStructureBlueprintFieldType.BOOLEAN, required=False, default_value=True,
),
}
- expected_structure2: Dict[str, ConceptStructureBlueprint] = {
+ expected_structure2: dict[str, ConceptStructureBlueprint] = {
"name": ConceptStructureBlueprint(definition="The name of the person", type=ConceptStructureBlueprintFieldType.TEXT, required=True),
"age": ConceptStructureBlueprint(definition="The age of the person", type=ConceptStructureBlueprintFieldType.NUMBER, required=True),
"active": ConceptStructureBlueprint(
- definition="Whether the person is active", type=ConceptStructureBlueprintFieldType.BOOLEAN, required=False, default_value=True
+ definition="Whether the person is active", type=ConceptStructureBlueprintFieldType.BOOLEAN, required=False, default_value=True,
),
}
@@ -76,7 +75,7 @@ class TestConceptFactory:
self,
domain: str,
concept_string_or_code: str,
- concept_codes_from_the_same_domain: Optional[List[str]],
+ concept_codes_from_the_same_domain: List[str] | None,
expected_result: List[str],
):
"""Test make_domain_and_concept_code_from_concept_string_or_code method with various inputs."""
@@ -97,7 +96,7 @@ class TestConceptFactory:
domain: str,
concept_code: str,
blueprint: ConceptBlueprint,
- concept_codes_from_the_same_domain: Optional[List[str]],
+ concept_codes_from_the_same_domain: list[str] | None,
expected_concept: Concept,
):
"""Test make_from_blueprint method with various blueprint configurations."""
diff --git a/tests/unit/pipelex/core/concepts/concept_factory/__init__.py b/tests/unit/pipelex/core/concepts/concept_factory/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/tests/unit/pipelex/core/concepts/concept_factory/data.py b/tests/unit/pipelex/core/concepts/concept_factory/data.py
new file mode 100644
index 000000000..930b8696a
--- /dev/null
+++ b/tests/unit/pipelex/core/concepts/concept_factory/data.py
@@ -0,0 +1,264 @@
+"""Test data for ConceptFactory tests."""
+
+from typing import ClassVar
+
+from pipelex.core.concepts.concept import Concept
+from pipelex.core.concepts.concept_blueprint import (
+ ConceptBlueprint,
+ ConceptStructureBlueprint,
+ ConceptStructureBlueprintFieldType,
+)
+from pipelex.core.concepts.concept_factory import DomainAndConceptCode
+from pipelex.core.concepts.concept_native import NATIVE_CONCEPTS_DATA, NativeConceptEnum
+from pipelex.core.domains.domain import SpecialDomain
+
+
+class TestCases:
+ """Test cases for ConceptFactory methods."""
+
+ # Test cases for make_refines method - only native concepts can be refined
+ MAKE_REFINES_TEST_CASES: ClassVar[list[tuple[str, ConceptBlueprint, str]]] = [
+ (
+ "native_concept_string",
+ ConceptBlueprint(definition="A concept that refines a native text concept", refines=NativeConceptEnum.TEXT.value),
+ f"{SpecialDomain.NATIVE.value}.{NativeConceptEnum.TEXT.value}",
+ ),
+ (
+ "fully_qualified_native_string",
+ ConceptBlueprint(
+ definition="A concept that refines a fully qualified native concept",
+ refines=f"{SpecialDomain.NATIVE.value}.{NativeConceptEnum.TEXT.value}",
+ ),
+ f"{SpecialDomain.NATIVE.value}.{NativeConceptEnum.TEXT.value}",
+ ),
+ ]
+
+ # Test cases for make_domain_and_concept_code_from_concept_string_or_concept_code method
+ MAKE_DOMAIN_AND_CONCEPT_CODE_TEST_CASES: ClassVar[list[tuple[str, str, list[str] | None, DomainAndConceptCode]]] = [
+ # Test case 1: Concept string with dot notation
+ ("my_domain", "other_domain.ConceptName", None, DomainAndConceptCode(domain="other_domain", concept_code="ConceptName")),
+ # Test case 2: Concept string with dot notation (ignores same domain codes)
+ ("my_domain", "other_domain.ConceptName", ["ConceptName"], DomainAndConceptCode(domain="other_domain", concept_code="ConceptName")),
+ # Test case 3: Native concept code (Text)
+ ("my_domain", "Text", None, DomainAndConceptCode(domain=SpecialDomain.NATIVE, concept_code="Text")),
+ # Test case 4: Native concept code (Image)
+ ("my_domain", "Image", None, DomainAndConceptCode(domain=SpecialDomain.NATIVE, concept_code="Image")),
+ # Test case 5: Native concept code (PDF)
+ ("my_domain", "PDF", None, DomainAndConceptCode(domain=SpecialDomain.NATIVE, concept_code="PDF")),
+ # Test case 6: Native concept code with same domain codes provided (native takes precedence)
+ ("my_domain", "Text", ["Text", "OtherConcept"], DomainAndConceptCode(domain=SpecialDomain.NATIVE, concept_code="Text")),
+ # Test case 7: Concept code from same domain
+ ("my_domain", "MyConcept", ["MyConcept", "OtherConcept"], DomainAndConceptCode(domain="my_domain", concept_code="MyConcept")),
+ # Test case 8: Concept code from same domain (case sensitive)
+ ("my_domain", "MyConcept", ["MyCon", "OtherConcept"], DomainAndConceptCode(domain=SpecialDomain.IMPLICIT, concept_code="MyConcept")),
+ # Test case 9: Unknown concept code (no same domain codes)
+ ("my_domain", "UnknownConcept", None, DomainAndConceptCode(domain=SpecialDomain.IMPLICIT, concept_code="UnknownConcept")),
+ # Test case 10: Unknown concept code (not in same domain codes)
+ (
+ "my_domain",
+ "UnknownConcept",
+ ["KnownConcept", "OtherConcept"],
+ DomainAndConceptCode(domain=SpecialDomain.IMPLICIT, concept_code="UnknownConcept"),
+ ),
+ # Test case 11: Empty same domain codes list
+ ("my_domain", "SomeConcept", [], DomainAndConceptCode(domain=SpecialDomain.IMPLICIT, concept_code="SomeConcept")),
+ # Test case 12: Different domain in concept string
+ ("my_domain", "another_domain.SomeConcept", ["SomeConcept"], DomainAndConceptCode(domain="another_domain", concept_code="SomeConcept")),
+ # Test case 13: All native concept codes
+ ("my_domain", "Dynamic", None, DomainAndConceptCode(domain=SpecialDomain.NATIVE, concept_code="Dynamic")),
+ ("my_domain", "TextAndImages", None, DomainAndConceptCode(domain=SpecialDomain.NATIVE, concept_code="TextAndImages")),
+ ("my_domain", "Number", None, DomainAndConceptCode(domain=SpecialDomain.NATIVE, concept_code="Number")),
+ ("my_domain", "LlmPrompt", None, DomainAndConceptCode(domain=SpecialDomain.NATIVE, concept_code="LlmPrompt")),
+ ("my_domain", "Page", None, DomainAndConceptCode(domain=SpecialDomain.NATIVE, concept_code="Page")),
+ ("my_domain", "Anything", None, DomainAndConceptCode(domain=SpecialDomain.NATIVE, concept_code="Anything")),
+ ]
+
+ # Test cases for make_from_blueprint method
+ MAKE_FROM_BLUEPRINT_TEST_CASES: ClassVar[list[tuple[str, str, str, ConceptBlueprint, list[str] | None, Concept]]] = [
+ # Test case 1: Simple blueprint with no structure or refines (goes to implicit)
+ (
+ "simple_blueprint",
+ "my_domain",
+ "SimpleConcept",
+ ConceptBlueprint(definition="A simple concept"),
+ None,
+ Concept(
+ domain=SpecialDomain.IMPLICIT,
+ code="SimpleConcept",
+ definition="A simple concept",
+ structure_class_name=NATIVE_CONCEPTS_DATA[NativeConceptEnum.TEXT].content_class_name,
+ refines=None,
+ ),
+ ),
+ # Test case 2: Blueprint with string structure (existing class, goes to implicit)
+ (
+ "string_structure_existing_class",
+ "my_domain",
+ "ConceptWithStructure",
+ ConceptBlueprint(definition="A concept with string structure", structure="TextContent"),
+ None,
+ Concept(
+ domain=SpecialDomain.IMPLICIT,
+ code="ConceptWithStructure",
+ definition="A concept with string structure",
+ structure_class_name=NATIVE_CONCEPTS_DATA[NativeConceptEnum.TEXT].content_class_name,
+ refines=None,
+ ),
+ ),
+ # Test case 3: Blueprint with refines (native concept, goes to implicit)
+ (
+ "blueprint_with_refines_text",
+ "my_domain",
+ "MyTextConcept",
+ ConceptBlueprint(definition="A concept that refines Text", refines="Text"),
+ None,
+ Concept(
+ domain=SpecialDomain.IMPLICIT,
+ code="MyTextConcept",
+ definition="A concept that refines Text",
+ structure_class_name=NATIVE_CONCEPTS_DATA[NativeConceptEnum.TEXT].content_class_name,
+ refines="native.Text",
+ ),
+ ),
+ # Test case 4: Blueprint with refines (fully qualified native concept, goes to implicit)
+ (
+ "blueprint_with_refines_native_image",
+ "my_domain",
+ "MyImageConcept",
+ ConceptBlueprint(definition="A concept that refines native.Image", refines="native.Image"),
+ None,
+ Concept(
+ domain=SpecialDomain.IMPLICIT,
+ code="MyImageConcept",
+ definition="A concept that refines native.Image",
+ structure_class_name=NATIVE_CONCEPTS_DATA[NativeConceptEnum.IMAGE].content_class_name,
+ refines="native.Image",
+ ),
+ ),
+ # Test case 5: Native concept code (should go to native domain)
+ (
+ "native_concept_code",
+ "my_domain",
+ "Text",
+ ConceptBlueprint(definition="Native text concept"),
+ None,
+ Concept(
+ domain=SpecialDomain.NATIVE,
+ code="Text",
+ definition="Native text concept",
+ structure_class_name=NATIVE_CONCEPTS_DATA[NativeConceptEnum.TEXT].content_class_name,
+ refines=None,
+ ),
+ ),
+ # Test case 6: Concept code from same domain
+ (
+ "same_domain_concept",
+ "my_domain",
+ "DomainConcept",
+ ConceptBlueprint(definition="A concept from same domain"),
+ ["DomainConcept", "OtherConcept"],
+ Concept(
+ domain="my_domain",
+ code="DomainConcept",
+ definition="A concept from same domain",
+ structure_class_name=NATIVE_CONCEPTS_DATA[NativeConceptEnum.TEXT].content_class_name,
+ refines=None,
+ ),
+ ),
+ # Test case 7: Unknown concept code (should go to implicit domain)
+ (
+ "implicit_concept",
+ "my_domain",
+ "UnknownConcept",
+ ConceptBlueprint(definition="An unknown concept"),
+ ["KnownConcept"],
+ Concept(
+ domain=SpecialDomain.IMPLICIT,
+ code="UnknownConcept",
+ definition="An unknown concept",
+ structure_class_name=NATIVE_CONCEPTS_DATA[NativeConceptEnum.TEXT].content_class_name,
+ refines=None,
+ ),
+ ),
+ # Test case 8: Blueprint with dict structure (implicit domain)
+ (
+ "dict_structure_implicit",
+ SpecialDomain.IMPLICIT,
+ "PersonConcept",
+ ConceptBlueprint(
+ definition="A person with structured data",
+ structure={
+ "name": "The person's name",
+ "age": ConceptStructureBlueprint(definition="The person's age", type=ConceptStructureBlueprintFieldType.NUMBER, required=True),
+ "active": ConceptStructureBlueprint(
+ definition="Whether the person is active", type=ConceptStructureBlueprintFieldType.BOOLEAN, required=False, default_value=True,
+ ),
+ },
+ ),
+ None,
+ Concept(
+ domain=SpecialDomain.IMPLICIT,
+ code="PersonConcept",
+ definition="A person with structured data",
+ structure_class_name="PersonConcept",
+ refines=None,
+ ),
+ ),
+ # Test case 9: Blueprint with dict structure (same domain)
+ (
+ "dict_structure_same_domain",
+ "my_domain",
+ "PersonConcept",
+ ConceptBlueprint(
+ definition="A person with structured data",
+ structure={
+ "name": ConceptStructureBlueprint(definition="The person's name", type=ConceptStructureBlueprintFieldType.TEXT, required=True),
+ "age": ConceptStructureBlueprint(definition="The person's age", type=ConceptStructureBlueprintFieldType.NUMBER, required=True),
+ "active": ConceptStructureBlueprint(
+ definition="Whether the person is active", type=ConceptStructureBlueprintFieldType.BOOLEAN, required=False, default_value=True,
+ ),
+ },
+ ),
+ ["PersonConcept", "OtherConcept"],
+ Concept(
+ domain="my_domain",
+ code="PersonConcept",
+ definition="A person with structured data",
+ structure_class_name="PersonConcept",
+ refines=None,
+ ),
+ ),
+ # Test case 10: Blueprint with refines (implicit domain)
+ (
+ "refines_implicit_domain",
+ "my_domain",
+ "ConceptWithRefines",
+ ConceptBlueprint(definition="A concept with refines", refines="Text"),
+ None,
+ Concept(
+ domain=SpecialDomain.IMPLICIT,
+ code="ConceptWithRefines",
+ definition="A concept with refines",
+ structure_class_name=NATIVE_CONCEPTS_DATA[NativeConceptEnum.TEXT].content_class_name,
+ refines="native.Text",
+ ),
+ ),
+ # Test case 11: Blueprint with refines (same domain)
+ (
+ "refines_same_domain",
+ "my_domain",
+ "ConceptWithRefines",
+ ConceptBlueprint(definition="A concept with refines", refines="native.Image"),
+ ["ConceptWithRefines", "OtherConcept"],
+ Concept(
+ domain="my_domain",
+ code="ConceptWithRefines",
+ definition="A concept with refines",
+ structure_class_name=NATIVE_CONCEPTS_DATA[NativeConceptEnum.IMAGE].content_class_name,
+ refines="native.Image",
+ ),
+ ),
+ ]
+
+ # Legacy alias for backward compatibility
+ TEST_CASES = MAKE_REFINES_TEST_CASES
diff --git a/tests/unit/pipelex/core/concepts/concept_factory/test_concept_factory.py b/tests/unit/pipelex/core/concepts/concept_factory/test_concept_factory.py
new file mode 100644
index 000000000..195a43805
--- /dev/null
+++ b/tests/unit/pipelex/core/concepts/concept_factory/test_concept_factory.py
@@ -0,0 +1,125 @@
+"""Tests for ConceptFactory methods."""
+
+
+import pytest
+
+from pipelex.core.concepts.concept import Concept
+from pipelex.core.concepts.concept_blueprint import (
+ ConceptBlueprint,
+ ConceptStructureBlueprint,
+ ConceptStructureBlueprintFieldType,
+)
+from pipelex.core.concepts.concept_factory import ConceptFactory
+
+from .data import TestCases
+
+
+class TestConceptFactory:
+ """Test ConceptFactory methods with various configurations."""
+
+ @pytest.mark.parametrize(
+ "test_name,blueprint,expected_result",
+ TestCases.MAKE_REFINES_TEST_CASES,
+ )
+ def test_make_refines(
+ self,
+ test_name: str,
+ blueprint: ConceptBlueprint,
+ expected_result: str,
+ ):
+ """Test make_refines method with different blueprint configurations."""
+ result = ConceptFactory.make_refines(blueprint=blueprint)
+ assert result == expected_result, f"Failed for test case: {test_name}"
+
+ def test_normalize_structure_blueprint(self):
+ """Test that mixed structure blueprints are properly normalized."""
+ mixed_structure_blueprint: dict[str, str | ConceptStructureBlueprint] = {
+ "name": "The name of the person",
+ "age": ConceptStructureBlueprint(definition="The age of the person", type=ConceptStructureBlueprintFieldType.NUMBER, required=True),
+ "active": ConceptStructureBlueprint(
+ definition="Whether the person is active", type=ConceptStructureBlueprintFieldType.BOOLEAN, required=False, default_value=True,
+ ),
+ }
+
+ expected_structure: dict[str, ConceptStructureBlueprint] = {
+ "name": ConceptStructureBlueprint(definition="The name of the person", type=ConceptStructureBlueprintFieldType.TEXT, required=True),
+ "age": ConceptStructureBlueprint(definition="The age of the person", type=ConceptStructureBlueprintFieldType.NUMBER, required=True),
+ "active": ConceptStructureBlueprint(
+ definition="Whether the person is active", type=ConceptStructureBlueprintFieldType.BOOLEAN, required=False, default_value=True,
+ ),
+ }
+
+ assert ConceptFactory.normalize_structure_blueprint(mixed_structure_blueprint) == expected_structure
+
+ mixed_structure_blueprint2: dict[str, str | ConceptStructureBlueprint] = {
+ "name": ConceptStructureBlueprint(definition="The name of the person", type=ConceptStructureBlueprintFieldType.TEXT, required=True),
+ "age": ConceptStructureBlueprint(definition="The age of the person", type=ConceptStructureBlueprintFieldType.NUMBER, required=True),
+ "active": ConceptStructureBlueprint(
+ definition="Whether the person is active", type=ConceptStructureBlueprintFieldType.BOOLEAN, required=False, default_value=True,
+ ),
+ }
+
+ expected_structure2: dict[str, ConceptStructureBlueprint] = {
+ "name": ConceptStructureBlueprint(definition="The name of the person", type=ConceptStructureBlueprintFieldType.TEXT, required=True),
+ "age": ConceptStructureBlueprint(definition="The age of the person", type=ConceptStructureBlueprintFieldType.NUMBER, required=True),
+ "active": ConceptStructureBlueprint(
+ definition="Whether the person is active", type=ConceptStructureBlueprintFieldType.BOOLEAN, required=False, default_value=True,
+ ),
+ }
+
+ assert ConceptFactory.normalize_structure_blueprint(mixed_structure_blueprint2) == expected_structure2
+
+ @pytest.mark.parametrize(
+ "domain,concept_string_or_code,concept_codes_from_the_same_domain,expected_result",
+ TestCases.MAKE_DOMAIN_AND_CONCEPT_CODE_TEST_CASES,
+ )
+ def test_make_domain_and_concept_code_from_concept_string_orcode(
+ self,
+ domain: str,
+ concept_string_or_code: str,
+ concept_codes_from_the_same_domain: list[str] | None,
+ expected_result: list[str],
+ ):
+ """Test make_domain_and_concept_code_from_concept_string_or_code method with various inputs."""
+ result = ConceptFactory.make_domain_and_concept_code_from_concept_string_or_code(
+ domain=domain,
+ concept_string_or_code=concept_string_or_code,
+ concept_codes_from_the_same_domain=concept_codes_from_the_same_domain,
+ )
+ assert result == expected_result
+
+ @pytest.mark.parametrize(
+ "test_name,domain,concept_code,blueprint,concept_codes_from_the_same_domain,expected_concept",
+ TestCases.MAKE_FROM_BLUEPRINT_TEST_CASES,
+ )
+ def test_make_from_blueprint(
+ self,
+ test_name: str,
+ domain: str,
+ concept_code: str,
+ blueprint: ConceptBlueprint,
+ concept_codes_from_the_same_domain: list[str] | None,
+ expected_concept: Concept,
+ ):
+ """Test make_from_blueprint method with various blueprint configurations."""
+ result = ConceptFactory.make_from_blueprint(
+ domain=domain,
+ concept_code=concept_code,
+ blueprint=blueprint,
+ concept_codes_from_the_same_domain=concept_codes_from_the_same_domain,
+ )
+
+ assert result == expected_concept, f"Concept mismatch for {test_name}"
+
+ def test_make_from_blueprint_with_invalid_structure_class(self):
+ """Test that make_from_blueprint raises StructureClassError for invalid structure class."""
+ blueprint = ConceptBlueprint(definition="A concept with invalid structure", structure="NonExistentStructureClass")
+
+ from pipelex.exceptions import StructureClassError
+
+ with pytest.raises(StructureClassError, match="is not a registered subclass of StuffContent"):
+ ConceptFactory.make_from_blueprint(
+ domain="my_domain",
+ concept_code="TestConcept",
+ blueprint=blueprint,
+ )
diff --git a/tests/unit/pipelex/core/concepts/concept_library/__init__.py b/tests/unit/pipelex/core/concepts/concept_library/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/tests/unit/pipelex/core/concepts/test_concept_blueprint.py b/tests/unit/pipelex/core/concepts/test_concept_blueprint.py
index 49e486db5..149d1f5a2 100644
--- a/tests/unit/pipelex/core/concepts/test_concept_blueprint.py
+++ b/tests/unit/pipelex/core/concepts/test_concept_blueprint.py
@@ -16,7 +16,7 @@ class TestConceptStructureBlueprintValidation:
"""Test that valid structure blueprints are accepted."""
# Valid text field with default
text_blueprint = ConceptStructureBlueprint(
- definition="A text field", type=ConceptStructureBlueprintFieldType.TEXT, default_value="default text"
+ definition="A text field", type=ConceptStructureBlueprintFieldType.TEXT, default_value="default text",
)
assert text_blueprint.default_value == "default text"
@@ -38,7 +38,7 @@ class TestConceptStructureBlueprintValidation:
# Valid list field with default
list_blueprint = ConceptStructureBlueprint(
- definition="A list field", type=ConceptStructureBlueprintFieldType.LIST, item_type="text", default_value=["item1", "item2"]
+ definition="A list field", type=ConceptStructureBlueprintFieldType.LIST, item_type="text", default_value=["item1", "item2"],
)
assert list_blueprint.default_value == ["item1", "item2"]
@@ -77,7 +77,7 @@ class TestConceptStructureBlueprintValidation:
# List field with non-list default
with pytest.raises(ConceptStructureBlueprintError, match="default_value type mismatch: expected list"):
ConceptStructureBlueprint(
- definition="A list field", type=ConceptStructureBlueprintFieldType.LIST, item_type="text", default_value="not a list"
+ definition="A list field", type=ConceptStructureBlueprintFieldType.LIST, item_type="text", default_value="not a list",
)
# Dict field with non-dict default
@@ -136,21 +136,21 @@ class TestConceptStructureBlueprintValidation:
# Boolean field with False default (should be valid)
bool_false_blueprint = ConceptStructureBlueprint(
- definition="Boolean field with False default", type=ConceptStructureBlueprintFieldType.BOOLEAN, default_value=False
+ definition="Boolean field with False default", type=ConceptStructureBlueprintFieldType.BOOLEAN, default_value=False,
)
assert bool_false_blueprint.default_value is False
# Integer field with 0 default (should be valid)
int_zero_blueprint = ConceptStructureBlueprint(
- definition="Integer field with 0 default", type=ConceptStructureBlueprintFieldType.INTEGER, default_value=0
+ definition="Integer field with 0 default", type=ConceptStructureBlueprintFieldType.INTEGER, default_value=0,
)
assert int_zero_blueprint.default_value == 0
# Empty list as default (should be valid)
empty_list_blueprint = ConceptStructureBlueprint(
- definition="List field with empty default", type=ConceptStructureBlueprintFieldType.LIST, item_type="text", default_value=[]
+ definition="List field with empty default", type=ConceptStructureBlueprintFieldType.LIST, item_type="text", default_value=[],
)
- assert empty_list_blueprint.default_value == []
+ assert not empty_list_blueprint.default_value
# Empty dict as default (should be valid)
empty_dict_blueprint = ConceptStructureBlueprint(
@@ -160,4 +160,4 @@ class TestConceptStructureBlueprintValidation:
value_type="text",
default_value={},
)
- assert empty_dict_blueprint.default_value == {}
+ assert not empty_dict_blueprint.default_value
diff --git a/tests/unit/pipelex/core/interpreter/__init__.py b/tests/unit/pipelex/core/interpreter/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/tests/unit/pipelex/core/interpreter/test_interpreter.py b/tests/unit/pipelex/core/interpreter/test_interpreter.py
index fb2c2be72..12bf546de 100644
--- a/tests/unit/pipelex/core/interpreter/test_interpreter.py
+++ b/tests/unit/pipelex/core/interpreter/test_interpreter.py
@@ -1,8 +1,8 @@
from pathlib import Path
-from typing import Type
import pytest
+from pipelex import pretty_print
from pipelex.core.bundles.pipelex_bundle_blueprint import PipelexBundleBlueprint
from pipelex.core.interpreter import PipelexInterpreter
from tests.unit.pipelex.core.test_data import InterpreterTestCases
@@ -10,7 +10,6 @@ from tests.unit.pipelex.core.test_data import InterpreterTestCases
class TestPipelexInterpreter:
def test_init_with_both_file_path_and_content(self, tmp_path: Path):
- """Test initialization with both file_path and file_content."""
test_file = tmp_path / "test.plx"
test_file.write_text("domain = 'test'\n[concept]\n")
content = "domain = 'other'\n[concept]\n"
@@ -21,9 +20,8 @@ class TestPipelexInterpreter:
@pytest.mark.parametrize("test_name,plx_content,expected_blueprint", InterpreterTestCases.VALID_TEST_CASES)
def test_make_pipelex_bundle_blueprint(self, test_name: str, plx_content: str, expected_blueprint: PipelexBundleBlueprint):
- """Test making blueprint from various valid PLX content."""
converter = PipelexInterpreter(file_content=plx_content)
- from pipelex import pretty_print
+
blueprint = converter.make_pipelex_bundle_blueprint()
pretty_print(blueprint, title=f"Blueprint {test_name}")
@@ -32,17 +30,14 @@ class TestPipelexInterpreter:
@pytest.mark.parametrize("test_name,plx_content,expected_blueprint", InterpreterTestCases.VALID_TEST_CASES)
def test_make_plx_content(self, test_name: str, plx_content: str, expected_blueprint: PipelexBundleBlueprint):
- """Test making blueprint from various valid PLX content."""
get_plx_content = PipelexInterpreter.make_plx_content(blueprint=expected_blueprint)
- from pipelex import pretty_print
pretty_print(get_plx_content, title=f"PLx content {test_name}")
pretty_print(plx_content, title=f"Expected PLX content {test_name}")
assert get_plx_content == plx_content
@pytest.mark.parametrize("test_name,invalid_plx_content,expected_exception", InterpreterTestCases.ERROR_TEST_CASES)
- def test_invalid_plx_should_raise_exception(self, test_name: str, invalid_plx_content: str, expected_exception: Type[Exception]):
- """Test that invalid PLX content raises appropriate exceptions."""
+ def test_invalid_plx_should_raise_exception(self, test_name: str, invalid_plx_content: str, expected_exception: type[Exception]):
converter = PipelexInterpreter(file_content=invalid_plx_content)
with pytest.raises(expected_exception):
diff --git a/tests/unit/pipelex/core/interpreter/test_interpreter_serialize.py b/tests/unit/pipelex/core/interpreter/test_interpreter_serialize.py
index 923b80aa1..97145efd8 100644
--- a/tests/unit/pipelex/core/interpreter/test_interpreter_serialize.py
+++ b/tests/unit/pipelex/core/interpreter/test_interpreter_serialize.py
@@ -1,6 +1,4 @@
-"""Unit tests for PipelexInterpreter serialize functions."""
-
-from typing import Any, Dict
+from typing import Any
from pipelex.core.concepts.concept_blueprint import ConceptBlueprint, ConceptStructureBlueprint, ConceptStructureBlueprintFieldType
from pipelex.core.interpreter import PipelexInterpreter
@@ -12,10 +10,7 @@ from pipelex.pipe_operators.ocr.pipe_ocr_blueprint import PipeOcrBlueprint
class TestSerializeInputRequirement:
- """Test _serialize_input_requirement function."""
-
def test_serialize_input_requirement_basic(self):
- """Test serializing basic InputRequirementBlueprint."""
input_req = InputRequirementBlueprint(concept="Text")
result = PipelexInterpreter.serialize_input_requirement(input_req)
@@ -23,7 +18,6 @@ class TestSerializeInputRequirement:
assert result == expected
def test_serialize_input_requirement_with_multiplicity_int(self):
- """Test serializing InputRequirementBlueprint with integer multiplicity."""
input_req = InputRequirementBlueprint(concept="Text", multiplicity=3)
result = PipelexInterpreter.serialize_input_requirement(input_req)
@@ -31,7 +25,6 @@ class TestSerializeInputRequirement:
assert result == expected
def test_serialize_input_requirement_with_multiplicity_bool(self):
- """Test serializing InputRequirementBlueprint with boolean multiplicity."""
input_req = InputRequirementBlueprint(concept="Text", multiplicity=True)
result = PipelexInterpreter.serialize_input_requirement(input_req)
@@ -40,11 +33,8 @@ class TestSerializeInputRequirement:
class TestSerializeInputs:
- """Test _serialize_inputs function."""
-
def test_serialize_inputs_mixed_types(self):
- """Test serializing mixed input types."""
- inputs: Dict[str, str | InputRequirementBlueprint] = {
+ inputs: dict[str, str | InputRequirementBlueprint] = {
"text": "Text",
"data": InputRequirementBlueprint(concept="Data", multiplicity=2),
"query": InputRequirementBlueprint(concept="Text"),
@@ -55,25 +45,21 @@ class TestSerializeInputs:
assert result == expected
def test_serialize_inputs_empty(self):
- """Test serializing empty inputs."""
- inputs: Dict[str, Any] = {}
+ inputs: dict[str, Any] = {}
result = PipelexInterpreter.serialize_inputs(inputs)
assert result == {}
class TestSerializeConceptStructure:
- """Test concept structure serialization functions."""
def test_serialize_concept_structure_field_string(self):
- """Test serializing string concept structure field."""
field_value = "Simple description"
result = PipelexInterpreter.serialize_concept_structure_field(field_value)
assert result == "Simple description"
def test_serialize_concept_structure_field_blueprint(self):
- """Test serializing ConceptStructureBlueprint field."""
field_value = ConceptStructureBlueprint(definition="A text field", type=ConceptStructureBlueprintFieldType.TEXT, required=True)
result = PipelexInterpreter.serialize_concept_structure_field(field_value)
@@ -81,7 +67,6 @@ class TestSerializeConceptStructure:
assert result == expected
def test_serialize_concept_structure_field_blueprint_with_all_fields(self):
- """Test serializing ConceptStructureBlueprint with all optional fields."""
field_value = ConceptStructureBlueprint(
definition="A complex field",
type=ConceptStructureBlueprintFieldType.DICT,
@@ -105,14 +90,12 @@ class TestSerializeConceptStructure:
assert result == expected
def test_serialize_concept_structure_string(self):
- """Test serializing string concept structure."""
structure = "Simple structure"
result = PipelexInterpreter.serialize_concept_structure(structure)
assert result == "Simple structure"
def test_serialize_concept_structure_dict(self):
- """Test serializing dict concept structure."""
structure = {
"name": "Person's name",
"age": ConceptStructureBlueprint(definition="Person's age", type=ConceptStructureBlueprintFieldType.INTEGER, required=True),
@@ -127,24 +110,20 @@ class TestSerializeConceptStructure:
class TestSerializeSingleConcept:
- """Test _serialize_single_concept function."""
def test_serialize_single_concept_string(self):
- """Test serializing string concept."""
concept = "Simple concept definition"
result = PipelexInterpreter.serialize_single_concept(concept)
assert result == "Simple concept definition"
def test_serialize_single_concept_simple_definition(self):
- """Test serializing ConceptBlueprint with just definition."""
concept = ConceptBlueprint(definition="A simple concept")
result = PipelexInterpreter.serialize_single_concept(concept)
assert result == "A simple concept"
def test_serialize_single_concept_with_refines(self):
- """Test serializing ConceptBlueprint with refines."""
concept = ConceptBlueprint(definition="A specialized text", refines="Text")
result = PipelexInterpreter.serialize_single_concept(concept)
@@ -152,7 +131,6 @@ class TestSerializeSingleConcept:
assert result == expected
def test_serialize_single_concept_with_structure(self):
- """Test serializing ConceptBlueprint with structure."""
concept = ConceptBlueprint(
definition="A person",
structure={
@@ -173,16 +151,12 @@ class TestSerializeSingleConcept:
class TestSerializeConcepts:
- """Test _serialize_concepts function."""
-
def test_serialize_concepts_empty(self):
- """Test serializing empty concepts."""
- result = PipelexInterpreter.serialize_concepts(None, "test_domain")
+ result = PipelexInterpreter.serialize_concepts(None)
assert result == {}
def test_serialize_concepts_mixed(self):
- """Test serializing mixed concept types."""
- concepts: Dict[str, ConceptBlueprint | str] = {
+ concepts: dict[str, ConceptBlueprint | str] = {
"SimpleText": "A simple text concept",
"PersonInfo": ConceptBlueprint(definition="Information about a person", refines="Text"),
"StructuredData": ConceptBlueprint(
@@ -193,7 +167,7 @@ class TestSerializeConcepts:
},
),
}
- result = PipelexInterpreter.serialize_concepts(concepts, "test_domain")
+ result = PipelexInterpreter.serialize_concepts(concepts)
expected = {
"SimpleText": "A simple text concept",
@@ -210,10 +184,8 @@ class TestSerializeConcepts:
class TestSerializeSubPipe:
- """Test _serialize_sub_pipe function."""
def test_serialize_sub_pipe_basic(self):
- """Test serializing basic sub pipe."""
sub_pipe = SubPipeBlueprint(pipe="test_pipe", result="test_result")
result = PipelexInterpreter.serialize_sub_pipe(sub_pipe)
@@ -221,7 +193,6 @@ class TestSerializeSubPipe:
assert result == expected
def test_serialize_sub_pipe_with_nb_output_and_batch(self):
- """Test serializing sub pipe with nb_output and batch fields."""
sub_pipe = SubPipeBlueprint(pipe="test_pipe", result="test_result", nb_output=3, batch_over="input_list", batch_as="current_item")
result = PipelexInterpreter.serialize_sub_pipe(sub_pipe)
@@ -229,7 +200,6 @@ class TestSerializeSubPipe:
assert result == expected
def test_serialize_sub_pipe_with_multiple_output_and_batch(self):
- """Test serializing sub pipe with multiple_output and batch fields."""
sub_pipe = SubPipeBlueprint(pipe="test_pipe", result="test_result", multiple_output=True, batch_over="input_list", batch_as="current_item")
result = PipelexInterpreter.serialize_sub_pipe(sub_pipe)
@@ -237,7 +207,6 @@ class TestSerializeSubPipe:
assert result == expected
def test_serialize_sub_pipe_skip_defaults(self):
- """Test that default values are not included in serialization."""
sub_pipe = SubPipeBlueprint(
pipe="test_pipe",
result="test_result",
@@ -251,18 +220,14 @@ class TestSerializeSubPipe:
class TestSerializePipeBlueprints:
- """Test individual pipe blueprint serialization functions."""
-
def test_serialize_llm_pipe_basic(self):
- """Test serializing basic PipeLLM blueprint."""
pipe = PipeLLMBlueprint(type="PipeLLM", definition="Generate text using LLM", output="Text", prompt_template="Generate a story")
- result = PipelexInterpreter.serialize_llm_pipe(pipe, "test_domain")
+ result = PipelexInterpreter.serialize_llm_pipe(pipe)
expected = {"type": "PipeLLM", "definition": "Generate text using LLM", "output": "Text", "prompt_template": "Generate a story"}
assert result == expected
def test_serialize_llm_pipe_with_inputs_and_options(self):
- """Test serializing PipeLLM blueprint with inputs and optional fields."""
pipe = PipeLLMBlueprint(
type="PipeLLM",
definition="Extract information",
@@ -272,7 +237,7 @@ class TestSerializePipeBlueprints:
prompt_template="Extract info about $topic from @text",
nb_output=2,
)
- result = PipelexInterpreter.serialize_llm_pipe(pipe, "test_domain")
+ result = PipelexInterpreter.serialize_llm_pipe(pipe)
expected = {
"type": "PipeLLM",
@@ -286,22 +251,20 @@ class TestSerializePipeBlueprints:
assert result == expected
def test_serialize_ocr_pipe_basic(self):
- """Test serializing basic PipeOcr blueprint."""
pipe = PipeOcrBlueprint(type="PipeOcr", definition="Extract text from PDF", output="Page", ocr="base_ocr_pypdfium2")
- result = PipelexInterpreter.serialize_ocr_pipe(pipe, "test_domain")
+ result = PipelexInterpreter.serialize_ocr_pipe(pipe)
expected = {"type": "PipeOcr", "definition": "Extract text from PDF", "output": "Page", "ocr": "base_ocr_pypdfium2"}
assert result == expected
def test_serialize_sequence_pipe_basic(self):
- """Test serializing basic PipeSequence blueprint."""
pipe = PipeSequenceBlueprint(
type="PipeSequence",
definition="Process in sequence",
output="Result",
steps=[SubPipeBlueprint(pipe="step1", result="result1"), SubPipeBlueprint(pipe="step2", result="result2")],
)
- result = PipelexInterpreter.serialize_sequence_pipe(pipe, "test_domain")
+ result = PipelexInterpreter.serialize_sequence_pipe(pipe)
expected = {
"type": "PipeSequence",
@@ -313,11 +276,8 @@ class TestSerializePipeBlueprints:
class TestAddCommonPipeFields:
- """Test _add_common_pipe_fields helper function."""
-
def test_add_common_pipe_fields_with_inputs(self):
- """Test adding common pipe fields when inputs exist."""
- result: Dict[str, Any] = {"type": "TestPipe"}
+ result: dict[str, Any] = {"type": "TestPipe"}
# Mock pipe with inputs
class MockPipe:
@@ -331,8 +291,7 @@ class TestAddCommonPipeFields:
assert result == expected
def test_add_common_pipe_fields_without_inputs(self):
- """Test adding common pipe fields when no inputs exist."""
- result: Dict[str, Any] = {"type": "TestPipe"}
+ result: dict[str, Any] = {"type": "TestPipe"}
# Mock pipe without inputs
class MockPipe:
@@ -346,8 +305,7 @@ class TestAddCommonPipeFields:
assert result == expected
def test_add_common_pipe_fields_no_inputs_attribute(self):
- """Test adding common pipe fields when pipe has no inputs attribute."""
- result: Dict[str, Any] = {"type": "TestPipe"}
+ result: dict[str, Any] = {"type": "TestPipe"}
# Mock pipe without inputs attribute
class MockPipe:
diff --git a/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_batch_plx.py b/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_batch_plx.py
index f8c27a00c..7e55a2efd 100644
--- a/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_batch_plx.py
+++ b/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_batch_plx.py
@@ -5,8 +5,6 @@ from pipelex.pipe_controllers.batch.pipe_batch_blueprint import PipeBatchBluepri
class TestPipelexInterpreterBatchPLX:
- """Test Batch pipe to PLX string conversion."""
-
@pytest.mark.parametrize(
"pipe_name,blueprint,expected_plx",
[
@@ -50,5 +48,5 @@ branch_pipe_code = "transform_item\"""",
)
def test_batch_pipe_to_plx_string(self, pipe_name: str, blueprint: PipeBatchBlueprint, expected_plx: str):
"""Test converting Batch pipe blueprint to PLX string."""
- result = PipelexInterpreter.batch_pipe_to_plx_string(pipe_name, blueprint, "test_domain")
+ result = PipelexInterpreter.batch_pipe_to_plx_string(pipe_name, blueprint)
assert result == expected_plx
diff --git a/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_concept_plx.py b/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_concept_plx.py
index 5aec6a2a3..3e319da41 100644
--- a/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_concept_plx.py
+++ b/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_concept_plx.py
@@ -5,8 +5,6 @@ from pipelex.core.interpreter import PipelexInterpreter
class TestPipelexInterpreterConceptPLX:
- """Test concept to PLX string conversion."""
-
@pytest.mark.parametrize(
"concepts,expected_plx",
[
@@ -93,5 +91,5 @@ structure = "ProductStructure\"""",
)
def test_concepts_to_plx_string(self, concepts: dict[str, ConceptBlueprint | str], expected_plx: str):
"""Test converting concepts dict to PLX string."""
- result = PipelexInterpreter.concepts_to_plx_string(concepts, "test_domain")
+ result = PipelexInterpreter.concepts_to_plx_string(concepts)
assert result == expected_plx
diff --git a/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_condition_plx.py b/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_condition_plx.py
index 216a8de5c..fd7098489 100644
--- a/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_condition_plx.py
+++ b/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_condition_plx.py
@@ -5,8 +5,6 @@ from pipelex.pipe_controllers.condition.pipe_condition_blueprint import PipeCond
class TestPipelexInterpreterConditionPLX:
- """Test Condition pipe to PLX string conversion."""
-
@pytest.mark.parametrize(
"pipe_name,blueprint,expected_plx",
[
@@ -50,5 +48,5 @@ expression_template = "{{ data.size if data.size > threshold else 'low' }}\"""",
)
def test_condition_pipe_to_plx_string(self, pipe_name: str, blueprint: PipeConditionBlueprint, expected_plx: str):
"""Test converting Condition pipe blueprint to PLX string."""
- result = PipelexInterpreter.condition_pipe_to_plx_string(pipe_name, blueprint, "test_domain")
+ result = PipelexInterpreter.condition_pipe_to_plx_string(pipe_name, blueprint)
assert result == expected_plx
diff --git a/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_func_plx.py b/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_func_plx.py
index 38e2cdb8f..80d6d53fb 100644
--- a/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_func_plx.py
+++ b/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_func_plx.py
@@ -5,8 +5,6 @@ from pipelex.pipe_operators.func.pipe_func_blueprint import PipeFuncBlueprint
class TestPipelexInterpreterFuncPLX:
- """Test Func pipe to PLX string conversion."""
-
@pytest.mark.parametrize(
"pipe_name,blueprint,expected_plx",
[
@@ -45,6 +43,5 @@ function_name = "transform_function\"""",
],
)
def test_func_pipe_to_plx_string(self, pipe_name: str, blueprint: PipeFuncBlueprint, expected_plx: str):
- """Test converting Func pipe blueprint to PLX string."""
- result = PipelexInterpreter.func_pipe_to_plx_string(pipe_name, blueprint, "test_domain")
+ result = PipelexInterpreter.func_pipe_to_plx_string(pipe_name, blueprint)
assert result == expected_plx
diff --git a/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_img_gen_plx.py b/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_img_gen_plx.py
index 663f44187..5b02034f4 100644
--- a/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_img_gen_plx.py
+++ b/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_img_gen_plx.py
@@ -6,8 +6,6 @@ from pipelex.pipe_operators.img_gen.pipe_img_gen_blueprint import PipeImgGenBlue
class TestPipelexInterpreterImgGenPLX:
- """Test ImgGen pipe to PLX string conversion."""
-
@pytest.mark.parametrize(
"pipe_name,blueprint,expected_plx",
[
@@ -48,6 +46,5 @@ seed = 12345""",
],
)
def test_img_gen_pipe_to_plx_string(self, pipe_name: str, blueprint: PipeImgGenBlueprint, expected_plx: str):
- """Test converting ImgGen pipe blueprint to PLX string."""
- result = PipelexInterpreter.img_gen_pipe_to_plx_string(pipe_name, blueprint, "test_domain")
+ result = PipelexInterpreter.img_gen_pipe_to_plx_string(pipe_name, blueprint)
assert result == expected_plx
diff --git a/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_input_spec.py b/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_input_spec.py
index 4a9f30432..ff135bbe1 100644
--- a/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_input_spec.py
+++ b/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_input_spec.py
@@ -7,10 +7,7 @@ from pipelex.core.pipes.pipe_input_blueprint import InputRequirementBlueprint
class TestPipelexInterpreterInputSpec:
- """Test input serialization for both simple strings and InputRequirementBlueprint objects."""
-
def test_inputs_to_plx_string_simple_strings(self):
- """Test converting simple string inputs to PLX format like { text = "Text", topic = "Text" }."""
# Simple string inputs (like "Text")
simple_inputs = {"text": "Text", "topic": "Text"}
@@ -20,7 +17,6 @@ class TestPipelexInterpreterInputSpec:
assert result == expected
def test_inputs_to_plx_string_input_requirement_blueprints(self):
- """Test converting InputRequirementBlueprint inputs to PLX format like { text = { concept = "Text", multiplicity = 1 } }."""
# InputRequirementBlueprint inputs
complex_inputs = {
"text": InputRequirementBlueprint(concept="Text", multiplicity=1),
@@ -53,6 +49,5 @@ class TestPipelexInterpreterInputSpec:
],
)
def test_serialize_input_requirement(self, input_req: InputRequirementBlueprint, expected: dict[str, Any]):
- """Test serializing a single InputRequirementBlueprint with various multiplicity values."""
result = PipelexInterpreter.serialize_input_requirement(input_req)
assert result == expected
diff --git a/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_jinja2_plx.py b/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_jinja2_plx.py
index 290efeb5f..525a17a9c 100644
--- a/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_jinja2_plx.py
+++ b/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_jinja2_plx.py
@@ -5,8 +5,6 @@ from pipelex.pipe_operators.jinja2.pipe_jinja2_blueprint import PipeJinja2Bluepr
class TestPipelexInterpreterJinja2PLX:
- """Test Jinja2 pipe to PLX string conversion."""
-
@pytest.mark.parametrize(
"pipe_name,blueprint,expected_plx",
[
@@ -45,6 +43,5 @@ jinja2_name = "data_formatter\"""",
],
)
def test_jinja2_pipe_to_plx_string(self, pipe_name: str, blueprint: PipeJinja2Blueprint, expected_plx: str):
- """Test converting Jinja2 pipe blueprint to PLX string."""
- result = PipelexInterpreter.jinja2_pipe_to_plx_string(pipe_name, blueprint, "test_domain")
+ result = PipelexInterpreter.jinja2_pipe_to_plx_string(pipe_name, blueprint)
assert result == expected_plx
diff --git a/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_jinja2_spacing.py b/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_jinja2_spacing.py
index 15e1b9d8c..41b8ec9f7 100644
--- a/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_jinja2_spacing.py
+++ b/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_jinja2_spacing.py
@@ -5,8 +5,6 @@ from pipelex.pipe_operators.jinja2.pipe_jinja2_blueprint import PipeJinja2Bluepr
class TestPipelexInterpreterJinja2PipePLX:
- """Test the jinja2_pipe_to_plx_string function directly."""
-
@pytest.mark.parametrize(
"pipe_name,blueprint,expected_plx",
[
@@ -75,6 +73,5 @@ jinja2_name = "greeting_template\"""",
],
)
def test_jinja2_pipe_to_plx_string(self, pipe_name: str, blueprint: PipeJinja2Blueprint, expected_plx: str):
- """Test converting PipeJinja2Blueprint to PLX string with various configurations."""
- result = PipelexInterpreter.jinja2_pipe_to_plx_string(pipe_name, blueprint, "test_domain")
+ result = PipelexInterpreter.jinja2_pipe_to_plx_string(pipe_name, blueprint)
assert result == expected_plx
diff --git a/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_llm_plx.py b/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_llm_plx.py
index 08d6641a7..5b9b0e6f4 100644
--- a/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_llm_plx.py
+++ b/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_llm_plx.py
@@ -6,8 +6,6 @@ from pipelex.pipe_operators.llm.pipe_llm_blueprint import PipeLLMBlueprint
class TestPipelexInterpreterLLMPLX:
- """Test LLM pipe to PLX string conversion."""
-
@pytest.mark.parametrize(
"pipe_name,blueprint,expected_plx",
[
@@ -89,6 +87,5 @@ prompt_template = "Generate ideas about: $topic\"""",
],
)
def test_llm_pipe_to_plx_string(self, pipe_name: str, blueprint: PipeLLMBlueprint, expected_plx: str):
- """Test converting LLM pipe blueprint to PLX string."""
- result = PipelexInterpreter.llm_pipe_to_plx_string(pipe_name, blueprint, "test_domain")
+ result = PipelexInterpreter.llm_pipe_to_plx_string(pipe_name, blueprint)
assert result == expected_plx
diff --git a/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_ocr_plx.py b/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_ocr_plx.py
index 54d83bf8d..c8f6b3fc9 100644
--- a/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_ocr_plx.py
+++ b/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_ocr_plx.py
@@ -5,8 +5,6 @@ from pipelex.pipe_operators.ocr.pipe_ocr_blueprint import PipeOcrBlueprint
class TestPipelexInterpreterOcrPLX:
- """Test OCR pipe to PLX string conversion."""
-
@pytest.mark.parametrize(
"pipe_name,blueprint,expected_plx",
[
@@ -45,6 +43,5 @@ ocr = "base_ocr_pypdfium2\"""",
],
)
def test_ocr_pipe_to_plx_string(self, pipe_name: str, blueprint: PipeOcrBlueprint, expected_plx: str):
- """Test converting OCR pipe blueprint to PLX string."""
- result = PipelexInterpreter.ocr_pipe_to_plx_string(pipe_name, blueprint, "test_domain")
+ result = PipelexInterpreter.ocr_pipe_to_plx_string(pipe_name, blueprint)
assert result == expected_plx
diff --git a/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_parallel_plx.py b/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_parallel_plx.py
index e2a086d3c..4ddbac40d 100644
--- a/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_parallel_plx.py
+++ b/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_parallel_plx.py
@@ -6,8 +6,6 @@ from pipelex.pipe_controllers.sub_pipe_blueprint import SubPipeBlueprint
class TestPipelexInterpreterParallelPLX:
- """Test Parallel pipe to PLX string conversion."""
-
@pytest.mark.parametrize(
"pipe_name,blueprint,expected_plx",
[
@@ -64,6 +62,5 @@ combined_output = "MergedResults\"""",
],
)
def test_parallel_pipe_to_plx_string(self, pipe_name: str, blueprint: PipeParallelBlueprint, expected_plx: str):
- """Test converting Parallel pipe blueprint to PLX string."""
- result = PipelexInterpreter.parallel_pipe_to_plx_string(pipe_name, blueprint, "test_domain")
+ result = PipelexInterpreter.parallel_pipe_to_plx_string(pipe_name, blueprint)
assert result == expected_plx
diff --git a/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_sequence_plx.py b/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_sequence_plx.py
index 1c0f95bdf..8c2e15b65 100644
--- a/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_sequence_plx.py
+++ b/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_sequence_plx.py
@@ -1,5 +1,3 @@
-"""Test PipelexInterpreter Sequence pipe to PLX string conversion."""
-
import pytest
from pipelex.core.interpreter import PipelexInterpreter
@@ -9,8 +7,6 @@ from pipelex.pipe_controllers.sub_pipe_blueprint import SubPipeBlueprint
class TestPipelexInterpreterSequencePLX:
- """Test Sequence pipe to PLX string conversion."""
-
@pytest.mark.parametrize(
"pipe_name,blueprint,expected_plx",
[
@@ -84,6 +80,5 @@ steps = [
],
)
def test_sequence_pipe_to_plx_string(self, pipe_name: str, blueprint: PipeSequenceBlueprint, expected_plx: str):
- """Test converting Sequence pipe blueprint to PLX string."""
- result = PipelexInterpreter.sequence_pipe_to_plx_string(pipe_name, blueprint, "test_domain")
+ result = PipelexInterpreter.sequence_pipe_to_plx_string(pipe_name, blueprint)
assert result == expected_plx
diff --git a/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_serialize_sub_pipe.py b/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_serialize_sub_pipe.py
index dac9c022e..95d72fa7c 100644
--- a/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_serialize_sub_pipe.py
+++ b/tests/unit/pipelex/core/interpreter/test_pipelex_interpreter_serialize_sub_pipe.py
@@ -1,6 +1,4 @@
-"""Test PipelexInterpreter serialize_sub_pipe function."""
-
-from typing import Any, Dict
+from typing import Any
import pytest
@@ -9,8 +7,6 @@ from pipelex.pipe_controllers.sub_pipe_blueprint import SubPipeBlueprint
class TestPipelexInterpreterSerializeSubPipe:
- """Test the serialize_sub_pipe function directly."""
-
@pytest.mark.parametrize(
"sub_pipe_blueprint,expected_result",
[
@@ -98,8 +94,7 @@ class TestPipelexInterpreterSerializeSubPipe:
),
],
)
- def test_serialize_sub_pipe(self, sub_pipe_blueprint: SubPipeBlueprint, expected_result: Dict[str, Any]):
- """Test serializing SubPipeBlueprint with various configurations."""
+ def test_serialize_sub_pipe(self, sub_pipe_blueprint: SubPipeBlueprint, expected_result: dict[str, Any]):
result = PipelexInterpreter.serialize_sub_pipe(sub_pipe_blueprint)
assert result == expected_result
@@ -154,6 +149,5 @@ class TestPipelexInterpreterSerializeSubPipe:
],
)
def test_sub_pipe_to_plx_string(self, sub_pipe_blueprint: SubPipeBlueprint, expected_plx_string: str):
- """Test converting SubPipeBlueprint directly to PLX inline table string."""
result = PipelexInterpreter.sub_pipe_to_plx_string(sub_pipe_blueprint)
assert result == expected_plx_string
diff --git a/tests/unit/pipelex/core/memory/__init__.py b/tests/unit/pipelex/core/memory/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/tests/unit/pipelex/core/memory/test_dry_working_memory.py b/tests/unit/pipelex/core/memory/test_dry_working_memory.py
index 83a6b5f53..89f923921 100644
--- a/tests/unit/pipelex/core/memory/test_dry_working_memory.py
+++ b/tests/unit/pipelex/core/memory/test_dry_working_memory.py
@@ -1,7 +1,3 @@
-"""Test dry working memory functionality."""
-
-from typing import List
-
import pytest
from pytest import FixtureRequest
@@ -17,13 +13,10 @@ from tests.test_pipelines.tricky_questions import ThoughtfulAnswer
@pytest.mark.dry_runnable
@pytest.mark.asyncio(loop_scope="class")
class TestDryWorkingMemory:
- """Test WorkingMemory dry run functionality."""
-
async def test_make_for_dry_run_with_page_content(
self,
request: FixtureRequest,
):
- """Test that make_for_dry_run creates appropriate mocks for PageContent."""
log.info("Testing dry run with PageContent")
# Test the specific inputs requested by the user
@@ -65,7 +58,6 @@ class TestDryWorkingMemory:
self,
request: FixtureRequest,
):
- """Test that make_for_dry_run creates appropriate mocks for structured content."""
log.info("Testing dry run with structured content (ThoughtfulAnswer)")
# Use ThoughtfulAnswer from tricky questions domain
@@ -88,9 +80,9 @@ class TestDryWorkingMemory:
]
dry_memory = WorkingMemoryFactory.make_for_dry_run(needed_inputs=needed_inputs)
+ expected_number_of_elements_in_memory = 2
- # Verify mocks were created
- assert len(dry_memory.root) == 2
+ assert len(dry_memory.root) == expected_number_of_elements_in_memory
assert dry_memory.get_optional_stuff("thoughtful_answer") is not None
assert dry_memory.get_optional_stuff("question") is not None
@@ -124,7 +116,6 @@ class TestDryWorkingMemory:
self,
request: FixtureRequest,
):
- """Test that make_for_dry_run falls back to TextContent when needed."""
log.info("Testing dry run with TextContent fallback")
needed_inputs = [
@@ -152,7 +143,6 @@ class TestDryWorkingMemory:
dry_memory = WorkingMemoryFactory.make_for_dry_run(needed_inputs=needed_inputs)
- # Verify mocks were created
assert len(dry_memory.root) == 2
assert dry_memory.get_optional_stuff("question_analysis") is not None
assert dry_memory.get_optional_stuff("conclusion") is not None
@@ -174,7 +164,6 @@ class TestDryWorkingMemory:
self,
request: FixtureRequest,
):
- """Test that make_for_dry_run handles mixed content types correctly."""
log.info("Testing dry run with mixed content types")
needed_inputs = [
@@ -207,7 +196,6 @@ class TestDryWorkingMemory:
dry_memory = WorkingMemoryFactory.make_for_dry_run(needed_inputs=needed_inputs)
- # Verify all mocks were created
assert len(dry_memory.root) == 3
# Verify structured content
@@ -234,10 +222,9 @@ class TestDryWorkingMemory:
self,
request: FixtureRequest,
):
- """Test that make_for_dry_run handles empty inputs gracefully."""
log.info("Testing dry run with empty inputs")
- needed_inputs: List[TypedNamedInputRequirement] = []
+ needed_inputs: list[TypedNamedInputRequirement] = []
dry_memory = WorkingMemoryFactory.make_for_dry_run(needed_inputs=needed_inputs)
@@ -251,7 +238,6 @@ class TestDryWorkingMemory:
self,
request: FixtureRequest,
):
- """Test make_for_dry_run with a realistic pipeline scenario from tricky questions."""
log.info("Testing dry run with realistic tricky questions pipeline scenario")
# Simulate the conclude_tricky_question_by_steps pipeline needs
diff --git a/tests/unit/pipelex/core/memory/test_working_memory.py b/tests/unit/pipelex/core/memory/test_working_memory.py
index 8158b9017..85e081f8b 100644
--- a/tests/unit/pipelex/core/memory/test_working_memory.py
+++ b/tests/unit/pipelex/core/memory/test_working_memory.py
@@ -1,4 +1,4 @@
-from typing import ClassVar, List, Tuple
+from typing import ClassVar
import pytest
@@ -12,8 +12,6 @@ from pipelex.core.stuffs.stuff_factory import StuffFactory
class TestWorkingMemoryData:
- """Test data for WorkingMemory tests."""
-
# Sample text content
SAMPLE_TEXT = """
The Dawn of Ultra-Rapid Transit: NextGen High-Speed Trains Redefine Travel
@@ -40,7 +38,7 @@ class TestWorkingMemoryData:
HTML_CONTENT_CASE = "html_content"
NUMBER_CONTENT_CASE = "number_content"
- TEST_CASES: ClassVar[List[Tuple[str, str]]] = [
+ TEST_CASES: ClassVar[list[tuple[str, str]]] = [
("Single text content", SINGLE_TEXT_CASE),
("Single image content", SINGLE_IMAGE_CASE),
("Single PDF content", SINGLE_PDF_CASE),
@@ -67,7 +65,7 @@ class TestWorkingMemory:
def single_image_memory(self) -> WorkingMemory:
"""Create WorkingMemory with single image content."""
return WorkingMemoryFactory.make_from_image(
- image_url=TestWorkingMemoryData.SAMPLE_IMAGE_URL, concept_string="gantt.GanttImage", name="gantt_chart_image"
+ image_url=TestWorkingMemoryData.SAMPLE_IMAGE_URL, concept_string="gantt.GanttImage", name="gantt_chart_image",
)
@pytest.fixture
@@ -129,7 +127,7 @@ class TestWorkingMemory:
TextContent(text="The quick brown fox jumps over the lazy dog"),
ImageContent(url=TestWorkingMemoryData.SAMPLE_IMAGE_URL),
NumberContent(number=42.5),
- ]
+ ],
)
complex_stuff = StuffFactory.make_stuff(
diff --git a/tests/unit/pipelex/core/pipes/data.py b/tests/unit/pipelex/core/pipes/data.py
index 27f4f22c1..145267a20 100644
--- a/tests/unit/pipelex/core/pipes/data.py
+++ b/tests/unit/pipelex/core/pipes/data.py
@@ -1,11 +1,7 @@
-"""Test data for make_output_multiplicity function tests."""
-
-from typing import Optional, Tuple
-
from pipelex.core.pipes.pipe_run_params import OutputMultiplicityResolution, PipeOutputMultiplicity
# Test cases format: (nb_output, multiple_output, expected_result, test_description)
-MAKE_OUTPUT_MULTIPLICITY_TEST_CASES: list[Tuple[Optional[int], Optional[bool], Optional[PipeOutputMultiplicity], str]] = [
+MAKE_OUTPUT_MULTIPLICITY_TEST_CASES: list[tuple[int | None, bool | None, PipeOutputMultiplicity | None, str]] = [
# nb_output, multiple_output, expected_result, description
# Test cases where nb_output takes precedence
(3, None, 3, "nb_output=3, multiple_output=None -> returns 3"),
@@ -31,7 +27,7 @@ MAKE_OUTPUT_MULTIPLICITY_TEST_CASES: list[Tuple[Optional[int], Optional[bool], O
# Test cases format: (base, override, expected_result, description)
OUTPUT_MULTIPLICITY_TO_APPLY_TEST_CASES: list[
- Tuple[Optional[PipeOutputMultiplicity], Optional[PipeOutputMultiplicity], OutputMultiplicityResolution, str]
+ tuple[PipeOutputMultiplicity | None, PipeOutputMultiplicity | None, OutputMultiplicityResolution, str]
] = [
# base, override, expected_result, description
# Override is None - use base value as-is
diff --git a/tests/unit/pipelex/core/pipes/test_pipe_run_params.py b/tests/unit/pipelex/core/pipes/test_pipe_run_params.py
index 0da380d08..3e96e7027 100644
--- a/tests/unit/pipelex/core/pipes/test_pipe_run_params.py
+++ b/tests/unit/pipelex/core/pipes/test_pipe_run_params.py
@@ -1,7 +1,3 @@
-"""Unit tests for pipe_run_params module."""
-
-from typing import Optional
-
import pytest
from pipelex.core.pipes.pipe_run_params import (
@@ -14,15 +10,13 @@ from tests.unit.pipelex.core.pipes.data import MAKE_OUTPUT_MULTIPLICITY_TEST_CAS
class TestMakeOutputMultiplicity:
- """Test cases for make_output_multiplicity function."""
-
@pytest.mark.parametrize(
"nb_output,multiple_output,expected_result,description",
MAKE_OUTPUT_MULTIPLICITY_TEST_CASES,
ids=[case[3] for case in MAKE_OUTPUT_MULTIPLICITY_TEST_CASES],
)
def test_make_output_multiplicity_all_cases(
- self, nb_output: Optional[int], multiple_output: Optional[bool], expected_result: Optional[PipeOutputMultiplicity], description: str
+ self, nb_output: int | None, multiple_output: bool | None, expected_result: PipeOutputMultiplicity | None, description: str,
):
"""Test make_output_multiplicity with all parameter combinations."""
result = make_output_multiplicity(nb_output=nb_output, multiple_output=multiple_output)
@@ -118,8 +112,8 @@ class TestOutputMultiplicityToApply:
)
def test_output_multiplicity_to_apply_all_cases(
self,
- base: Optional[PipeOutputMultiplicity],
- override: Optional[PipeOutputMultiplicity],
+ base: PipeOutputMultiplicity | None,
+ override: PipeOutputMultiplicity | None,
expected_result: OutputMultiplicityResolution,
description: str,
):
@@ -138,19 +132,19 @@ class TestOutputMultiplicityToApply:
assert result.specific_output_count is None
# Base True -> multiple output, LLM decides
- result = output_multiplicity_to_apply(True, None)
+ result = output_multiplicity_to_apply(base_multiplicity=True, override_multiplicity=None)
assert result.resolved_multiplicity is True
assert result.enable_multiple_outputs is True
assert result.specific_output_count is None
# Base int -> multiple output with specific count
- result = output_multiplicity_to_apply(3, None)
+ result = output_multiplicity_to_apply(base_multiplicity=3, override_multiplicity=None)
assert result.resolved_multiplicity == 3
assert result.enable_multiple_outputs is True
assert result.specific_output_count == 3
# Base False -> single output
- result = output_multiplicity_to_apply(False, None)
+ result = output_multiplicity_to_apply(base_multiplicity=False, override_multiplicity=None)
assert result.resolved_multiplicity is False
assert result.enable_multiple_outputs is False
assert result.specific_output_count is None
@@ -161,7 +155,7 @@ class TestOutputMultiplicityToApply:
test_bases = [None, True, False, 1, 5, 10, -1]
for base in test_bases:
- result = output_multiplicity_to_apply(base, False)
+ result = output_multiplicity_to_apply(base_multiplicity=base, override_multiplicity=False)
assert result.resolved_multiplicity is False, f"Failed for base={base}: resolved_multiplicity should be False"
assert result.enable_multiple_outputs is False, f"Failed for base={base}: enable_multiple_outputs should be False"
assert result.specific_output_count is None, f"Failed for base={base}: specific_output_count should be None"
@@ -169,26 +163,26 @@ class TestOutputMultiplicityToApply:
def test_override_true_enables_multiple_outputs(self):
"""Test that override=True enables multiple outputs, preserving int base values."""
# Base None -> preserve None but enable multiple
- result = output_multiplicity_to_apply(None, True)
+ result = output_multiplicity_to_apply(base_multiplicity=None, override_multiplicity=True)
assert result.resolved_multiplicity is None
assert result.enable_multiple_outputs is True
assert result.specific_output_count is None
# Base False -> enable multiple, LLM decides count
- result = output_multiplicity_to_apply(False, True)
+ result = output_multiplicity_to_apply(base_multiplicity=False, override_multiplicity=True)
assert result.resolved_multiplicity is True
assert result.enable_multiple_outputs is True
assert result.specific_output_count is None
# Base True -> keep True
- result = output_multiplicity_to_apply(True, True)
+ result = output_multiplicity_to_apply(base_multiplicity=True, override_multiplicity=True)
assert result.resolved_multiplicity is True
assert result.enable_multiple_outputs is True
assert result.specific_output_count is None
# Base int -> preserve count, enable multiple
for base_int in [1, 3, 5, 10]:
- result = output_multiplicity_to_apply(base_int, True)
+ result = output_multiplicity_to_apply(base_multiplicity=base_int, override_multiplicity=True)
assert result.resolved_multiplicity == base_int
assert result.enable_multiple_outputs is True
assert result.specific_output_count == base_int
@@ -199,14 +193,14 @@ class TestOutputMultiplicityToApply:
override_int = 5
for base in test_bases:
- result = output_multiplicity_to_apply(base, override_int)
+ result = output_multiplicity_to_apply(base_multiplicity=base, override_multiplicity=override_int)
assert result.resolved_multiplicity == override_int, f"Failed for base={base}: resolved_multiplicity should be {override_int}"
assert result.enable_multiple_outputs is True, f"Failed for base={base}: enable_multiple_outputs should be True"
assert result.specific_output_count == override_int, f"Failed for base={base}: specific_output_count should be {override_int}"
def test_return_basemodel_structure(self):
"""Test that the function always returns an OutputMultiplicityResolution with correct types."""
- result = output_multiplicity_to_apply(None, None)
+ result = output_multiplicity_to_apply(base_multiplicity=None, override_multiplicity=None)
assert isinstance(result, OutputMultiplicityResolution)
# Test field types
@@ -217,29 +211,29 @@ class TestOutputMultiplicityToApply:
def test_edge_cases(self):
"""Test edge cases and boundary conditions."""
# Negative numbers
- result = output_multiplicity_to_apply(-1, None)
+ result = output_multiplicity_to_apply(base_multiplicity=-1, override_multiplicity=None)
assert result.resolved_multiplicity == -1
assert result.enable_multiple_outputs is True
assert result.specific_output_count == -1
- result = output_multiplicity_to_apply(3, -2)
+ result = output_multiplicity_to_apply(base_multiplicity=3, override_multiplicity=-2)
assert result.resolved_multiplicity == -2
assert result.enable_multiple_outputs is True
assert result.specific_output_count == -2
# Zero values
- result = output_multiplicity_to_apply(0, None)
+ result = output_multiplicity_to_apply(base_multiplicity=0, override_multiplicity=None)
assert result.resolved_multiplicity == 0
assert result.enable_multiple_outputs is True
assert result.specific_output_count == 0
- result = output_multiplicity_to_apply(True, 0)
+ result = output_multiplicity_to_apply(base_multiplicity=True, override_multiplicity=0)
assert result.resolved_multiplicity == 0
assert result.enable_multiple_outputs is True
assert result.specific_output_count == 0
# Large numbers
- result = output_multiplicity_to_apply(1000, None)
+ result = output_multiplicity_to_apply(base_multiplicity=1000, override_multiplicity=None)
assert result.resolved_multiplicity == 1000
assert result.enable_multiple_outputs is True
assert result.specific_output_count == 1000
@@ -247,40 +241,40 @@ class TestOutputMultiplicityToApply:
def test_override_precedence_logic(self):
"""Test the precedence logic where override always takes precedence when provided."""
# Override False beats any base
- result = output_multiplicity_to_apply(True, False)
+ result = output_multiplicity_to_apply(base_multiplicity=True, override_multiplicity=False)
assert result.resolved_multiplicity is False
assert result.enable_multiple_outputs is False
assert result.specific_output_count is None
- result = output_multiplicity_to_apply(5, False)
+ result = output_multiplicity_to_apply(base_multiplicity=5, override_multiplicity=False)
assert result.resolved_multiplicity is False
assert result.enable_multiple_outputs is False
assert result.specific_output_count is None
# Override int beats any base
- result = output_multiplicity_to_apply(True, 3)
+ result = output_multiplicity_to_apply(base_multiplicity=True, override_multiplicity=3)
assert result.resolved_multiplicity == 3
assert result.enable_multiple_outputs is True
assert result.specific_output_count == 3
- result = output_multiplicity_to_apply(7, 3)
+ result = output_multiplicity_to_apply(base_multiplicity=7, override_multiplicity=3)
assert result.resolved_multiplicity == 3
assert result.enable_multiple_outputs is True
assert result.specific_output_count == 3
# Override True with int base preserves int
- result = output_multiplicity_to_apply(5, True)
+ result = output_multiplicity_to_apply(base_multiplicity=5, override_multiplicity=True)
assert result.resolved_multiplicity == 5
assert result.enable_multiple_outputs is True
assert result.specific_output_count == 5
# Override True with bool base becomes True
- result = output_multiplicity_to_apply(True, True)
+ result = output_multiplicity_to_apply(base_multiplicity=True, override_multiplicity=True)
assert result.resolved_multiplicity is True
assert result.enable_multiple_outputs is True
assert result.specific_output_count is None
- result = output_multiplicity_to_apply(False, True)
+ result = output_multiplicity_to_apply(base_multiplicity=False, override_multiplicity=True)
assert result.resolved_multiplicity is True
assert result.enable_multiple_outputs is True
assert result.specific_output_count is None
@@ -288,25 +282,25 @@ class TestOutputMultiplicityToApply:
def test_complex_scenarios(self):
"""Test complex real-world scenarios."""
# Scenario: Pipe has base multiplicity=3, runtime wants to disable it
- result = output_multiplicity_to_apply(3, False)
+ result = output_multiplicity_to_apply(base_multiplicity=3, override_multiplicity=False)
assert result.resolved_multiplicity is False
assert result.enable_multiple_outputs is False
assert result.specific_output_count is None
# Scenario: Pipe has no base multiplicity, runtime wants 5 outputs
- result = output_multiplicity_to_apply(None, 5)
+ result = output_multiplicity_to_apply(base_multiplicity=None, override_multiplicity=5)
assert result.resolved_multiplicity == 5
assert result.enable_multiple_outputs is True
assert result.specific_output_count == 5
# Scenario: Pipe has base True, runtime wants specific count
- result = output_multiplicity_to_apply(True, 2)
+ result = output_multiplicity_to_apply(base_multiplicity=True, override_multiplicity=2)
assert result.resolved_multiplicity == 2
assert result.enable_multiple_outputs is True
assert result.specific_output_count == 2
# Scenario: Pipe has base count, runtime wants to enable but let LLM decide
- result = output_multiplicity_to_apply(4, True)
+ result = output_multiplicity_to_apply(base_multiplicity=4, override_multiplicity=True)
assert result.resolved_multiplicity == 4 # Preserves base count
assert result.enable_multiple_outputs is True
assert result.specific_output_count == 4
diff --git a/tests/unit/pipelex/core/stuffs/__init__.py b/tests/unit/pipelex/core/stuffs/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/tests/unit/pipelex/core/stuffs/test_stuff_content_factory.py b/tests/unit/pipelex/core/stuffs/test_stuff_content_factory.py
index 8e6574b9f..860d9968a 100644
--- a/tests/unit/pipelex/core/stuffs/test_stuff_content_factory.py
+++ b/tests/unit/pipelex/core/stuffs/test_stuff_content_factory.py
@@ -1,4 +1,4 @@
-from typing import Any, ClassVar, Dict, List, Tuple
+from typing import Any, ClassVar, Dict
import pytest
@@ -29,24 +29,24 @@ class TestCases:
}
# Test cases for registered class (using actual registered class)
- REGISTERED_CLASS_BLUEPRINT: ClassVar[Dict[str, Any]] = {
+ REGISTERED_CLASS_BLUEPRINT: ClassVar[dict[str, Any]] = {
"concept_string": "test.MockRegisteredContent",
"content": {"title": "Test Question", "description": "What are aerodynamic features?"},
}
# Test cases for unregistered class (creates implicit concept, returns TextContent)
- UNREGISTERED_STRING_BLUEPRINT: ClassVar[Dict[str, Any]] = {
+ UNREGISTERED_STRING_BLUEPRINT: ClassVar[dict[str, Any]] = {
"concept_string": "unknown.NonExistentConcept",
"content": "This should create implicit concept and return TextContent",
}
# Test cases for unregistered class with dict content
- UNREGISTERED_DICT_BLUEPRINT: ClassVar[Dict[str, Any]] = {
+ UNREGISTERED_DICT_BLUEPRINT: ClassVar[dict[str, Any]] = {
"concept_string": "unknown.NonExistentConcept",
"content": {"text": "Dict content for implicit concept"},
}
- TEST_BLUEPRINTS: ClassVar[List[Tuple[str, Dict[str, Any]]]] = [
+ TEST_BLUEPRINTS: ClassVar[list[tuple[str, dict[str, Any]]]] = [
("text_string", TEXT_STRING_BLUEPRINT),
("text_dict", TEXT_DICT_BLUEPRINT),
("text_no_prefix", TEXT_NO_PREFIX_BLUEPRINT),
@@ -79,7 +79,7 @@ class TestStuffContentFactory:
description: str
result = StuffContentFactory.make_content_from_value(
- stuff_content_subclass=MockStructuredContent, value={"title": "Test Title", "description": "Test Description"}
+ stuff_content_subclass=MockStructuredContent, value={"title": "Test Title", "description": "Test Description"},
)
assert isinstance(result, MockStructuredContent)
@@ -89,7 +89,7 @@ class TestStuffContentFactory:
def test_make_stuffcontent_from_concept_code_required_text_content(self):
"""Test required method with native.Text concept (should work)."""
result = StuffContentFactory.make_stuff_content_from_concept_required(
- concept=ConceptFactory.make_native_concept(native_concept_data=NATIVE_CONCEPTS_DATA[NativeConceptEnum.TEXT]), value="Test text content"
+ concept=ConceptFactory.make_native_concept(native_concept_data=NATIVE_CONCEPTS_DATA[NativeConceptEnum.TEXT]), value="Test text content",
)
assert isinstance(result, TextContent)
@@ -108,7 +108,7 @@ class TestStuffContentFactory:
def test_make_stuffcontent_from_concept_code_with_fallback_text_success(self):
"""Test fallback method with native.Text concept."""
result = StuffContentFactory.make_stuff_content_from_concept_with_fallback(
- concept=ConceptFactory.make_native_concept(native_concept_data=NATIVE_CONCEPTS_DATA[NativeConceptEnum.TEXT]), value="Test text content"
+ concept=ConceptFactory.make_native_concept(native_concept_data=NATIVE_CONCEPTS_DATA[NativeConceptEnum.TEXT]), value="Test text content",
)
assert isinstance(result, TextContent)
@@ -138,14 +138,14 @@ class TestStuffContentFactory:
"test_name, blueprint",
TestCases.TEST_BLUEPRINTS,
)
- def test_blueprint_scenarios(self, test_name: str, blueprint: Dict[str, Any]):
+ def test_blueprint_scenarios(self, test_name: str, blueprint: dict[str, Any]):
"""Test various blueprint scenarios with parametrized test cases."""
content = blueprint["content"]
if test_name.startswith("text_"):
# Test native.Text concept with required method
result = StuffContentFactory.make_stuff_content_from_concept_required(
- concept=ConceptFactory.make_native_concept(native_concept_data=NATIVE_CONCEPTS_DATA[NativeConceptEnum.TEXT]), value=content
+ concept=ConceptFactory.make_native_concept(native_concept_data=NATIVE_CONCEPTS_DATA[NativeConceptEnum.TEXT]), value=content,
)
assert isinstance(result, TextContent)
if isinstance(content, str):
@@ -162,7 +162,7 @@ class TestStuffContentFactory:
# The dict will be passed through model_validate which should fail for TextContent
try:
result_required = StuffContentFactory.make_stuff_content_from_concept_required(
- concept=ConceptFactory.make_native_concept(native_concept_data=NATIVE_CONCEPTS_DATA[NativeConceptEnum.TEXT]), value=content
+ concept=ConceptFactory.make_native_concept(native_concept_data=NATIVE_CONCEPTS_DATA[NativeConceptEnum.TEXT]), value=content,
)
# If it succeeds, it should be TextContent (due to implicit concept)
assert isinstance(result_required, TextContent)
@@ -173,7 +173,7 @@ class TestStuffContentFactory:
# Test fallback method - same behavior expected
try:
result_fallback = StuffContentFactory.make_stuff_content_from_concept_with_fallback(
- concept=ConceptFactory.make_native_concept(native_concept_data=NATIVE_CONCEPTS_DATA[NativeConceptEnum.TEXT]), value=content
+ concept=ConceptFactory.make_native_concept(native_concept_data=NATIVE_CONCEPTS_DATA[NativeConceptEnum.TEXT]), value=content,
)
# If it succeeds, it should be TextContent (due to implicit concept)
assert isinstance(result_fallback, TextContent)
@@ -186,12 +186,12 @@ class TestStuffContentFactory:
# Both methods should work and return TextContent due to implicit concept creation
result_required = StuffContentFactory.make_stuff_content_from_concept_required(
- concept=ConceptFactory.make_native_concept(native_concept_data=NATIVE_CONCEPTS_DATA[NativeConceptEnum.TEXT]), value=content
+ concept=ConceptFactory.make_native_concept(native_concept_data=NATIVE_CONCEPTS_DATA[NativeConceptEnum.TEXT]), value=content,
)
assert isinstance(result_required, TextContent)
result_fallback = StuffContentFactory.make_stuff_content_from_concept_with_fallback(
- concept=ConceptFactory.make_native_concept(native_concept_data=NATIVE_CONCEPTS_DATA[NativeConceptEnum.TEXT]), value=content
+ concept=ConceptFactory.make_native_concept(native_concept_data=NATIVE_CONCEPTS_DATA[NativeConceptEnum.TEXT]), value=content,
)
assert isinstance(result_fallback, TextContent)
diff --git a/tests/unit/pipelex/core/stuffs/test_stuff_factory_make_stuff_from_content.py b/tests/unit/pipelex/core/stuffs/test_stuff_factory_make_stuff_from_content.py
index 5bbe352fd..623b7eaf7 100644
--- a/tests/unit/pipelex/core/stuffs/test_stuff_factory_make_stuff_from_content.py
+++ b/tests/unit/pipelex/core/stuffs/test_stuff_factory_make_stuff_from_content.py
@@ -1,4 +1,4 @@
-from typing import Any, ClassVar, Dict, List
+from typing import Any, ClassVar, Dict
import pytest
from pytest_mock import MockerFixture
@@ -15,24 +15,19 @@ from pipelex.core.stuffs.stuff_factory import StuffFactory, StuffFactoryError
class MockCustomContent(StructuredContent):
- """Mock content class for testing custom concepts."""
-
title: str
description: str
class TestData:
- """Test data for various input scenarios."""
-
- # Basic test data
TEXT_STRING: ClassVar[str] = "This is a test text"
CONCEPT_NAME: ClassVar[str] = "TestConcept"
- SEARCH_DOMAINS: ClassVar[List[str]] = ["test", "native"]
+ SEARCH_DOMAINS: ClassVar[list[str]] = ["test", "native"]
STUFF_NAME: ClassVar[str] = "test_stuff"
STUFF_CODE: ClassVar[str] = "test123"
# ListContent test data
- TEXT_LIST_ITEMS: ClassVar[List[TextContent]] = [
+ TEXT_LIST_ITEMS: ClassVar[list[TextContent]] = [
TextContent(text="Item 1"),
TextContent(text="Item 2"),
]
@@ -42,24 +37,22 @@ class TestData:
NATIVE_TEXT_DICT: ClassVar[Dict[str, Any]] = {"concept": NativeConceptEnum.TEXT, "content": {"text": "Native text content"}}
# Dictionary test data - custom concept with concept field
- CUSTOM_CONCEPT_DICT: ClassVar[Dict[str, Any]] = {
+ CUSTOM_CONCEPT_DICT: ClassVar[dict[str, Any]] = {
"concept": "test.CustomConcept",
"content": {"title": "Test Title", "description": "Test Description"},
}
# Dictionary test data - custom concept with concept_code field
- CUSTOM_CONCEPT_CODE_DICT: ClassVar[Dict[str, Any]] = {"concept_code": "test.AnotherConcept", "content": {"data": "test data"}}
+ CUSTOM_CONCEPT_CODE_DICT: ClassVar[dict[str, Any]] = {"concept_code": "test.AnotherConcept", "content": {"data": "test data"}}
# Dictionary test data - missing concept
- MISSING_CONCEPT_DICT: ClassVar[Dict[str, Any]] = {"content": {"data": "test data"}}
+ MISSING_CONCEPT_DICT: ClassVar[dict[str, Any]] = {"content": {"data": "test data"}}
# Dictionary test data - missing content
- MISSING_CONTENT_DICT: ClassVar[Dict[str, Any]] = {"concept": "test.SomeConcept"}
+ MISSING_CONTENT_DICT: ClassVar[dict[str, Any]] = {"concept": "test.SomeConcept"}
class TestMakeStuffFromStuffContentUsingSearchDomains:
- """Unit tests for make_stuff_from_stuff_content_using_search_domains method."""
-
def test_listcontent_with_valid_items(self, mocker: MockerFixture):
"""Test ListContent with valid items."""
list_content: ListContent[TextContent] = ListContent(items=TestData.TEXT_LIST_ITEMS)
@@ -280,7 +273,7 @@ class TestMakeStuffFromStuffContentUsingSearchDomains:
assert result == mock_stuff
mock_provider.get_required_concept.assert_called_with(concept_string="test.CustomConcept")
mock_content_factory.make_stuff_content_from_concept_with_fallback.assert_called_once_with(
- concept=mock_concept, value={"title": "Test Title", "description": "Test Description"}
+ concept=mock_concept, value={"title": "Test Title", "description": "Test Description"},
)
def test_dict_with_concept_code_field(self, mocker: MockerFixture):
diff --git a/tests/unit/pipelex/core/test_data/__init__.py b/tests/unit/pipelex/core/test_data/__init__.py
index fcbd717bb..6c1d15983 100644
--- a/tests/unit/pipelex/core/test_data/__init__.py
+++ b/tests/unit/pipelex/core/test_data/__init__.py
@@ -1,10 +1,7 @@
-"""Organized test data for PipelexInterpreter tests."""
-
from typing import ClassVar, List, Tuple, Type, Union
from pipelex.core.bundles.pipelex_bundle_blueprint import PipelexBundleBlueprint
-# Import all test data from organized modules
from .complex.multi_feature import COMPLEX_TEST_CASES
from .concepts.refining_concepts import REFINING_CONCEPT_TEST_CASES
from .concepts.simple_concepts import SIMPLE_CONCEPT_TEST_CASES
@@ -26,7 +23,7 @@ class InterpreterTestCases:
"""Test cases for PipelexInterpreter with PLX content and expected blueprints."""
# Aggregate all valid test cases from organized modules
- VALID_TEST_CASES: ClassVar[List[Tuple[str, str, PipelexBundleBlueprint]]] = [
+ VALID_TEST_CASES: ClassVar[list[tuple[str, str, PipelexBundleBlueprint]]] = [
# Domain tests
*DOMAIN_TEST_CASES,
# Concept tests
@@ -49,4 +46,4 @@ class InterpreterTestCases:
]
# Error test cases
- ERROR_TEST_CASES: ClassVar[List[Tuple[str, str, Union[Type[Exception], Tuple[Type[Exception], ...]]]]] = ERROR_TEST_CASES
+ ERROR_TEST_CASES: ClassVar[list[tuple[str, str, type[Exception] | tuple[type[Exception], ...]]]] = ERROR_TEST_CASES
diff --git a/tests/unit/pipelex/core/test_data/complex/multi_feature.py b/tests/unit/pipelex/core/test_data/complex/multi_feature.py
index 85f8225d9..04641cb16 100644
--- a/tests/unit/pipelex/core/test_data/complex/multi_feature.py
+++ b/tests/unit/pipelex/core/test_data/complex/multi_feature.py
@@ -1,5 +1,3 @@
-"""Complex test cases combining multiple features."""
-
from pipelex.core.bundles.pipelex_bundle_blueprint import PipelexBundleBlueprint
from pipelex.pipe_controllers.sequence.pipe_sequence_blueprint import PipeSequenceBlueprint
from pipelex.pipe_controllers.sub_pipe_blueprint import SubPipeBlueprint
diff --git a/tests/unit/pipelex/core/test_data/concepts/refining_concepts.py b/tests/unit/pipelex/core/test_data/concepts/refining_concepts.py
index da3d9170c..fc23dd0dc 100644
--- a/tests/unit/pipelex/core/test_data/concepts/refining_concepts.py
+++ b/tests/unit/pipelex/core/test_data/concepts/refining_concepts.py
@@ -1,5 +1,3 @@
-"""Refining concept test cases."""
-
from pipelex.core.bundles.pipelex_bundle_blueprint import PipelexBundleBlueprint
from pipelex.core.concepts.concept_blueprint import ConceptBlueprint
diff --git a/tests/unit/pipelex/core/test_data/concepts/simple_concepts.py b/tests/unit/pipelex/core/test_data/concepts/simple_concepts.py
index ff4f54df1..41fe3202b 100644
--- a/tests/unit/pipelex/core/test_data/concepts/simple_concepts.py
+++ b/tests/unit/pipelex/core/test_data/concepts/simple_concepts.py
@@ -1,5 +1,3 @@
-"""Simple concept test cases."""
-
from pipelex.core.bundles.pipelex_bundle_blueprint import PipelexBundleBlueprint
SIMPLE_CONCEPTS = (
diff --git a/tests/unit/pipelex/core/test_data/concepts/structured_concepts.py b/tests/unit/pipelex/core/test_data/concepts/structured_concepts.py
index 61c1f6fc4..b52a65e39 100644
--- a/tests/unit/pipelex/core/test_data/concepts/structured_concepts.py
+++ b/tests/unit/pipelex/core/test_data/concepts/structured_concepts.py
@@ -1,5 +1,3 @@
-"""Structured concept test cases."""
-
from pipelex.core.bundles.pipelex_bundle_blueprint import PipelexBundleBlueprint
from pipelex.core.concepts.concept_blueprint import ConceptBlueprint, ConceptStructureBlueprint, ConceptStructureBlueprintFieldType
@@ -29,10 +27,10 @@ birthdate = { type = "date", definition = "The birthdate of the person", require
structure={
"name": "The name of the person",
"age": ConceptStructureBlueprint(
- type=ConceptStructureBlueprintFieldType.NUMBER, definition="The age of the person", required=True
+ type=ConceptStructureBlueprintFieldType.NUMBER, definition="The age of the person", required=True,
),
"birthdate": ConceptStructureBlueprint(
- type=ConceptStructureBlueprintFieldType.DATE, definition="The birthdate of the person", required=True
+ type=ConceptStructureBlueprintFieldType.DATE, definition="The birthdate of the person", required=True,
),
},
),
diff --git a/tests/unit/pipelex/core/test_data/domain/simple_domains.py b/tests/unit/pipelex/core/test_data/domain/simple_domains.py
index 3dc8c2fd5..9e391cc6b 100644
--- a/tests/unit/pipelex/core/test_data/domain/simple_domains.py
+++ b/tests/unit/pipelex/core/test_data/domain/simple_domains.py
@@ -1,5 +1,3 @@
-"""Simple domain test cases."""
-
from pipelex.core.bundles.pipelex_bundle_blueprint import PipelexBundleBlueprint
SIMPLE_DOMAIN = (
diff --git a/tests/unit/pipelex/core/test_data/errors/invalid_plx.py b/tests/unit/pipelex/core/test_data/errors/invalid_plx.py
index 4d71aab81..ec70c8d7a 100644
--- a/tests/unit/pipelex/core/test_data/errors/invalid_plx.py
+++ b/tests/unit/pipelex/core/test_data/errors/invalid_plx.py
@@ -1,13 +1,8 @@
-"""Invalid PLX test cases focused on PLX structure and PipelexBundleBlueprint validation."""
-
-from typing import List, Tuple, Type, Union
-
from pydantic import ValidationError
from pipelex.core.domains.exceptions import DomainError
from pipelex.core.interpreter import PLXDecodeError
-# PLX Syntax Errors
INVALID_PLX_SYNTAX = (
"invalid_plx_syntax",
"""domain = "test_domain"
@@ -207,7 +202,7 @@ definition = "Domain with invalid array syntax"
)
# Export all error test cases
-ERROR_TEST_CASES: List[Tuple[str, str, Union[Type[Exception], Tuple[Type[Exception], ...]]]] = [
+ERROR_TEST_CASES: list[tuple[str, str, type[Exception] | tuple[type[Exception], ...]]] = [
# PLX Syntax Errors
INVALID_PLX_SYNTAX,
MALFORMED_SECTION,
diff --git a/tests/unit/pipelex/core/test_data/pipes/controllers/batch/pipe_batch.py b/tests/unit/pipelex/core/test_data/pipes/controllers/batch/pipe_batch.py
index 5aa459c10..0cc0c1226 100644
--- a/tests/unit/pipelex/core/test_data/pipes/controllers/batch/pipe_batch.py
+++ b/tests/unit/pipelex/core/test_data/pipes/controllers/batch/pipe_batch.py
@@ -1,5 +1,3 @@
-"""PipeBatch test cases."""
-
from pipelex.core.bundles.pipelex_bundle_blueprint import PipelexBundleBlueprint
from pipelex.pipe_controllers.batch.pipe_batch_blueprint import PipeBatchBlueprint
diff --git a/tests/unit/pipelex/core/test_data/pipes/controllers/condition/pipe_condition.py b/tests/unit/pipelex/core/test_data/pipes/controllers/condition/pipe_condition.py
index 150f9d445..d83deab06 100644
--- a/tests/unit/pipelex/core/test_data/pipes/controllers/condition/pipe_condition.py
+++ b/tests/unit/pipelex/core/test_data/pipes/controllers/condition/pipe_condition.py
@@ -1,5 +1,3 @@
-"""PipeCondition test cases."""
-
from pipelex.core.bundles.pipelex_bundle_blueprint import PipelexBundleBlueprint
from pipelex.pipe_controllers.condition.pipe_condition_blueprint import PipeConditionBlueprint, PipeConditionPipeMapBlueprint
@@ -28,7 +26,7 @@ expression = "input_data.category"
{
"small": "process_small",
"large": "process_large",
- }
+ },
),
),
},
diff --git a/tests/unit/pipelex/core/test_data/pipes/controllers/parallel/pipe_parallel.py b/tests/unit/pipelex/core/test_data/pipes/controllers/parallel/pipe_parallel.py
index 1c4382957..1056a0abc 100644
--- a/tests/unit/pipelex/core/test_data/pipes/controllers/parallel/pipe_parallel.py
+++ b/tests/unit/pipelex/core/test_data/pipes/controllers/parallel/pipe_parallel.py
@@ -1,5 +1,3 @@
-"""PipeParallel test cases."""
-
from pipelex.core.bundles.pipelex_bundle_blueprint import PipelexBundleBlueprint
from pipelex.pipe_controllers.parallel.pipe_parallel_blueprint import PipeParallelBlueprint
from pipelex.pipe_controllers.sub_pipe_blueprint import SubPipeBlueprint
diff --git a/tests/unit/pipelex/core/test_data/pipes/controllers/sequence/pipe_sequence.py b/tests/unit/pipelex/core/test_data/pipes/controllers/sequence/pipe_sequence.py
index 0267d74b7..0875c8edb 100644
--- a/tests/unit/pipelex/core/test_data/pipes/controllers/sequence/pipe_sequence.py
+++ b/tests/unit/pipelex/core/test_data/pipes/controllers/sequence/pipe_sequence.py
@@ -1,5 +1,3 @@
-"""PipeSequence test cases."""
-
from pipelex.core.bundles.pipelex_bundle_blueprint import PipelexBundleBlueprint
from pipelex.pipe_controllers.sequence.pipe_sequence_blueprint import PipeSequenceBlueprint
from pipelex.pipe_controllers.sub_pipe_blueprint import SubPipeBlueprint
diff --git a/tests/unit/pipelex/core/test_data/pipes/operators/func/pipe_func.py b/tests/unit/pipelex/core/test_data/pipes/operators/func/pipe_func.py
index 9130e6a26..2a17137a4 100644
--- a/tests/unit/pipelex/core/test_data/pipes/operators/func/pipe_func.py
+++ b/tests/unit/pipelex/core/test_data/pipes/operators/func/pipe_func.py
@@ -1,5 +1,3 @@
-"""PipeFunc test cases."""
-
from pipelex.core.bundles.pipelex_bundle_blueprint import PipelexBundleBlueprint
from pipelex.pipe_operators.func.pipe_func_blueprint import PipeFuncBlueprint
diff --git a/tests/unit/pipelex/core/test_data/pipes/operators/img_gen/pipe_img_gen.py b/tests/unit/pipelex/core/test_data/pipes/operators/img_gen/pipe_img_gen.py
index 51c194749..7247803fb 100644
--- a/tests/unit/pipelex/core/test_data/pipes/operators/img_gen/pipe_img_gen.py
+++ b/tests/unit/pipelex/core/test_data/pipes/operators/img_gen/pipe_img_gen.py
@@ -1,5 +1,3 @@
-"""PipeImgGen test cases."""
-
from pipelex.core.bundles.pipelex_bundle_blueprint import PipelexBundleBlueprint
from pipelex.core.concepts.concept_native import NativeConceptEnum
from pipelex.pipe_operators.img_gen.pipe_img_gen_blueprint import PipeImgGenBlueprint
diff --git a/tests/unit/pipelex/core/test_data/pipes/operators/jinja2/pipe_jinja2.py b/tests/unit/pipelex/core/test_data/pipes/operators/jinja2/pipe_jinja2.py
index 8e5b3f984..4781fe0e8 100644
--- a/tests/unit/pipelex/core/test_data/pipes/operators/jinja2/pipe_jinja2.py
+++ b/tests/unit/pipelex/core/test_data/pipes/operators/jinja2/pipe_jinja2.py
@@ -1,5 +1,3 @@
-"""PipeJinja2 test cases."""
-
from pipelex.core.bundles.pipelex_bundle_blueprint import PipelexBundleBlueprint
from pipelex.core.concepts.concept_native import NativeConceptEnum
from pipelex.pipe_operators.jinja2.pipe_jinja2_blueprint import PipeJinja2Blueprint
diff --git a/tests/unit/pipelex/core/test_data/pipes/operators/llm/pipe_llm.py b/tests/unit/pipelex/core/test_data/pipes/operators/llm/pipe_llm.py
index 9e5eb9816..112a341ac 100644
--- a/tests/unit/pipelex/core/test_data/pipes/operators/llm/pipe_llm.py
+++ b/tests/unit/pipelex/core/test_data/pipes/operators/llm/pipe_llm.py
@@ -1,5 +1,3 @@
-"""PipeLLM test cases."""
-
from pipelex.core.bundles.pipelex_bundle_blueprint import PipelexBundleBlueprint
from pipelex.core.concepts.concept_native import NativeConceptEnum
from pipelex.core.pipes.pipe_input_blueprint import InputRequirementBlueprint
diff --git a/tests/unit/pipelex/core/test_data/pipes/operators/ocr/pipe_ocr.py b/tests/unit/pipelex/core/test_data/pipes/operators/ocr/pipe_ocr.py
index 72d099884..5a860f26e 100644
--- a/tests/unit/pipelex/core/test_data/pipes/operators/ocr/pipe_ocr.py
+++ b/tests/unit/pipelex/core/test_data/pipes/operators/ocr/pipe_ocr.py
@@ -1,5 +1,3 @@
-"""PipeOcr test cases."""
-
from pipelex.core.bundles.pipelex_bundle_blueprint import PipelexBundleBlueprint
from pipelex.core.concepts.concept_native import NativeConceptEnum
from pipelex.pipe_operators.ocr.pipe_ocr_blueprint import PipeOcrBlueprint
diff --git a/tests/unit/pipelex/create/__init__.py b/tests/unit/pipelex/create/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/tests/unit/pipelex/create/test_structured_output_generator.py b/tests/unit/pipelex/create/test_structured_output_generator.py
index 38cce78f7..1204c0f7a 100644
--- a/tests/unit/pipelex/create/test_structured_output_generator.py
+++ b/tests/unit/pipelex/create/test_structured_output_generator.py
@@ -1,7 +1,3 @@
-"""Tests for structured output generator."""
-
-from typing import Dict, Union
-
from pipelex import pretty_print
from pipelex.core.concepts.concept_blueprint import ConceptStructureBlueprint, ConceptStructureBlueprintFieldType
from pipelex.core.concepts.concept_factory import ConceptFactory
@@ -15,7 +11,7 @@ class TestStructureGenerator:
"name": ConceptStructureBlueprint(definition="Name field", type=ConceptStructureBlueprintFieldType.TEXT, required=True),
"age": ConceptStructureBlueprint(definition="Age field", type=ConceptStructureBlueprintFieldType.INTEGER, required=False),
"active": ConceptStructureBlueprint(
- definition="Active status", type=ConceptStructureBlueprintFieldType.BOOLEAN, required=False, default_value=True
+ definition="Active status", type=ConceptStructureBlueprintFieldType.BOOLEAN, required=False, default_value=True,
),
}
@@ -44,13 +40,13 @@ class TestStructureGenerator:
"""Test generation with complex types like lists and dicts."""
structure_blueprint = {
"tags": ConceptStructureBlueprint(
- definition="List of tags", type=ConceptStructureBlueprintFieldType.LIST, item_type="text", required=False
+ definition="List of tags", type=ConceptStructureBlueprintFieldType.LIST, item_type="text", required=False,
),
"metadata": ConceptStructureBlueprint(
- definition="Metadata dictionary", type=ConceptStructureBlueprintFieldType.DICT, key_type="text", value_type="text", required=False
+ definition="Metadata dictionary", type=ConceptStructureBlueprintFieldType.DICT, key_type="text", value_type="text", required=False,
),
"scores": ConceptStructureBlueprint(
- definition="List of scores", type=ConceptStructureBlueprintFieldType.LIST, item_type="number", required=True
+ definition="List of scores", type=ConceptStructureBlueprintFieldType.LIST, item_type="number", required=True,
),
}
@@ -106,7 +102,7 @@ class TestStructureGenerator:
def test_empty_structure(self):
"""Test generation of structure with no fields."""
- structure_blueprint: Dict[str, ConceptStructureBlueprint] = {}
+ structure_blueprint: dict[str, ConceptStructureBlueprint] = {}
result = StructureGenerator().generate_from_structure_blueprint("EmptyModel", structure_blueprint)
@@ -157,7 +153,7 @@ class TestStructureGenerator:
def test_generate_from_blueprint_dict_function(self):
"""Test the convenience function for generating from blueprint dict."""
structure_blueprint = {
- "value": ConceptStructureBlueprint(definition="Test value", type=ConceptStructureBlueprintFieldType.TEXT, required=True)
+ "value": ConceptStructureBlueprint(definition="Test value", type=ConceptStructureBlueprintFieldType.TEXT, required=True),
}
result = StructureGenerator().generate_from_structure_blueprint("ConvenienceTest", structure_blueprint)
@@ -187,10 +183,10 @@ class TestStructureGenerator:
"integer_field": ConceptStructureBlueprint(definition="Integer field", type=ConceptStructureBlueprintFieldType.INTEGER, required=False),
"boolean_field": ConceptStructureBlueprint(definition="Boolean field", type=ConceptStructureBlueprintFieldType.BOOLEAN, required=False),
"list_field": ConceptStructureBlueprint(
- definition="List field", type=ConceptStructureBlueprintFieldType.LIST, item_type="text", required=False
+ definition="List field", type=ConceptStructureBlueprintFieldType.LIST, item_type="text", required=False,
),
"dict_field": ConceptStructureBlueprint(
- definition="Dict field", type=ConceptStructureBlueprintFieldType.DICT, key_type="text", value_type="integer", required=False
+ definition="Dict field", type=ConceptStructureBlueprintFieldType.DICT, key_type="text", value_type="integer", required=False,
),
}
@@ -249,13 +245,13 @@ class TestStructureGenerator:
"""Test fields with default values."""
structure_blueprint = {
"name": ConceptStructureBlueprint(
- definition="Person name", type=ConceptStructureBlueprintFieldType.TEXT, required=False, default_value="Anonymous"
+ definition="Person name", type=ConceptStructureBlueprintFieldType.TEXT, required=False, default_value="Anonymous",
),
"age": ConceptStructureBlueprint(
- definition="Person age", type=ConceptStructureBlueprintFieldType.INTEGER, required=False, default_value=0
+ definition="Person age", type=ConceptStructureBlueprintFieldType.INTEGER, required=False, default_value=0,
),
"active": ConceptStructureBlueprint(
- definition="Is active", type=ConceptStructureBlueprintFieldType.BOOLEAN, required=False, default_value=True
+ definition="Is active", type=ConceptStructureBlueprintFieldType.BOOLEAN, required=False, default_value=True,
),
}
@@ -284,13 +280,13 @@ class TestStructureGenerator:
"""Test nested list types with different item types."""
structure_blueprint = {
"text_list": ConceptStructureBlueprint(
- definition="List of text items", type=ConceptStructureBlueprintFieldType.LIST, item_type="text", required=False
+ definition="List of text items", type=ConceptStructureBlueprintFieldType.LIST, item_type="text", required=False,
),
"number_list": ConceptStructureBlueprint(
- definition="List of numbers", type=ConceptStructureBlueprintFieldType.LIST, item_type="number", required=True
+ definition="List of numbers", type=ConceptStructureBlueprintFieldType.LIST, item_type="number", required=True,
),
"integer_list": ConceptStructureBlueprint(
- definition="List of integers", type=ConceptStructureBlueprintFieldType.LIST, item_type="integer", required=False
+ definition="List of integers", type=ConceptStructureBlueprintFieldType.LIST, item_type="integer", required=False,
),
}
@@ -368,16 +364,16 @@ class TestStructureGenerator:
"id": ConceptStructureBlueprint(definition="Unique identifier", type=ConceptStructureBlueprintFieldType.INTEGER, required=True),
"name": ConceptStructureBlueprint(definition="Display name", type=ConceptStructureBlueprintFieldType.TEXT, required=True),
"tags": ConceptStructureBlueprint(
- definition="Associated tags", type=ConceptStructureBlueprintFieldType.LIST, item_type="text", required=False
+ definition="Associated tags", type=ConceptStructureBlueprintFieldType.LIST, item_type="text", required=False,
),
"metadata": ConceptStructureBlueprint(
- definition="Additional metadata", type=ConceptStructureBlueprintFieldType.DICT, key_type="text", value_type="text", required=False
+ definition="Additional metadata", type=ConceptStructureBlueprintFieldType.DICT, key_type="text", value_type="text", required=False,
),
"active": ConceptStructureBlueprint(
- definition="Whether item is active", type=ConceptStructureBlueprintFieldType.BOOLEAN, required=False, default_value=True
+ definition="Whether item is active", type=ConceptStructureBlueprintFieldType.BOOLEAN, required=False, default_value=True,
),
"priority": ConceptStructureBlueprint(
- definition="Priority level", choices=["low", "medium", "high", "urgent"], required=False, default_value="medium"
+ definition="Priority level", choices=["low", "medium", "high", "urgent"], required=False, default_value="medium",
),
}
@@ -408,11 +404,11 @@ class TestStructureGenerator:
def test_mixed_structure_blueprint_normalization(self):
"""Test that mixed structure blueprints (strings and ConceptStructureBlueprint objects) are properly normalized."""
# Create a mixed structure blueprint similar to what would come from TOML parsing
- mixed_structure_blueprint: Dict[str, Union[str, ConceptStructureBlueprint]] = {
+ mixed_structure_blueprint: dict[str, str | ConceptStructureBlueprint] = {
"name": "The name of the person", # Simple string definition
"age": ConceptStructureBlueprint(definition="The age of the person", type=ConceptStructureBlueprintFieldType.NUMBER, required=True),
"birthdate": ConceptStructureBlueprint(
- definition="The birthdate of the person", type=ConceptStructureBlueprintFieldType.DATE, required=True
+ definition="The birthdate of the person", type=ConceptStructureBlueprintFieldType.DATE, required=True,
),
}
diff --git a/tests/unit/pipelex/libraries/pipelines/builder/__init__.py b/tests/unit/pipelex/libraries/pipelines/builder/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/tests/unit/pipelex/libraries/pipelines/builder/concept/__init__.py b/tests/unit/pipelex/libraries/pipelines/builder/concept/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/tests/unit/pipelex/libraries/pipelines/builder/concept/test_data.py b/tests/unit/pipelex/libraries/pipelines/builder/concept/test_data.py
index 1bc960393..27152474f 100644
--- a/tests/unit/pipelex/libraries/pipelines/builder/concept/test_data.py
+++ b/tests/unit/pipelex/libraries/pipelines/builder/concept/test_data.py
@@ -1,12 +1,10 @@
-from typing import ClassVar, List, Tuple
+from typing import ClassVar
from pipelex.core.concepts.concept_blueprint import ConceptBlueprint, ConceptStructureBlueprint, ConceptStructureBlueprintFieldType
from pipelex.libraries.pipelines.builder.concept.concept_spec import ConceptSpec, ConceptStructureSpec, ConceptStructureSpecFieldType
class ConceptBlueprintTestCases:
- """Test cases for ConceptBlueprint.to_blueprint conversion."""
-
SIMPLE_CONCEPT = (
"simple_concept",
ConceptSpec(
@@ -148,7 +146,7 @@ class ConceptBlueprintTestCases:
),
)
- TEST_CASES: ClassVar[List[Tuple[str, ConceptSpec, ConceptBlueprint]]] = [
+ TEST_CASES: ClassVar[list[tuple[str, ConceptSpec, ConceptBlueprint]]] = [
SIMPLE_CONCEPT,
CONCEPT_WITH_REFINES,
CONCEPT_WITH_TEXT_FIELD,
diff --git a/tests/unit/pipelex/libraries/pipelines/builder/pipe/__init__.py b/tests/unit/pipelex/libraries/pipelines/builder/pipe/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_controllers/pipe_batch/__init__.py b/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_controllers/pipe_batch/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_controllers/pipe_batch/test_data.py b/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_controllers/pipe_batch/test_data.py
index 98b0a8016..eee197946 100644
--- a/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_controllers/pipe_batch/test_data.py
+++ b/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_controllers/pipe_batch/test_data.py
@@ -1,4 +1,4 @@
-from typing import ClassVar, List, Tuple
+from typing import ClassVar
from pipelex.core.pipes.pipe_input_blueprint import InputRequirementBlueprint
from pipelex.libraries.pipelines.builder.pipe.inputs_spec import InputRequirementSpec
@@ -16,7 +16,6 @@ class PipeBatchTestCases:
output="ProcessedItems",
branch_pipe_code="process_item",
),
- "test_domain",
PipeBatchBlueprint(
definition="Process items in batch",
inputs={"items": InputRequirementBlueprint(concept="ItemList")},
@@ -40,7 +39,6 @@ class PipeBatchTestCases:
input_list_name="data_list",
input_item_name="current_data",
),
- "test_domain",
PipeBatchBlueprint(
definition="Batch with custom names",
inputs={"data": InputRequirementBlueprint(concept="DataList")},
@@ -53,7 +51,7 @@ class PipeBatchTestCases:
),
)
- TEST_CASES: ClassVar[List[Tuple[str, PipeBatchSpec, str, PipeBatchBlueprint]]] = [
+ TEST_CASES: ClassVar[list[tuple[str, PipeBatchSpec, PipeBatchBlueprint]]] = [
SIMPLE_BATCH,
BATCH_WITH_NAMES,
]
diff --git a/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_controllers/pipe_batch/test_pipe_batch.py b/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_controllers/pipe_batch/test_pipe_batch.py
index b469ef32e..eec8f555d 100644
--- a/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_controllers/pipe_batch/test_pipe_batch.py
+++ b/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_controllers/pipe_batch/test_pipe_batch.py
@@ -8,14 +8,13 @@ from .test_data import PipeBatchTestCases
class TestPipeBatchBlueprintConversion:
@pytest.mark.parametrize(
- "test_name,pipe_spec,domain,expected_blueprint",
+ "test_name,pipe_spec,expected_blueprint",
PipeBatchTestCases.TEST_CASES,
)
def test_pipe_batch_spec_to_blueprint(
self,
test_name: str,
pipe_spec: PipeBatchSpec,
- domain: str,
expected_blueprint: PipeBatchBlueprint,
):
result = pipe_spec.to_blueprint()
diff --git a/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_controllers/pipe_condition/__init__.py b/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_controllers/pipe_condition/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_controllers/pipe_condition/test_data.py b/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_controllers/pipe_condition/test_data.py
index 09f189046..fa7f84917 100644
--- a/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_controllers/pipe_condition/test_data.py
+++ b/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_controllers/pipe_condition/test_data.py
@@ -1,4 +1,4 @@
-from typing import ClassVar, List, Tuple
+from typing import ClassVar
from pipelex.core.pipes.pipe_input_blueprint import InputRequirementBlueprint
from pipelex.libraries.pipelines.builder.pipe.inputs_spec import InputRequirementSpec
@@ -19,10 +19,9 @@ class PipeConditionTestCases:
root={
"active": "process_active",
"inactive": "process_inactive",
- }
+ },
),
),
- "test_domain",
PipeConditionBlueprint(
definition="Choose pipe based on condition",
inputs={"data": InputRequirementBlueprint(concept="Data")},
@@ -35,7 +34,7 @@ class PipeConditionTestCases:
root={
"active": "process_active",
"inactive": "process_inactive",
- }
+ },
),
default_pipe_code=None,
add_alias_from_expression_to=None,
@@ -55,12 +54,11 @@ class PipeConditionTestCases:
"A": "process_a",
"B": "process_b",
"C": "process_c",
- }
+ },
),
default_pipe_code="process_default",
add_alias_from_expression_to="category_result",
),
- "test_domain",
PipeConditionBlueprint(
definition="Conditional with template",
inputs={"item": InputRequirementBlueprint(concept="Item")},
@@ -74,14 +72,14 @@ class PipeConditionTestCases:
"A": "process_a",
"B": "process_b",
"C": "process_c",
- }
+ },
),
default_pipe_code="process_default",
add_alias_from_expression_to="category_result",
),
)
- TEST_CASES: ClassVar[List[Tuple[str, PipeConditionSpec, str, PipeConditionBlueprint]]] = [
+ TEST_CASES: ClassVar[list[tuple[str, PipeConditionSpec, PipeConditionBlueprint]]] = [
SIMPLE_CONDITION,
CONDITION_WITH_TEMPLATE,
]
diff --git a/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_controllers/pipe_condition/test_pipe_condition.py b/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_controllers/pipe_condition/test_pipe_condition.py
index ba32686fb..6820e806b 100644
--- a/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_controllers/pipe_condition/test_pipe_condition.py
+++ b/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_controllers/pipe_condition/test_pipe_condition.py
@@ -8,14 +8,13 @@ from .test_data import PipeConditionTestCases
class TestPipeConditionBlueprintConversion:
@pytest.mark.parametrize(
- "test_name,pipe_spec,domain,expected_blueprint",
+ "test_name,pipe_spec,expected_blueprint",
PipeConditionTestCases.TEST_CASES,
)
def test_pipe_condition_spec_to_blueprint(
self,
test_name: str,
pipe_spec: PipeConditionSpec,
- domain: str,
expected_blueprint: PipeConditionBlueprint,
):
result = pipe_spec.to_blueprint()
diff --git a/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_controllers/pipe_parallel/__init__.py b/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_controllers/pipe_parallel/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_controllers/pipe_parallel/test_data.py b/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_controllers/pipe_parallel/test_data.py
index ed5579415..25023d6eb 100644
--- a/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_controllers/pipe_parallel/test_data.py
+++ b/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_controllers/pipe_parallel/test_data.py
@@ -1,4 +1,4 @@
-from typing import ClassVar, List, Tuple
+from typing import ClassVar
from pipelex.core.pipes.pipe_input_blueprint import InputRequirementBlueprint
from pipelex.libraries.pipelines.builder.pipe.inputs_spec import InputRequirementSpec
@@ -22,7 +22,6 @@ class PipeParallelTestCases:
SubPipeSpec(the_pipe_code="validate_data", result="validation"),
],
),
- "test_domain",
PipeParallelBlueprint(
definition="Run pipes in parallel",
inputs={"data": InputRequirementBlueprint(concept="Data")},
@@ -53,7 +52,6 @@ class PipeParallelTestCases:
add_each_output=False,
combined_output="MergedData",
),
- "test_domain",
PipeParallelBlueprint(
definition="Parallel with combined output",
inputs={"input": InputRequirementBlueprint(concept="Input")},
@@ -69,7 +67,7 @@ class PipeParallelTestCases:
),
)
- TEST_CASES: ClassVar[List[Tuple[str, PipeParallelSpec, str, PipeParallelBlueprint]]] = [
+ TEST_CASES: ClassVar[list[tuple[str, PipeParallelSpec, PipeParallelBlueprint]]] = [
SIMPLE_PARALLEL,
PARALLEL_WITH_COMBINED,
]
diff --git a/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_controllers/pipe_parallel/test_pipe_parallel.py b/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_controllers/pipe_parallel/test_pipe_parallel.py
index cf3315971..04d999a7f 100644
--- a/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_controllers/pipe_parallel/test_pipe_parallel.py
+++ b/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_controllers/pipe_parallel/test_pipe_parallel.py
@@ -8,14 +8,13 @@ from .test_data import PipeParallelTestCases
class TestPipeParallelBlueprintConversion:
@pytest.mark.parametrize(
- "test_name,pipe_spec,domain,expected_blueprint",
+ "test_name,pipe_spec,expected_blueprint",
PipeParallelTestCases.TEST_CASES,
)
def test_pipe_parallel_spec_to_blueprint(
self,
test_name: str,
pipe_spec: PipeParallelSpec,
- domain: str,
expected_blueprint: PipeParallelBlueprint,
):
result = pipe_spec.to_blueprint()
diff --git a/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_controllers/pipe_sequence/__init__.py b/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_controllers/pipe_sequence/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_controllers/pipe_sequence/test_data.py b/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_controllers/pipe_sequence/test_data.py
index 3448c51d9..75ee231b8 100644
--- a/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_controllers/pipe_sequence/test_data.py
+++ b/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_controllers/pipe_sequence/test_data.py
@@ -1,4 +1,4 @@
-from typing import ClassVar, List, Tuple
+from typing import ClassVar
from pipelex.core.pipes.pipe_input_blueprint import InputRequirementBlueprint
from pipelex.libraries.pipelines.builder.pipe.inputs_spec import InputRequirementSpec
@@ -22,7 +22,6 @@ class PipeSequenceTestCases:
SubPipeSpec(the_pipe_code="step3", result="final_result"),
],
),
- "test_domain",
PipeSequenceBlueprint(
definition="A sequence of operations",
inputs={"input_data": InputRequirementBlueprint(concept="Text")},
@@ -54,7 +53,6 @@ class PipeSequenceTestCases:
),
],
),
- "test_domain",
PipeSequenceBlueprint(
definition="Sequence with batch",
inputs={"items": InputRequirementBlueprint(concept="ItemList")},
@@ -73,7 +71,7 @@ class PipeSequenceTestCases:
),
)
- TEST_CASES: ClassVar[List[Tuple[str, PipeSequenceSpec, str, PipeSequenceBlueprint]]] = [
+ TEST_CASES: ClassVar[list[tuple[str, PipeSequenceSpec, PipeSequenceBlueprint]]] = [
SIMPLE_SEQUENCE,
SEQUENCE_WITH_BATCH,
]
diff --git a/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_controllers/pipe_sequence/test_pipe_sequence.py b/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_controllers/pipe_sequence/test_pipe_sequence.py
index 4d57af598..dbd99537c 100644
--- a/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_controllers/pipe_sequence/test_pipe_sequence.py
+++ b/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_controllers/pipe_sequence/test_pipe_sequence.py
@@ -8,14 +8,13 @@ from .test_data import PipeSequenceTestCases
class TestPipeSequenceBlueprintConversion:
@pytest.mark.parametrize(
- "test_name,pipe_spec,domain,expected_blueprint",
+ "test_name,pipe_spec,expected_blueprint",
PipeSequenceTestCases.TEST_CASES,
)
def test_pipe_sequence_spec_to_blueprint(
self,
test_name: str,
pipe_spec: PipeSequenceSpec,
- domain: str,
expected_blueprint: PipeSequenceBlueprint,
):
result = pipe_spec.to_blueprint()
diff --git a/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_operators/pipe_func/__init__.py b/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_operators/pipe_func/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_operators/pipe_func/test_data.py b/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_operators/pipe_func/test_data.py
index b184452fa..4bfdf9bc6 100644
--- a/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_operators/pipe_func/test_data.py
+++ b/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_operators/pipe_func/test_data.py
@@ -1,4 +1,4 @@
-from typing import ClassVar, List, Tuple
+from typing import ClassVar
from pipelex.core.pipes.pipe_input_blueprint import InputRequirementBlueprint
from pipelex.libraries.pipelines.builder.pipe.inputs_spec import InputRequirementSpec
@@ -16,7 +16,6 @@ class PipeFuncTestCases:
output="ProcessedData",
function_name="process_data",
),
- "test_domain",
PipeFuncBlueprint(
definition="Execute a function",
inputs={"data": InputRequirementBlueprint(concept="Data")},
@@ -36,7 +35,6 @@ class PipeFuncTestCases:
output="GeneratedData",
function_name="generate_data",
),
- "test_domain",
PipeFuncBlueprint(
definition="Generate data",
inputs=None,
@@ -47,7 +45,7 @@ class PipeFuncTestCases:
),
)
- TEST_CASES: ClassVar[List[Tuple[str, PipeFuncSpec, str, PipeFuncBlueprint]]] = [
+ TEST_CASES: ClassVar[list[tuple[str, PipeFuncSpec, PipeFuncBlueprint]]] = [
SIMPLE_FUNC,
FUNC_NO_INPUTS,
]
diff --git a/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_operators/pipe_func/test_pipe_func.py b/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_operators/pipe_func/test_pipe_func.py
index b77d93727..b735e19c8 100644
--- a/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_operators/pipe_func/test_pipe_func.py
+++ b/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_operators/pipe_func/test_pipe_func.py
@@ -8,14 +8,13 @@ from .test_data import PipeFuncTestCases
class TestPipeFuncBlueprintConversion:
@pytest.mark.parametrize(
- "test_name,pipe_spec,domain,expected_blueprint",
+ "test_name,pipe_spec,expected_blueprint",
PipeFuncTestCases.TEST_CASES,
)
def test_pipe_func_spec_to_blueprint(
self,
test_name: str,
pipe_spec: PipeFuncSpec,
- domain: str,
expected_blueprint: PipeFuncBlueprint,
):
result = pipe_spec.to_blueprint()
diff --git a/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_operators/pipe_img/__init__.py b/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_operators/pipe_img/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_operators/pipe_img/test_data.py b/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_operators/pipe_img/test_data.py
index 6920cc151..e239f230b 100644
--- a/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_operators/pipe_img/test_data.py
+++ b/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_operators/pipe_img/test_data.py
@@ -1,4 +1,4 @@
-from typing import ClassVar, List, Tuple
+from typing import ClassVar
from pipelex.cogt.img_gen.img_gen_job_components import AspectRatio
from pipelex.core.pipes.pipe_input_blueprint import InputRequirementBlueprint
@@ -65,7 +65,7 @@ class PipeImgGenTestCases:
),
)
- TEST_CASES: ClassVar[List[Tuple[str, PipeImgGenSpec, PipeImgGenBlueprint]]] = [
+ TEST_CASES: ClassVar[list[tuple[str, PipeImgGenSpec, PipeImgGenBlueprint]]] = [
SIMPLE_IMG_GEN,
IMG_GEN_WITH_OPTIONS,
]
diff --git a/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_operators/pipe_img/test_pipe_img.py b/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_operators/pipe_img/test_pipe_img.py
index e54b5360f..09f8c6aa9 100644
--- a/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_operators/pipe_img/test_pipe_img.py
+++ b/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_operators/pipe_img/test_pipe_img.py
@@ -18,8 +18,4 @@ class TestPipeImgGenBlueprintConversion:
expected_blueprint: PipeImgGenBlueprint,
):
result = pipe_spec.to_blueprint()
- from pipelex import pretty_print
-
- pretty_print(result, title=f"Result {test_name}")
- pretty_print(expected_blueprint, title=f"Expected {test_name}")
assert result == expected_blueprint
diff --git a/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_operators/pipe_jinja2/__init__.py b/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_operators/pipe_jinja2/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_operators/pipe_jinja2/test_data.py b/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_operators/pipe_jinja2/test_data.py
index df261e843..32a2a4d96 100644
--- a/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_operators/pipe_jinja2/test_data.py
+++ b/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_operators/pipe_jinja2/test_data.py
@@ -1,4 +1,4 @@
-from typing import ClassVar, List, Tuple
+from typing import ClassVar
from pipelex.core.pipes.pipe_input_blueprint import InputRequirementBlueprint
from pipelex.libraries.pipelines.builder.pipe.inputs_spec import InputRequirementSpec
@@ -18,7 +18,6 @@ class PipeJinja2TestCases:
output="RenderedText",
jinja2="Hello {{ data.name }}!",
),
- "test_domain",
PipeJinja2Blueprint(
definition="Render a template",
inputs={"data": InputRequirementBlueprint(concept="Data")},
@@ -48,7 +47,6 @@ class PipeJinja2TestCases:
template_category=Jinja2TemplateCategory.MARKDOWN,
extra_context={"version": "1.0"},
),
- "test_domain",
PipeJinja2Blueprint(
definition="Template with prompting style",
inputs={"input": InputRequirementBlueprint(concept="Input")},
@@ -66,7 +64,7 @@ class PipeJinja2TestCases:
),
)
- TEST_CASES: ClassVar[List[Tuple[str, PipeJinja2Spec, str, PipeJinja2Blueprint]]] = [
+ TEST_CASES: ClassVar[list[tuple[str, PipeJinja2Spec, PipeJinja2Blueprint]]] = [
SIMPLE_JINJA2,
JINJA2_WITH_STYLE,
]
diff --git a/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_operators/pipe_jinja2/test_pipe_jinja2.py b/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_operators/pipe_jinja2/test_pipe_jinja2.py
index d471ae3e4..a0afd4e26 100644
--- a/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_operators/pipe_jinja2/test_pipe_jinja2.py
+++ b/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_operators/pipe_jinja2/test_pipe_jinja2.py
@@ -8,14 +8,13 @@ from .test_data import PipeJinja2TestCases
class TestPipeJinja2BlueprintConversion:
@pytest.mark.parametrize(
- "test_name,pipe_spec,domain,expected_blueprint",
+ "test_name,pipe_spec,expected_blueprint",
PipeJinja2TestCases.TEST_CASES,
)
def test_pipe_jinja2_spec_to_blueprint(
self,
test_name: str,
pipe_spec: PipeJinja2Spec,
- domain: str,
expected_blueprint: PipeJinja2Blueprint,
):
result = pipe_spec.to_blueprint()
diff --git a/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_operators/pipe_llm/__init__.py b/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_operators/pipe_llm/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_operators/pipe_llm/test_data.py b/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_operators/pipe_llm/test_data.py
index d0a1e5441..e3a6d25a7 100644
--- a/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_operators/pipe_llm/test_data.py
+++ b/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_operators/pipe_llm/test_data.py
@@ -1,4 +1,4 @@
-from typing import ClassVar, List, Tuple
+from typing import ClassVar
from pipelex.cogt.llm.llm_setting import LLMSetting
from pipelex.core.pipes.pipe_input_blueprint import InputRequirementBlueprint
@@ -17,7 +17,6 @@ class PipeLLMTestCases:
output="Text",
prompt_template="Write about $topic",
),
- "test_domain",
PipeLLMBlueprint(
type="PipeLLM",
definition="Generate text",
@@ -36,7 +35,6 @@ class PipeLLMTestCases:
output="Text",
prompt_template="Generate something interesting",
),
- "test_domain",
PipeLLMBlueprint(
type="PipeLLM",
definition="Generate without inputs",
@@ -55,7 +53,6 @@ class PipeLLMTestCases:
prompt_template="Generate text",
llm="llm_to_reason",
),
- "test_domain",
PipeLLMBlueprint(
type="PipeLLM",
definition="Generate with preset",
@@ -79,7 +76,6 @@ class PipeLLMTestCases:
max_tokens=None, # "auto" is handled at conversion to core
),
),
- "test_domain",
PipeLLMBlueprint(
type="PipeLLM",
definition="Generate with settings",
@@ -103,7 +99,6 @@ class PipeLLMTestCases:
system_prompt="You are a data analyst",
prompt_template="Analyze: @data",
),
- "test_domain",
PipeLLMBlueprint(
type="PipeLLM",
definition="Generate with system prompt",
@@ -124,7 +119,6 @@ class PipeLLMTestCases:
prompt_template="Generate items",
multiple_output=True,
),
- "test_domain",
PipeLLMBlueprint(
type="PipeLLM",
definition="Generate multiple items",
@@ -145,7 +139,6 @@ class PipeLLMTestCases:
prompt_template="Generate items",
nb_output=5,
),
- "test_domain",
PipeLLMBlueprint(
type="PipeLLM",
definition="Generate exactly 5 items",
@@ -174,7 +167,6 @@ class PipeLLMTestCases:
prompt_template_to_structure="Structure the output",
system_prompt_to_structure="You are a data structurer",
),
- "test_domain",
PipeLLMBlueprint(
type="PipeLLM",
definition="Extract structured data",
@@ -192,7 +184,7 @@ class PipeLLMTestCases:
),
)
- TEST_CASES: ClassVar[List[Tuple[str, PipeLLMSpec, str, PipeLLMBlueprint]]] = [
+ TEST_CASES: ClassVar[list[tuple[str, PipeLLMSpec, PipeLLMBlueprint]]] = [
SIMPLE_LLM,
LLM_NO_INPUTS,
LLM_WITH_PRESET,
diff --git a/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_operators/pipe_llm/test_pipe_llm.py b/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_operators/pipe_llm/test_pipe_llm.py
index 48f075d7a..88288672e 100644
--- a/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_operators/pipe_llm/test_pipe_llm.py
+++ b/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_operators/pipe_llm/test_pipe_llm.py
@@ -8,9 +8,9 @@ from .test_data import PipeLLMTestCases
class TestPipeLLMBlueprintConversion:
@pytest.mark.parametrize(
- "test_name,pipe_spec,domain,expected_blueprint",
+ "test_name,pipe_spec,expected_blueprint",
PipeLLMTestCases.TEST_CASES,
)
- def test_pipe_llm_spec_to_blueprint(self, test_name: str, pipe_spec: PipeLLMSpec, domain: str, expected_blueprint: PipeLLMBlueprint):
+ def test_pipe_llm_spec_to_blueprint(self, test_name: str, pipe_spec: PipeLLMSpec, expected_blueprint: PipeLLMBlueprint):
result = pipe_spec.to_blueprint()
assert result == expected_blueprint
diff --git a/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_operators/pipe_ocr/__init__.py b/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_operators/pipe_ocr/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_operators/pipe_ocr/test_data.py b/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_operators/pipe_ocr/test_data.py
index 2e5fc5b05..06786ebdc 100644
--- a/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_operators/pipe_ocr/test_data.py
+++ b/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_operators/pipe_ocr/test_data.py
@@ -1,4 +1,4 @@
-from typing import ClassVar, List, Tuple
+from typing import ClassVar
from pipelex.core.pipes.pipe_input_blueprint import InputRequirementBlueprint
from pipelex.libraries.pipelines.builder.pipe.inputs_spec import InputRequirementSpec
@@ -16,7 +16,6 @@ class PipeOcrTestCases:
output="ExtractedText",
ocr="mistral-pixtral",
),
- "test_domain",
PipeOcrBlueprint(
definition="Extract text from image",
inputs={"ocr_input": InputRequirementBlueprint(concept="Image")},
@@ -40,7 +39,6 @@ class PipeOcrTestCases:
page_views=True,
page_views_dpi=300,
),
- "test_domain",
PipeOcrBlueprint(
definition="OCR with page options",
inputs={"ocr_input": InputRequirementBlueprint(concept="PDF")},
@@ -51,7 +49,7 @@ class PipeOcrTestCases:
),
)
- TEST_CASES: ClassVar[List[Tuple[str, PipeOcrSpec, str, PipeOcrBlueprint]]] = [
+ TEST_CASES: ClassVar[list[tuple[str, PipeOcrSpec, PipeOcrBlueprint]]] = [
SIMPLE_OCR,
OCR_WITH_OPTIONS,
]
diff --git a/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_operators/pipe_ocr/test_pipe_ocr.py b/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_operators/pipe_ocr/test_pipe_ocr.py
index b33043418..f4cdfdafb 100644
--- a/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_operators/pipe_ocr/test_pipe_ocr.py
+++ b/tests/unit/pipelex/libraries/pipelines/builder/pipe/pipe_operators/pipe_ocr/test_pipe_ocr.py
@@ -8,14 +8,13 @@ from .test_data import PipeOcrTestCases
class TestPipeOcrBlueprintConversion:
@pytest.mark.parametrize(
- "test_name,pipe_spec,domain,expected_blueprint",
+ "test_name,pipe_spec,expected_blueprint",
PipeOcrTestCases.TEST_CASES,
)
def test_pipe_ocr_spec_to_blueprint(
self,
test_name: str,
pipe_spec: PipeOcrSpec,
- domain: str,
expected_blueprint: PipeOcrBlueprint,
):
result = pipe_spec.to_blueprint()
diff --git a/tests/unit/pipelex/libraries/pipelines/builder/pipe/test_data_inputs.py b/tests/unit/pipelex/libraries/pipelines/builder/pipe/test_data_inputs.py
index dd84d0c7b..77f1856e8 100644
--- a/tests/unit/pipelex/libraries/pipelines/builder/pipe/test_data_inputs.py
+++ b/tests/unit/pipelex/libraries/pipelines/builder/pipe/test_data_inputs.py
@@ -1,4 +1,4 @@
-from typing import ClassVar, List, Tuple
+from typing import ClassVar
from pipelex.core.pipes.pipe_input_blueprint import InputRequirementBlueprint
from pipelex.libraries.pipelines.builder.pipe.inputs_spec import InputRequirementSpec
@@ -33,7 +33,7 @@ class InputRequirementTestCases:
InputRequirementBlueprint(concept="domain.ConceptName"),
)
- TEST_CASES: ClassVar[List[Tuple[str, InputRequirementSpec, str, InputRequirementBlueprint]]] = [
+ TEST_CASES: ClassVar[list[tuple[str, InputRequirementSpec, str, InputRequirementBlueprint]]] = [
SIMPLE_TEXT_INPUT,
IMAGE_INPUT,
CUSTOM_CONCEPT_INPUT,
diff --git a/tests/unit/pipelex/libraries/pipelines/builder/pipe/test_data_pipe.py b/tests/unit/pipelex/libraries/pipelines/builder/pipe/test_data_pipe.py
index 0769cb741..f52961f78 100644
--- a/tests/unit/pipelex/libraries/pipelines/builder/pipe/test_data_pipe.py
+++ b/tests/unit/pipelex/libraries/pipelines/builder/pipe/test_data_pipe.py
@@ -1,4 +1,4 @@
-from typing import ClassVar, List, Tuple
+from typing import ClassVar
from pipelex.core.pipes.pipe_blueprint import PipeBlueprint
from pipelex.core.pipes.pipe_input_blueprint import InputRequirementBlueprint
@@ -16,7 +16,6 @@ class PipeBlueprintTestCases:
inputs={"input": "Text"},
output="ProcessedText",
),
- "test_domain",
PipeBlueprint(
type="PipeLLM",
category="PipeOperator",
@@ -38,7 +37,6 @@ class PipeBlueprintTestCases:
},
output="Result",
),
- "test_domain",
PipeBlueprint(
type="PipeFunc",
category="PipeOperator",
@@ -60,7 +58,6 @@ class PipeBlueprintTestCases:
inputs=None,
output="GeneratedData",
),
- "test_domain",
PipeBlueprint(
type="PipeFunc",
category="PipeOperator",
@@ -70,7 +67,7 @@ class PipeBlueprintTestCases:
),
)
- TEST_CASES: ClassVar[List[Tuple[str, PipeSpec, str, PipeBlueprint]]] = [
+ TEST_CASES: ClassVar[list[tuple[str, PipeSpec, PipeBlueprint]]] = [
SIMPLE_PIPE,
PIPE_WITH_INPUT_REQUIREMENTS,
PIPE_NO_INPUTS,
diff --git a/tests/unit/pipelex/libraries/pipelines/builder/pipe/test_data_sub_pipe.py b/tests/unit/pipelex/libraries/pipelines/builder/pipe/test_data_sub_pipe.py
index 46bfdd378..0938419e1 100644
--- a/tests/unit/pipelex/libraries/pipelines/builder/pipe/test_data_sub_pipe.py
+++ b/tests/unit/pipelex/libraries/pipelines/builder/pipe/test_data_sub_pipe.py
@@ -1,4 +1,4 @@
-from typing import ClassVar, List, Tuple
+from typing import ClassVar
from pipelex.libraries.pipelines.builder.pipe.sub_pipe_spec import SubPipeSpec
from pipelex.pipe_controllers.sub_pipe_blueprint import SubPipeBlueprint
@@ -29,7 +29,7 @@ class SubPipeTestCases:
SubPipeBlueprint(pipe="process_item", result="processed_items", batch_over="input_list", batch_as="current_item"),
)
- TEST_CASES: ClassVar[List[Tuple[str, SubPipeSpec, SubPipeBlueprint]]] = [
+ TEST_CASES: ClassVar[list[tuple[str, SubPipeSpec, SubPipeBlueprint]]] = [
SIMPLE_SUB_PIPE,
SUB_PIPE_WITH_MULTIPLE_OUTPUT,
SUB_PIPE_WITH_FIXED_OUTPUT,
diff --git a/tests/unit/pipelex/libraries/pipelines/builder/pipe/test_inputs_blueprint.py b/tests/unit/pipelex/libraries/pipelines/builder/pipe/test_inputs_blueprint.py
index 4657f1d67..53cfdfb3e 100644
--- a/tests/unit/pipelex/libraries/pipelines/builder/pipe/test_inputs_blueprint.py
+++ b/tests/unit/pipelex/libraries/pipelines/builder/pipe/test_inputs_blueprint.py
@@ -12,7 +12,7 @@ class TestInputRequirementBlueprintConversion:
InputRequirementTestCases.TEST_CASES,
)
def test_input_requirement_spec_to_blueprint(
- self, test_name: str, input_spec: InputRequirementSpec, domain: str, expected_blueprint: InputRequirementBlueprint
+ self, test_name: str, input_spec: InputRequirementSpec, domain: str, expected_blueprint: InputRequirementBlueprint,
):
result = input_spec.to_blueprint()
assert result == expected_blueprint
diff --git a/tests/unit/pipelex/libraries/pipelines/builder/pipe/test_pipe_blueprint.py b/tests/unit/pipelex/libraries/pipelines/builder/pipe/test_pipe_blueprint.py
index 191b33820..ab0d3b66e 100644
--- a/tests/unit/pipelex/libraries/pipelines/builder/pipe/test_pipe_blueprint.py
+++ b/tests/unit/pipelex/libraries/pipelines/builder/pipe/test_pipe_blueprint.py
@@ -8,14 +8,13 @@ from .test_data_pipe import PipeBlueprintTestCases
class TestPipeBlueprintConversion:
@pytest.mark.parametrize(
- "test_name,pipe_spec,domain,expected_blueprint",
+ "test_name,pipe_spec,expected_blueprint",
PipeBlueprintTestCases.TEST_CASES,
)
def test_pipe_blueprint_to_core(
self,
test_name: str,
pipe_spec: PipeSpec,
- domain: str,
expected_blueprint: PipeBlueprint,
):
result = pipe_spec.to_blueprint()
diff --git a/tests/unit/pipelex/migration/test_migrate_v0_1_0_to_v0_2_0.py b/tests/unit/pipelex/migration/test_migrate_v0_1_0_to_v0_2_0.py
index 086ae324c..51309e346 100644
--- a/tests/unit/pipelex/migration/test_migrate_v0_1_0_to_v0_2_0.py
+++ b/tests/unit/pipelex/migration/test_migrate_v0_1_0_to_v0_2_0.py
@@ -226,7 +226,7 @@ refines = "Text"
assert changes[1]["new_line"] == 'definition = "Another concept"'
def test_migrate_file_success(
- self, migrator: TOMLMigrator, tmp_path: Path, sample_old_syntax_content: str, sample_new_syntax_content: str
+ self, migrator: TOMLMigrator, tmp_path: Path, sample_old_syntax_content: str, sample_new_syntax_content: str,
) -> None:
"""Test successful file migration."""
test_file = tmp_path / "test.toml"
@@ -266,7 +266,7 @@ refines = "Text"
assert changes_count == 0
def test_migrate_file_without_backup(
- self, migrator: TOMLMigrator, tmp_path: Path, sample_old_syntax_content: str, sample_new_syntax_content: str
+ self, migrator: TOMLMigrator, tmp_path: Path, sample_old_syntax_content: str, sample_new_syntax_content: str,
) -> None:
"""Test file migration without creating backup."""
test_file = tmp_path / "test.toml"
diff --git a/tests/unit/pipelex/tools/config/__init__.py b/tests/unit/pipelex/tools/config/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/tests/unit/pipelex/tools/misc/__init__.py b/tests/unit/pipelex/tools/misc/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/tests/unit/pipelex/tools/misc/test_dict_utils.py b/tests/unit/pipelex/tools/misc/test_dict_utils.py
index f24765c01..0f0c2c44a 100644
--- a/tests/unit/pipelex/tools/misc/test_dict_utils.py
+++ b/tests/unit/pipelex/tools/misc/test_dict_utils.py
@@ -1,14 +1,10 @@
-"""Unit tests for dict_utils module."""
-
-from typing import Any, Dict
+from typing import Any
from pipelex.core.concepts.concept_native import NativeConceptEnum
from pipelex.tools.misc.dict_utils import apply_to_strings_in_list, apply_to_strings_recursive, insert_before
class TestDictUtils:
- """Test class for dict_utils functions."""
-
def test_insert_before_basic(self) -> None:
"""Test basic insert_before functionality."""
original = {"a": 1, "c": 3}
@@ -94,7 +90,7 @@ class TestDictUtils:
def test_apply_to_strings_recursive_empty_structures(self) -> None:
"""Test apply_to_strings_recursive with empty dictionaries and lists."""
- data: Dict[str, Any] = {"empty_dict": {}, "empty_list": [], "mixed": {"inner_empty": {}, "inner_list": []}}
+ data: dict[str, Any] = {"empty_dict": {}, "empty_list": [], "mixed": {"inner_empty": {}, "inner_list": []}}
def dummy_transform(s: str) -> str:
return s.upper()
diff --git a/tests/unit/pipelex/tools/misc/test_filetype_utils.py b/tests/unit/pipelex/tools/misc/test_filetype_utils.py
index 8ef4afadb..15799d2c2 100644
--- a/tests/unit/pipelex/tools/misc/test_filetype_utils.py
+++ b/tests/unit/pipelex/tools/misc/test_filetype_utils.py
@@ -131,7 +131,7 @@ class TestDetectFileTypeFromBase64:
def test_detect_file_type_from_base64_string_success(self, mocker: MockerFixture):
# Mock detect_file_type_from_bytes
mock_detect_bytes = mocker.patch(
- "pipelex.tools.misc.filetype_utils.detect_file_type_from_bytes", return_value=FileType(extension="gif", mime="image/gif")
+ "pipelex.tools.misc.filetype_utils.detect_file_type_from_bytes", return_value=FileType(extension="gif", mime="image/gif"),
)
# GIF header encoded in base64
@@ -148,7 +148,7 @@ class TestDetectFileTypeFromBase64:
def test_detect_file_type_from_base64_bytes_success(self, mocker: MockerFixture):
# Mock detect_file_type_from_bytes
mock_detect_bytes = mocker.patch(
- "pipelex.tools.misc.filetype_utils.detect_file_type_from_bytes", return_value=FileType(extension="jpg", mime="image/jpeg")
+ "pipelex.tools.misc.filetype_utils.detect_file_type_from_bytes", return_value=FileType(extension="jpg", mime="image/jpeg"),
)
# JPEG header encoded in base64
@@ -165,7 +165,7 @@ class TestDetectFileTypeFromBase64:
def test_detect_file_type_from_base64_data_url_success(self, mocker: MockerFixture):
# Mock detect_file_type_from_bytes
mock_detect_bytes = mocker.patch(
- "pipelex.tools.misc.filetype_utils.detect_file_type_from_bytes", return_value=FileType(extension="png", mime="image/png")
+ "pipelex.tools.misc.filetype_utils.detect_file_type_from_bytes", return_value=FileType(extension="png", mime="image/png"),
)
# PNG header encoded in base64
@@ -183,7 +183,7 @@ class TestDetectFileTypeFromBase64:
def test_detect_file_type_from_base64_data_url_with_whitespace(self, mocker: MockerFixture):
# Mock detect_file_type_from_bytes
mock_detect_bytes = mocker.patch(
- "pipelex.tools.misc.filetype_utils.detect_file_type_from_bytes", return_value=FileType(extension="txt", mime="text/plain")
+ "pipelex.tools.misc.filetype_utils.detect_file_type_from_bytes", return_value=FileType(extension="txt", mime="text/plain"),
)
test_bytes = b"hello world"
@@ -215,7 +215,7 @@ class TestDetectFileTypeFromBase64:
def test_detect_file_type_from_base64_data_url_no_comma(self, mocker: MockerFixture):
# Test data URL without comma (should be treated as regular base64)
mocker.patch(
- "pipelex.tools.misc.filetype_utils.detect_file_type_from_bytes", return_value=FileType(extension="bin", mime="application/octet-stream")
+ "pipelex.tools.misc.filetype_utils.detect_file_type_from_bytes", return_value=FileType(extension="bin", mime="application/octet-stream"),
)
# This will be treated as a base64 string since there's no comma
diff --git a/tests/unit/pipelex/tools/misc/test_find_files_in_dir.py b/tests/unit/pipelex/tools/misc/test_find_files_in_dir.py
index 5473b60f5..eee007216 100644
--- a/tests/unit/pipelex/tools/misc/test_find_files_in_dir.py
+++ b/tests/unit/pipelex/tools/misc/test_find_files_in_dir.py
@@ -5,8 +5,6 @@ from pipelex.tools.misc.file_utils import find_files_in_dir
class TestFindFilesInDir:
- """Test the find_files_in_dir helper function."""
-
def test_find_files_non_recursive(self):
"""Test finding files non-recursively."""
with tempfile.TemporaryDirectory() as temp_dir:
@@ -45,7 +43,8 @@ class TestFindFilesInDir:
# Find Python files recursively
files = find_files_in_dir(temp_dir, "*.py", is_recursive=True)
- assert len(files) == 3
+ expected_files_length = 3
+ assert len(files) == expected_files_length
file_names = [f.name for f in files]
assert "file1.py" in file_names
assert "file2.py" in file_names
diff --git a/tests/unit/pipelex/tools/misc/test_markdown.py b/tests/unit/pipelex/tools/misc/test_markdown.py
index ff04d563b..943269d56 100644
--- a/tests/unit/pipelex/tools/misc/test_markdown.py
+++ b/tests/unit/pipelex/tools/misc/test_markdown.py
@@ -1,4 +1,4 @@
-from typing import Any, Dict, List
+from typing import Any
from pydantic import BaseModel
@@ -8,9 +8,9 @@ from pipelex.tools.misc.markdown_utils import convert_to_markdown
class Article(BaseModel):
title: str
author: str
- tags: List[str]
+ tags: list[str]
content: str
- metadata: Dict[str, Any]
+ metadata: dict[str, Any]
def test_convert_md():
diff --git a/tests/unit/pipelex/tools/misc/test_path_utils.py b/tests/unit/pipelex/tools/misc/test_path_utils.py
index 4c280bc91..9741b0f73 100644
--- a/tests/unit/pipelex/tools/misc/test_path_utils.py
+++ b/tests/unit/pipelex/tools/misc/test_path_utils.py
@@ -1,3 +1,5 @@
+import os
+
import pytest
from pytest_mock import MockerFixture
@@ -71,7 +73,6 @@ class TestInterpretPathOrUrl:
def test_interpret_path_with_current_os_separator(self, mocker: MockerFixture):
# Test with the current OS path separator
- import os
path_with_sep = f"user{os.sep}file.txt"
result = interpret_path_or_url(path_with_sep)
@@ -109,7 +110,7 @@ class TestClarifyPathOrUrl:
assert file_path == "/C:/Users/user/file.txt"
assert url is None
- def test_clarify_file_uri_with_spaces(self):
+ def test_clarify_file_uri_with_spaces(self) -> None:
file_path, url = clarify_path_or_url("file:///home/user/my%20file.txt")
assert file_path == "/home/user/my file.txt"
assert url is None
@@ -150,12 +151,12 @@ class TestClarifyPathOrUrl:
assert url is None
def test_clarify_base_64_raises_not_implemented(self, mocker: MockerFixture):
+ # Mock interpret_path_or_url to return BASE_64
+ mocker.patch(
+ "pipelex.tools.misc.path_utils.interpret_path_or_url",
+ return_value=InterpretedPathOrUrl.BASE_64,
+ )
with pytest.raises(NotImplementedError, match="Base 64 is not supported yet by clarify_path_or_url"):
- # Mock interpret_path_or_url to return BASE_64
- mocker.patch(
- "pipelex.tools.misc.path_utils.interpret_path_or_url",
- return_value=InterpretedPathOrUrl.BASE_64,
- )
clarify_path_or_url("some_base64_data")
def test_clarify_uses_interpret_path_or_url(self, mocker: MockerFixture):
diff --git a/tests/unit/pipelex/tools/misc/test_pretty.py b/tests/unit/pipelex/tools/misc/test_pretty.py
index 12a269093..31c91172f 100644
--- a/tests/unit/pipelex/tools/misc/test_pretty.py
+++ b/tests/unit/pipelex/tools/misc/test_pretty.py
@@ -1,6 +1,5 @@
import re
import textwrap
-from typing import Dict, List, Optional, Union
import pytest
from pydantic import BaseModel, Field
@@ -13,22 +12,22 @@ class Address(BaseModel):
street: str
city: str
country: str
- postal_code: Optional[str] = None
+ postal_code: str | None = None
class UserPreferences(BaseModel):
theme: str = "dark"
notifications: bool = True
- tags: List[str] = Field(default_factory=list)
+ tags: list[str] = Field(default_factory=list)
class ComplexUser(BaseModel):
name: str
age: int
- email: Optional[str]
- addresses: List[Address]
+ email: str | None
+ addresses: list[Address]
preferences: UserPreferences
- metadata: Dict[str, Union[str, int, bool]] = Field(default_factory=dict)
+ metadata: dict[str, str | int | bool] = Field(default_factory=dict)
def remove_ansi_escape_codes(text: str) -> str:
diff --git a/tests/unit/pipelex/tools/misc/test_string_utils.py b/tests/unit/pipelex/tools/misc/test_string_utils.py
index 80c3e7366..9d76d99ff 100644
--- a/tests/unit/pipelex/tools/misc/test_string_utils.py
+++ b/tests/unit/pipelex/tools/misc/test_string_utils.py
@@ -17,7 +17,8 @@ from pipelex.tools.misc.string_utils import (
class BadStr:
def __str__(self) -> str: # pyright: ignore[reportImplicitOverride] pragma: no cover - used only for raising
- raise RuntimeError("boom")
+ msg = "boom"
+ raise RuntimeError(msg)
@pytest.mark.parametrize(
@@ -262,7 +263,6 @@ class TestMatchesWildcardPattern:
],
)
def test_matches_wildcard_pattern(self, text: str, pattern: str, expected: bool) -> None:
- """Test wildcard pattern matching with comprehensive test cases."""
assert matches_wildcard_pattern(text, pattern) is expected
def test_matches_wildcard_pattern_model_routing_examples(self) -> None:
diff --git a/tests/unit/pipelex/tools/secrets/__init__.py b/tests/unit/pipelex/tools/secrets/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/tests/unit/pipelex/tools/secrets/test_secrets_utils.py b/tests/unit/pipelex/tools/secrets/test_secrets_utils.py
index d344f1337..86218178f 100644
--- a/tests/unit/pipelex/tools/secrets/test_secrets_utils.py
+++ b/tests/unit/pipelex/tools/secrets/test_secrets_utils.py
@@ -3,7 +3,7 @@ from typing import Any
import pytest
from pytest_mock import MockerFixture
-
+from pipelex.tools.secrets.secrets_errors import SecretNotFoundError
from pipelex.tools.secrets.secrets_provider_abstract import SecretsProviderAbstract
from pipelex.tools.secrets.secrets_utils import UnknownVarPrefixError, VarFallbackPatternError, VarNotFoundError, substitute_vars
@@ -16,10 +16,7 @@ def mock_secrets_provider(mocker: MockerFixture) -> Any:
class TestSubstituteVars:
- """Test cases for substitute_vars function."""
-
def test_simple_secret_substitution(self, mocker: MockerFixture, mock_secrets_provider: Any) -> None:
- """Test basic secret substitution using default behavior."""
mock_secrets_provider.get_secret.return_value = "secret_value"
mocker.patch("pipelex.tools.secrets.secrets_utils.get_secrets_provider", return_value=mock_secrets_provider)
@@ -29,14 +26,12 @@ class TestSubstituteVars:
mock_secrets_provider.get_secret.assert_called_once_with(secret_id="API_KEY")
def test_explicit_env_substitution(self, mocker: MockerFixture) -> None:
- """Test explicit environment variable substitution."""
mocker.patch.dict(os.environ, {"TEST_VAR": "env_value"})
result = substitute_vars("Value: ${env:TEST_VAR}")
assert result == "Value: env_value"
def test_explicit_secret_substitution(self, mocker: MockerFixture, mock_secrets_provider: Any) -> None:
- """Test explicit secret substitution."""
mock_secrets_provider.get_secret.return_value = "secret_value"
mocker.patch("pipelex.tools.secrets.secrets_utils.get_secrets_provider", return_value=mock_secrets_provider)
@@ -46,7 +41,6 @@ class TestSubstituteVars:
mock_secrets_provider.get_secret.assert_called_once_with(secret_id="API_KEY")
def test_fallback_env_to_secret(self, mocker: MockerFixture, mock_secrets_provider: Any) -> None:
- """Test fallback from env to secret when env var doesn't exist."""
mock_secrets_provider.get_secret.return_value = "secret_fallback"
mocker.patch("pipelex.tools.secrets.secrets_utils.get_secrets_provider", return_value=mock_secrets_provider)
@@ -56,7 +50,6 @@ class TestSubstituteVars:
mock_secrets_provider.get_secret.assert_called_once_with(secret_id="FALLBACK_KEY")
def test_env_takes_precedence_in_fallback(self, mocker: MockerFixture, mock_secrets_provider: Any) -> None:
- """Test that env var takes precedence when both env and secret are available."""
mocker.patch.dict(os.environ, {"EXISTING_VAR": "env_value"})
mocker.patch("pipelex.tools.secrets.secrets_utils.get_secrets_provider", return_value=mock_secrets_provider)
@@ -133,9 +126,6 @@ class TestSubstituteVars:
substitute_vars("Value: ${MISSING_SECRET}")
def test_fallback_both_missing_raises_error(self, mocker: MockerFixture, mock_secrets_provider: Any) -> None:
- """Test that when both env and secret are missing, error is raised."""
- from pipelex.tools.secrets.secrets_errors import SecretNotFoundError
-
mock_secrets_provider.get_secret.side_effect = SecretNotFoundError("Secret not found")
mocker.patch("pipelex.tools.secrets.secrets_utils.get_secrets_provider", return_value=mock_secrets_provider)
diff --git a/tests/unit/pipelex/tools/templating/__init__.py b/tests/unit/pipelex/tools/templating/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/tests/unit/pipelex/tools/templating/test_template_preprocessor.py b/tests/unit/pipelex/tools/templating/test_template_preprocessor.py
index 16be6635c..76f56f99b 100644
--- a/tests/unit/pipelex/tools/templating/test_template_preprocessor.py
+++ b/tests/unit/pipelex/tools/templating/test_template_preprocessor.py
@@ -2,8 +2,6 @@ from pipelex.tools.templating.template_preprocessor import preprocess_template
class TestTemplatePreprocessor:
- """Test the template preprocessing functionality."""
-
def test_at_variable_pattern(self):
"""Test basic @variable pattern replacement."""
template = "@expense\n@invoices"
diff --git a/tests/unit/pipelex/tools/test_func_registry.py b/tests/unit/pipelex/tools/test_func_registry.py
index a49068dab..17f308cbb 100644
--- a/tests/unit/pipelex/tools/test_func_registry.py
+++ b/tests/unit/pipelex/tools/test_func_registry.py
@@ -1,5 +1,6 @@
import logging
-from typing import Any, Callable, Dict, List, Tuple
+from collections.abc import Callable
+from typing import Any
import pytest
from pytest import LogCaptureFixture
@@ -53,10 +54,10 @@ def no_type_hints(working_memory): # type: ignore
# Type alias for test parameters
-TestParams = Tuple[str, Callable[..., Any], bool]
+TestParams = tuple[str, Callable[..., Any], bool]
# Test data
-TEST_CASES: List[TestParams] = [
+TEST_CASES: list[TestParams] = [
# Valid cases
("valid_function", valid_function, True),
("valid_async_function", valid_async_function, True),
@@ -84,8 +85,6 @@ def registry():
class TestFuncRegistry:
@pytest.mark.parametrize("test_name, func, is_eligible", TEST_CASES)
def test_function_eligibility_and_registration(self, registry: FuncRegistry, test_name: str, func: Callable[..., Any], is_eligible: bool):
- """Test function eligibility check and registration behavior."""
-
# Test eligibility check directly
actual_eligibility = registry.is_eligible_function(func)
assert actual_eligibility == is_eligible, f"Eligibility check failed for {test_name}: expected {is_eligible}, got {actual_eligibility}"
@@ -131,7 +130,7 @@ class TestFuncRegistry:
registry.unregister_function_by_name("non_existent")
def test_register_functions_dict_with_ineligible_functions(self, registry: FuncRegistry):
- functions: Dict[str, Callable[..., Any]] = {"func1": sample_function, "func2": another_function}
+ functions: dict[str, Callable[..., Any]] = {"func1": sample_function, "func2": another_function}
registry.register_functions_dict(functions)
# Ineligible functions should be silently skipped
assert not registry.has_function("func1")
@@ -139,7 +138,7 @@ class TestFuncRegistry:
assert len(registry.root) == 0
def test_register_functions_list_with_ineligible_functions(self, registry: FuncRegistry):
- functions: List[Callable[..., Any]] = [sample_function, another_function]
+ functions: list[Callable[..., Any]] = [sample_function, another_function]
registry.register_functions(functions)
# Ineligible functions should be silently skipped
assert not registry.has_function("sample_function")
@@ -164,17 +163,17 @@ class TestFuncRegistry:
def test_get_required_function_with_signature(self, registry: FuncRegistry):
registry.register_function(valid_function)
- func = registry.get_required_function_with_signature("valid_function", valid_function)
+ func = registry.get_required_function_with_signature("valid_function")
assert func is valid_function
def test_get_required_function_with_signature_not_found(self, registry: FuncRegistry):
with pytest.raises(FuncRegistryError, match="not found in registry"):
- registry.get_required_function_with_signature("non_existent", sample_function)
+ registry.get_required_function_with_signature("non_existent")
def test_get_required_function_with_signature_not_callable(self, registry: FuncRegistry):
- registry.root["not_a_function"] = "a string" # type: ignore
+ registry.root["not_a_function"] = "a string" # type: ignore
with pytest.raises(FuncRegistryError, match="is not a callable function"):
- registry.get_required_function_with_signature("not_a_function", sample_function)
+ registry.get_required_function_with_signature("not_a_function")
def test_set_logger(self, registry: FuncRegistry, caplog: LogCaptureFixture):
"""Test setting a custom logger"""
diff --git a/tests/unit/pipelex/tools/test_func_registry_utils.py b/tests/unit/pipelex/tools/test_func_registry_utils.py
index b1f55469d..bb0696603 100644
--- a/tests/unit/pipelex/tools/test_func_registry_utils.py
+++ b/tests/unit/pipelex/tools/test_func_registry_utils.py
@@ -1,38 +1,30 @@
-#!/usr/bin/env python3
-"""Test script for FuncRegistryUtils"""
-
import tempfile
from pathlib import Path
-from typing import ClassVar, List, Tuple
+from typing import ClassVar
import pytest
from pydantic import Field
-from pipelex.core.stuffs.stuff_content import StructuredContent
+from pipelex.core.memory.working_memory import WorkingMemory
+from pipelex.core.stuffs.stuff_content import StructuredContent, TextContent
from pipelex.tools.func_registry import func_registry
from pipelex.tools.func_registry_utils import FuncRegistryUtils
class FilePath(StructuredContent):
- """A path to a file in the codebase."""
-
path: str = Field(description="Path to the file")
class CodebaseFileContent(StructuredContent):
- """Content of a codebase file."""
-
file_path: str = Field(description="Path to the codebase file")
file_content: str = Field(description="Content of the codebase file")
class TestCases:
- """Test cases for function eligibility."""
-
VALID_ASYNC_FUNCTION = """
async def read_file_content(working_memory: WorkingMemory) -> ListContent[CodebaseFileContent]:
'''Read the content of related codebase files.'''
-
+
file_paths_list = working_memory.get_stuff_as_list("related_file_paths", item_type=FilePath)
codebase_files: List[CodebaseFileContent] = []
@@ -88,7 +80,7 @@ def invalid_function_no_type_hints(working_memory):
pass
"""
- TEST_CASES: ClassVar[List[Tuple[str, str, List[str], List[str]]]] = [
+ TEST_CASES: ClassVar[list[tuple[str, str, list[str], list[str]]]] = [
("valid_async_function", VALID_ASYNC_FUNCTION, ["read_file_content"], []),
("valid_sync_function", VALID_SYNC_FUNCTION, ["sync_function"], []),
(
@@ -108,8 +100,6 @@ def invalid_function_no_type_hints(working_memory):
class TestFuncRegistryUtils:
- """Test class for FuncRegistryUtils."""
-
@pytest.mark.parametrize(
"test_name, function_code, expected_registered, expected_not_registered",
TestCases.TEST_CASES,
@@ -118,11 +108,9 @@ class TestFuncRegistryUtils:
self,
test_name: str,
function_code: str,
- expected_registered: List[str],
- expected_not_registered: List[str],
+ expected_registered: list[str],
+ expected_not_registered: list[str],
):
- """Test that only eligible functions are registered (both sync and async are now eligible)."""
-
# Create a temporary directory and file with test functions
with tempfile.TemporaryDirectory() as temp_dir:
test_file = Path(temp_dir) / "test_functions.py"
@@ -171,7 +159,6 @@ class CodebaseFileContent(StructuredContent):
def test_recursive_folder_search(self):
"""Test that recursive folder search works correctly."""
-
with tempfile.TemporaryDirectory() as temp_dir:
# Create nested directory structure
nested_dir = Path(temp_dir) / "subdir"
@@ -216,18 +203,12 @@ async def nested_function(working_memory: WorkingMemory) -> TextContent:
assert not func_registry.has_function("nested_function"), "nested_function should NOT be registered"
def test_eligibility_check_directly(self):
- """Test the eligibility check method directly."""
-
- # Create test functions dynamically for direct testing
- from pipelex.core.memory.working_memory import WorkingMemory
- from pipelex.core.stuffs.stuff_content import TextContent
-
# Valid async function
- async def valid_async_function(working_memory: WorkingMemory) -> TextContent:
+ async def valid_async_function(working_memory: WorkingMemory) -> TextContent: # noqa: ARG001
return TextContent(text="test")
# Valid sync function
- def valid_sync_function(working_memory: WorkingMemory) -> TextContent:
+ def valid_sync_function(working_memory: WorkingMemory) -> TextContent: # noqa: ARG001
return TextContent(text="test")
# Test eligibility
@@ -235,20 +216,14 @@ async def nested_function(working_memory: WorkingMemory) -> TextContent:
assert func_registry.is_eligible_function(valid_sync_function), "Valid sync function should be eligible"
def test_register_function_checks_eligibility(self):
- """Test that register_function checks eligibility before registering."""
-
- from pipelex.core.memory.working_memory import WorkingMemory
- from pipelex.core.stuffs.stuff_content import TextContent
-
- # Clear registry
func_registry.teardown()
# Valid async function
- async def valid_async_function(working_memory: WorkingMemory) -> TextContent:
+ async def valid_async_function(working_memory: WorkingMemory) -> TextContent: # noqa: ARG001
return TextContent(text="valid")
# Valid sync function
- def valid_sync_function(working_memory: WorkingMemory) -> TextContent:
+ def valid_sync_function(working_memory: WorkingMemory) -> TextContent: # noqa: ARG001
return TextContent(text="valid")
# Try to register both functions
@@ -262,14 +237,14 @@ async def nested_function(working_memory: WorkingMemory) -> TextContent:
# Test register_functions method as well
func_registry.teardown()
- async def another_valid_async_function(working_memory: WorkingMemory) -> TextContent:
+ async def another_valid_async_function(working_memory: WorkingMemory) -> TextContent: # noqa: ARG001
return TextContent(text="another_valid_async")
- def another_valid_sync_function(working_memory: WorkingMemory) -> TextContent:
+ def another_valid_sync_function(working_memory: WorkingMemory) -> TextContent: # noqa: ARG001
return TextContent(text="another_valid_sync")
# Invalid function (wrong parameter name)
- def invalid_function(other_param: WorkingMemory) -> TextContent:
+ def invalid_function(other_param: WorkingMemory) -> TextContent: # noqa: ARG001
return TextContent(text="invalid")
# Register multiple functions at once - invalid functions should be silently skipped
diff --git a/tests/unit/pipelex/tools/test_testing.py b/tests/unit/pipelex/tools/test_testing.py
index 8866810d9..a0632b494 100644
--- a/tests/unit/pipelex/tools/test_testing.py
+++ b/tests/unit/pipelex/tools/test_testing.py
@@ -9,4 +9,5 @@ def test_testing():
case RunMode.UNIT_TEST:
assert not is_env_var_set(key="GITHUB_ACTIONS") and not is_env_var_set(key="CI")
case _:
- raise RuntimeError(f"Invalid run mode: {runtime_manager.run_mode}")
+ msg = f"Invalid run mode: {runtime_manager.run_mode}"
+ raise RuntimeError(msg)
diff --git a/tests/unit/pipelex/tools/typing/__init__.py b/tests/unit/pipelex/tools/typing/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/tests/unit/pipelex/tools/typing/test_module_inspector.py b/tests/unit/pipelex/tools/typing/test_module_inspector.py
index 686affe16..a075e7dee 100644
--- a/tests/unit/pipelex/tools/typing/test_module_inspector.py
+++ b/tests/unit/pipelex/tools/typing/test_module_inspector.py
@@ -1,9 +1,7 @@
-# SPDX-FileCopyrightText: © 2025 Evotis S.A.S.
-# SPDX-License-Identifier: Apache-2.0
-
import os
import shutil
import sys
+import types
from pathlib import Path
import pytest
@@ -33,11 +31,11 @@ class TestImportModuleFromFile:
# Clean up nested directories and __pycache__ directories
directories_to_remove = ["nested/subdir", "nested", "__pycache__", "nested/__pycache__", "nested/subdir/__pycache__"]
for dir_path in directories_to_remove:
- if os.path.exists(dir_path):
+ if Path(dir_path).exists():
shutil.rmtree(dir_path)
# Clean up sys.modules entries for test modules
- modules_to_remove = [name for name in sys.modules.keys() if name.startswith("test_module")]
+ modules_to_remove = [name for name in sys.modules if name.startswith("test_module")]
for module_name in modules_to_remove:
del sys.modules[module_name]
@@ -57,7 +55,7 @@ class TestClass:
assert hasattr(module, "TestClass")
assert module.test_function() == "test_value"
- def test_import_non_python_file_raises_error(self, tmp_path: Path):
+ def test_import_non_python_file_raises_error(self):
"""Test that importing a non-Python file raises ModuleFileError."""
test_file_path = "test_file.txt"
with open(test_file_path, "w") as test_file:
@@ -66,13 +64,13 @@ class TestClass:
import_module_from_file(test_file_path)
assert "is not a Python file" in str(excinfo.value)
- def test_import_nonexistent_file_raises_error(self, tmp_path: Path):
+ def test_import_nonexistent_file_raises_error(self):
"""Test that importing a nonexistent file raises FileNotFoundError."""
nonexistent_file_path = "nonexistent.py"
with pytest.raises(FileNotFoundError):
import_module_from_file(nonexistent_file_path)
- def test_import_file_with_syntax_error_raises_error(self, tmp_path: Path):
+ def test_import_file_with_syntax_error_raises_error(self):
"""Test that importing a file with syntax errors raises SyntaxError."""
test_file_path = "syntax_error.py"
with open(test_file_path, "w") as test_file:
@@ -83,7 +81,7 @@ def test_function(
with pytest.raises(SyntaxError):
import_module_from_file(test_file_path)
- def test_import_file_with_import_error_raises_error(self, tmp_path: Path):
+ def test_import_file_with_import_error_raises_error(self):
"""Test that importing a file with import errors raises ImportError."""
test_file_path = "import_error.py"
with open(test_file_path, "w") as test_file:
@@ -93,7 +91,7 @@ import nonexistent_module
with pytest.raises(ImportError):
import_module_from_file(test_file_path)
- def test_module_added_to_sys_modules(self, tmp_path: Path):
+ def test_module_added_to_sys_modules(self):
"""Test that imported module is added to sys.modules."""
test_file_path = "test_module.py"
with open(test_file_path, "w") as test_file:
@@ -108,7 +106,7 @@ def test_function():
assert expected_module_name in sys.modules
assert sys.modules[expected_module_name] is module
- def test_module_name_with_path_separators(self, tmp_path: Path):
+ def test_module_name_with_path_separators(self):
"""Test that module name is correctly formatted with path separators (should use file basename)."""
test_file_path = "nested/subdir/test_module.py"
# Create nested directories
@@ -126,8 +124,6 @@ def test_function():
class TestFindClassesInModule:
def test_find_all_classes_no_base_class(self):
"""Test finding all classes when no base class is specified."""
- import types
-
test_module = types.ModuleType("test_module")
class ClassA:
@@ -143,14 +139,13 @@ class TestFindClassesInModule:
setattr(test_module, "ClassB", ClassB)
setattr(test_module, "some_function", lambda: None)
classes = find_classes_in_module(test_module, base_class=None, include_imported=False)
- assert len(classes) == 2
+ expected_number_of_classes = 2
+ assert len(classes) == expected_number_of_classes
assert ClassA in classes
assert ClassB in classes
def test_find_classes_with_base_class(self):
"""Test finding classes that inherit from a specific base class."""
- import types
-
test_module = types.ModuleType("test_module")
class BaseClass:
@@ -170,15 +165,14 @@ class TestFindClassesInModule:
setattr(test_module, "SubClass", SubClass)
setattr(test_module, "UnrelatedClass", UnrelatedClass)
classes = find_classes_in_module(test_module, base_class=BaseClass, include_imported=False)
- assert len(classes) == 2 # BaseClass and SubClass
+ expected_number_of_classes = 2
+ assert len(classes) == expected_number_of_classes
assert BaseClass in classes
assert SubClass in classes
assert UnrelatedClass not in classes
def test_find_classes_exclude_imported(self):
"""Test that imported classes are excluded when include_imported=False."""
- import types
-
test_module = types.ModuleType("test_module")
class LocalClass:
@@ -192,14 +186,13 @@ class TestFindClassesInModule:
setattr(test_module, "LocalClass", LocalClass)
setattr(test_module, "ImportedClass", ImportedClass)
classes = find_classes_in_module(test_module, base_class=None, include_imported=False)
- assert len(classes) == 1
+ expected_number_of_classes = 1
+ assert len(classes) == expected_number_of_classes
assert LocalClass in classes
assert ImportedClass not in classes
def test_find_classes_include_imported(self):
"""Test that imported classes are included when include_imported=True."""
- import types
-
test_module = types.ModuleType("test_module")
class LocalClass:
@@ -213,22 +206,20 @@ class TestFindClassesInModule:
setattr(test_module, "LocalClass", LocalClass)
setattr(test_module, "ImportedClass", ImportedClass)
classes = find_classes_in_module(test_module, base_class=None, include_imported=True)
- assert len(classes) == 2
+ expected_number_of_classes = 2
+ assert len(classes) == expected_number_of_classes
assert LocalClass in classes
assert ImportedClass in classes
def test_find_classes_empty_module(self):
"""Test finding classes in an empty module."""
- import types
-
test_module = types.ModuleType("test_module")
classes = find_classes_in_module(test_module, base_class=None, include_imported=False)
- assert len(classes) == 0
+ expected_number_of_classes = 0
+ assert len(classes) == expected_number_of_classes
def test_find_classes_with_functions_and_variables(self):
"""Test that functions and variables are not included in class search."""
- import types
-
test_module = types.ModuleType("test_module")
class TestClass:
@@ -244,13 +235,12 @@ class TestFindClassesInModule:
setattr(test_module, "some_variable", 42)
setattr(test_module, "some_string", "hello")
classes = find_classes_in_module(test_module, base_class=None, include_imported=False)
- assert len(classes) == 1
+ expected_number_of_classes = 1
+ assert len(classes) == expected_number_of_classes
assert TestClass in classes
def test_find_classes_with_nested_classes(self):
"""Test finding classes including nested classes."""
- import types
-
test_module = types.ModuleType("test_module")
class OuterClass:
@@ -260,13 +250,12 @@ class TestFindClassesInModule:
OuterClass.__module__ = test_module.__name__
setattr(test_module, "OuterClass", OuterClass)
classes = find_classes_in_module(test_module, base_class=None, include_imported=False)
- assert len(classes) == 1
+ expected_number_of_classes = 1
+ assert len(classes) == expected_number_of_classes
assert OuterClass in classes
def test_find_classes_with_builtin_types(self):
"""Test finding classes including user-defined types (not builtins, which can't be assigned __module__)."""
- import types
-
test_module = types.ModuleType("test_module")
class MyClassA:
@@ -280,14 +269,12 @@ class TestFindClassesInModule:
setattr(test_module, "MyClassA", MyClassA)
setattr(test_module, "MyClassB", MyClassB)
classes = find_classes_in_module(test_module, base_class=None, include_imported=False)
- assert len(classes) == 2
+ expected_number_of_classes = 2
+ assert len(classes) == expected_number_of_classes
assert MyClassA in classes
assert MyClassB in classes
def test_find_classes_with_base_class_and_imported(self):
- """Test finding classes with base class filter and imported classes."""
- import types
-
test_module = types.ModuleType("test_module")
class BaseClass:
@@ -306,12 +293,14 @@ class TestFindClassesInModule:
setattr(test_module, "LocalSubClass", LocalSubClass)
setattr(test_module, "ImportedSubClass", ImportedSubClass)
classes = find_classes_in_module(test_module, base_class=BaseClass, include_imported=False)
- assert len(classes) == 2 # BaseClass and LocalSubClass
+ expected_number_of_classes = 2
+ assert len(classes) == expected_number_of_classes
assert BaseClass in classes
assert LocalSubClass in classes
assert ImportedSubClass not in classes
classes = find_classes_in_module(test_module, base_class=BaseClass, include_imported=True)
- assert len(classes) == 3 # BaseClass, LocalSubClass, and ImportedSubClass
+ expected_number_of_classes = 3
+ assert len(classes) == expected_number_of_classes
assert BaseClass in classes
assert LocalSubClass in classes
assert ImportedSubClass in classes
diff --git a/tests/unit/pipelex/tools/typing/test_pydantic_utils.py b/tests/unit/pipelex/tools/typing/test_pydantic_utils.py
index 0ed7b8d90..26b010083 100644
--- a/tests/unit/pipelex/tools/typing/test_pydantic_utils.py
+++ b/tests/unit/pipelex/tools/typing/test_pydantic_utils.py
@@ -1,4 +1,4 @@
-from typing import Any, Dict, cast
+from typing import Any, cast
import pytest
from pydantic import BaseModel, ConfigDict, Field, ValidationError
@@ -34,14 +34,14 @@ def test_serialize_model_with_hidden_fields():
parent = ParentModel(parent_name="bar", child=child)
# Test default model_dump() shows everything
- model_dump_result: Dict[str, Any] = parent.model_dump()
- child_dict = cast(Dict[str, Any], model_dump_result["child"])
+ model_dump_result: dict[str, Any] = parent.model_dump()
+ child_dict = cast("dict[str, Any]", model_dump_result["child"])
assert "child_secret" in child_dict
assert child_dict["child_secret"] == "secret_foo"
# Test our custom serializer omits hidden fields (child_secret)
- serialized_result = cast(Dict[str, Any], serialize_model(parent))
- child_serialized = cast(Dict[str, Any], serialized_result["child"])
+ serialized_result = cast("dict[str, Any]", serialize_model(parent))
+ child_serialized = cast("dict[str, Any]", serialized_result["child"])
assert "child_secret" not in child_serialized
assert serialized_result == {"parent_name": "bar", "child": {"child_name": "foo"}}
diff --git a/tests/unit/pipelex/tools/typing/test_structure_printer.py b/tests/unit/pipelex/tools/typing/test_structure_printer.py
index de79872b0..233c230bf 100644
--- a/tests/unit/pipelex/tools/typing/test_structure_printer.py
+++ b/tests/unit/pipelex/tools/typing/test_structure_printer.py
@@ -1,8 +1,8 @@
from datetime import datetime
-from typing import List, Literal, Optional
+from typing import Literal, Optional
+import pytest
from pydantic import BaseModel, Field, field_validator
-from pytest import FixtureRequest
from pipelex.core.stuffs.stuff_content import ListContent, StructuredContent, TextContent
from pipelex.tools.typing.structure_printer import StructurePrinter
@@ -34,7 +34,6 @@ class MusicGenre(StrEnum):
class SimpleTextContent(TextContent):
"""A simple text content class"""
- pass
class MusicCategoryContent(StructuredContent):
@@ -79,26 +78,26 @@ class PersonContent(StructuredContent):
name: str
age: int
address: AddressContent = Field(description="Address of the person")
- documents: List[DocumentTypeContent]
- priority: Optional[Priority] = None
+ documents: list[DocumentTypeContent]
+ priority: Priority | None = None
class ComplexListContent(ListContent[PersonContent]):
"""List content with complex items"""
- items: List[PersonContent]
+ items: list[PersonContent]
class GanttTaskDetails(StructuredContent):
"""Do not include timezone in the dates."""
name: str
- start_date: Optional[datetime] = None
- end_date: Optional[datetime] = None
+ start_date: datetime | None = None
+ end_date: datetime | None = None
@field_validator("start_date", "end_date")
@classmethod
- def remove_tzinfo(cls, v: Optional[datetime]) -> Optional[datetime]:
+ def remove_tzinfo(cls, v: datetime | None) -> datetime | None:
if v is not None:
return v.replace(tzinfo=None)
return v
@@ -106,25 +105,24 @@ class GanttTaskDetails(StructuredContent):
class Milestone(StructuredContent):
name: str
- date: Optional[datetime]
+ date: datetime | None
@field_validator("date")
@classmethod
- def remove_tzinfo(cls, v: Optional[datetime]) -> Optional[datetime]:
+ def remove_tzinfo(cls, v: datetime | None) -> datetime | None:
if v is not None:
return v.replace(tzinfo=None)
return v
class GanttChart(StructuredContent):
- tasks: Optional[List[GanttTaskDetails]] = None
- milestones: Optional[List[Milestone]] = None
+ tasks: list[GanttTaskDetails] | None = None
+ milestones: list[Milestone] | None = None
class TestStructurePrinter:
- """Tests for the type inspector functionality"""
-
- def test_simple_text_content(self, request: FixtureRequest):
+ @pytest.mark.usefixtures("request")
+ def test_simple_text_content(self):
"""Test structure of simple text content"""
result = StructurePrinter().get_type_structure(SimpleTextContent)
expected = [
@@ -134,7 +132,8 @@ class TestStructurePrinter:
]
assert result == expected, f"Expected:\n{''.join(expected)}\n\nGot:\n{''.join(result)}"
- def test_simple_structured_content(self, request: FixtureRequest):
+ @pytest.mark.usefixtures("request")
+ def test_simple_structured_content(self):
"""Test structure of simple structured content"""
result = StructurePrinter().get_type_structure(SimpleStructuredContent)
expected = [
@@ -146,7 +145,8 @@ class TestStructurePrinter:
]
assert result == expected, f"Expected:\n{''.join(expected)}\n\nGot:\n{''.join(result)}"
- def test_enum_content(self, request: FixtureRequest):
+ @pytest.mark.usefixtures("request")
+ def test_enum_content(self):
"""Test structure of content with enum"""
result = StructurePrinter().get_type_structure(DocumentTypeContent)
expected = [
@@ -160,7 +160,8 @@ class TestStructurePrinter:
]
assert result == expected, f"Expected:\n{''.join(expected)}\n\nGot:\n{''.join(result)}"
- def test_nested_content(self, request: FixtureRequest):
+ @pytest.mark.usefixtures("request")
+ def test_nested_content(self):
"""Test structure of nested content"""
result = StructurePrinter().get_type_structure(PersonContent)
expected = [
@@ -170,7 +171,7 @@ class TestStructurePrinter:
" age: int",
" address: AddressContent # Address of the person",
" documents: List[DocumentTypeContent]",
- " priority: Optional[Priority] = None",
+ " priority: Priority | None = None",
"",
"class AddressContent(StructuredContent):",
' """Nested address content"""',
@@ -190,9 +191,13 @@ class TestStructurePrinter:
' HIGH = "HIGH"',
' LOW = "LOW"',
]
+ from pipelex import pretty_print
+ pretty_print(result, "results")
+ pretty_print(expected, "expected")
assert result == expected, f"Expected:\n{''.join(expected)}\n\nGot:\n{''.join(result)}"
- def test_list_content(self, request: FixtureRequest):
+ @pytest.mark.usefixtures("request")
+ def test_list_content(self):
"""Test structure of list content"""
result = StructurePrinter().get_type_structure(ComplexListContent)
expected = [
@@ -206,7 +211,7 @@ class TestStructurePrinter:
" age: int",
" address: AddressContent # Address of the person",
" documents: List[DocumentTypeContent]",
- " priority: Optional[Priority] = None",
+ " priority: Priority | None = None",
"",
"class AddressContent(StructuredContent):",
' """Nested address content"""',
@@ -228,7 +233,8 @@ class TestStructurePrinter:
]
assert result == expected, f"Expected:\n{''.join(expected)}\n\nGot:\n{''.join(result)}"
- def test_model_with_field_description(self, request: FixtureRequest):
+ @pytest.mark.usefixtures("request")
+ def test_model_with_field_description(self):
"""Test structure of a model with field descriptions"""
class Person(BaseModel):
@@ -249,7 +255,8 @@ class TestStructurePrinter:
]
assert result == expected, f"Expected:\n{''.join(expected)}\n\nGot:\n{''.join(result)}"
- def test_model_with_docstring_and_field_description(self, request: FixtureRequest):
+ @pytest.mark.usefixtures("request")
+ def test_model_with_docstring_and_field_description(self):
"""Test structure of a model with both docstring and field descriptions"""
class TaskContent(StructuredContent):
@@ -261,7 +268,7 @@ class TestStructurePrinter:
title: str = Field(description="The title of the task")
description: str = Field(description="Detailed description of what needs to be done")
- is_completed: bool = Field(False, description="Whether the task is completed")
+ is_completed: bool = Field(default=False, description="Whether the task is completed")
result = StructurePrinter().get_type_structure(TaskContent)
expected = [
@@ -277,7 +284,8 @@ class TestStructurePrinter:
]
assert result == expected, f"Expected:\n{''.join(expected)}\n\nGot:\n{''.join(result)}"
- def test_literal_field_content(self, request: FixtureRequest):
+ @pytest.mark.usefixtures("request")
+ def test_literal_field_content(self) -> None:
"""Test structure of content with Literal field"""
result = StructurePrinter().get_type_structure(MusicCategoryContent)
expected = [
@@ -301,23 +309,22 @@ class TestStructurePrinter:
]
assert result == expected, f"Expected:\n{''.join(expected)}\n\nGot:\n{''.join(result)}"
- def test_gantt_chart_content(self, request: FixtureRequest):
- """Test structure of Gantt chart content with datetime validators"""
-
+ @pytest.mark.usefixtures("request")
+ def test_gantt_chart_content(self):
result = StructurePrinter().get_type_structure(GanttChart, base_class=StructuredContent)
expected = [
"class GanttChart(StructuredContent):",
- " tasks: Optional[List[GanttTaskDetails]] = None",
- " milestones: Optional[List[Milestone]] = None",
+ " tasks: List[GanttTaskDetails] | None = None",
+ " milestones: List[Milestone] | None = None",
"",
"class GanttTaskDetails(StructuredContent):",
' """Do not include timezone in the dates."""',
" name: str",
- " start_date: Optional[datetime] = None",
- " end_date: Optional[datetime] = None",
+ " start_date: datetime | None = None",
+ " end_date: datetime | None = None",
"",
"class Milestone(StructuredContent):",
" name: str",
- " date: Optional[datetime] = None",
+ " date: datetime | None = None",
]
- assert result == expected, f"Expected:\n{''.join(expected)}\n\nGot:\n{''.join(result)}"
+ assert result == expected
diff --git a/uv.lock b/uv.lock
index 6b7cb9cb5..f24620d7f 100644
--- a/uv.lock
+++ b/uv.lock
@@ -182,7 +182,7 @@ wheels = [
[[package]]
name = "anthropic"
-version = "0.68.0"
+version = "0.68.1"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "anyio" },
@@ -194,9 +194,9 @@ dependencies = [
{ name = "sniffio" },
{ name = "typing-extensions" },
]
-sdist = { url = "https://files.pythonhosted.org/packages/64/46/da44bf087ddaf3f7dbe4808c00c7cde466fe68c4fc9fbebdfc231f4ea205/anthropic-0.68.0.tar.gz", hash = "sha256:507e9b5f627d1b249128ff15b21855e718fa4ed8dabc787d0e68860a4b32a7a8", size = 471584, upload-time = "2025-09-17T15:20:19.509Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/43/b9/5e736a924dabe0473b212597ae83a124e09818fac672998bb5960a64e9b1/anthropic-0.68.1.tar.gz", hash = "sha256:5fc01019dfffbc39e87a24f4d7cfe34e83dda4fff7b5371f33639f982f0c8dba", size = 471965, upload-time = "2025-09-26T16:27:30.547Z" }
wheels = [
- { url = "https://files.pythonhosted.org/packages/60/32/2d7553184b05bdbec61dd600014a55b9028408aee6128b25cb6f20e3002c/anthropic-0.68.0-py3-none-any.whl", hash = "sha256:ac579ea5eca22a7165b1042e6af57c4bf556e51afae3ca80e24768d4756b78c0", size = 325199, upload-time = "2025-09-17T15:20:17.452Z" },
+ { url = "https://files.pythonhosted.org/packages/c9/5c/e871e40ba11338502569a3f5cb176650f99f7b4ecf915f4d38fe2b8cff37/anthropic-0.68.1-py3-none-any.whl", hash = "sha256:40fc545c4781ebc45d8e7849240c4001f40d61d18a17e5363c0acb10afefc33b", size = 325214, upload-time = "2025-09-26T16:27:29.049Z" },
]
[[package]]
@@ -214,6 +214,18 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/15/b3/9b1a8074496371342ec1e796a96f99c82c945a339cd81a8e73de28b4cf9e/anyio-4.11.0-py3-none-any.whl", hash = "sha256:0287e96f4d26d4149305414d4e3bc32f0dcd0862365a4bddea19d7a1ec38c4fc", size = 109097, upload-time = "2025-09-23T09:19:10.601Z" },
]
+[[package]]
+name = "astroid"
+version = "3.3.11"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+ { name = "typing-extensions", marker = "python_full_version < '3.11'" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/18/74/dfb75f9ccd592bbedb175d4a32fc643cf569d7c218508bfbd6ea7ef9c091/astroid-3.3.11.tar.gz", hash = "sha256:1e5a5011af2920c7c67a53f65d536d65bfa7116feeaf2354d8b94f29573bb0ce", size = 400439, upload-time = "2025-07-13T18:04:23.177Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/af/0f/3b8fdc946b4d9cc8cc1e8af42c4e409468c84441b933d037e101b3d72d86/astroid-3.3.11-py3-none-any.whl", hash = "sha256:54c760ae8322ece1abd213057c4b5bba7c49818853fc901ef09719a60dbf9dec", size = 275612, upload-time = "2025-07-13T18:04:21.07Z" },
+]
+
[[package]]
name = "async-timeout"
version = "5.0.1"
@@ -289,16 +301,16 @@ wheels = [
[[package]]
name = "boto3-stubs"
-version = "1.40.39"
+version = "1.40.40"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "botocore-stubs" },
{ name = "types-s3transfer" },
{ name = "typing-extensions", marker = "python_full_version < '3.12'" },
]
-sdist = { url = "https://files.pythonhosted.org/packages/ad/17/ec5698d04ea4d4a285d581c0d9b0152c6f1cd89e335b211a372bef659b0e/boto3_stubs-1.40.39.tar.gz", hash = "sha256:320315400d9a1e717014b30362b8453f0957a09dc331c5aec6128c32313ca8e2", size = 100837, upload-time = "2025-09-25T19:26:03.075Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/d5/09/598565add9282520403e74bcc385301fa3dd3eacbc6e14a5b5b43d641fb6/boto3_stubs-1.40.40.tar.gz", hash = "sha256:68a49cb1d11280b9b0443766af32c5aa7f71d1ef4ddff8dfdbf6fd64c12283bf", size = 100830, upload-time = "2025-09-26T19:29:55.021Z" }
wheels = [
- { url = "https://files.pythonhosted.org/packages/2e/36/620718640c7e28645da3059348800e1185485597661d10095f6624dad836/boto3_stubs-1.40.39-py3-none-any.whl", hash = "sha256:1f5319c486f792ff869efb255cd43b61c072c7d478e9025e08b8f98e6e16c1e4", size = 69687, upload-time = "2025-09-25T19:25:53.38Z" },
+ { url = "https://files.pythonhosted.org/packages/6a/d7/f263878305e43adaa5de36f7cc49a9b4cf3c1d2142613651f3d7e3023f41/boto3_stubs-1.40.40-py3-none-any.whl", hash = "sha256:b52d9491a0f701c60d6d54ba310019d4293afc5929710e4c9d69c11d02a01344", size = 69690, upload-time = "2025-09-26T19:29:49.296Z" },
]
[[package]]
@@ -706,6 +718,15 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/6e/c6/ac0b6c1e2d138f1002bcf799d330bd6d85084fece321e662a14223794041/Deprecated-1.2.18-py2.py3-none-any.whl", hash = "sha256:bd5011788200372a32418f888e326a09ff80d0214bd961147cfed01b5c018eec", size = 9998, upload-time = "2025-01-27T10:46:09.186Z" },
]
+[[package]]
+name = "dill"
+version = "0.4.0"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/12/80/630b4b88364e9a8c8c5797f4602d0f76ef820909ee32f0bacb9f90654042/dill-0.4.0.tar.gz", hash = "sha256:0633f1d2df477324f53a895b02c901fb961bdbf65a17122586ea7019292cbcf0", size = 186976, upload-time = "2025-04-16T00:41:48.867Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/50/3d/9373ad9c56321fdab5b41197068e1d8c25883b3fea29dd361f9b55116869/dill-0.4.0-py3-none-any.whl", hash = "sha256:44f54bf6412c2c8464c14e8243eb163690a9800dbe2c367330883b19c7561049", size = 119668, upload-time = "2025-04-16T00:41:47.671Z" },
+]
+
[[package]]
name = "diskcache"
version = "5.6.3"
@@ -941,7 +962,7 @@ wheels = [
[[package]]
name = "google-genai"
-version = "1.39.0"
+version = "1.39.1"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "anyio" },
@@ -953,9 +974,9 @@ dependencies = [
{ name = "typing-extensions" },
{ name = "websockets" },
]
-sdist = { url = "https://files.pythonhosted.org/packages/ee/30/eda6ec8d47946ddf25fc193d8a9be6f29296f6659ab4a482607a2cf32552/google_genai-1.39.0.tar.gz", hash = "sha256:995fbe76f3f094ed3a4122f5e5f34e3601b774aa025110a2b013a9a493eb2f81", size = 244435, upload-time = "2025-09-25T21:18:49.286Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/f4/3e/25b88bda07ca237043f1be45d13c49ffbc73f9edf45d3232345802f67197/google_genai-1.39.1.tar.gz", hash = "sha256:4721704b43d170fc3f1b1cb5494bee1a7f7aae20de3a5383cdf6a129139df80b", size = 244631, upload-time = "2025-09-26T20:56:19.5Z" }
wheels = [
- { url = "https://files.pythonhosted.org/packages/c8/93/781ea98fd1dbf8ffaf78c494e59b6364b086e6cbc7dfbc1b750d92fbbe9e/google_genai-1.39.0-py3-none-any.whl", hash = "sha256:eaba325728ea8b90d6111ff009954f526ee10091badfabc31b84dbfcecdf2d02", size = 244544, upload-time = "2025-09-25T21:18:47.508Z" },
+ { url = "https://files.pythonhosted.org/packages/cb/c3/12c1f386184d2fcd694b73adeabc3714a5ed65c01cc183b4e3727a26b9d1/google_genai-1.39.1-py3-none-any.whl", hash = "sha256:6ca36c7e40db6fcba7049dfdd102c86da326804f34403bd7d90fa613a45e5a78", size = 244681, upload-time = "2025-09-26T20:56:17.527Z" },
]
[[package]]
@@ -1051,6 +1072,15 @@ google-genai = [
{ name = "jsonref" },
]
+[[package]]
+name = "isort"
+version = "6.0.1"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/b8/21/1e2a441f74a653a144224d7d21afe8f4169e6c7c20bb13aec3a2dc3815e0/isort-6.0.1.tar.gz", hash = "sha256:1cb5df28dfbc742e490c5e41bad6da41b805b0a8be7bc93cd0fb2a8a890ac450", size = 821955, upload-time = "2025-02-26T21:13:16.955Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/c1/11/114d0a5f4dabbdcedc1125dee0888514c3c3b16d3e9facad87ed96fad97c/isort-6.0.1-py3-none-any.whl", hash = "sha256:2dc5d7f65c9678d94c88dfc29161a320eec67328bc97aad576874cb4be1e9615", size = 94186, upload-time = "2025-02-26T21:13:14.911Z" },
+]
+
[[package]]
name = "jinja2"
version = "3.1.6"
@@ -1259,6 +1289,15 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/4f/65/6079a46068dfceaeabb5dcad6d674f5f5c61a6fa5673746f42a9f4c233b3/MarkupSafe-3.0.2-cp313-cp313t-win_amd64.whl", hash = "sha256:e444a31f8db13eb18ada366ab3cf45fd4b31e4db1236a4448f68778c1d1a5a2f", size = 15739, upload-time = "2024-10-18T15:21:42.784Z" },
]
+[[package]]
+name = "mccabe"
+version = "0.7.0"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/e7/ff/0ffefdcac38932a54d2b5eed4e0ba8a408f215002cd178ad1df0f2806ff8/mccabe-0.7.0.tar.gz", hash = "sha256:348e0240c33b60bbdf4e523192ef919f28cb2c3d7d5c7794f74009290f236325", size = 9658, upload-time = "2022-01-24T01:14:51.113Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/27/1a/1f68f9ba0c207934b35b86a8ca3aad8395a3d6dd7921c0686e23853ff5a9/mccabe-0.7.0-py2.py3-none-any.whl", hash = "sha256:6c2d30ab6be0e4a46919781807b4f0d834ebdd6c6e3dca0bda5a15f863427b6e", size = 7350, upload-time = "2022-01-24T01:14:49.62Z" },
+]
+
[[package]]
name = "mdurl"
version = "0.1.2"
@@ -1838,16 +1877,16 @@ wheels = [
[[package]]
name = "pandas-stubs"
-version = "2.3.2.250827"
+version = "2.3.2.250926"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" },
{ name = "numpy", version = "2.3.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" },
{ name = "types-pytz" },
]
-sdist = { url = "https://files.pythonhosted.org/packages/03/7b/8d2076a76ddf35806319798037056e4bbdcacdc832fb7c95b517f4c03fb2/pandas_stubs-2.3.2.250827.tar.gz", hash = "sha256:bcc2d49a2766325e4a1a492c3eeda879e9521bb5e26e69e2bbf13e80e7ef569a", size = 100032, upload-time = "2025-08-27T23:18:12.802Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/1b/3b/32be58a125db39d0b5f62cc93795f32b5bb2915bd5c4a46f0e35171985e2/pandas_stubs-2.3.2.250926.tar.gz", hash = "sha256:c64b9932760ceefb96a3222b953e6a251321a9832a28548be6506df473a66406", size = 102147, upload-time = "2025-09-26T19:50:39.522Z" }
wheels = [
- { url = "https://files.pythonhosted.org/packages/a1/b8/dc820157be5aa9527f1f7ffe81737ee4d1cf0924081e1bfbd680530dde41/pandas_stubs-2.3.2.250827-py3-none-any.whl", hash = "sha256:3d613013b4189147a9a6bb18d8bec1e5b137de091496e9b9ff9f137ec3e223a9", size = 157775, upload-time = "2025-08-27T23:18:11.083Z" },
+ { url = "https://files.pythonhosted.org/packages/40/96/1e4a035eaf4dce9610aac6e43026d0c6baa05773daf6d21e635a4fe19e21/pandas_stubs-2.3.2.250926-py3-none-any.whl", hash = "sha256:81121818453dcfe00f45c852f4dceee043640b813830f6e7bd084a4ef7ff7270", size = 159995, upload-time = "2025-09-26T19:50:38.241Z" },
]
[[package]]
@@ -1963,7 +2002,7 @@ wheels = [
[[package]]
name = "pipelex"
-version = "0.10.3"
+version = "0.10.2"
source = { editable = "." }
dependencies = [
{ name = "aiofiles" },
@@ -2008,6 +2047,7 @@ dev = [
{ name = "cocode" },
{ name = "mypy" },
{ name = "pandas-stubs" },
+ { name = "pylint" },
{ name = "pyright" },
{ name = "pytest" },
{ name = "pytest-asyncio" },
@@ -2080,8 +2120,9 @@ requires-dist = [
{ name = "pillow", specifier = ">=11.2.1" },
{ name = "polyfactory", specifier = ">=2.21.0" },
{ name = "pydantic", specifier = "==2.10.6" },
+ { name = "pylint", marker = "extra == 'dev'", specifier = ">=3.3.8" },
{ name = "pypdfium2", specifier = ">=4.30.0,!=4.30.1" },
- { name = "pyright", marker = "extra == 'dev'", specifier = "==1.1.398" },
+ { name = "pyright", marker = "extra == 'dev'", specifier = ">=1.1.405" },
{ name = "pytest", marker = "extra == 'dev'", specifier = ">=8.3.3" },
{ name = "pytest-asyncio", marker = "extra == 'dev'", specifier = ">=0.24.0" },
{ name = "pytest-cov", marker = "extra == 'dev'", specifier = ">=6.1.1" },
@@ -2389,6 +2430,25 @@ crypto = [
{ name = "cryptography" },
]
+[[package]]
+name = "pylint"
+version = "3.3.8"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+ { name = "astroid" },
+ { name = "colorama", marker = "sys_platform == 'win32'" },
+ { name = "dill" },
+ { name = "isort" },
+ { name = "mccabe" },
+ { name = "platformdirs" },
+ { name = "tomli", marker = "python_full_version < '3.11'" },
+ { name = "tomlkit" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/9d/58/1f614a84d3295c542e9f6e2c764533eea3f318f4592dc1ea06c797114767/pylint-3.3.8.tar.gz", hash = "sha256:26698de19941363037e2937d3db9ed94fb3303fdadf7d98847875345a8bb6b05", size = 1523947, upload-time = "2025-08-09T09:12:57.234Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/2d/1a/711e93a7ab6c392e349428ea56e794a3902bb4e0284c1997cff2d7efdbc1/pylint-3.3.8-py3-none-any.whl", hash = "sha256:7ef94aa692a600e82fabdd17102b73fc226758218c97473c7ad67bd4cb905d83", size = 523153, upload-time = "2025-08-09T09:12:54.836Z" },
+]
+
[[package]]
name = "pymdown-extensions"
version = "10.16.1"
@@ -2461,15 +2521,15 @@ wheels = [
[[package]]
name = "pyright"
-version = "1.1.398"
+version = "1.1.405"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "nodeenv" },
{ name = "typing-extensions" },
]
-sdist = { url = "https://files.pythonhosted.org/packages/24/d6/48740f1d029e9fc4194880d1ad03dcf0ba3a8f802e0e166b8f63350b3584/pyright-1.1.398.tar.gz", hash = "sha256:357a13edd9be8082dc73be51190913e475fa41a6efb6ec0d4b7aab3bc11638d8", size = 3892675, upload-time = "2025-03-26T10:06:06.063Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/fb/6c/ba4bbee22e76af700ea593a1d8701e3225080956753bee9750dcc25e2649/pyright-1.1.405.tar.gz", hash = "sha256:5c2a30e1037af27eb463a1cc0b9f6d65fec48478ccf092c1ac28385a15c55763", size = 4068319, upload-time = "2025-09-04T03:37:06.776Z" }
wheels = [
- { url = "https://files.pythonhosted.org/packages/58/e0/5283593f61b3c525d6d7e94cfb6b3ded20b3df66e953acaf7bb4f23b3f6e/pyright-1.1.398-py3-none-any.whl", hash = "sha256:0a70bfd007d9ea7de1cf9740e1ad1a40a122592cfe22a3f6791b06162ad08753", size = 5780235, upload-time = "2025-03-26T10:06:03.994Z" },
+ { url = "https://files.pythonhosted.org/packages/d5/1a/524f832e1ff1962a22a1accc775ca7b143ba2e9f5924bb6749dce566784a/pyright-1.1.405-py3-none-any.whl", hash = "sha256:a2cb13700b5508ce8e5d4546034cb7ea4aedb60215c6c33f56cec7f53996035a", size = 5905038, upload-time = "2025-09-04T03:37:04.913Z" },
]
[[package]]