Build AI methods with your coding agent, then run them anywhere: as an MCP for chatbots, as a webapp for people, or via API for your software.
dev to release/v0.2.8 (#5)
Thomas Hebrard19e36e05f565
diff --git a/.env.example b/.env.exampleindex d4a212441..73218e151 100644--- a/.env.example+++ b/.env.example@@ -9,14 +9,19 @@ AWS_REGION=AZURE_OPENAI_ENDPOINT=AZURE_OPENAI_API_VERSION=AZURE_OPENAI_API_KEY=-AZURE_OPENAI_API_KEY_METHOD=--GCP_CREDENTIALS_FILE_PATH=ANTHROPIC_API_KEY=MISTRAL_API_KEY=PERPLEXITY_API_KEY=+PERPLEXITY_API_ENDPOINT=https://api.perplexity.aiFAL_API_KEY=++GCP_PROJECT_ID=+GCP_REGION=+GCP_CREDENTIALS_FILE_PATH=path/to/credentials.json++CUSTOM_ENDPOINT_BASE_URL=+CUSTOM_ENDPOINT_API_KEY=\ No newline at end of filediff --git a/.github/assets/demo_thumbnail.png b/.github/assets/demo_thumbnail.pngnew file mode 100644index 000000000..8b49e62ccBinary files /dev/null and b/.github/assets/demo_thumbnail.png differdiff --git a/.gitignore b/.gitignoreindex 196c6b21f..49aa0da4f 100644--- a/.gitignore+++ b/.gitignore@@ -48,3 +48,7 @@ pipelex_super.toml# temporary filestemp/++gcp_credentials.json++/pipelex.toml\ No newline at end of filediff --git a/Makefile b/Makefileindex 517cae739..33aa17546 100644--- a/Makefile+++ b/Makefile@@ -124,14 +124,14 @@ env: check-uvfiinit: env- $(call PRINT_TITLE,"Running `pipelex init` with overwrite")- pipelex init --overwrite+ $(call PRINT_TITLE,"Running `pipelex init`")+ pipelex initinstall: env$(call PRINT_TITLE,"Installing dependencies")@. $(VIRTUAL_ENV)/bin/activate && \uv sync --all-extras && \- pipelex init --overwrite && \+ pipelex init && \echo "Installed Pipelex dependencies in ${VIRTUAL_ENV} with all extras and initialized Pipelex";lock: envdiff --git a/README.md b/README.mdindex df9a84f0b..f349c0de0 100644--- a/README.md+++ b/README.md@@ -5,7 +5,8 @@<p align="center">Pipelex is an open‑source dev tool based on a simple declarative language<br/>that lets you define replicable, structured, composable LLM pipelines.</p><div>- <a href="https://github.com/Pipelex/pipelex/blob/main/doc/Documentation.md"><strong>Docs</strong></a> -+ <a href="https://www.pipelex.com/demo"><strong>Demo</strong></a> -+ <a href="https://github.com/Pipelex/pipelex/blob/main/doc/Documentation.md"><strong>Documentation</strong></a> -<a href="https://github.com/Pipelex/pipelex/issues"><strong>Report Bug</strong></a> -<a href="https://github.com/Pipelex/pipelex/discussions"><strong>Feature Request</strong></a></div>@@ -23,6 +24,10 @@<br/></div>+<div align="center">+ <a href="https://www.pipelex.com/demo"><strong>Checkout our demo!</strong></a>+</div>+# 📑 Table of Contents- [Introduction](#introduction)diff --git a/pipelex/__init__.py b/pipelex/__init__.pyindex 191fc7df8..7c5ab0fec 100644--- a/pipelex/__init__.py+++ b/pipelex/__init__.py@@ -5,3 +5,11 @@ __all__ = ["log","pretty_print",]++# ------------------------------------------------------------+# Keep <project>/pipelex_libraries on sys.path for every installer (Fix for uv)+# ------------------------------------------------------------++from ._bootstrap_user_libs import activate as _px_bootstrap_user_libs++_px_bootstrap_user_libs()diff --git a/pipelex/_bootstrap_user_libs.py b/pipelex/_bootstrap_user_libs.pynew file mode 100644index 000000000..b782846da--- /dev/null+++ b/pipelex/_bootstrap_user_libs.py@@ -0,0 +1,30 @@+import sys+from pathlib import Path++# ------------------------------------------------------------------------+# Public helper: run once, keep user libs import-able+# ------------------------------------------------------------------------+++def activate() -> None:+ # 1) Re-create Poetry’s behaviour: put <cwd> itself on sys.path+ root = Path.cwd()+ root_str = str(root)+ if root_str not in sys.path:+ sys.path.insert(0, root_str)++ # 2) If a pipelex_libraries folder already exists, also touch __init__.py+ # (helps IDE & static type-checkers) and ensure the *exact* folder is+ # on sys.path. When it does **not** exist yet, nothing else to do —+ # as soon as the user runs `pipelex init`, the directory will be+ # created *inside* <root>, which is already import-able thanks to (1).+ for parent in (root, *root.parents):+ lib_dir = parent / "pipelex_libraries"+ if lib_dir.is_dir():+ # 1) make it a *real* package so editors & type-checkers see it+ (lib_dir / "__init__.py").touch(exist_ok=True)+ # 2) put it at the front of sys.path exactly once+ lib_path = str(lib_dir)+ if lib_path not in sys.path:+ sys.path.insert(0, lib_path)+ breakdiff --git a/pipelex/cli/_cli.py b/pipelex/cli/_cli.pyindex 262e69979..87ceb08bd 100644--- a/pipelex/cli/_cli.py+++ b/pipelex/cli/_cli.py@@ -1,3 +1,5 @@+import os+import shutilfrom typing import Annotated, Optionalimport typer@@ -6,6 +8,7 @@ from typer.core import TyperGroupfrom typing_extensions import overridefrom pipelex import log, pretty_print+from pipelex.exceptions import PipelexCLIError, PipelexConfigErrorfrom pipelex.libraries.library_config import LibraryConfigfrom pipelex.pipelex import Pipelexfrom pipelex.tools.config.manager import config_manager@@ -34,12 +37,21 @@ def init(overwrite: Annotated[bool, typer.Option("--overwrite", "-o", help="Warning: If set, existing files will be overwritten.")] = False,) -> None:"""Initialize pipelex configuration in the current directory."""- if overwrite:- typer.echo("Overwriting existing pipelex library files.")-- # Duplicate pipelines and other libraries from the base libraryLibraryConfig.export_libraries(overwrite=overwrite)+ pipelex_init_path = os.path.join(config_manager.pipelex_root_dir, "pipelex_init.toml")+ target_config_path = os.path.join(config_manager.local_root_dir, "pipelex.toml")++ if os.path.exists(target_config_path) and not overwrite:+ typer.echo("Warning: pipelex.toml already exists. Use --overwrite to force creation.")+ return++ try:+ shutil.copy2(pipelex_init_path, target_config_path)+ typer.echo(f"Created pipelex.toml at {target_config_path}")+ except Exception as e:+ raise PipelexCLIError(f"Failed to create pipelex.toml: {e}")+@app.command()def run_setup() -> None:@@ -56,8 +68,7 @@ def show_config() -> None:final_config = config_manager.load_config()pretty_print(final_config, title=f"Pipelex configuration for project: {config_manager.get_project_name()}")except Exception as e:- typer.echo(f"Error: {e}", err=True)- raise typer.Exit(1)+ raise PipelexConfigError(f"Error loading configuration: {e}")def main() -> None:diff --git a/pipelex/cogt/config_cogt.py b/pipelex/cogt/config_cogt.pyindex b2b6754ee..d9002dd0e 100644--- a/pipelex/cogt/config_cogt.py+++ b/pipelex/cogt/config_cogt.py@@ -10,7 +10,7 @@ from pipelex.cogt.plugin.anthropic.anthropic_config import AnthropicConfigfrom pipelex.cogt.plugin.bedrock.bedrock_config import BedrockConfigfrom pipelex.cogt.plugin.mistral.mistral_config import MistralConfigfrom pipelex.cogt.plugin.openai.azure_openai_config import AzureOpenAIConfig-from pipelex.cogt.plugin.openai.custom_openai_config import CustomOpenAIConfig+from pipelex.cogt.plugin.openai.custom_endpoint_config import CustomEndpointConfigfrom pipelex.cogt.plugin.openai.openai_config import OpenAIOpenAIConfigfrom pipelex.cogt.plugin.openai.perplexity_config import PerplexityConfigfrom pipelex.cogt.plugin.openai.vertexai_config import VertexAIConfig@@ -55,7 +55,7 @@ class LLMConfig(ConfigModel):mistral_config: MistralConfigopenai_openai_config: OpenAIOpenAIConfigperplexity_config: PerplexityConfig- custom_openai_config: CustomOpenAIConfig+ custom_endpoint_config: CustomEndpointConfiginstructor_config: InstructorConfigllm_job_config: LLMJobConfigdiff --git a/pipelex/cogt/llm/llm_models/llm_model_library.py b/pipelex/cogt/llm/llm_models/llm_model_library.pyindex 9b77916e1..d0d15d42c 100644--- a/pipelex/cogt/llm/llm_models/llm_model_library.py+++ b/pipelex/cogt/llm/llm_models/llm_model_library.py@@ -50,7 +50,7 @@ class LLMModelLibrary(LLMModelProviderAbstract, RootModel[LLMModelLibraryRoot]):llm_model = LLMModel.model_validate(complete_llm_model_dict)self.root.append(llm_model)- log.info(f"Loaded {len(self.root)} llm models")+ log.debug(f"Loaded {len(self.root)} llm models")@overridedef teardown(self):diff --git a/pipelex/cogt/plugin/anthropic/anthropic_config.py b/pipelex/cogt/plugin/anthropic/anthropic_config.pyindex e9882b81c..7c77361ac 100644--- a/pipelex/cogt/plugin/anthropic/anthropic_config.py+++ b/pipelex/cogt/plugin/anthropic/anthropic_config.py@@ -19,24 +19,28 @@ class AnthropicKeyMethod(StrEnum):ENV = "env"+ANTHROPIC_API_KEY_VAR_NAME = "ANTHROPIC_API_KEY"++class AnthropicConfig(ConfigModel):api_key_method: AnthropicKeyMethod = Field(strict=False)- api_key_secret_name: str- api_key_env_var_name: str++ def configure(self, secrets_provider: SecretsProviderAbstract) -> str:+ return self.get_api_key(secrets_provider=secrets_provider)def get_api_key(self, secrets_provider: SecretsProviderAbstract) -> str:match self.api_key_method:case AnthropicKeyMethod.ENV:log.verbose("Getting Anthropic API key from environment.")try:- key_from_env = get_required_env(self.api_key_env_var_name)+ key_from_env = get_required_env(ANTHROPIC_API_KEY_VAR_NAME)return key_from_envexcept EnvVarNotFoundError as exc:raise AnthropicCredentialsError(f"Error getting Anthropic API key from environment: {exc}") from exccase AnthropicKeyMethod.SECRET_PROVIDER:log.verbose("Getting Anthropic API key from secrets provider.")try:- key_from_service_provider = secrets_provider.get_secret(secret_id=self.api_key_secret_name)+ key_from_service_provider = secrets_provider.get_secret(secret_id=ANTHROPIC_API_KEY_VAR_NAME)except SecretNotFoundError as exc:raise AnthropicCredentialsError("Error getting Anthropic API key from secrets provider.") from excreturn key_from_service_providerdiff --git a/pipelex/cogt/plugin/bedrock/bedrock_config.py b/pipelex/cogt/plugin/bedrock/bedrock_config.pyindex 3a268d769..3eda2380c 100644--- a/pipelex/cogt/plugin/bedrock/bedrock_config.py+++ b/pipelex/cogt/plugin/bedrock/bedrock_config.py@@ -2,7 +2,16 @@ from enum import StrEnumfrom pydantic import Field+from pipelex import log+from pipelex.cogt.exceptions import CogtErrorfrom pipelex.tools.config.models import ConfigModel+from pipelex.tools.environment import EnvVarNotFoundError, get_required_env+from pipelex.tools.secrets.secrets_errors import SecretNotFoundError+from pipelex.tools.secrets.secrets_provider_abstract import SecretsProviderAbstract+++class BedrockCredentialsError(CogtError):+ passclass BedrockClientMethod(StrEnum):@@ -10,6 +19,24 @@ class BedrockClientMethod(StrEnum):AIBOTO3 = "aioboto3"+AWS_REGION_VAR_NAME = "AWS_REGION"++class BedrockConfig(ConfigModel):- aws_region: strclient_method: BedrockClientMethod = Field(strict=False)++ def configure(self, secrets_provider: SecretsProviderAbstract) -> str:+ """Configure and return AWS region."""+ return self._get_region(secrets_provider=secrets_provider)++ def _get_region(self, secrets_provider: SecretsProviderAbstract) -> str:+ """Get AWS region from environment or secrets provider."""+ log.debug("Getting AWS region.")+ try:+ return get_required_env(AWS_REGION_VAR_NAME)+ except EnvVarNotFoundError:+ log.verbose("AWS region not found in environment, trying secrets provider.")+ try:+ return secrets_provider.get_secret(secret_id=AWS_REGION_VAR_NAME)+ except SecretNotFoundError as exc:+ raise BedrockCredentialsError("Error getting AWS region from secrets provider.") from excdiff --git a/pipelex/cogt/plugin/bedrock/bedrock_factory.py b/pipelex/cogt/plugin/bedrock/bedrock_factory.pyindex 3b9ddef6d..a4800f5e2 100644--- a/pipelex/cogt/plugin/bedrock/bedrock_factory.py+++ b/pipelex/cogt/plugin/bedrock/bedrock_factory.py@@ -6,6 +6,7 @@ from pipelex.cogt.plugin.bedrock.bedrock_client_protocol import BedrockClientProfrom pipelex.cogt.plugin.bedrock.bedrock_config import BedrockClientMethodfrom pipelex.cogt.plugin.bedrock.bedrock_message import BedrockContentItem, BedrockImage, BedrockMessage, BedrockSource, ImageFormatfrom pipelex.config import get_config+from pipelex.hub import get_secrets_providerclass BedrockFactory:@@ -16,7 +17,7 @@ class BedrockFactory:@classmethoddef make_bedrock_client(cls) -> BedrockClientProtocol:bedrock_config = get_config().cogt.llm_config.bedrock_config- aws_region = bedrock_config.aws_region+ aws_region = bedrock_config.configure(secrets_provider=get_secrets_provider())client_method = bedrock_config.client_methodbedrock_async_client: BedrockClientProtocollog.verbose(f"Using '{client_method}' for BedrockClient")diff --git a/pipelex/cogt/plugin/mistral/mistral_config.py b/pipelex/cogt/plugin/mistral/mistral_config.pyindex 84df9b29f..078ed7f45 100644--- a/pipelex/cogt/plugin/mistral/mistral_config.py+++ b/pipelex/cogt/plugin/mistral/mistral_config.py@@ -1,5 +1,11 @@+from enum import StrEnum++from pydantic import Field++from pipelex import logfrom pipelex.cogt.exceptions import CogtErrorfrom pipelex.tools.config.models import ConfigModel+from pipelex.tools.environment import EnvVarNotFoundError, get_required_envfrom pipelex.tools.secrets.secrets_errors import SecretNotFoundErrorfrom pipelex.tools.secrets.secrets_provider_abstract import SecretsProviderAbstract@@ -8,11 +14,27 @@ class MistralCredentialsError(CogtError):pass+class MistralKeyMethod(StrEnum):+ SECRET_PROVIDER = "secret_provider"+ ENV = "env"++class MistralConfig(ConfigModel):- api_key_secret_name: str+ api_key_method: MistralKeyMethod = Field(strict=False)def api_key(self, secrets_provider: SecretsProviderAbstract) -> str:- try:- return secrets_provider.get_secret(secret_id=self.api_key_secret_name)- except SecretNotFoundError as exc:- raise MistralCredentialsError("Error getting Mistral API key from secrets provider.") from exc+ match self.api_key_method:+ case MistralKeyMethod.ENV:+ log.debug("Using Mistral API key from environment.")+ try:+ key_from_env = get_required_env("MISTRAL_API_KEY")+ return key_from_env+ except EnvVarNotFoundError as exc:+ raise MistralCredentialsError(f"Error getting Mistral API key from environment: {exc}") from exc+ case MistralKeyMethod.SECRET_PROVIDER:+ log.verbose("Using Mistral API key from secrets provider.")+ try:+ key_from_secrets_provider = secrets_provider.get_secret(secret_id="MISTRAL_API_KEY")+ return key_from_secrets_provider+ except SecretNotFoundError as exc:+ raise MistralCredentialsError("Error getting Mistral API key from secrets provider.") from excdiff --git a/pipelex/cogt/plugin/openai/azure_openai_config.py b/pipelex/cogt/plugin/openai/azure_openai_config.pyindex 74f5f0516..78585ce59 100644--- a/pipelex/cogt/plugin/openai/azure_openai_config.py+++ b/pipelex/cogt/plugin/openai/azure_openai_config.py@@ -1,4 +1,5 @@from enum import StrEnum+from typing import Tuplefrom pydantic import Field@@ -19,19 +20,60 @@ class AzureOpenAIKeyMethod(StrEnum):ENV = "env"+AZURE_OPENAI_API_KEY_VAR_NAME = "AZURE_OPENAI_API_KEY"+AZURE_OPENAI_API_ENDPOINT_VAR_NAME = "AZURE_OPENAI_API_ENDPOINT"+AZURE_OPENAI_API_VERSION_VAR_NAME = "AZURE_OPENAI_API_VERSION"++class AzureOpenAIConfig(ConfigModel):- api_endpoint: str- api_version: strapi_key_method: AzureOpenAIKeyMethod = Field(strict=False)- api_key_env_var_name: str- api_key_secret_name: str- def get_api_key(self, secrets_provider: SecretsProviderAbstract) -> str:+ def configure(self, secrets_provider: SecretsProviderAbstract) -> Tuple[str, str, str]:+ """Configure and return endpoint, version, and API key."""+ api_endpoint = self._get_api_endpoint(secrets_provider=secrets_provider)+ api_version = self._get_api_version(secrets_provider=secrets_provider)+ api_key = self._get_api_key(secrets_provider=secrets_provider)+ return api_endpoint, api_version, api_key++ def _get_api_endpoint(self, secrets_provider: SecretsProviderAbstract) -> str:+ """Get API endpoint from environment or secrets provider."""+ match self.api_key_method:+ case AzureOpenAIKeyMethod.ENV:+ log.debug("Getting Azure OpenAI API endpoint from environment.")+ try:+ return get_required_env(AZURE_OPENAI_API_ENDPOINT_VAR_NAME)+ except EnvVarNotFoundError as exc:+ raise AzureOpenAICredentialsError(f"Error getting Azure OpenAI API endpoint from environment: {exc}") from exc+ case AzureOpenAIKeyMethod.SECRET_PROVIDER:+ log.verbose("Getting Azure OpenAI API endpoint from secrets provider.")+ try:+ return secrets_provider.get_secret(secret_id=AZURE_OPENAI_API_ENDPOINT_VAR_NAME)+ except SecretNotFoundError as exc:+ raise AzureOpenAICredentialsError("Error getting Azure OpenAI API endpoint from secrets provider.") from exc++ def _get_api_version(self, secrets_provider: SecretsProviderAbstract) -> str:+ """Get API version from environment or secrets provider."""+ match self.api_key_method:+ case AzureOpenAIKeyMethod.ENV:+ log.debug("Getting Azure OpenAI API version from environment.")+ try:+ return get_required_env(AZURE_OPENAI_API_VERSION_VAR_NAME)+ except EnvVarNotFoundError as exc:+ raise AzureOpenAICredentialsError(f"Error getting Azure OpenAI API version from environment: {exc}") from exc+ case AzureOpenAIKeyMethod.SECRET_PROVIDER:+ log.verbose("Getting Azure OpenAI API version from secrets provider.")+ try:+ return secrets_provider.get_secret(secret_id=AZURE_OPENAI_API_VERSION_VAR_NAME)+ except SecretNotFoundError as exc:+ raise AzureOpenAICredentialsError("Error getting Azure OpenAI API version from secrets provider.") from exc++ def _get_api_key(self, secrets_provider: SecretsProviderAbstract) -> str:+ """Get API key from environment or secrets provider."""match self.api_key_method:case AzureOpenAIKeyMethod.ENV:log.debug("Using Azure OpenAI API key from environment.")try:- key_from_env = get_required_env(self.api_key_env_var_name)+ key_from_env = get_required_env(AZURE_OPENAI_API_KEY_VAR_NAME)return key_from_envexcept EnvVarNotFoundError as exc:raise AzureOpenAICredentialsError(f"Error getting Azure OpenAI API key from environment: {exc}") from exc@@ -39,7 +81,7 @@ class AzureOpenAIConfig(ConfigModel):log.verbose("Using Azure OpenAI API key from secrets provider.")# TODO: make it automatically select the right key for the right azure resourcetry:- key_from_secrets_provider = secrets_provider.get_secret(secret_id=self.api_key_secret_name)+ key_from_secrets_provider = secrets_provider.get_secret(secret_id=AZURE_OPENAI_API_KEY_VAR_NAME)except SecretNotFoundError as exc:raise AzureOpenAICredentialsError("Error getting Azure OpenAI API key from secrets provider.") from excreturn key_from_secrets_providerdiff --git a/pipelex/cogt/plugin/openai/custom_endpoint_config.py b/pipelex/cogt/plugin/openai/custom_endpoint_config.pynew file mode 100644index 000000000..bbdff710c--- /dev/null+++ b/pipelex/cogt/plugin/openai/custom_endpoint_config.py@@ -0,0 +1,74 @@+from enum import StrEnum+from typing import Optional, Tuple++from pydantic import Field++from pipelex import log+from pipelex.cogt.exceptions import CogtError+from pipelex.tools.config.models import ConfigModel+from pipelex.tools.environment import EnvVarNotFoundError, get_required_env+from pipelex.tools.secrets.secrets_errors import SecretNotFoundError+from pipelex.tools.secrets.secrets_provider_abstract import SecretsProviderAbstract+++class CustomEndpointCredentialsError(CogtError):+ pass+++class OpenAIKeyMethod(StrEnum):+ SECRET_PROVIDER = "secret_provider"+ ENV = "env"+++CUSTOM_ENDPOINT_API_KEY_VAR_NAME = "CUSTOM_ENDPOINT_API_KEY"+CUSTOM_ENDPOINT_BASE_URL_VAR_NAME = "CUSTOM_ENDPOINT_BASE_URL"+++class CustomEndpointConfig(ConfigModel):+ """Configuration for custom OpenAI-compatible endpoints (e.g., Ollama, LM Studio, etc.)"""++ api_key_method: OpenAIKeyMethod = Field(strict=False)++ def configure(self, secrets_provider: Optional[SecretsProviderAbstract] = None) -> Tuple[str, Optional[str]]:+ """Configure and return base URL and optional API key."""+ base_url = self._get_base_url(secrets_provider=secrets_provider)+ api_key = self._get_api_key(secrets_provider=secrets_provider)+ return base_url, api_key++ def _get_base_url(self, secrets_provider: Optional[SecretsProviderAbstract]) -> str:+ """Get base URL from environment or secrets provider."""+ match self.api_key_method:+ case OpenAIKeyMethod.ENV:+ log.debug("Getting Custom OpenAI base URL from environment.")+ try:+ return get_required_env(CUSTOM_ENDPOINT_BASE_URL_VAR_NAME)+ except EnvVarNotFoundError as exc:+ raise CustomEndpointCredentialsError(f"Error getting Custom OpenAI base URL from environment: {exc}") from exc+ case OpenAIKeyMethod.SECRET_PROVIDER:+ if not secrets_provider:+ raise CustomEndpointCredentialsError("Secrets provider is required when using SECRET_PROVIDER method")+ log.verbose("Getting Custom OpenAI base URL from secrets provider.")+ try:+ return secrets_provider.get_secret(secret_id=CUSTOM_ENDPOINT_BASE_URL_VAR_NAME)+ except SecretNotFoundError as exc:+ raise CustomEndpointCredentialsError("Error getting Custom OpenAI base URL from secrets provider.") from exc++ def _get_api_key(self, secrets_provider: Optional[SecretsProviderAbstract] = None) -> Optional[str]:+ """Get API key if configured, otherwise return None for services that don't require authentication"""+ match self.api_key_method:+ case OpenAIKeyMethod.ENV:+ log.debug("Using Custom OpenAI API key from environment.")+ try:+ return get_required_env(CUSTOM_ENDPOINT_API_KEY_VAR_NAME)+ except EnvVarNotFoundError:+ log.debug("No Custom OpenAI API key found in environment - using None.")+ return None+ case OpenAIKeyMethod.SECRET_PROVIDER:+ if not secrets_provider:+ raise CustomEndpointCredentialsError("Secrets provider is required when using SECRET_PROVIDER method")+ log.verbose("Using Custom OpenAI API key from secrets provider.")+ try:+ return secrets_provider.get_secret(secret_id=CUSTOM_ENDPOINT_API_KEY_VAR_NAME)+ except SecretNotFoundError:+ log.debug("No Custom OpenAI API key found in secrets provider - using None.")+ return Nonediff --git a/pipelex/cogt/plugin/openai/custom_openai_config.py b/pipelex/cogt/plugin/openai/custom_openai_config.pydeleted file mode 100644index 7f2c4393e..000000000--- a/pipelex/cogt/plugin/openai/custom_openai_config.py+++ /dev/null@@ -1,52 +0,0 @@-from typing import Optional--from pipelex import log-from pipelex.cogt.exceptions import CogtError-from pipelex.tools.config.models import ConfigModel-from pipelex.tools.environment import EnvVarNotFoundError, get_required_env-from pipelex.tools.secrets.secrets_errors import SecretNotFoundError-from pipelex.tools.secrets.secrets_provider_abstract import SecretsProviderAbstract---class CustomOpenAICredentialsError(CogtError):- pass---class CustomOpenAIConfig(ConfigModel):- """Configuration for custom OpenAI-compatible endpoints (e.g., Ollama, LM Studio, etc.)"""-- base_url: str- api_key: Optional[str] = None- # Option to load API key from environment or secrets if needed- api_key_env_var_name: Optional[str] = None- api_key_secret_name: Optional[str] = None-- def get_api_key(self, secrets_provider: Optional[SecretsProviderAbstract] = None) -> Optional[str]:- """Get API key if configured, otherwise return None for services that don't require authentication"""-- # If api_key is directly set, use it- if self.api_key:- log.debug("Using Custom OpenAI API key from config.")- return self.api_key-- # Try to get from environment variable if configured- if self.api_key_env_var_name:- log.debug("Using Custom OpenAI API key from environment.")- try:- key_from_env = get_required_env(self.api_key_env_var_name)- return key_from_env- except EnvVarNotFoundError as exc:- raise CustomOpenAICredentialsError(f"Error getting Custom OpenAI API key from environment: {exc}") from exc-- # Try to get from secrets provider if configured- if self.api_key_secret_name and secrets_provider:- log.verbose("Using Custom OpenAI API key from secrets provider.")- try:- key_from_secrets_provider = secrets_provider.get_secret(secret_id=self.api_key_secret_name)- return key_from_secrets_provider- except SecretNotFoundError as exc:- raise CustomOpenAICredentialsError("Error getting Custom OpenAI API key from secrets provider.") from exc-- # No API key configured - return None (for services that don't require auth)- log.debug("No Custom OpenAI API key configured - using None.")- return Nonediff --git a/pipelex/cogt/plugin/openai/openai_config.py b/pipelex/cogt/plugin/openai/openai_config.pyindex cff8ece50..443508dc9 100644--- a/pipelex/cogt/plugin/openai/openai_config.py+++ b/pipelex/cogt/plugin/openai/openai_config.py@@ -19,24 +19,25 @@ class OpenAIKeyMethod(StrEnum):ENV = "env"+OPENAI_API_KEY_VAR_NAME = "OPENAI_API_KEY"++class OpenAIOpenAIConfig(ConfigModel):api_key_method: OpenAIKeyMethod = Field(strict=False)- api_key_secret_name: str- api_key_env_var_name: strdef get_api_key(self, secrets_provider: SecretsProviderAbstract) -> str:match self.api_key_method:case OpenAIKeyMethod.ENV:log.debug("Using OpenAI API key from environment.")try:- key_from_env = get_required_env(self.api_key_env_var_name)+ key_from_env = get_required_env(OPENAI_API_KEY_VAR_NAME)return key_from_envexcept EnvVarNotFoundError as exc:raise OpenAICredentialsError(f"Error getting OpenAI API key from environment: {exc}") from exccase OpenAIKeyMethod.SECRET_PROVIDER:log.verbose("Using OpenAI API key from secrets provider.")try:- key_from_service_provider = secrets_provider.get_secret(secret_id=self.api_key_secret_name)+ key_from_service_provider = secrets_provider.get_secret(secret_id=OPENAI_API_KEY_VAR_NAME)except SecretNotFoundError as exc:raise OpenAICredentialsError("Error getting OpenAI API key from secrets provider.") from excreturn key_from_service_providerdiff --git a/pipelex/cogt/plugin/openai/openai_factory.py b/pipelex/cogt/plugin/openai/openai_factory.pyindex 8c56ef004..b602b6b00 100644--- a/pipelex/cogt/plugin/openai/openai_factory.py+++ b/pipelex/cogt/plugin/openai/openai_factory.py@@ -36,20 +36,17 @@ class OpenAIFactory:match llm_platform:case LLMPlatform.AZURE_OPENAI:azure_openai_config = get_config().cogt.llm_config.azure_openai_config- api_key = azure_openai_config.get_api_key(secrets_provider=get_secrets_provider())- azure_endpoint = azure_openai_config.api_endpoint- api_version = azure_openai_config.api_version+ endpoint, api_version, api_key = azure_openai_config.configure(secrets_provider=get_secrets_provider())- log.verbose(f"Making AsyncAzureOpenAI client with endpoint: {azure_endpoint}, api_version: {api_version}")+ log.verbose(f"Making AsyncAzureOpenAI client with endpoint: {endpoint}, api_version: {api_version}")the_client = openai.AsyncAzureOpenAI(- azure_endpoint=azure_endpoint,+ azure_endpoint=endpoint,api_key=api_key,api_version=api_version,)case LLMPlatform.PERPLEXITY:perplexity_config = get_config().cogt.llm_config.perplexity_config- api_key = perplexity_config.get_api_key(secrets_provider=get_secrets_provider())- endpoint = perplexity_config.api_endpoint+ endpoint, api_key = perplexity_config.configure(secrets_provider=get_secrets_provider())log.verbose(f"Making perplexity AsyncOpenAI client with endpoint: {endpoint}")the_client = openai.AsyncOpenAI(@@ -62,10 +59,7 @@ class OpenAIFactory:the_client = openai.AsyncOpenAI(api_key=api_key)case LLMPlatform.VERTEXAI_OPENAI:vertexai_config = get_config().cogt.llm_config.vertexai_config- project_id = vertexai_config.project_id- region = vertexai_config.region- api_key = vertexai_config.get_api_key()- endpoint = f"https://{region}-aiplatform.googleapis.com/v1beta1/projects/{project_id}/locations/{region}/endpoints/openapi"+ endpoint, api_key = vertexai_config.configure(secrets_provider=get_secrets_provider())log.verbose(f"Making vertex AsyncOpenAI client with endpoint: {endpoint}")the_client = openai.AsyncOpenAI(@@ -73,9 +67,8 @@ class OpenAIFactory:base_url=endpoint,)case LLMPlatform.CUSTOM_OPENAI:- custom_openai_config = get_config().cogt.llm_config.custom_openai_config- api_key = custom_openai_config.get_api_key(secrets_provider=get_secrets_provider())- base_url = custom_openai_config.base_url+ custom_endpoint_config = get_config().cogt.llm_config.custom_endpoint_config+ base_url, api_key = custom_endpoint_config.configure(secrets_provider=get_secrets_provider())log.verbose(f"Making custom AsyncOpenAI client with base_url: {base_url}")the_client = openai.AsyncOpenAI(diff --git a/pipelex/cogt/plugin/openai/perplexity_config.py b/pipelex/cogt/plugin/openai/perplexity_config.pyindex b52430afc..49aeba20c 100644--- a/pipelex/cogt/plugin/openai/perplexity_config.py+++ b/pipelex/cogt/plugin/openai/perplexity_config.py@@ -1,4 +1,5 @@from enum import StrEnum+from typing import Tuplefrom pydantic import Field@@ -19,25 +20,49 @@ class PerplexityKeyMethod(StrEnum):ENV = "env"+PERPLEXITY_API_KEY_VAR_NAME = "PERPLEXITY_API_KEY"+PERPLEXITY_API_ENDPOINT_VAR_NAME = "PERPLEXITY_API_ENDPOINT"++class PerplexityConfig(ConfigModel):- api_endpoint: strapi_key_method: PerplexityKeyMethod = Field(strict=False)- api_key_env_var_name: str- api_key_secret_name: str- def get_api_key(self, secrets_provider: SecretsProviderAbstract) -> str:+ def configure(self, secrets_provider: SecretsProviderAbstract) -> Tuple[str, str]:+ """Configure and return endpoint and API key."""+ api_endpoint = self._get_api_endpoint(secrets_provider=secrets_provider)+ api_key = self._get_api_key(secrets_provider=secrets_provider)+ return api_endpoint, api_key++ def _get_api_endpoint(self, secrets_provider: SecretsProviderAbstract) -> str:+ """Get API endpoint from environment or secrets provider."""+ match self.api_key_method:+ case PerplexityKeyMethod.ENV:+ log.debug("Getting Perplexity API endpoint from environment.")+ try:+ return get_required_env(PERPLEXITY_API_ENDPOINT_VAR_NAME)+ except EnvVarNotFoundError as exc:+ raise PerplexityCredentialsError(f"Error getting Perplexity API endpoint from environment: {exc}") from exc+ case PerplexityKeyMethod.SECRET_PROVIDER:+ log.verbose("Getting Perplexity API endpoint from secrets provider.")+ try:+ return secrets_provider.get_secret(secret_id=PERPLEXITY_API_ENDPOINT_VAR_NAME)+ except SecretNotFoundError as exc:+ raise PerplexityCredentialsError("Error getting Perplexity API endpoint from secrets provider.") from exc++ def _get_api_key(self, secrets_provider: SecretsProviderAbstract) -> str:+ """Get API key from environment or secrets provider."""match self.api_key_method:case PerplexityKeyMethod.ENV:log.debug("Using Perplexity API key from environment.")try:- key_from_env = get_required_env(self.api_key_env_var_name)+ key_from_env = get_required_env(PERPLEXITY_API_KEY_VAR_NAME)return key_from_envexcept EnvVarNotFoundError as exc:- raise PerplexityCredentialsError(f"Error getting Azure OpenAI API key from environment: {exc}") from exc+ raise PerplexityCredentialsError(f"Error getting Perplexity API key from environment: {exc}") from exccase PerplexityKeyMethod.SECRET_PROVIDER:log.verbose("Using Perplexity API key from secrets provider.")try:- key_from_secrets_provider = secrets_provider.get_secret(secret_id=self.api_key_secret_name)+ key_from_secrets_provider = secrets_provider.get_secret(secret_id=PERPLEXITY_API_KEY_VAR_NAME)except SecretNotFoundError as exc:raise PerplexityCredentialsError("Error getting Perplexity API key from secrets provider.") from excreturn key_from_secrets_providerdiff --git a/pipelex/cogt/plugin/openai/vertexai_config.py b/pipelex/cogt/plugin/openai/vertexai_config.pyindex 061e8315e..3b8d01bde 100644--- a/pipelex/cogt/plugin/openai/vertexai_config.py+++ b/pipelex/cogt/plugin/openai/vertexai_config.py@@ -1,10 +1,15 @@from enum import StrEnum-from typing import Any, Dict+from typing import Any, Dict, Tuple+from pydantic import Field++from pipelex import logfrom pipelex.cogt.exceptions import CogtError, MissingDependencyErrorfrom pipelex.tools.config.models import ConfigModelfrom pipelex.tools.environment import EnvVarNotFoundError, get_required_envfrom pipelex.tools.misc.json_utils import load_json_dict_from_path+from pipelex.tools.secrets.secrets_errors import SecretNotFoundError+from pipelex.tools.secrets.secrets_provider_abstract import SecretsProviderAbstractclass VertexAICredentialsError(CogtError):@@ -16,11 +21,43 @@ class VertexAIKeyMethod(StrEnum):ENV = "env"+GCP_PROJECT_ID_VAR_NAME = "GCP_PROJECT_ID"+GCP_REGION_VAR_NAME = "GCP_REGION"+GCP_CREDENTIALS_FILE_PATH_VAR_NAME = "GCP_CREDENTIALS_FILE_PATH"++class VertexAIConfig(ConfigModel):- project_id: str- region: str+ api_key_method: VertexAIKeyMethod = Field(strict=False)- def get_api_key(self) -> str:+ def configure(self, secrets_provider: SecretsProviderAbstract) -> Tuple[str, str]:+ """Configure and return project ID, region, and API key."""+ project_id, region = self._get_project_settings(secrets_provider=secrets_provider)+ api_key = self._get_credentials()+ endpoint = f"https://{region}-aiplatform.googleapis.com/v1beta1/projects/{project_id}/locations/{region}/endpoints/openapi"+ return endpoint, api_key++ def _get_project_settings(self, secrets_provider: SecretsProviderAbstract) -> Tuple[str, str]:+ """Get project ID and region from environment or secrets provider."""+ match self.api_key_method:+ case VertexAIKeyMethod.ENV:+ log.debug("Getting GCP project settings from environment.")+ try:+ project_id = get_required_env(GCP_PROJECT_ID_VAR_NAME)+ region = get_required_env(GCP_REGION_VAR_NAME)+ return project_id, region+ except EnvVarNotFoundError as exc:+ raise VertexAICredentialsError(f"Error getting GCP project settings from environment: {exc}") from exc+ case VertexAIKeyMethod.SECRET_PROVIDER:+ log.verbose("Getting GCP project settings from secrets provider.")+ try:+ project_id = secrets_provider.get_secret(secret_id=GCP_PROJECT_ID_VAR_NAME)+ region = secrets_provider.get_secret(secret_id=GCP_REGION_VAR_NAME)+ return project_id, region+ except SecretNotFoundError as exc:+ raise VertexAICredentialsError("Error getting GCP project settings from secrets provider.") from exc++ def _get_credentials(self) -> str:+ """Get GCP credentials and return access token."""try:from google.auth.transport.requests import Requestfrom google.oauth2.service_account import Credentials@@ -33,10 +70,19 @@ class VertexAIConfig(ConfigModel):(such as Azure OpenAI, OpenAI, anthropic or bedrock).",) from exc- try:- gcp_credentials_file_path = get_required_env("GCP_CREDENTIALS_FILE_PATH")- except EnvVarNotFoundError as exc:- raise VertexAICredentialsError("GCP_CREDENTIALS_FILE_PATH environment variable not found") from exc+ match self.api_key_method:+ case VertexAIKeyMethod.ENV:+ log.debug("Getting GCP credentials from environment.")+ try:+ gcp_credentials_file_path = get_required_env(GCP_CREDENTIALS_FILE_PATH_VAR_NAME)+ except EnvVarNotFoundError as exc:+ raise VertexAICredentialsError(f"{GCP_CREDENTIALS_FILE_PATH_VAR_NAME} environment variable not found") from exc+ case VertexAIKeyMethod.SECRET_PROVIDER:+ log.verbose("Getting GCP credentials from secrets provider.")+ try:+ gcp_credentials_file_path = get_required_env(GCP_CREDENTIALS_FILE_PATH_VAR_NAME)+ except EnvVarNotFoundError as exc:+ raise VertexAICredentialsError(f"{GCP_CREDENTIALS_FILE_PATH_VAR_NAME} environment variable not found") from exctry:credentials_dict: Dict[str, Any] = load_json_dict_from_path(path=gcp_credentials_file_path)diff --git a/pipelex/core/working_memory.py b/pipelex/core/working_memory.pyindex f1a8467cf..a3ab9cafe 100644--- a/pipelex/core/working_memory.py+++ b/pipelex/core/working_memory.py@@ -274,6 +274,10 @@ class WorkingMemory(BaseModel):"""return self.get_stuff(name=name).as_list_of_fixed_content_type(item_type=item_type)+ def get_list_stuff_first_item_as(self, name: str, item_type: Type[StuffContentType]) -> StuffContentType:+ """Get stuff content as ListContent with items of type StuffContentType then return the first item."""+ return self.get_stuff_as_list(name=name, item_type=item_type).items[0]+def get_stuff_as_text(self, name: str) -> TextContent:"""Get stuff content as TextContent if applicable."""return self.get_stuff(name=name).as_text@@ -317,6 +321,10 @@ class WorkingMemory(BaseModel):"""return self.get_stuff_as_list(name=MAIN_STUFF_NAME, item_type=item_type)+ def main_list_stuff_first_item_as(self, item_type: Type[StuffContentType]) -> StuffContentType:+ """Get main stuff content as first list item of type StuffContentType."""+ return self.get_list_stuff_first_item_as(name=MAIN_STUFF_NAME, item_type=item_type)+@propertydef main_stuff_as_text(self) -> TextContent:"""Get main stuff content as TextContent if applicable."""diff --git a/pipelex/exceptions.py b/pipelex/exceptions.pyindex 3b7dae1f7..be6b4ec3f 100644--- a/pipelex/exceptions.py+++ b/pipelex/exceptions.py@@ -1,3 +1,5 @@+from click import ClickException+from pipelex.tools.exceptions import RootException@@ -5,6 +7,12 @@ class PipelexError(RootException):pass+class PipelexCLIError(PipelexError, ClickException):+ """Raised when there's an error in CLI usage or operation."""++ pass++class PipelexConfigError(PipelexError):passdiff --git a/pipelex/pipelex.py b/pipelex/pipelex.pyindex 90fbb7ed7..f5c3ed47f 100644--- a/pipelex/pipelex.py+++ b/pipelex/pipelex.py@@ -82,7 +82,6 @@ class Pipelex:mission_tracker: Optional[MissionTracker] = None,activity_manager: Optional[ActivityManagerProtocol] = None,) -> None:- print(f"{PACKAGE_NAME} version {PACKAGE_VERSION} init started...", flush=True)self.pipelex_hub = pipelex_hub or PipelexHub()set_pipelex_hub(self.pipelex_hub)@@ -105,7 +104,7 @@ class Pipelex:project_name=get_config().project_name or "unknown_project",log_config=get_config().pipelex.log_config,)- log.info("Logs are configured")+ log.debug("Logs are configured")# toolsself.template_provider = template_provider or TemplateLibrary()@@ -150,7 +149,7 @@ class Pipelex:self.pipelex_hub.set_activity_manager(activity_manager=self.activity_manager)Pipelex._pipelex_instance = self- log.info(f"{PACKAGE_NAME} version {PACKAGE_VERSION} init done")+ log.debug(f"{PACKAGE_NAME} version {PACKAGE_VERSION} init done")def setup(self,@@ -233,5 +232,5 @@ class Pipelex:pipelex_instance = cls()pipelex_instance.setup(structure_classes=structure_classes)pipelex_instance.finish_setup()- log.info(f"Boot sequence completed for {get_config().project_name} and session id {get_config().session_id}")+ log.info(f"Pipelex {PACKAGE_VERSION} initialized.")return pipelex_instancediff --git a/pipelex/pipelex.toml b/pipelex/pipelex.tomlindex 46a79073c..677d357f2 100644--- a/pipelex/pipelex.toml+++ b/pipelex/pipelex.toml@@ -45,10 +45,6 @@ instructor = "INFO"urllib3-connectionpool = "INFO"urllib3-util-retry = "INFO"-kajson = "INFO"-kajson-decoder-sandbox = "INFO"-kajson-encoder-sandbox = "INFO"-pipelex = "INFO"@@ -65,14 +61,7 @@ tracebacks_suppress = []keywords_to_hilight = [][pipelex.aws_config]-api_key_method = "env" # "env", "secret_provider"-aws_region = "needs_to_be_set"--aws_access_key_id_env_var_name = "AWS_ACCESS_KEY_ID"-aws_secret_access_key_env_var_name = "AWS_SECRET_ACCESS_KEY"-aws_region_env_var_name = "AWS_REGION"-aws_access_key_id_secret_name = "AWS_ACCESS_KEY_ID"-aws_secret_access_key_secret_name = "AWS_SECRET_ACCESS_KEY"+api_key_method = "env"[cogt]@@ -108,43 +97,30 @@ is_streaming_enabled = false# "gpt-4o-mini" = "openai"[cogt.llm_config.openai_openai_config]-api_key_method = "env" # "env", "secret_provider"-api_key_env_var_name = "OPENAI_API_KEY"-api_key_secret_name = "OPENAI_API_KEY"+api_key_method = "env"[cogt.llm_config.azure_openai_config]-api_endpoint = "needs_to_be_set"-api_version = "2024-05-01-preview"-api_key_method = "env" # "env", "secret_provider"-api_key_env_var_name = "AZURE_OPENAI_API_KEY"-api_key_secret_name = "AZURE_OPENAI_API_KEY"+api_key_method = "env"# TODO: handle multiple azure openai accounts with different resource groups and account names for various llm model[cogt.llm_config.perplexity_config]-api_endpoint = "https://api.perplexity.ai"-api_key_method = "env" # "env", "secret_provider"-api_key_env_var_name = "PERPLEXITY_API_KEY"-api_key_secret_name = "PERPLEXITY_API_KEY"+api_key_method = "env"[cogt.llm_config.vertexai_config]-project_id = "needs_to_be_set"-region = "needs_to_be_set"+api_key_method = "env"[cogt.llm_config.mistral_config]-api_key_secret_name = "MISTRAL_API_KEY"+api_key_method = "env"[cogt.llm_config.bedrock_config]client_method = "aioboto3"-aws_region = "needs_to_be_set"[cogt.llm_config.anthropic_config]-api_key_method = "env" # "env", "secret_provider"-api_key_env_var_name = "ANTHROPIC_API_KEY"-api_key_secret_name = "ANTHROPIC_API_KEY"+api_key_method = "env"-[cogt.llm_config.custom_openai_config]-base_url = "http://localhost:11434/v1"+[cogt.llm_config.custom_endpoint_config]+api_key_method = "env"##################################################################################################### Image generation configdiff --git a/pipelex/pipelex_init.toml b/pipelex/pipelex_init.tomlnew file mode 100644index 000000000..0db9a3045--- /dev/null+++ b/pipelex/pipelex_init.toml@@ -0,0 +1,39 @@+[pipelex.aws_config]+api_key_method = "env"+# The possible values are "env" and "secret_provider".+# "env" means means that the env var are stored in your .env file.+# "secret_provider" means that the env var are stored in your Secret Manager (See the doc for injecting a secret provider).++[cogt]+[cogt.llm_config.preferred_platforms]+# This overrrides the defaults set for any llm handle+"gpt-4o-mini" = "openai"++[cogt.llm_config.openai_openai_config]+api_key_method = "env"++[cogt.llm_config.azure_openai_config]+api_key_method = "env"++[cogt.llm_config.perplexity_config]+api_key_method = "env"++[cogt.llm_config.vertexai_config]+api_key_method = "env"++[cogt.llm_config.mistral_config]+api_key_method = "env"++[cogt.llm_config.anthropic_config]+api_key_method = "env"++[cogt.llm_config.bedrock_config]+client_method = "aioboto3"++####################################################################################################+# OCR config+####################################################################################################++[cogt.ocr_config]+ocr_handles = ["mistral/mistral-ocr-latest"]+page_output_text_file_name = "page_text.md"diff --git a/pipelex/tools/aws/aws_config.py b/pipelex/tools/aws/aws_config.pyindex da99bd73c..d2a2b0304 100644--- a/pipelex/tools/aws/aws_config.py+++ b/pipelex/tools/aws/aws_config.py@@ -6,7 +6,7 @@ from pydantic import Fieldfrom pipelex import logfrom pipelex.hub import get_secretfrom pipelex.tools.config.models import ConfigModel-from pipelex.tools.environment import EnvVarNotFoundError, get_optional_env, get_required_env+from pipelex.tools.environment import EnvVarNotFoundError, get_required_envfrom pipelex.tools.exceptions import ToolExceptionfrom pipelex.tools.secrets.secrets_errors import SecretNotFoundError@@ -20,42 +20,37 @@ class AwsKeyMethod(StrEnum):ENV = "env"-class AwsConfig(ConfigModel):- api_key_method: AwsKeyMethod = Field(strict=False)- aws_region: str+AWS_ACCESS_KEY_ID_VAR_NAME = "AWS_ACCESS_KEY_ID"+AWS_SECRET_ACCESS_KEY_VAR_NAME = "AWS_SECRET_ACCESS_KEY"+AWS_REGION_VAR_NAME = "AWS_REGION"- aws_access_key_id_env_var_name: str- aws_secret_access_key_env_var_name: str- aws_region_env_var_name: str- aws_access_key_id_secret_name: str- aws_secret_access_key_secret_name: str+class AwsConfig(ConfigModel):+ api_key_method: AwsKeyMethod = Field(strict=False)def get_aws_access_keys(self) -> Tuple[str, str, str]:return self.get_aws_access_keys_with_method(api_key_method=self.api_key_method)def get_aws_access_keys_with_method(self, api_key_method: AwsKeyMethod) -> Tuple[str, str, str]:- aws_access_key_id: str- aws_secret_access_key: str- aws_region: strmatch api_key_method:case AwsKeyMethod.ENV:log.debug("Getting AWS access keys from environment (key id and secret access key).")try:- aws_access_key_id = get_required_env(self.aws_access_key_id_env_var_name)- aws_secret_access_key = get_required_env(self.aws_secret_access_key_env_var_name)+ aws_access_key_id = get_required_env(AWS_ACCESS_KEY_ID_VAR_NAME)+ aws_secret_access_key = get_required_env(AWS_SECRET_ACCESS_KEY_VAR_NAME)+ aws_region = get_required_env(AWS_REGION_VAR_NAME)except EnvVarNotFoundError as exc:raise AwsCredentialsError(f"Error getting AWS access keys from environment: {exc}") from exclog.debug("Getting AWS region from environment (priority override) or from aws_config.")- aws_region = get_optional_env(self.aws_region_env_var_name) or self.aws_region+case AwsKeyMethod.SECRET_PROVIDER:log.debug("Getting AWS secret access key from secrets provider (key id and secret access key).")try:- aws_access_key_id = get_secret(self.aws_access_key_id_secret_name)- aws_secret_access_key = get_secret(self.aws_secret_access_key_secret_name)+ aws_access_key_id = get_secret(AWS_ACCESS_KEY_ID_VAR_NAME)+ aws_secret_access_key = get_secret(AWS_SECRET_ACCESS_KEY_VAR_NAME)+ aws_region = get_secret(AWS_REGION_VAR_NAME)except SecretNotFoundError as exc:raise AwsCredentialsError("Error getting AWS access keys from secrets provider.") from exclog.debug("Getting AWS region from environment (priority override) or from aws_config.")- aws_region = get_optional_env(self.aws_region_env_var_name) or self.aws_regionreturn aws_access_key_id, aws_secret_access_key, aws_regiondiff --git a/pipelex/tools/log/log.py b/pipelex/tools/log/log.pyindex fd45d7b58..317904b86 100644--- a/pipelex/tools/log/log.py+++ b/pipelex/tools/log/log.py@@ -7,7 +7,7 @@ from pipelex.tools.log.log_config import ()from pipelex.tools.log.log_dispatch import LogDispatchfrom pipelex.tools.log.log_formatter import EmojiLogFormatter, LevelAndEmojiLogFormatter-from pipelex.tools.log.log_levels import LOGGING_LEVEL_DEV, LOGGING_LEVEL_VERBOSE+from pipelex.tools.log.log_levels import LOGGING_LEVEL_DEV, LOGGING_LEVEL_OFF, LOGGING_LEVEL_VERBOSEclass Log:@@ -108,7 +108,6 @@ class Log:self.poor_handler.setFormatter(LevelAndEmojiLogFormatter())poor_loggers = set(log_config.poor_loggers + [log_config.generic_poor_logger])for logger_name in poor_loggers:- # print(f"Configuring poor logger: {logger_name}")logger = logging.getLogger(logger_name)logger.setLevel(log_config.default_log_level.int_logging_level)logger.addHandler(self.poor_handler)@@ -119,17 +118,6 @@ class Log:self.verbose("Logs configured")self.verbose(f"Config set for {project_name}")- def set_rich_log_formatter(self, formatter: logging.Formatter):- """- Set the formatter for the rich log handler.-- Args:- formatter (logging.Formatter): The formatter to use for rich logging.- """- if self.rich_handler is None:- raise RuntimeError("Rich log handler is not set.")- self.rich_handler.setFormatter(formatter)-def set_poor_log_formatter(self, formatter: logging.Formatter):"""Set the formatter for the poor log handler.@@ -176,11 +164,10 @@ class Log:Args:level_name (str): The name of the log level."""- print(f"Setting default logs to '{level_name=}'")if level_name.upper() == LogLevel.DEV:level = LOGGING_LEVEL_DEVelif level_name.upper() == LogLevel.OFF:- level = LOGGING_LEVEL_DEV+ level = LOGGING_LEVEL_OFFelse:level = getattr(logging, level_name.upper())self.set_level_by_int(level)@@ -188,7 +175,6 @@ class Log:def set_level(self, level: LogLevel):"""Set the default log level for all loggers.-Args:level (LogLevel): The log level to set."""@@ -204,7 +190,6 @@ class Log:level (LogLevel): The log level to set for the package."""logger_name = package_name.replace("-", ".")- # print(f"Setting {logger_name=} to {level=}")logging.getLogger(logger_name).setLevel(level.int_logging_level)def set_levels_for_packages(self, package_log_levels: Dict[str, LogLevel]):diff --git a/pipelex/tools/log/log_dispatch.py b/pipelex/tools/log/log_dispatch.pyindex 47b8162b2..db50ec652 100644--- a/pipelex/tools/log/log_dispatch.py+++ b/pipelex/tools/log/log_dispatch.py@@ -279,7 +279,6 @@ class LogDispatch:breakfinally:del frame_info-logger = logging.getLogger(log_origin_name)logger.log(level=severity, msg=message, stacklevel=5)finally:diff --git a/pipelex/tools/log/log_levels.py b/pipelex/tools/log/log_levels.pyindex 1445601a6..6d5dd2b5c 100644--- a/pipelex/tools/log/log_levels.py+++ b/pipelex/tools/log/log_levels.py@@ -9,6 +9,7 @@ logging.addLevelName(LOGGING_LEVEL_DEV, LOGGING_LEVEL_DEV_NAME)LOGGING_LEVEL_VERBOSE = 5LOGGING_LEVEL_VERBOSE_NAME = "VERBOSE"logging.addLevelName(LOGGING_LEVEL_VERBOSE, LOGGING_LEVEL_VERBOSE_NAME)+LOGGING_LEVEL_OFF = 999class LogLevel(StrEnum):@@ -39,7 +40,7 @@ class LogLevel(StrEnum):case LogLevel.CRITICAL:return logging.CRITICALcase LogLevel.OFF:- return int(logging.CRITICAL) + 1+ return LOGGING_LEVEL_OFF@staticmethoddef from_int(logging_level: int) -> "LogLevel":@@ -47,7 +48,7 @@ class LogLevel(StrEnum):return LogLevel.VERBOSEelif logging_level == LOGGING_LEVEL_DEV:return LogLevel.DEV- elif logging_level > int(logging.CRITICAL):+ elif logging_level == LOGGING_LEVEL_OFF or logging_level > logging.CRITICAL:return LogLevel.OFFelse:return LogLevel(logging.getLevelName(logging_level))diff --git a/pyproject.toml b/pyproject.tomlindex e58e31eed..453aa7b13 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -35,7 +35,7 @@ dependencies = ["rich>=13.8.1","shortuuid>=1.0.13","toml>=0.10.2",- "typer>=0.12.5",+ "typer>=0.16.0","typing-extensions>=4.12.2","yattag>=1.15.2"]diff --git a/uv.lock b/uv.lockindex e48abf5a9..d7328815d 100644--- a/uv.lock+++ b/uv.lock@@ -834,7 +834,7 @@ wheels = [[[package]]name = "pipelex"-version = "0.2.8"+version = "0.2.9"source = { editable = "." }dependencies = [{ name = "aiofiles" },@@ -937,7 +937,7 @@ requires-dist = [{ name = "ruff", marker = "extra == 'dev'", specifier = ">=0.6.8" },{ name = "shortuuid", specifier = ">=1.0.13" },{ name = "toml", specifier = ">=0.10.2" },- { name = "typer", specifier = ">=0.12.5" },+ { name = "typer", specifier = ">=0.16.0" },{ name = "types-aioboto3", extras = ["bedrock", "bedrock-runtime"], marker = "extra == 'dev'", specifier = ">=13.4.0" },{ name = "types-aiofiles", marker = "extra == 'dev'", specifier = ">=24.1.0.20240626" },{ name = "types-beautifulsoup4", marker = "extra == 'dev'", specifier = ">=4.12.0.20240907" },@@ -1365,7 +1365,7 @@ wheels = [[[package]]name = "typer"-version = "0.15.4"+version = "0.16.0"source = { registry = "https://pypi.org/simple" }dependencies = [{ name = "click" },@@ -1373,9 +1373,9 @@ dependencies = [{ name = "shellingham" },{ name = "typing-extensions" },]-sdist = { url = "https://files.pythonhosted.org/packages/6c/89/c527e6c848739be8ceb5c44eb8208c52ea3515c6cf6406aa61932887bf58/typer-0.15.4.tar.gz", hash = "sha256:89507b104f9b6a0730354f27c39fae5b63ccd0c95b1ce1f1a6ba0cfd329997c3", size = 101559, upload-time = "2025-05-14T16:34:57.704Z" }+sdist = { url = "https://files.pythonhosted.org/packages/c5/8c/7d682431efca5fd290017663ea4588bf6f2c6aad085c7f108c5dbc316e70/typer-0.16.0.tar.gz", hash = "sha256:af377ffaee1dbe37ae9440cb4e8f11686ea5ce4e9bae01b84ae7c63b87f1dd3b", size = 102625, upload-time = "2025-05-26T14:30:31.824Z" }wheels = [- { url = "https://files.pythonhosted.org/packages/c9/62/d4ba7afe2096d5659ec3db8b15d8665bdcb92a3c6ff0b95e99895b335a9c/typer-0.15.4-py3-none-any.whl", hash = "sha256:eb0651654dcdea706780c466cf06d8f174405a659ffff8f163cfbfee98c0e173", size = 45258, upload-time = "2025-05-14T16:34:55.583Z" },+ { url = "https://files.pythonhosted.org/packages/76/42/3efaf858001d2c2913de7f354563e3a3a2f0decae3efe98427125a8f441e/typer-0.16.0-py3-none-any.whl", hash = "sha256:1f79bed11d4d02d4310e3c1b7ba594183bcedb0ac73b27a9e5f28f6fb5b98855", size = 46317, upload-time = "2025-05-26T14:30:30.523Z" },][[package]]