Skip to content

Build AI methods with your coding agent, then run them anywhere: as an MCP for chatbots, as a webapp for people, or via API for your software.

dev to release/v0.2.8 (#5)

Thomas Hebrard19e36e05f565
diff --git a/.env.example b/.env.example
index d4a212441..73218e151 100644
--- a/.env.example
+++ b/.env.example
@@ -9,14 +9,19 @@ AWS_REGION=
AZURE_OPENAI_ENDPOINT=
AZURE_OPENAI_API_VERSION=
AZURE_OPENAI_API_KEY=
-AZURE_OPENAI_API_KEY_METHOD=
-
-GCP_CREDENTIALS_FILE_PATH=
ANTHROPIC_API_KEY=
MISTRAL_API_KEY=
PERPLEXITY_API_KEY=
+PERPLEXITY_API_ENDPOINT=https://api.perplexity.ai
FAL_API_KEY=
+
+GCP_PROJECT_ID=
+GCP_REGION=
+GCP_CREDENTIALS_FILE_PATH=path/to/credentials.json
+
+CUSTOM_ENDPOINT_BASE_URL=
+CUSTOM_ENDPOINT_API_KEY=
\ No newline at end of file
diff --git a/.github/assets/demo_thumbnail.png b/.github/assets/demo_thumbnail.png
new file mode 100644
index 000000000..8b49e62cc
Binary files /dev/null and b/.github/assets/demo_thumbnail.png differ
diff --git a/.gitignore b/.gitignore
index 196c6b21f..49aa0da4f 100644
--- a/.gitignore
+++ b/.gitignore
@@ -48,3 +48,7 @@ pipelex_super.toml
# temporary files
temp/
+
+gcp_credentials.json
+
+/pipelex.toml
\ No newline at end of file
diff --git a/Makefile b/Makefile
index 517cae739..33aa17546 100644
--- a/Makefile
+++ b/Makefile
@@ -124,14 +124,14 @@ env: check-uv
fi
init: env
- $(call PRINT_TITLE,"Running `pipelex init` with overwrite")
- pipelex init --overwrite
+ $(call PRINT_TITLE,"Running `pipelex init`")
+ pipelex init
install: env
$(call PRINT_TITLE,"Installing dependencies")
@. $(VIRTUAL_ENV)/bin/activate && \
uv sync --all-extras && \
- pipelex init --overwrite && \
+ pipelex init && \
echo "Installed Pipelex dependencies in ${VIRTUAL_ENV} with all extras and initialized Pipelex";
lock: env
diff --git a/README.md b/README.md
index df9a84f0b..f349c0de0 100644
--- a/README.md
+++ b/README.md
@@ -5,7 +5,8 @@
<p align="center">Pipelex is an open‑source dev tool based on a simple declarative language<br/>that lets you define replicable, structured, composable LLM pipelines.</p>
<div>
- <a href="https://github.com/Pipelex/pipelex/blob/main/doc/Documentation.md"><strong>Docs</strong></a> -
+ <a href="https://www.pipelex.com/demo"><strong>Demo</strong></a> -
+ <a href="https://github.com/Pipelex/pipelex/blob/main/doc/Documentation.md"><strong>Documentation</strong></a> -
<a href="https://github.com/Pipelex/pipelex/issues"><strong>Report Bug</strong></a> -
<a href="https://github.com/Pipelex/pipelex/discussions"><strong>Feature Request</strong></a>
</div>
@@ -23,6 +24,10 @@
<br/>
</div>
+<div align="center">
+ <a href="https://www.pipelex.com/demo"><strong>Checkout our demo!</strong></a>
+</div>
+
# 📑 Table of Contents
- [Introduction](#introduction)
diff --git a/pipelex/__init__.py b/pipelex/__init__.py
index 191fc7df8..7c5ab0fec 100644
--- a/pipelex/__init__.py
+++ b/pipelex/__init__.py
@@ -5,3 +5,11 @@ __all__ = [
"log",
"pretty_print",
]
+
+# ------------------------------------------------------------
+# Keep <project>/pipelex_libraries on sys.path for every installer (Fix for uv)
+# ------------------------------------------------------------
+
+from ._bootstrap_user_libs import activate as _px_bootstrap_user_libs
+
+_px_bootstrap_user_libs()
diff --git a/pipelex/_bootstrap_user_libs.py b/pipelex/_bootstrap_user_libs.py
new file mode 100644
index 000000000..b782846da
--- /dev/null
+++ b/pipelex/_bootstrap_user_libs.py
@@ -0,0 +1,30 @@
+import sys
+from pathlib import Path
+
+# ------------------------------------------------------------------------
+# Public helper: run once, keep user libs import-able
+# ------------------------------------------------------------------------
+
+
+def activate() -> None:
+ # 1) Re-create Poetry’s behaviour: put <cwd> itself on sys.path
+ root = Path.cwd()
+ root_str = str(root)
+ if root_str not in sys.path:
+ sys.path.insert(0, root_str)
+
+ # 2) If a pipelex_libraries folder already exists, also touch __init__.py
+ # (helps IDE & static type-checkers) and ensure the *exact* folder is
+ # on sys.path. When it does **not** exist yet, nothing else to do —
+ # as soon as the user runs `pipelex init`, the directory will be
+ # created *inside* <root>, which is already import-able thanks to (1).
+ for parent in (root, *root.parents):
+ lib_dir = parent / "pipelex_libraries"
+ if lib_dir.is_dir():
+ # 1) make it a *real* package so editors & type-checkers see it
+ (lib_dir / "__init__.py").touch(exist_ok=True)
+ # 2) put it at the front of sys.path exactly once
+ lib_path = str(lib_dir)
+ if lib_path not in sys.path:
+ sys.path.insert(0, lib_path)
+ break
diff --git a/pipelex/cli/_cli.py b/pipelex/cli/_cli.py
index 262e69979..87ceb08bd 100644
--- a/pipelex/cli/_cli.py
+++ b/pipelex/cli/_cli.py
@@ -1,3 +1,5 @@
+import os
+import shutil
from typing import Annotated, Optional
import typer
@@ -6,6 +8,7 @@ from typer.core import TyperGroup
from typing_extensions import override
from pipelex import log, pretty_print
+from pipelex.exceptions import PipelexCLIError, PipelexConfigError
from pipelex.libraries.library_config import LibraryConfig
from pipelex.pipelex import Pipelex
from pipelex.tools.config.manager import config_manager
@@ -34,12 +37,21 @@ def init(
overwrite: Annotated[bool, typer.Option("--overwrite", "-o", help="Warning: If set, existing files will be overwritten.")] = False,
) -> None:
"""Initialize pipelex configuration in the current directory."""
- if overwrite:
- typer.echo("Overwriting existing pipelex library files.")
-
- # Duplicate pipelines and other libraries from the base library
LibraryConfig.export_libraries(overwrite=overwrite)
+ pipelex_init_path = os.path.join(config_manager.pipelex_root_dir, "pipelex_init.toml")
+ target_config_path = os.path.join(config_manager.local_root_dir, "pipelex.toml")
+
+ if os.path.exists(target_config_path) and not overwrite:
+ typer.echo("Warning: pipelex.toml already exists. Use --overwrite to force creation.")
+ return
+
+ try:
+ shutil.copy2(pipelex_init_path, target_config_path)
+ typer.echo(f"Created pipelex.toml at {target_config_path}")
+ except Exception as e:
+ raise PipelexCLIError(f"Failed to create pipelex.toml: {e}")
+
@app.command()
def run_setup() -> None:
@@ -56,8 +68,7 @@ def show_config() -> None:
final_config = config_manager.load_config()
pretty_print(final_config, title=f"Pipelex configuration for project: {config_manager.get_project_name()}")
except Exception as e:
- typer.echo(f"Error: {e}", err=True)
- raise typer.Exit(1)
+ raise PipelexConfigError(f"Error loading configuration: {e}")
def main() -> None:
diff --git a/pipelex/cogt/config_cogt.py b/pipelex/cogt/config_cogt.py
index b2b6754ee..d9002dd0e 100644
--- a/pipelex/cogt/config_cogt.py
+++ b/pipelex/cogt/config_cogt.py
@@ -10,7 +10,7 @@ from pipelex.cogt.plugin.anthropic.anthropic_config import AnthropicConfig
from pipelex.cogt.plugin.bedrock.bedrock_config import BedrockConfig
from pipelex.cogt.plugin.mistral.mistral_config import MistralConfig
from pipelex.cogt.plugin.openai.azure_openai_config import AzureOpenAIConfig
-from pipelex.cogt.plugin.openai.custom_openai_config import CustomOpenAIConfig
+from pipelex.cogt.plugin.openai.custom_endpoint_config import CustomEndpointConfig
from pipelex.cogt.plugin.openai.openai_config import OpenAIOpenAIConfig
from pipelex.cogt.plugin.openai.perplexity_config import PerplexityConfig
from pipelex.cogt.plugin.openai.vertexai_config import VertexAIConfig
@@ -55,7 +55,7 @@ class LLMConfig(ConfigModel):
mistral_config: MistralConfig
openai_openai_config: OpenAIOpenAIConfig
perplexity_config: PerplexityConfig
- custom_openai_config: CustomOpenAIConfig
+ custom_endpoint_config: CustomEndpointConfig
instructor_config: InstructorConfig
llm_job_config: LLMJobConfig
diff --git a/pipelex/cogt/llm/llm_models/llm_model_library.py b/pipelex/cogt/llm/llm_models/llm_model_library.py
index 9b77916e1..d0d15d42c 100644
--- a/pipelex/cogt/llm/llm_models/llm_model_library.py
+++ b/pipelex/cogt/llm/llm_models/llm_model_library.py
@@ -50,7 +50,7 @@ class LLMModelLibrary(LLMModelProviderAbstract, RootModel[LLMModelLibraryRoot]):
llm_model = LLMModel.model_validate(complete_llm_model_dict)
self.root.append(llm_model)
- log.info(f"Loaded {len(self.root)} llm models")
+ log.debug(f"Loaded {len(self.root)} llm models")
@override
def teardown(self):
diff --git a/pipelex/cogt/plugin/anthropic/anthropic_config.py b/pipelex/cogt/plugin/anthropic/anthropic_config.py
index e9882b81c..7c77361ac 100644
--- a/pipelex/cogt/plugin/anthropic/anthropic_config.py
+++ b/pipelex/cogt/plugin/anthropic/anthropic_config.py
@@ -19,24 +19,28 @@ class AnthropicKeyMethod(StrEnum):
ENV = "env"
+ANTHROPIC_API_KEY_VAR_NAME = "ANTHROPIC_API_KEY"
+
+
class AnthropicConfig(ConfigModel):
api_key_method: AnthropicKeyMethod = Field(strict=False)
- api_key_secret_name: str
- api_key_env_var_name: str
+
+ def configure(self, secrets_provider: SecretsProviderAbstract) -> str:
+ return self.get_api_key(secrets_provider=secrets_provider)
def get_api_key(self, secrets_provider: SecretsProviderAbstract) -> str:
match self.api_key_method:
case AnthropicKeyMethod.ENV:
log.verbose("Getting Anthropic API key from environment.")
try:
- key_from_env = get_required_env(self.api_key_env_var_name)
+ key_from_env = get_required_env(ANTHROPIC_API_KEY_VAR_NAME)
return key_from_env
except EnvVarNotFoundError as exc:
raise AnthropicCredentialsError(f"Error getting Anthropic API key from environment: {exc}") from exc
case AnthropicKeyMethod.SECRET_PROVIDER:
log.verbose("Getting Anthropic API key from secrets provider.")
try:
- key_from_service_provider = secrets_provider.get_secret(secret_id=self.api_key_secret_name)
+ key_from_service_provider = secrets_provider.get_secret(secret_id=ANTHROPIC_API_KEY_VAR_NAME)
except SecretNotFoundError as exc:
raise AnthropicCredentialsError("Error getting Anthropic API key from secrets provider.") from exc
return key_from_service_provider
diff --git a/pipelex/cogt/plugin/bedrock/bedrock_config.py b/pipelex/cogt/plugin/bedrock/bedrock_config.py
index 3a268d769..3eda2380c 100644
--- a/pipelex/cogt/plugin/bedrock/bedrock_config.py
+++ b/pipelex/cogt/plugin/bedrock/bedrock_config.py
@@ -2,7 +2,16 @@ from enum import StrEnum
from pydantic import Field
+from pipelex import log
+from pipelex.cogt.exceptions import CogtError
from pipelex.tools.config.models import ConfigModel
+from pipelex.tools.environment import EnvVarNotFoundError, get_required_env
+from pipelex.tools.secrets.secrets_errors import SecretNotFoundError
+from pipelex.tools.secrets.secrets_provider_abstract import SecretsProviderAbstract
+
+
+class BedrockCredentialsError(CogtError):
+ pass
class BedrockClientMethod(StrEnum):
@@ -10,6 +19,24 @@ class BedrockClientMethod(StrEnum):
AIBOTO3 = "aioboto3"
+AWS_REGION_VAR_NAME = "AWS_REGION"
+
+
class BedrockConfig(ConfigModel):
- aws_region: str
client_method: BedrockClientMethod = Field(strict=False)
+
+ def configure(self, secrets_provider: SecretsProviderAbstract) -> str:
+ """Configure and return AWS region."""
+ return self._get_region(secrets_provider=secrets_provider)
+
+ def _get_region(self, secrets_provider: SecretsProviderAbstract) -> str:
+ """Get AWS region from environment or secrets provider."""
+ log.debug("Getting AWS region.")
+ try:
+ return get_required_env(AWS_REGION_VAR_NAME)
+ except EnvVarNotFoundError:
+ log.verbose("AWS region not found in environment, trying secrets provider.")
+ try:
+ return secrets_provider.get_secret(secret_id=AWS_REGION_VAR_NAME)
+ except SecretNotFoundError as exc:
+ raise BedrockCredentialsError("Error getting AWS region from secrets provider.") from exc
diff --git a/pipelex/cogt/plugin/bedrock/bedrock_factory.py b/pipelex/cogt/plugin/bedrock/bedrock_factory.py
index 3b9ddef6d..a4800f5e2 100644
--- a/pipelex/cogt/plugin/bedrock/bedrock_factory.py
+++ b/pipelex/cogt/plugin/bedrock/bedrock_factory.py
@@ -6,6 +6,7 @@ from pipelex.cogt.plugin.bedrock.bedrock_client_protocol import BedrockClientPro
from pipelex.cogt.plugin.bedrock.bedrock_config import BedrockClientMethod
from pipelex.cogt.plugin.bedrock.bedrock_message import BedrockContentItem, BedrockImage, BedrockMessage, BedrockSource, ImageFormat
from pipelex.config import get_config
+from pipelex.hub import get_secrets_provider
class BedrockFactory:
@@ -16,7 +17,7 @@ class BedrockFactory:
@classmethod
def make_bedrock_client(cls) -> BedrockClientProtocol:
bedrock_config = get_config().cogt.llm_config.bedrock_config
- aws_region = bedrock_config.aws_region
+ aws_region = bedrock_config.configure(secrets_provider=get_secrets_provider())
client_method = bedrock_config.client_method
bedrock_async_client: BedrockClientProtocol
log.verbose(f"Using '{client_method}' for BedrockClient")
diff --git a/pipelex/cogt/plugin/mistral/mistral_config.py b/pipelex/cogt/plugin/mistral/mistral_config.py
index 84df9b29f..078ed7f45 100644
--- a/pipelex/cogt/plugin/mistral/mistral_config.py
+++ b/pipelex/cogt/plugin/mistral/mistral_config.py
@@ -1,5 +1,11 @@
+from enum import StrEnum
+
+from pydantic import Field
+
+from pipelex import log
from pipelex.cogt.exceptions import CogtError
from pipelex.tools.config.models import ConfigModel
+from pipelex.tools.environment import EnvVarNotFoundError, get_required_env
from pipelex.tools.secrets.secrets_errors import SecretNotFoundError
from pipelex.tools.secrets.secrets_provider_abstract import SecretsProviderAbstract
@@ -8,11 +14,27 @@ class MistralCredentialsError(CogtError):
pass
+class MistralKeyMethod(StrEnum):
+ SECRET_PROVIDER = "secret_provider"
+ ENV = "env"
+
+
class MistralConfig(ConfigModel):
- api_key_secret_name: str
+ api_key_method: MistralKeyMethod = Field(strict=False)
def api_key(self, secrets_provider: SecretsProviderAbstract) -> str:
- try:
- return secrets_provider.get_secret(secret_id=self.api_key_secret_name)
- except SecretNotFoundError as exc:
- raise MistralCredentialsError("Error getting Mistral API key from secrets provider.") from exc
+ match self.api_key_method:
+ case MistralKeyMethod.ENV:
+ log.debug("Using Mistral API key from environment.")
+ try:
+ key_from_env = get_required_env("MISTRAL_API_KEY")
+ return key_from_env
+ except EnvVarNotFoundError as exc:
+ raise MistralCredentialsError(f"Error getting Mistral API key from environment: {exc}") from exc
+ case MistralKeyMethod.SECRET_PROVIDER:
+ log.verbose("Using Mistral API key from secrets provider.")
+ try:
+ key_from_secrets_provider = secrets_provider.get_secret(secret_id="MISTRAL_API_KEY")
+ return key_from_secrets_provider
+ except SecretNotFoundError as exc:
+ raise MistralCredentialsError("Error getting Mistral API key from secrets provider.") from exc
diff --git a/pipelex/cogt/plugin/openai/azure_openai_config.py b/pipelex/cogt/plugin/openai/azure_openai_config.py
index 74f5f0516..78585ce59 100644
--- a/pipelex/cogt/plugin/openai/azure_openai_config.py
+++ b/pipelex/cogt/plugin/openai/azure_openai_config.py
@@ -1,4 +1,5 @@
from enum import StrEnum
+from typing import Tuple
from pydantic import Field
@@ -19,19 +20,60 @@ class AzureOpenAIKeyMethod(StrEnum):
ENV = "env"
+AZURE_OPENAI_API_KEY_VAR_NAME = "AZURE_OPENAI_API_KEY"
+AZURE_OPENAI_API_ENDPOINT_VAR_NAME = "AZURE_OPENAI_API_ENDPOINT"
+AZURE_OPENAI_API_VERSION_VAR_NAME = "AZURE_OPENAI_API_VERSION"
+
+
class AzureOpenAIConfig(ConfigModel):
- api_endpoint: str
- api_version: str
api_key_method: AzureOpenAIKeyMethod = Field(strict=False)
- api_key_env_var_name: str
- api_key_secret_name: str
- def get_api_key(self, secrets_provider: SecretsProviderAbstract) -> str:
+ def configure(self, secrets_provider: SecretsProviderAbstract) -> Tuple[str, str, str]:
+ """Configure and return endpoint, version, and API key."""
+ api_endpoint = self._get_api_endpoint(secrets_provider=secrets_provider)
+ api_version = self._get_api_version(secrets_provider=secrets_provider)
+ api_key = self._get_api_key(secrets_provider=secrets_provider)
+ return api_endpoint, api_version, api_key
+
+ def _get_api_endpoint(self, secrets_provider: SecretsProviderAbstract) -> str:
+ """Get API endpoint from environment or secrets provider."""
+ match self.api_key_method:
+ case AzureOpenAIKeyMethod.ENV:
+ log.debug("Getting Azure OpenAI API endpoint from environment.")
+ try:
+ return get_required_env(AZURE_OPENAI_API_ENDPOINT_VAR_NAME)
+ except EnvVarNotFoundError as exc:
+ raise AzureOpenAICredentialsError(f"Error getting Azure OpenAI API endpoint from environment: {exc}") from exc
+ case AzureOpenAIKeyMethod.SECRET_PROVIDER:
+ log.verbose("Getting Azure OpenAI API endpoint from secrets provider.")
+ try:
+ return secrets_provider.get_secret(secret_id=AZURE_OPENAI_API_ENDPOINT_VAR_NAME)
+ except SecretNotFoundError as exc:
+ raise AzureOpenAICredentialsError("Error getting Azure OpenAI API endpoint from secrets provider.") from exc
+
+ def _get_api_version(self, secrets_provider: SecretsProviderAbstract) -> str:
+ """Get API version from environment or secrets provider."""
+ match self.api_key_method:
+ case AzureOpenAIKeyMethod.ENV:
+ log.debug("Getting Azure OpenAI API version from environment.")
+ try:
+ return get_required_env(AZURE_OPENAI_API_VERSION_VAR_NAME)
+ except EnvVarNotFoundError as exc:
+ raise AzureOpenAICredentialsError(f"Error getting Azure OpenAI API version from environment: {exc}") from exc
+ case AzureOpenAIKeyMethod.SECRET_PROVIDER:
+ log.verbose("Getting Azure OpenAI API version from secrets provider.")
+ try:
+ return secrets_provider.get_secret(secret_id=AZURE_OPENAI_API_VERSION_VAR_NAME)
+ except SecretNotFoundError as exc:
+ raise AzureOpenAICredentialsError("Error getting Azure OpenAI API version from secrets provider.") from exc
+
+ def _get_api_key(self, secrets_provider: SecretsProviderAbstract) -> str:
+ """Get API key from environment or secrets provider."""
match self.api_key_method:
case AzureOpenAIKeyMethod.ENV:
log.debug("Using Azure OpenAI API key from environment.")
try:
- key_from_env = get_required_env(self.api_key_env_var_name)
+ key_from_env = get_required_env(AZURE_OPENAI_API_KEY_VAR_NAME)
return key_from_env
except EnvVarNotFoundError as exc:
raise AzureOpenAICredentialsError(f"Error getting Azure OpenAI API key from environment: {exc}") from exc
@@ -39,7 +81,7 @@ class AzureOpenAIConfig(ConfigModel):
log.verbose("Using Azure OpenAI API key from secrets provider.")
# TODO: make it automatically select the right key for the right azure resource
try:
- key_from_secrets_provider = secrets_provider.get_secret(secret_id=self.api_key_secret_name)
+ key_from_secrets_provider = secrets_provider.get_secret(secret_id=AZURE_OPENAI_API_KEY_VAR_NAME)
except SecretNotFoundError as exc:
raise AzureOpenAICredentialsError("Error getting Azure OpenAI API key from secrets provider.") from exc
return key_from_secrets_provider
diff --git a/pipelex/cogt/plugin/openai/custom_endpoint_config.py b/pipelex/cogt/plugin/openai/custom_endpoint_config.py
new file mode 100644
index 000000000..bbdff710c
--- /dev/null
+++ b/pipelex/cogt/plugin/openai/custom_endpoint_config.py
@@ -0,0 +1,74 @@
+from enum import StrEnum
+from typing import Optional, Tuple
+
+from pydantic import Field
+
+from pipelex import log
+from pipelex.cogt.exceptions import CogtError
+from pipelex.tools.config.models import ConfigModel
+from pipelex.tools.environment import EnvVarNotFoundError, get_required_env
+from pipelex.tools.secrets.secrets_errors import SecretNotFoundError
+from pipelex.tools.secrets.secrets_provider_abstract import SecretsProviderAbstract
+
+
+class CustomEndpointCredentialsError(CogtError):
+ pass
+
+
+class OpenAIKeyMethod(StrEnum):
+ SECRET_PROVIDER = "secret_provider"
+ ENV = "env"
+
+
+CUSTOM_ENDPOINT_API_KEY_VAR_NAME = "CUSTOM_ENDPOINT_API_KEY"
+CUSTOM_ENDPOINT_BASE_URL_VAR_NAME = "CUSTOM_ENDPOINT_BASE_URL"
+
+
+class CustomEndpointConfig(ConfigModel):
+ """Configuration for custom OpenAI-compatible endpoints (e.g., Ollama, LM Studio, etc.)"""
+
+ api_key_method: OpenAIKeyMethod = Field(strict=False)
+
+ def configure(self, secrets_provider: Optional[SecretsProviderAbstract] = None) -> Tuple[str, Optional[str]]:
+ """Configure and return base URL and optional API key."""
+ base_url = self._get_base_url(secrets_provider=secrets_provider)
+ api_key = self._get_api_key(secrets_provider=secrets_provider)
+ return base_url, api_key
+
+ def _get_base_url(self, secrets_provider: Optional[SecretsProviderAbstract]) -> str:
+ """Get base URL from environment or secrets provider."""
+ match self.api_key_method:
+ case OpenAIKeyMethod.ENV:
+ log.debug("Getting Custom OpenAI base URL from environment.")
+ try:
+ return get_required_env(CUSTOM_ENDPOINT_BASE_URL_VAR_NAME)
+ except EnvVarNotFoundError as exc:
+ raise CustomEndpointCredentialsError(f"Error getting Custom OpenAI base URL from environment: {exc}") from exc
+ case OpenAIKeyMethod.SECRET_PROVIDER:
+ if not secrets_provider:
+ raise CustomEndpointCredentialsError("Secrets provider is required when using SECRET_PROVIDER method")
+ log.verbose("Getting Custom OpenAI base URL from secrets provider.")
+ try:
+ return secrets_provider.get_secret(secret_id=CUSTOM_ENDPOINT_BASE_URL_VAR_NAME)
+ except SecretNotFoundError as exc:
+ raise CustomEndpointCredentialsError("Error getting Custom OpenAI base URL from secrets provider.") from exc
+
+ def _get_api_key(self, secrets_provider: Optional[SecretsProviderAbstract] = None) -> Optional[str]:
+ """Get API key if configured, otherwise return None for services that don't require authentication"""
+ match self.api_key_method:
+ case OpenAIKeyMethod.ENV:
+ log.debug("Using Custom OpenAI API key from environment.")
+ try:
+ return get_required_env(CUSTOM_ENDPOINT_API_KEY_VAR_NAME)
+ except EnvVarNotFoundError:
+ log.debug("No Custom OpenAI API key found in environment - using None.")
+ return None
+ case OpenAIKeyMethod.SECRET_PROVIDER:
+ if not secrets_provider:
+ raise CustomEndpointCredentialsError("Secrets provider is required when using SECRET_PROVIDER method")
+ log.verbose("Using Custom OpenAI API key from secrets provider.")
+ try:
+ return secrets_provider.get_secret(secret_id=CUSTOM_ENDPOINT_API_KEY_VAR_NAME)
+ except SecretNotFoundError:
+ log.debug("No Custom OpenAI API key found in secrets provider - using None.")
+ return None
diff --git a/pipelex/cogt/plugin/openai/custom_openai_config.py b/pipelex/cogt/plugin/openai/custom_openai_config.py
deleted file mode 100644
index 7f2c4393e..000000000
--- a/pipelex/cogt/plugin/openai/custom_openai_config.py
+++ /dev/null
@@ -1,52 +0,0 @@
-from typing import Optional
-
-from pipelex import log
-from pipelex.cogt.exceptions import CogtError
-from pipelex.tools.config.models import ConfigModel
-from pipelex.tools.environment import EnvVarNotFoundError, get_required_env
-from pipelex.tools.secrets.secrets_errors import SecretNotFoundError
-from pipelex.tools.secrets.secrets_provider_abstract import SecretsProviderAbstract
-
-
-class CustomOpenAICredentialsError(CogtError):
- pass
-
-
-class CustomOpenAIConfig(ConfigModel):
- """Configuration for custom OpenAI-compatible endpoints (e.g., Ollama, LM Studio, etc.)"""
-
- base_url: str
- api_key: Optional[str] = None
- # Option to load API key from environment or secrets if needed
- api_key_env_var_name: Optional[str] = None
- api_key_secret_name: Optional[str] = None
-
- def get_api_key(self, secrets_provider: Optional[SecretsProviderAbstract] = None) -> Optional[str]:
- """Get API key if configured, otherwise return None for services that don't require authentication"""
-
- # If api_key is directly set, use it
- if self.api_key:
- log.debug("Using Custom OpenAI API key from config.")
- return self.api_key
-
- # Try to get from environment variable if configured
- if self.api_key_env_var_name:
- log.debug("Using Custom OpenAI API key from environment.")
- try:
- key_from_env = get_required_env(self.api_key_env_var_name)
- return key_from_env
- except EnvVarNotFoundError as exc:
- raise CustomOpenAICredentialsError(f"Error getting Custom OpenAI API key from environment: {exc}") from exc
-
- # Try to get from secrets provider if configured
- if self.api_key_secret_name and secrets_provider:
- log.verbose("Using Custom OpenAI API key from secrets provider.")
- try:
- key_from_secrets_provider = secrets_provider.get_secret(secret_id=self.api_key_secret_name)
- return key_from_secrets_provider
- except SecretNotFoundError as exc:
- raise CustomOpenAICredentialsError("Error getting Custom OpenAI API key from secrets provider.") from exc
-
- # No API key configured - return None (for services that don't require auth)
- log.debug("No Custom OpenAI API key configured - using None.")
- return None
diff --git a/pipelex/cogt/plugin/openai/openai_config.py b/pipelex/cogt/plugin/openai/openai_config.py
index cff8ece50..443508dc9 100644
--- a/pipelex/cogt/plugin/openai/openai_config.py
+++ b/pipelex/cogt/plugin/openai/openai_config.py
@@ -19,24 +19,25 @@ class OpenAIKeyMethod(StrEnum):
ENV = "env"
+OPENAI_API_KEY_VAR_NAME = "OPENAI_API_KEY"
+
+
class OpenAIOpenAIConfig(ConfigModel):
api_key_method: OpenAIKeyMethod = Field(strict=False)
- api_key_secret_name: str
- api_key_env_var_name: str
def get_api_key(self, secrets_provider: SecretsProviderAbstract) -> str:
match self.api_key_method:
case OpenAIKeyMethod.ENV:
log.debug("Using OpenAI API key from environment.")
try:
- key_from_env = get_required_env(self.api_key_env_var_name)
+ key_from_env = get_required_env(OPENAI_API_KEY_VAR_NAME)
return key_from_env
except EnvVarNotFoundError as exc:
raise OpenAICredentialsError(f"Error getting OpenAI API key from environment: {exc}") from exc
case OpenAIKeyMethod.SECRET_PROVIDER:
log.verbose("Using OpenAI API key from secrets provider.")
try:
- key_from_service_provider = secrets_provider.get_secret(secret_id=self.api_key_secret_name)
+ key_from_service_provider = secrets_provider.get_secret(secret_id=OPENAI_API_KEY_VAR_NAME)
except SecretNotFoundError as exc:
raise OpenAICredentialsError("Error getting OpenAI API key from secrets provider.") from exc
return key_from_service_provider
diff --git a/pipelex/cogt/plugin/openai/openai_factory.py b/pipelex/cogt/plugin/openai/openai_factory.py
index 8c56ef004..b602b6b00 100644
--- a/pipelex/cogt/plugin/openai/openai_factory.py
+++ b/pipelex/cogt/plugin/openai/openai_factory.py
@@ -36,20 +36,17 @@ class OpenAIFactory:
match llm_platform:
case LLMPlatform.AZURE_OPENAI:
azure_openai_config = get_config().cogt.llm_config.azure_openai_config
- api_key = azure_openai_config.get_api_key(secrets_provider=get_secrets_provider())
- azure_endpoint = azure_openai_config.api_endpoint
- api_version = azure_openai_config.api_version
+ endpoint, api_version, api_key = azure_openai_config.configure(secrets_provider=get_secrets_provider())
- log.verbose(f"Making AsyncAzureOpenAI client with endpoint: {azure_endpoint}, api_version: {api_version}")
+ log.verbose(f"Making AsyncAzureOpenAI client with endpoint: {endpoint}, api_version: {api_version}")
the_client = openai.AsyncAzureOpenAI(
- azure_endpoint=azure_endpoint,
+ azure_endpoint=endpoint,
api_key=api_key,
api_version=api_version,
)
case LLMPlatform.PERPLEXITY:
perplexity_config = get_config().cogt.llm_config.perplexity_config
- api_key = perplexity_config.get_api_key(secrets_provider=get_secrets_provider())
- endpoint = perplexity_config.api_endpoint
+ endpoint, api_key = perplexity_config.configure(secrets_provider=get_secrets_provider())
log.verbose(f"Making perplexity AsyncOpenAI client with endpoint: {endpoint}")
the_client = openai.AsyncOpenAI(
@@ -62,10 +59,7 @@ class OpenAIFactory:
the_client = openai.AsyncOpenAI(api_key=api_key)
case LLMPlatform.VERTEXAI_OPENAI:
vertexai_config = get_config().cogt.llm_config.vertexai_config
- project_id = vertexai_config.project_id
- region = vertexai_config.region
- api_key = vertexai_config.get_api_key()
- endpoint = f"https://{region}-aiplatform.googleapis.com/v1beta1/projects/{project_id}/locations/{region}/endpoints/openapi"
+ endpoint, api_key = vertexai_config.configure(secrets_provider=get_secrets_provider())
log.verbose(f"Making vertex AsyncOpenAI client with endpoint: {endpoint}")
the_client = openai.AsyncOpenAI(
@@ -73,9 +67,8 @@ class OpenAIFactory:
base_url=endpoint,
)
case LLMPlatform.CUSTOM_OPENAI:
- custom_openai_config = get_config().cogt.llm_config.custom_openai_config
- api_key = custom_openai_config.get_api_key(secrets_provider=get_secrets_provider())
- base_url = custom_openai_config.base_url
+ custom_endpoint_config = get_config().cogt.llm_config.custom_endpoint_config
+ base_url, api_key = custom_endpoint_config.configure(secrets_provider=get_secrets_provider())
log.verbose(f"Making custom AsyncOpenAI client with base_url: {base_url}")
the_client = openai.AsyncOpenAI(
diff --git a/pipelex/cogt/plugin/openai/perplexity_config.py b/pipelex/cogt/plugin/openai/perplexity_config.py
index b52430afc..49aeba20c 100644
--- a/pipelex/cogt/plugin/openai/perplexity_config.py
+++ b/pipelex/cogt/plugin/openai/perplexity_config.py
@@ -1,4 +1,5 @@
from enum import StrEnum
+from typing import Tuple
from pydantic import Field
@@ -19,25 +20,49 @@ class PerplexityKeyMethod(StrEnum):
ENV = "env"
+PERPLEXITY_API_KEY_VAR_NAME = "PERPLEXITY_API_KEY"
+PERPLEXITY_API_ENDPOINT_VAR_NAME = "PERPLEXITY_API_ENDPOINT"
+
+
class PerplexityConfig(ConfigModel):
- api_endpoint: str
api_key_method: PerplexityKeyMethod = Field(strict=False)
- api_key_env_var_name: str
- api_key_secret_name: str
- def get_api_key(self, secrets_provider: SecretsProviderAbstract) -> str:
+ def configure(self, secrets_provider: SecretsProviderAbstract) -> Tuple[str, str]:
+ """Configure and return endpoint and API key."""
+ api_endpoint = self._get_api_endpoint(secrets_provider=secrets_provider)
+ api_key = self._get_api_key(secrets_provider=secrets_provider)
+ return api_endpoint, api_key
+
+ def _get_api_endpoint(self, secrets_provider: SecretsProviderAbstract) -> str:
+ """Get API endpoint from environment or secrets provider."""
+ match self.api_key_method:
+ case PerplexityKeyMethod.ENV:
+ log.debug("Getting Perplexity API endpoint from environment.")
+ try:
+ return get_required_env(PERPLEXITY_API_ENDPOINT_VAR_NAME)
+ except EnvVarNotFoundError as exc:
+ raise PerplexityCredentialsError(f"Error getting Perplexity API endpoint from environment: {exc}") from exc
+ case PerplexityKeyMethod.SECRET_PROVIDER:
+ log.verbose("Getting Perplexity API endpoint from secrets provider.")
+ try:
+ return secrets_provider.get_secret(secret_id=PERPLEXITY_API_ENDPOINT_VAR_NAME)
+ except SecretNotFoundError as exc:
+ raise PerplexityCredentialsError("Error getting Perplexity API endpoint from secrets provider.") from exc
+
+ def _get_api_key(self, secrets_provider: SecretsProviderAbstract) -> str:
+ """Get API key from environment or secrets provider."""
match self.api_key_method:
case PerplexityKeyMethod.ENV:
log.debug("Using Perplexity API key from environment.")
try:
- key_from_env = get_required_env(self.api_key_env_var_name)
+ key_from_env = get_required_env(PERPLEXITY_API_KEY_VAR_NAME)
return key_from_env
except EnvVarNotFoundError as exc:
- raise PerplexityCredentialsError(f"Error getting Azure OpenAI API key from environment: {exc}") from exc
+ raise PerplexityCredentialsError(f"Error getting Perplexity API key from environment: {exc}") from exc
case PerplexityKeyMethod.SECRET_PROVIDER:
log.verbose("Using Perplexity API key from secrets provider.")
try:
- key_from_secrets_provider = secrets_provider.get_secret(secret_id=self.api_key_secret_name)
+ key_from_secrets_provider = secrets_provider.get_secret(secret_id=PERPLEXITY_API_KEY_VAR_NAME)
except SecretNotFoundError as exc:
raise PerplexityCredentialsError("Error getting Perplexity API key from secrets provider.") from exc
return key_from_secrets_provider
diff --git a/pipelex/cogt/plugin/openai/vertexai_config.py b/pipelex/cogt/plugin/openai/vertexai_config.py
index 061e8315e..3b8d01bde 100644
--- a/pipelex/cogt/plugin/openai/vertexai_config.py
+++ b/pipelex/cogt/plugin/openai/vertexai_config.py
@@ -1,10 +1,15 @@
from enum import StrEnum
-from typing import Any, Dict
+from typing import Any, Dict, Tuple
+from pydantic import Field
+
+from pipelex import log
from pipelex.cogt.exceptions import CogtError, MissingDependencyError
from pipelex.tools.config.models import ConfigModel
from pipelex.tools.environment import EnvVarNotFoundError, get_required_env
from pipelex.tools.misc.json_utils import load_json_dict_from_path
+from pipelex.tools.secrets.secrets_errors import SecretNotFoundError
+from pipelex.tools.secrets.secrets_provider_abstract import SecretsProviderAbstract
class VertexAICredentialsError(CogtError):
@@ -16,11 +21,43 @@ class VertexAIKeyMethod(StrEnum):
ENV = "env"
+GCP_PROJECT_ID_VAR_NAME = "GCP_PROJECT_ID"
+GCP_REGION_VAR_NAME = "GCP_REGION"
+GCP_CREDENTIALS_FILE_PATH_VAR_NAME = "GCP_CREDENTIALS_FILE_PATH"
+
+
class VertexAIConfig(ConfigModel):
- project_id: str
- region: str
+ api_key_method: VertexAIKeyMethod = Field(strict=False)
- def get_api_key(self) -> str:
+ def configure(self, secrets_provider: SecretsProviderAbstract) -> Tuple[str, str]:
+ """Configure and return project ID, region, and API key."""
+ project_id, region = self._get_project_settings(secrets_provider=secrets_provider)
+ api_key = self._get_credentials()
+ endpoint = f"https://{region}-aiplatform.googleapis.com/v1beta1/projects/{project_id}/locations/{region}/endpoints/openapi"
+ return endpoint, api_key
+
+ def _get_project_settings(self, secrets_provider: SecretsProviderAbstract) -> Tuple[str, str]:
+ """Get project ID and region from environment or secrets provider."""
+ match self.api_key_method:
+ case VertexAIKeyMethod.ENV:
+ log.debug("Getting GCP project settings from environment.")
+ try:
+ project_id = get_required_env(GCP_PROJECT_ID_VAR_NAME)
+ region = get_required_env(GCP_REGION_VAR_NAME)
+ return project_id, region
+ except EnvVarNotFoundError as exc:
+ raise VertexAICredentialsError(f"Error getting GCP project settings from environment: {exc}") from exc
+ case VertexAIKeyMethod.SECRET_PROVIDER:
+ log.verbose("Getting GCP project settings from secrets provider.")
+ try:
+ project_id = secrets_provider.get_secret(secret_id=GCP_PROJECT_ID_VAR_NAME)
+ region = secrets_provider.get_secret(secret_id=GCP_REGION_VAR_NAME)
+ return project_id, region
+ except SecretNotFoundError as exc:
+ raise VertexAICredentialsError("Error getting GCP project settings from secrets provider.") from exc
+
+ def _get_credentials(self) -> str:
+ """Get GCP credentials and return access token."""
try:
from google.auth.transport.requests import Request
from google.oauth2.service_account import Credentials
@@ -33,10 +70,19 @@ class VertexAIConfig(ConfigModel):
(such as Azure OpenAI, OpenAI, anthropic or bedrock).",
) from exc
- try:
- gcp_credentials_file_path = get_required_env("GCP_CREDENTIALS_FILE_PATH")
- except EnvVarNotFoundError as exc:
- raise VertexAICredentialsError("GCP_CREDENTIALS_FILE_PATH environment variable not found") from exc
+ match self.api_key_method:
+ case VertexAIKeyMethod.ENV:
+ log.debug("Getting GCP credentials from environment.")
+ try:
+ gcp_credentials_file_path = get_required_env(GCP_CREDENTIALS_FILE_PATH_VAR_NAME)
+ except EnvVarNotFoundError as exc:
+ raise VertexAICredentialsError(f"{GCP_CREDENTIALS_FILE_PATH_VAR_NAME} environment variable not found") from exc
+ case VertexAIKeyMethod.SECRET_PROVIDER:
+ log.verbose("Getting GCP credentials from secrets provider.")
+ try:
+ gcp_credentials_file_path = get_required_env(GCP_CREDENTIALS_FILE_PATH_VAR_NAME)
+ except EnvVarNotFoundError as exc:
+ raise VertexAICredentialsError(f"{GCP_CREDENTIALS_FILE_PATH_VAR_NAME} environment variable not found") from exc
try:
credentials_dict: Dict[str, Any] = load_json_dict_from_path(path=gcp_credentials_file_path)
diff --git a/pipelex/core/working_memory.py b/pipelex/core/working_memory.py
index f1a8467cf..a3ab9cafe 100644
--- a/pipelex/core/working_memory.py
+++ b/pipelex/core/working_memory.py
@@ -274,6 +274,10 @@ class WorkingMemory(BaseModel):
"""
return self.get_stuff(name=name).as_list_of_fixed_content_type(item_type=item_type)
+ def get_list_stuff_first_item_as(self, name: str, item_type: Type[StuffContentType]) -> StuffContentType:
+ """Get stuff content as ListContent with items of type StuffContentType then return the first item."""
+ return self.get_stuff_as_list(name=name, item_type=item_type).items[0]
+
def get_stuff_as_text(self, name: str) -> TextContent:
"""Get stuff content as TextContent if applicable."""
return self.get_stuff(name=name).as_text
@@ -317,6 +321,10 @@ class WorkingMemory(BaseModel):
"""
return self.get_stuff_as_list(name=MAIN_STUFF_NAME, item_type=item_type)
+ def main_list_stuff_first_item_as(self, item_type: Type[StuffContentType]) -> StuffContentType:
+ """Get main stuff content as first list item of type StuffContentType."""
+ return self.get_list_stuff_first_item_as(name=MAIN_STUFF_NAME, item_type=item_type)
+
@property
def main_stuff_as_text(self) -> TextContent:
"""Get main stuff content as TextContent if applicable."""
diff --git a/pipelex/exceptions.py b/pipelex/exceptions.py
index 3b7dae1f7..be6b4ec3f 100644
--- a/pipelex/exceptions.py
+++ b/pipelex/exceptions.py
@@ -1,3 +1,5 @@
+from click import ClickException
+
from pipelex.tools.exceptions import RootException
@@ -5,6 +7,12 @@ class PipelexError(RootException):
pass
+class PipelexCLIError(PipelexError, ClickException):
+ """Raised when there's an error in CLI usage or operation."""
+
+ pass
+
+
class PipelexConfigError(PipelexError):
pass
diff --git a/pipelex/pipelex.py b/pipelex/pipelex.py
index 90fbb7ed7..f5c3ed47f 100644
--- a/pipelex/pipelex.py
+++ b/pipelex/pipelex.py
@@ -82,7 +82,6 @@ class Pipelex:
mission_tracker: Optional[MissionTracker] = None,
activity_manager: Optional[ActivityManagerProtocol] = None,
) -> None:
- print(f"{PACKAGE_NAME} version {PACKAGE_VERSION} init started...", flush=True)
self.pipelex_hub = pipelex_hub or PipelexHub()
set_pipelex_hub(self.pipelex_hub)
@@ -105,7 +104,7 @@ class Pipelex:
project_name=get_config().project_name or "unknown_project",
log_config=get_config().pipelex.log_config,
)
- log.info("Logs are configured")
+ log.debug("Logs are configured")
# tools
self.template_provider = template_provider or TemplateLibrary()
@@ -150,7 +149,7 @@ class Pipelex:
self.pipelex_hub.set_activity_manager(activity_manager=self.activity_manager)
Pipelex._pipelex_instance = self
- log.info(f"{PACKAGE_NAME} version {PACKAGE_VERSION} init done")
+ log.debug(f"{PACKAGE_NAME} version {PACKAGE_VERSION} init done")
def setup(
self,
@@ -233,5 +232,5 @@ class Pipelex:
pipelex_instance = cls()
pipelex_instance.setup(structure_classes=structure_classes)
pipelex_instance.finish_setup()
- log.info(f"Boot sequence completed for {get_config().project_name} and session id {get_config().session_id}")
+ log.info(f"Pipelex {PACKAGE_VERSION} initialized.")
return pipelex_instance
diff --git a/pipelex/pipelex.toml b/pipelex/pipelex.toml
index 46a79073c..677d357f2 100644
--- a/pipelex/pipelex.toml
+++ b/pipelex/pipelex.toml
@@ -45,10 +45,6 @@ instructor = "INFO"
urllib3-connectionpool = "INFO"
urllib3-util-retry = "INFO"
-kajson = "INFO"
-kajson-decoder-sandbox = "INFO"
-kajson-encoder-sandbox = "INFO"
-
pipelex = "INFO"
@@ -65,14 +61,7 @@ tracebacks_suppress = []
keywords_to_hilight = []
[pipelex.aws_config]
-api_key_method = "env" # "env", "secret_provider"
-aws_region = "needs_to_be_set"
-
-aws_access_key_id_env_var_name = "AWS_ACCESS_KEY_ID"
-aws_secret_access_key_env_var_name = "AWS_SECRET_ACCESS_KEY"
-aws_region_env_var_name = "AWS_REGION"
-aws_access_key_id_secret_name = "AWS_ACCESS_KEY_ID"
-aws_secret_access_key_secret_name = "AWS_SECRET_ACCESS_KEY"
+api_key_method = "env"
[cogt]
@@ -108,43 +97,30 @@ is_streaming_enabled = false
# "gpt-4o-mini" = "openai"
[cogt.llm_config.openai_openai_config]
-api_key_method = "env" # "env", "secret_provider"
-api_key_env_var_name = "OPENAI_API_KEY"
-api_key_secret_name = "OPENAI_API_KEY"
+api_key_method = "env"
[cogt.llm_config.azure_openai_config]
-api_endpoint = "needs_to_be_set"
-api_version = "2024-05-01-preview"
-api_key_method = "env" # "env", "secret_provider"
-api_key_env_var_name = "AZURE_OPENAI_API_KEY"
-api_key_secret_name = "AZURE_OPENAI_API_KEY"
+api_key_method = "env"
# TODO: handle multiple azure openai accounts with different resource groups and account names for various llm model
[cogt.llm_config.perplexity_config]
-api_endpoint = "https://api.perplexity.ai"
-api_key_method = "env" # "env", "secret_provider"
-api_key_env_var_name = "PERPLEXITY_API_KEY"
-api_key_secret_name = "PERPLEXITY_API_KEY"
+api_key_method = "env"
[cogt.llm_config.vertexai_config]
-project_id = "needs_to_be_set"
-region = "needs_to_be_set"
+api_key_method = "env"
[cogt.llm_config.mistral_config]
-api_key_secret_name = "MISTRAL_API_KEY"
+api_key_method = "env"
[cogt.llm_config.bedrock_config]
client_method = "aioboto3"
-aws_region = "needs_to_be_set"
[cogt.llm_config.anthropic_config]
-api_key_method = "env" # "env", "secret_provider"
-api_key_env_var_name = "ANTHROPIC_API_KEY"
-api_key_secret_name = "ANTHROPIC_API_KEY"
+api_key_method = "env"
-[cogt.llm_config.custom_openai_config]
-base_url = "http://localhost:11434/v1"
+[cogt.llm_config.custom_endpoint_config]
+api_key_method = "env"
####################################################################################################
# Image generation config
diff --git a/pipelex/pipelex_init.toml b/pipelex/pipelex_init.toml
new file mode 100644
index 000000000..0db9a3045
--- /dev/null
+++ b/pipelex/pipelex_init.toml
@@ -0,0 +1,39 @@
+[pipelex.aws_config]
+api_key_method = "env"
+# The possible values are "env" and "secret_provider".
+# "env" means means that the env var are stored in your .env file.
+# "secret_provider" means that the env var are stored in your Secret Manager (See the doc for injecting a secret provider).
+
+[cogt]
+[cogt.llm_config.preferred_platforms]
+# This overrrides the defaults set for any llm handle
+"gpt-4o-mini" = "openai"
+
+[cogt.llm_config.openai_openai_config]
+api_key_method = "env"
+
+[cogt.llm_config.azure_openai_config]
+api_key_method = "env"
+
+[cogt.llm_config.perplexity_config]
+api_key_method = "env"
+
+[cogt.llm_config.vertexai_config]
+api_key_method = "env"
+
+[cogt.llm_config.mistral_config]
+api_key_method = "env"
+
+[cogt.llm_config.anthropic_config]
+api_key_method = "env"
+
+[cogt.llm_config.bedrock_config]
+client_method = "aioboto3"
+
+####################################################################################################
+# OCR config
+####################################################################################################
+
+[cogt.ocr_config]
+ocr_handles = ["mistral/mistral-ocr-latest"]
+page_output_text_file_name = "page_text.md"
diff --git a/pipelex/tools/aws/aws_config.py b/pipelex/tools/aws/aws_config.py
index da99bd73c..d2a2b0304 100644
--- a/pipelex/tools/aws/aws_config.py
+++ b/pipelex/tools/aws/aws_config.py
@@ -6,7 +6,7 @@ from pydantic import Field
from pipelex import log
from pipelex.hub import get_secret
from pipelex.tools.config.models import ConfigModel
-from pipelex.tools.environment import EnvVarNotFoundError, get_optional_env, get_required_env
+from pipelex.tools.environment import EnvVarNotFoundError, get_required_env
from pipelex.tools.exceptions import ToolException
from pipelex.tools.secrets.secrets_errors import SecretNotFoundError
@@ -20,42 +20,37 @@ class AwsKeyMethod(StrEnum):
ENV = "env"
-class AwsConfig(ConfigModel):
- api_key_method: AwsKeyMethod = Field(strict=False)
- aws_region: str
+AWS_ACCESS_KEY_ID_VAR_NAME = "AWS_ACCESS_KEY_ID"
+AWS_SECRET_ACCESS_KEY_VAR_NAME = "AWS_SECRET_ACCESS_KEY"
+AWS_REGION_VAR_NAME = "AWS_REGION"
- aws_access_key_id_env_var_name: str
- aws_secret_access_key_env_var_name: str
- aws_region_env_var_name: str
- aws_access_key_id_secret_name: str
- aws_secret_access_key_secret_name: str
+class AwsConfig(ConfigModel):
+ api_key_method: AwsKeyMethod = Field(strict=False)
def get_aws_access_keys(self) -> Tuple[str, str, str]:
return self.get_aws_access_keys_with_method(api_key_method=self.api_key_method)
def get_aws_access_keys_with_method(self, api_key_method: AwsKeyMethod) -> Tuple[str, str, str]:
- aws_access_key_id: str
- aws_secret_access_key: str
- aws_region: str
match api_key_method:
case AwsKeyMethod.ENV:
log.debug("Getting AWS access keys from environment (key id and secret access key).")
try:
- aws_access_key_id = get_required_env(self.aws_access_key_id_env_var_name)
- aws_secret_access_key = get_required_env(self.aws_secret_access_key_env_var_name)
+ aws_access_key_id = get_required_env(AWS_ACCESS_KEY_ID_VAR_NAME)
+ aws_secret_access_key = get_required_env(AWS_SECRET_ACCESS_KEY_VAR_NAME)
+ aws_region = get_required_env(AWS_REGION_VAR_NAME)
except EnvVarNotFoundError as exc:
raise AwsCredentialsError(f"Error getting AWS access keys from environment: {exc}") from exc
log.debug("Getting AWS region from environment (priority override) or from aws_config.")
- aws_region = get_optional_env(self.aws_region_env_var_name) or self.aws_region
+
case AwsKeyMethod.SECRET_PROVIDER:
log.debug("Getting AWS secret access key from secrets provider (key id and secret access key).")
try:
- aws_access_key_id = get_secret(self.aws_access_key_id_secret_name)
- aws_secret_access_key = get_secret(self.aws_secret_access_key_secret_name)
+ aws_access_key_id = get_secret(AWS_ACCESS_KEY_ID_VAR_NAME)
+ aws_secret_access_key = get_secret(AWS_SECRET_ACCESS_KEY_VAR_NAME)
+ aws_region = get_secret(AWS_REGION_VAR_NAME)
except SecretNotFoundError as exc:
raise AwsCredentialsError("Error getting AWS access keys from secrets provider.") from exc
log.debug("Getting AWS region from environment (priority override) or from aws_config.")
- aws_region = get_optional_env(self.aws_region_env_var_name) or self.aws_region
return aws_access_key_id, aws_secret_access_key, aws_region
diff --git a/pipelex/tools/log/log.py b/pipelex/tools/log/log.py
index fd45d7b58..317904b86 100644
--- a/pipelex/tools/log/log.py
+++ b/pipelex/tools/log/log.py
@@ -7,7 +7,7 @@ from pipelex.tools.log.log_config import (
)
from pipelex.tools.log.log_dispatch import LogDispatch
from pipelex.tools.log.log_formatter import EmojiLogFormatter, LevelAndEmojiLogFormatter
-from pipelex.tools.log.log_levels import LOGGING_LEVEL_DEV, LOGGING_LEVEL_VERBOSE
+from pipelex.tools.log.log_levels import LOGGING_LEVEL_DEV, LOGGING_LEVEL_OFF, LOGGING_LEVEL_VERBOSE
class Log:
@@ -108,7 +108,6 @@ class Log:
self.poor_handler.setFormatter(LevelAndEmojiLogFormatter())
poor_loggers = set(log_config.poor_loggers + [log_config.generic_poor_logger])
for logger_name in poor_loggers:
- # print(f"Configuring poor logger: {logger_name}")
logger = logging.getLogger(logger_name)
logger.setLevel(log_config.default_log_level.int_logging_level)
logger.addHandler(self.poor_handler)
@@ -119,17 +118,6 @@ class Log:
self.verbose("Logs configured")
self.verbose(f"Config set for {project_name}")
- def set_rich_log_formatter(self, formatter: logging.Formatter):
- """
- Set the formatter for the rich log handler.
-
- Args:
- formatter (logging.Formatter): The formatter to use for rich logging.
- """
- if self.rich_handler is None:
- raise RuntimeError("Rich log handler is not set.")
- self.rich_handler.setFormatter(formatter)
-
def set_poor_log_formatter(self, formatter: logging.Formatter):
"""
Set the formatter for the poor log handler.
@@ -176,11 +164,10 @@ class Log:
Args:
level_name (str): The name of the log level.
"""
- print(f"Setting default logs to '{level_name=}'")
if level_name.upper() == LogLevel.DEV:
level = LOGGING_LEVEL_DEV
elif level_name.upper() == LogLevel.OFF:
- level = LOGGING_LEVEL_DEV
+ level = LOGGING_LEVEL_OFF
else:
level = getattr(logging, level_name.upper())
self.set_level_by_int(level)
@@ -188,7 +175,6 @@ class Log:
def set_level(self, level: LogLevel):
"""
Set the default log level for all loggers.
-
Args:
level (LogLevel): The log level to set.
"""
@@ -204,7 +190,6 @@ class Log:
level (LogLevel): The log level to set for the package.
"""
logger_name = package_name.replace("-", ".")
- # print(f"Setting {logger_name=} to {level=}")
logging.getLogger(logger_name).setLevel(level.int_logging_level)
def set_levels_for_packages(self, package_log_levels: Dict[str, LogLevel]):
diff --git a/pipelex/tools/log/log_dispatch.py b/pipelex/tools/log/log_dispatch.py
index 47b8162b2..db50ec652 100644
--- a/pipelex/tools/log/log_dispatch.py
+++ b/pipelex/tools/log/log_dispatch.py
@@ -279,7 +279,6 @@ class LogDispatch:
break
finally:
del frame_info
-
logger = logging.getLogger(log_origin_name)
logger.log(level=severity, msg=message, stacklevel=5)
finally:
diff --git a/pipelex/tools/log/log_levels.py b/pipelex/tools/log/log_levels.py
index 1445601a6..6d5dd2b5c 100644
--- a/pipelex/tools/log/log_levels.py
+++ b/pipelex/tools/log/log_levels.py
@@ -9,6 +9,7 @@ logging.addLevelName(LOGGING_LEVEL_DEV, LOGGING_LEVEL_DEV_NAME)
LOGGING_LEVEL_VERBOSE = 5
LOGGING_LEVEL_VERBOSE_NAME = "VERBOSE"
logging.addLevelName(LOGGING_LEVEL_VERBOSE, LOGGING_LEVEL_VERBOSE_NAME)
+LOGGING_LEVEL_OFF = 999
class LogLevel(StrEnum):
@@ -39,7 +40,7 @@ class LogLevel(StrEnum):
case LogLevel.CRITICAL:
return logging.CRITICAL
case LogLevel.OFF:
- return int(logging.CRITICAL) + 1
+ return LOGGING_LEVEL_OFF
@staticmethod
def from_int(logging_level: int) -> "LogLevel":
@@ -47,7 +48,7 @@ class LogLevel(StrEnum):
return LogLevel.VERBOSE
elif logging_level == LOGGING_LEVEL_DEV:
return LogLevel.DEV
- elif logging_level > int(logging.CRITICAL):
+ elif logging_level == LOGGING_LEVEL_OFF or logging_level > logging.CRITICAL:
return LogLevel.OFF
else:
return LogLevel(logging.getLevelName(logging_level))
diff --git a/pyproject.toml b/pyproject.toml
index e58e31eed..453aa7b13 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -35,7 +35,7 @@ dependencies = [
"rich>=13.8.1",
"shortuuid>=1.0.13",
"toml>=0.10.2",
- "typer>=0.12.5",
+ "typer>=0.16.0",
"typing-extensions>=4.12.2",
"yattag>=1.15.2"
]
diff --git a/uv.lock b/uv.lock
index e48abf5a9..d7328815d 100644
--- a/uv.lock
+++ b/uv.lock
@@ -834,7 +834,7 @@ wheels = [
[[package]]
name = "pipelex"
-version = "0.2.8"
+version = "0.2.9"
source = { editable = "." }
dependencies = [
{ name = "aiofiles" },
@@ -937,7 +937,7 @@ requires-dist = [
{ name = "ruff", marker = "extra == 'dev'", specifier = ">=0.6.8" },
{ name = "shortuuid", specifier = ">=1.0.13" },
{ name = "toml", specifier = ">=0.10.2" },
- { name = "typer", specifier = ">=0.12.5" },
+ { name = "typer", specifier = ">=0.16.0" },
{ name = "types-aioboto3", extras = ["bedrock", "bedrock-runtime"], marker = "extra == 'dev'", specifier = ">=13.4.0" },
{ name = "types-aiofiles", marker = "extra == 'dev'", specifier = ">=24.1.0.20240626" },
{ name = "types-beautifulsoup4", marker = "extra == 'dev'", specifier = ">=4.12.0.20240907" },
@@ -1365,7 +1365,7 @@ wheels = [
[[package]]
name = "typer"
-version = "0.15.4"
+version = "0.16.0"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "click" },
@@ -1373,9 +1373,9 @@ dependencies = [
{ name = "shellingham" },
{ name = "typing-extensions" },
]
-sdist = { url = "https://files.pythonhosted.org/packages/6c/89/c527e6c848739be8ceb5c44eb8208c52ea3515c6cf6406aa61932887bf58/typer-0.15.4.tar.gz", hash = "sha256:89507b104f9b6a0730354f27c39fae5b63ccd0c95b1ce1f1a6ba0cfd329997c3", size = 101559, upload-time = "2025-05-14T16:34:57.704Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/c5/8c/7d682431efca5fd290017663ea4588bf6f2c6aad085c7f108c5dbc316e70/typer-0.16.0.tar.gz", hash = "sha256:af377ffaee1dbe37ae9440cb4e8f11686ea5ce4e9bae01b84ae7c63b87f1dd3b", size = 102625, upload-time = "2025-05-26T14:30:31.824Z" }
wheels = [
- { url = "https://files.pythonhosted.org/packages/c9/62/d4ba7afe2096d5659ec3db8b15d8665bdcb92a3c6ff0b95e99895b335a9c/typer-0.15.4-py3-none-any.whl", hash = "sha256:eb0651654dcdea706780c466cf06d8f174405a659ffff8f163cfbfee98c0e173", size = 45258, upload-time = "2025-05-14T16:34:55.583Z" },
+ { url = "https://files.pythonhosted.org/packages/76/42/3efaf858001d2c2913de7f354563e3a3a2f0decae3efe98427125a8f441e/typer-0.16.0-py3-none-any.whl", hash = "sha256:1f79bed11d4d02d4310e3c1b7ba594183bcedb0ac73b27a9e5f28f6fb5b98855", size = 46317, upload-time = "2025-05-26T14:30:30.523Z" },
]
[[package]]