Skip to content

Build AI methods with your coding agent, then run them anywhere: as an MCP for chatbots, as a webapp for people, or via API for your software.

Rename plugins package, move its config, move constants to config (#38)

Louis Choquel3593a8df6e3b
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 3700e4b4d..d9fe8fc99 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -20,12 +20,14 @@
- **Gemini update** - Added latest `gemini-2.5-pro` to the lineup
- **Image generation enhancements** - Better quality controls, improved background handling options, auto-adapts to different models: Flux, SDXL and now gpt-image-1
-### Changed
+### Refactored
+
+- Moved subpackage `plugin` to the same level as `cogt` within **pipelex** for better visibility
+- Major cleanup in the unit tests, hierarchy significantly flattened
+- Strengthened error handling throughout inference flows and template preprocessing
+- Added `make test-quiet` (shorthand `tq`) to Makefile to run tests without capturing outputs (i.e. without pytest `-s` option)
+- Stopped using Fixtures for `pipe_router` and `content_generator`: we're now always getting the singleton from `pipelex.hub`
-- **Error handling** - Strengthened throughout inference flows with better error messages across template preprocessing
-- **Test organization** - Migrated cogt tests to `tests/pipelex` for shared fixtures (especially imgg_handle)
-- **Development tooling** - Swapped pytest-pretty for pytest-sugar - because readable test names > pretty tables
-- **Python support** - Expanded from Python 3.11-only to Python 3.11, 3.12, and 3.13
### Fixed
@@ -33,9 +35,19 @@
### Dependencies
+- Added **pytest-xdist** to run unit tests in parallel on multiple CPUs. Not yet integrated into the Makefile, so run it manually with `pytest -n auto` (without inference) or `pytest -n auto -m "inference"` (inference only).
+- Swapped pytest-pretty for pytest-sugar - because readable test names > pretty tables
- Updated instructor to v1.8.3
- All dependencies tested against Python 3.11, 3.12, and 3.13
+### Tests
+
+- TestTemplatePreprocessor
+- TestImggByOpenAIGpt
+- TestImageGeneration
+- TestPipeImgg
+
+
## [v0.2.9] - 2025-05-30
- Include `pyproject.toml` inside the project build.
diff --git a/pipelex/cogt/config_cogt.py b/pipelex/cogt/config_cogt.py
index a4a05145f..11e475638 100644
--- a/pipelex/cogt/config_cogt.py
+++ b/pipelex/cogt/config_cogt.py
@@ -6,15 +6,6 @@ from pipelex.cogt.imgg.imgg_handle import ImggHandle
from pipelex.cogt.imgg.imgg_job_components import ImggJobConfig, ImggJobParams, ImggJobParamsDefaults
from pipelex.cogt.llm.llm_job_components import LLMJobConfig
from pipelex.cogt.llm.llm_models.llm_platform import LLMPlatform
-from pipelex.plugin.anthropic.anthropic_config import AnthropicConfig
-from pipelex.plugin.bedrock.bedrock_config import BedrockConfig
-from pipelex.plugin.mistral.mistral_config import MistralConfig
-from pipelex.plugin.openai.azure_openai_config import AzureOpenAIConfig
-from pipelex.plugin.openai.custom_endpoint_config import CustomEndpointConfig
-from pipelex.plugin.openai.openai_config import OpenAIOpenAIConfig
-from pipelex.plugin.openai.perplexity_config import PerplexityConfig
-from pipelex.plugin.openai.vertexai_config import VertexAIConfig
-from pipelex.plugin.openai.xai_config import XaiConfig
from pipelex.tools.config.models import ConfigModel
@@ -48,17 +39,6 @@ class InstructorConfig(ConfigModel):
class LLMConfig(ConfigModel):
preferred_platforms: Dict[str, LLMPlatform]
-
- anthropic_config: AnthropicConfig
- azure_openai_config: AzureOpenAIConfig
- bedrock_config: BedrockConfig
- vertexai_config: VertexAIConfig
- mistral_config: MistralConfig
- openai_openai_config: OpenAIOpenAIConfig
- perplexity_config: PerplexityConfig
- xai_config: XaiConfig
- custom_endpoint_config: CustomEndpointConfig
-
instructor_config: InstructorConfig
llm_job_config: LLMJobConfig
diff --git a/pipelex/cogt/imgg/imgg_worker_factory.py b/pipelex/cogt/imgg/imgg_worker_factory.py
index ab174a1ad..16fcb56ba 100644
--- a/pipelex/cogt/imgg/imgg_worker_factory.py
+++ b/pipelex/cogt/imgg/imgg_worker_factory.py
@@ -8,7 +8,7 @@ from pipelex.cogt.inference.inference_report_delegate import InferenceReportDele
from pipelex.cogt.llm.llm_models.llm_platform import LLMPlatform
from pipelex.cogt.plugin_manager import PluginHandle
from pipelex.hub import get_plugin_manager, get_secret
-from pipelex.plugin.openai.openai_imgg_worker import OpenAIImggWorker
+from pipelex.plugins.openai.openai_imgg_worker import OpenAIImggWorker
from pipelex.tools.secrets.secrets_errors import SecretNotFoundError
@@ -39,7 +39,7 @@ class ImggWorkerFactory:
"fal-client", "fal", "The fal-client SDK is required to use FAL models (generation of images)."
) from exc
- from pipelex.plugin.fal.fal_imgg_worker import FalImggWorker
+ from pipelex.plugins.fal.fal_imgg_worker import FalImggWorker
imgg_sdk_instance = plugin_manager.get_imgg_sdk_instance(imgg_sdk_handle=imgg_sdk_handle) or plugin_manager.set_imgg_sdk_instance(
imgg_sdk_handle=imgg_sdk_handle,
@@ -52,7 +52,7 @@ class ImggWorkerFactory:
report_delegate=report_delegate,
)
case ImggPlatform.OPENAI:
- from pipelex.plugin.openai.openai_factory import OpenAIFactory
+ from pipelex.plugins.openai.openai_factory import OpenAIFactory
imgg_sdk_instance = plugin_manager.get_llm_sdk_instance(llm_sdk_handle=imgg_sdk_handle) or plugin_manager.set_llm_sdk_instance(
llm_sdk_handle=imgg_sdk_handle,
diff --git a/pipelex/cogt/llm/llm_worker_factory.py b/pipelex/cogt/llm/llm_worker_factory.py
index 445e64554..d48a2eb44 100644
--- a/pipelex/cogt/llm/llm_worker_factory.py
+++ b/pipelex/cogt/llm/llm_worker_factory.py
@@ -22,13 +22,13 @@ class LLMWorkerFactory:
llm_worker: LLMWorkerAbstract
match llm_engine.llm_platform:
case LLMPlatform.OPENAI | LLMPlatform.AZURE_OPENAI | LLMPlatform.PERPLEXITY | LLMPlatform.XAI:
- from pipelex.plugin.openai.openai_factory import OpenAIFactory
+ from pipelex.plugins.openai.openai_factory import OpenAIFactory
structure_method: Optional[StructureMethod] = None
if get_config().cogt.llm_config.instructor_config.is_openai_structured_output_enabled:
structure_method = StructureMethod.INSTRUCTOR_OPENAI_STRUCTURED
- from pipelex.plugin.openai.openai_llm_worker import OpenAILLMWorker
+ from pipelex.plugins.openai.openai_llm_worker import OpenAILLMWorker
llm_sdk_instance = plugin_manager.get_llm_sdk_instance(llm_sdk_handle=llm_sdk_handle) or plugin_manager.set_llm_sdk_instance(
llm_sdk_handle=llm_sdk_handle,
@@ -47,8 +47,8 @@ class LLMWorkerFactory:
except ImportError as exc:
raise MissingDependencyError("google-auth-oauthlib", "google", "This dependency is required to connect to google.") from exc
- from pipelex.plugin.openai.openai_factory import OpenAIFactory
- from pipelex.plugin.openai.openai_llm_worker import OpenAILLMWorker
+ from pipelex.plugins.openai.openai_factory import OpenAIFactory
+ from pipelex.plugins.openai.openai_llm_worker import OpenAILLMWorker
llm_sdk_instance = plugin_manager.get_llm_sdk_instance(llm_sdk_handle=llm_sdk_handle) or plugin_manager.set_llm_sdk_instance(
llm_sdk_handle=llm_sdk_handle,
@@ -62,8 +62,8 @@ class LLMWorkerFactory:
report_delegate=report_delegate,
)
case LLMPlatform.CUSTOM_OPENAI:
- from pipelex.plugin.openai.openai_factory import OpenAIFactory
- from pipelex.plugin.openai.openai_llm_worker import OpenAILLMWorker
+ from pipelex.plugins.openai.openai_factory import OpenAIFactory
+ from pipelex.plugins.openai.openai_llm_worker import OpenAILLMWorker
llm_sdk_instance = plugin_manager.get_llm_sdk_instance(llm_sdk_handle=llm_sdk_handle) or plugin_manager.set_llm_sdk_instance(
llm_sdk_handle=llm_sdk_handle,
@@ -88,8 +88,8 @@ class LLMWorkerFactory:
(eg: bedrock-anthropic-claude)",
) from exc
- from pipelex.plugin.anthropic.anthropic_factory import AnthropicFactory
- from pipelex.plugin.anthropic.anthropic_llm_worker import AnthropicLLMWorker
+ from pipelex.plugins.anthropic.anthropic_factory import AnthropicFactory
+ from pipelex.plugins.anthropic.anthropic_llm_worker import AnthropicLLMWorker
llm_sdk_instance = plugin_manager.get_llm_sdk_instance(llm_sdk_handle=llm_sdk_handle) or plugin_manager.set_llm_sdk_instance(
llm_sdk_handle=llm_sdk_handle,
@@ -114,8 +114,8 @@ class LLMWorkerFactory:
(eg: bedrock-mistral-large)",
) from exc
- from pipelex.plugin.mistral.mistral_factory import MistralFactory
- from pipelex.plugin.mistral.mistral_llm_worker import MistralLLMWorker
+ from pipelex.plugins.mistral.mistral_factory import MistralFactory
+ from pipelex.plugins.mistral.mistral_llm_worker import MistralLLMWorker
llm_sdk_instance = plugin_manager.get_llm_sdk_instance(llm_sdk_handle=llm_sdk_handle) or plugin_manager.set_llm_sdk_instance(
llm_sdk_handle=llm_sdk_handle,
@@ -137,8 +137,8 @@ class LLMWorkerFactory:
"boto3,aioboto3", "bedrock", "The boto3 and aioboto3 SDKs are required to use Bedrock models."
) from exc
- from pipelex.plugin.bedrock.bedrock_factory import BedrockFactory
- from pipelex.plugin.bedrock.bedrock_llm_worker import BedrockLLMWorker
+ from pipelex.plugins.bedrock.bedrock_factory import BedrockFactory
+ from pipelex.plugins.bedrock.bedrock_llm_worker import BedrockLLMWorker
llm_sdk_instance = plugin_manager.get_llm_sdk_instance(llm_sdk_handle=llm_sdk_handle) or plugin_manager.set_llm_sdk_instance(
llm_sdk_handle=llm_sdk_handle,
diff --git a/pipelex/cogt/ocr/ocr_worker_factory.py b/pipelex/cogt/ocr/ocr_worker_factory.py
index 06a85f14d..8d8b90ae5 100644
--- a/pipelex/cogt/ocr/ocr_worker_factory.py
+++ b/pipelex/cogt/ocr/ocr_worker_factory.py
@@ -29,8 +29,8 @@ class OcrWorkerFactory:
"The mistralai SDK is required to use Mistral OCR models through the mistralai client.",
) from exc
- from pipelex.plugin.mistral.mistral_factory import MistralFactory
- from pipelex.plugin.mistral.mistral_ocr_worker import MistralOcrWorker
+ from pipelex.plugins.mistral.mistral_factory import MistralFactory
+ from pipelex.plugins.mistral.mistral_ocr_worker import MistralOcrWorker
ocr_sdk_instance = plugin_manager.get_ocr_sdk_instance(ocr_sdk_handle=ocr_sdk_handle) or plugin_manager.set_ocr_sdk_instance(
ocr_sdk_handle=ocr_sdk_handle,
diff --git a/pipelex/config.py b/pipelex/config.py
index a873a63d5..d114b8804 100644
--- a/pipelex/config.py
+++ b/pipelex/config.py
@@ -8,6 +8,7 @@ from pipelex.exceptions import PipelexError
from pipelex.hub import get_required_config
from pipelex.libraries.library_config import LibraryConfig
from pipelex.mission.track.tracker_config import TrackerConfig
+from pipelex.plugins.plugins_config import PluginsConfig
from pipelex.tools.aws.aws_config import AwsConfig
from pipelex.tools.config.models import ConfigModel, ConfigRoot
from pipelex.tools.log.log_config import LogConfig
@@ -65,6 +66,7 @@ class Pipelex(ConfigModel):
class PipelexConfig(ConfigRoot):
session_id: str = shortuuid.uuid()
cogt: Cogt
+ plugins: PluginsConfig
pipelex: Pipelex
diff --git a/pipelex/pipelex.toml b/pipelex/pipelex.toml
index b4768b23b..1004cb6b8 100644
--- a/pipelex/pipelex.toml
+++ b/pipelex/pipelex.toml
@@ -63,6 +63,10 @@ keywords_to_hilight = []
[pipelex.aws_config]
api_key_method = "env"
+####################################################################################################
+# Cogt inference config
+####################################################################################################
+
[cogt]
[cogt.inference_manager_config]
@@ -88,44 +92,10 @@ is_openai_structured_output_enabled = false
max_retries = 3
is_streaming_enabled = false
-####################################################################################################
-# Config to use LLM Platforms
-####################################################################################################
-
[cogt.llm_config.preferred_platforms]
# These overrride the defaults set for any llm handle
# "gpt-4o-mini" = "openai"
-[cogt.llm_config.openai_openai_config]
-api_key_method = "env"
-
-[cogt.llm_config.azure_openai_config]
-api_key_method = "env"
-
-# TODO: handle multiple azure openai accounts with different resource groups and account names for various llm model
-
-[cogt.llm_config.perplexity_config]
-api_key_method = "env"
-
-[cogt.llm_config.xai_config]
-api_key_method = "env"
-
-[cogt.llm_config.vertexai_config]
-api_key_method = "env"
-
-[cogt.llm_config.mistral_config]
-api_key_method = "env"
-
-[cogt.llm_config.bedrock_config]
-client_method = "aioboto3"
-
-[cogt.llm_config.anthropic_config]
-claude_4_reduced_tokens_limit = 8192 # use "unlimited" to enable the full 32/64K tokens Opus/Sonet but it raises streaming/timeout issues
-api_key_method = "env"
-
-[cogt.llm_config.custom_endpoint_config]
-api_key_method = "env"
-
####################################################################################################
# Image generation config
####################################################################################################
@@ -138,9 +108,10 @@ default_imgg_handle = "fal-ai/flux-pro/v1.1-ultra" # use "fal-ai/fast-lightning
is_sync_mode = false
[cogt.imgg_config.imgg_param_defaults]
-aspect_ratio = "square" # "square", "landscape_4_3", "landscape_16_9", "landscape_21_9", "portrait_3_4", "portrait_9_16", "portrait_9_21"
-# nb_steps = 1 # 28 for Flux, one of [1,2,4,8] for SDXL Lightning
+# quality and nb_steps cannot be set at the same time
+# nb_steps = 1 # 28 is a good default for Flux, possible values are [1,2,4,8] for SDXL Lightning
quality = "low"
+aspect_ratio = "square" # "square", "landscape_4_3", "landscape_16_9", "landscape_21_9", "portrait_3_4", "portrait_9_16", "portrait_9_21"
guidance_scale = 3.5
background = "auto"
is_moderated = true
@@ -148,7 +119,6 @@ safety_tolerance = 5
is_raw = false
output_format = "jpg"
seed = "auto"
-# seed = 1
####################################################################################################
# OCR config
@@ -158,6 +128,47 @@ seed = "auto"
ocr_handles = ["mistral/mistral-ocr-latest"]
page_output_text_file_name = "page_text.md"
+####################################################################################################
+# Plugins config
+####################################################################################################
+
+[plugins]
+
+[plugins.openai_config]
+image_output_compression = 100
+api_key_method = "env"
+
+[plugins.azure_openai_config]
+api_key_method = "env"
+
+# TODO: handle multiple azure openai accounts with different resource groups and account names for various llm model
+
+[plugins.perplexity_config]
+api_key_method = "env"
+
+[plugins.xai_config]
+api_key_method = "env"
+
+[plugins.vertexai_config]
+api_key_method = "env"
+
+[plugins.mistral_config]
+api_key_method = "env"
+
+[plugins.bedrock_config]
+client_method = "aioboto3"
+
+[plugins.anthropic_config]
+claude_4_reduced_tokens_limit = 8192 # use "unlimited" to enable the full 32/64K tokens Opus/Sonet but it raises streaming/timeout issues
+api_key_method = "env"
+
+[plugins.custom_endpoint_config]
+api_key_method = "env"
+
+[plugins.fal_config]
+flux_map_quality_to_steps = { "low" = 14, "medium" = 28, "high" = 56 }
+sdxl_lightning_map_quality_to_steps = { "low" = 2, "medium" = 4, "high" = 8 }
+
####################################################################################################
# Pipelex prompting config
####################################################################################################
diff --git a/pipelex/plugin/__init__.py b/pipelex/plugins/__init__.py
similarity index 100%
rename from pipelex/plugin/__init__.py
rename to pipelex/plugins/__init__.py
diff --git a/pipelex/plugin/anthropic/__init__.py b/pipelex/plugins/anthropic/__init__.py
similarity index 100%
rename from pipelex/plugin/anthropic/__init__.py
rename to pipelex/plugins/anthropic/__init__.py
diff --git a/pipelex/plugin/anthropic/anthropic_config.py b/pipelex/plugins/anthropic/anthropic_config.py
similarity index 100%
rename from pipelex/plugin/anthropic/anthropic_config.py
rename to pipelex/plugins/anthropic/anthropic_config.py
diff --git a/pipelex/plugin/anthropic/anthropic_factory.py b/pipelex/plugins/anthropic/anthropic_factory.py
similarity index 99%
rename from pipelex/plugin/anthropic/anthropic_factory.py
rename to pipelex/plugins/anthropic/anthropic_factory.py
index 968744c03..4c318c7f5 100644
--- a/pipelex/plugin/anthropic/anthropic_factory.py
+++ b/pipelex/plugins/anthropic/anthropic_factory.py
@@ -43,7 +43,7 @@ class AnthropicFactory:
# TODO: also support Anthropic with VertexAI
match llm_platform:
case LLMPlatform.ANTHROPIC:
- anthropic_config = get_config().cogt.llm_config.anthropic_config
+ anthropic_config = get_config().plugins.anthropic_config
api_key = anthropic_config.get_api_key(secrets_provider=get_secrets_provider())
return AsyncAnthropic(api_key=api_key)
case LLMPlatform.BEDROCK_ANTHROPIC:
diff --git a/pipelex/plugin/anthropic/anthropic_llm_worker.py b/pipelex/plugins/anthropic/anthropic_llm_worker.py
similarity index 97%
rename from pipelex/plugin/anthropic/anthropic_llm_worker.py
rename to pipelex/plugins/anthropic/anthropic_llm_worker.py
index 57d2461c2..f13621a50 100644
--- a/pipelex/plugin/anthropic/anthropic_llm_worker.py
+++ b/pipelex/plugins/anthropic/anthropic_llm_worker.py
@@ -14,7 +14,7 @@ from pipelex.cogt.llm.llm_models.llm_platform import LLMPlatform
from pipelex.cogt.llm.llm_worker_abstract import LLMWorkerAbstract
from pipelex.cogt.llm.structured_output import StructureMethod
from pipelex.config import get_config
-from pipelex.plugin.anthropic.anthropic_factory import AnthropicFactory
+from pipelex.plugins.anthropic.anthropic_factory import AnthropicFactory
from pipelex.tools.typing.pydantic_utils import BaseModelTypeVar
@@ -64,7 +64,7 @@ class AnthropicLLMWorker(LLMWorkerAbstract):
# TODO: implement streaming behind the scenes to avoid timeout/streaming errors with Claude 4 and high tokens
def _adapt_max_tokens(self, max_tokens: Optional[int]) -> int:
max_tokens = max_tokens or self.default_max_tokens
- if claude_4_tokens_limit := get_config().cogt.llm_config.anthropic_config.claude_4_tokens_limit:
+ if claude_4_tokens_limit := get_config().plugins.anthropic_config.claude_4_tokens_limit:
if max_tokens > claude_4_tokens_limit:
max_tokens = claude_4_tokens_limit
log.warning(f"Max tokens is greater than the claude 4 reduced tokens limit, reducing to {max_tokens}")
diff --git a/pipelex/plugin/anthropic/anthropic_llms.py b/pipelex/plugins/anthropic/anthropic_llms.py
similarity index 94%
rename from pipelex/plugin/anthropic/anthropic_llms.py
rename to pipelex/plugins/anthropic/anthropic_llms.py
index 5975ddffa..92b722d48 100644
--- a/pipelex/plugin/anthropic/anthropic_llms.py
+++ b/pipelex/plugins/anthropic/anthropic_llms.py
@@ -5,7 +5,7 @@ from anthropic.types import ModelInfo
from pipelex.cogt.exceptions import LLMModelProviderError, LLMSDKError
from pipelex.cogt.llm.llm_models.llm_platform import LLMPlatform
-from pipelex.plugin.anthropic.anthropic_factory import AnthropicFactory
+from pipelex.plugins.anthropic.anthropic_factory import AnthropicFactory
async def anthropic_list_anthropic_models(llm_platform: LLMPlatform) -> List[ModelInfo]:
diff --git a/pipelex/plugin/bedrock/__init__.py b/pipelex/plugins/bedrock/__init__.py
similarity index 100%
rename from pipelex/plugin/bedrock/__init__.py
rename to pipelex/plugins/bedrock/__init__.py
diff --git a/pipelex/plugin/bedrock/bedrock_client_aioboto3.py b/pipelex/plugins/bedrock/bedrock_client_aioboto3.py
similarity index 92%
rename from pipelex/plugin/bedrock/bedrock_client_aioboto3.py
rename to pipelex/plugins/bedrock/bedrock_client_aioboto3.py
index ee0f5bfd7..2c46a3173 100644
--- a/pipelex/plugin/bedrock/bedrock_client_aioboto3.py
+++ b/pipelex/plugins/bedrock/bedrock_client_aioboto3.py
@@ -6,8 +6,8 @@ from typing_extensions import override
from pipelex import log
from pipelex.cogt.llm.token_category import NbTokensByCategoryDict, TokenCategory
-from pipelex.plugin.bedrock.bedrock_client_protocol import BedrockClientProtocol
-from pipelex.plugin.bedrock.bedrock_message import BedrockMessageDictList
+from pipelex.plugins.bedrock.bedrock_client_protocol import BedrockClientProtocol
+from pipelex.plugins.bedrock.bedrock_message import BedrockMessageDictList
class BedrockClientAioboto3(BedrockClientProtocol):
diff --git a/pipelex/plugin/bedrock/bedrock_client_boto3.py b/pipelex/plugins/bedrock/bedrock_client_boto3.py
similarity index 91%
rename from pipelex/plugin/bedrock/bedrock_client_boto3.py
rename to pipelex/plugins/bedrock/bedrock_client_boto3.py
index 7c64fabf4..4a45edee6 100644
--- a/pipelex/plugin/bedrock/bedrock_client_boto3.py
+++ b/pipelex/plugins/bedrock/bedrock_client_boto3.py
@@ -6,8 +6,8 @@ from typing_extensions import override
from pipelex import log
from pipelex.cogt.llm.token_category import NbTokensByCategoryDict, TokenCategory
-from pipelex.plugin.bedrock.bedrock_client_protocol import BedrockClientProtocol
-from pipelex.plugin.bedrock.bedrock_message import BedrockMessageDictList
+from pipelex.plugins.bedrock.bedrock_client_protocol import BedrockClientProtocol
+from pipelex.plugins.bedrock.bedrock_message import BedrockMessageDictList
class BedrockClientBoto3(BedrockClientProtocol):
diff --git a/pipelex/plugin/bedrock/bedrock_client_protocol.py b/pipelex/plugins/bedrock/bedrock_client_protocol.py
similarity index 85%
rename from pipelex/plugin/bedrock/bedrock_client_protocol.py
rename to pipelex/plugins/bedrock/bedrock_client_protocol.py
index c94dd13f1..f025a4c4d 100644
--- a/pipelex/plugin/bedrock/bedrock_client_protocol.py
+++ b/pipelex/plugins/bedrock/bedrock_client_protocol.py
@@ -1,7 +1,7 @@
from typing import Optional, Protocol, Tuple, runtime_checkable
from pipelex.cogt.llm.token_category import NbTokensByCategoryDict
-from pipelex.plugin.bedrock.bedrock_message import BedrockMessageDictList
+from pipelex.plugins.bedrock.bedrock_message import BedrockMessageDictList
@runtime_checkable
diff --git a/pipelex/plugin/bedrock/bedrock_config.py b/pipelex/plugins/bedrock/bedrock_config.py
similarity index 100%
rename from pipelex/plugin/bedrock/bedrock_config.py
rename to pipelex/plugins/bedrock/bedrock_config.py
diff --git a/pipelex/plugin/bedrock/bedrock_factory.py b/pipelex/plugins/bedrock/bedrock_factory.py
similarity index 81%
rename from pipelex/plugin/bedrock/bedrock_factory.py
rename to pipelex/plugins/bedrock/bedrock_factory.py
index 4572c24e0..b12dca2bd 100644
--- a/pipelex/plugin/bedrock/bedrock_factory.py
+++ b/pipelex/plugins/bedrock/bedrock_factory.py
@@ -4,9 +4,9 @@ from pipelex.cogt.image.prompt_image import PromptImageBytes
from pipelex.cogt.llm.llm_job import LLMJob
from pipelex.config import get_config
from pipelex.hub import get_secrets_provider
-from pipelex.plugin.bedrock.bedrock_client_protocol import BedrockClientProtocol
-from pipelex.plugin.bedrock.bedrock_config import BedrockClientMethod
-from pipelex.plugin.bedrock.bedrock_message import BedrockContentItem, BedrockImage, BedrockMessage, BedrockSource, ImageFormat
+from pipelex.plugins.bedrock.bedrock_client_protocol import BedrockClientProtocol
+from pipelex.plugins.bedrock.bedrock_config import BedrockClientMethod
+from pipelex.plugins.bedrock.bedrock_message import BedrockContentItem, BedrockImage, BedrockMessage, BedrockSource, ImageFormat
class BedrockFactory:
@@ -16,18 +16,18 @@ class BedrockFactory:
@classmethod
def make_bedrock_client(cls) -> BedrockClientProtocol:
- bedrock_config = get_config().cogt.llm_config.bedrock_config
+ bedrock_config = get_config().plugins.bedrock_config
aws_region = bedrock_config.configure(secrets_provider=get_secrets_provider())
client_method = bedrock_config.client_method
bedrock_async_client: BedrockClientProtocol
log.verbose(f"Using '{client_method}' for BedrockClient")
match client_method:
case BedrockClientMethod.AIBOTO3:
- from pipelex.plugin.bedrock.bedrock_client_aioboto3 import BedrockClientAioboto3
+ from pipelex.plugins.bedrock.bedrock_client_aioboto3 import BedrockClientAioboto3
bedrock_async_client = BedrockClientAioboto3(aws_region=aws_region)
case BedrockClientMethod.BOTO3:
- from pipelex.plugin.bedrock.bedrock_client_boto3 import BedrockClientBoto3
+ from pipelex.plugins.bedrock.bedrock_client_boto3 import BedrockClientBoto3
bedrock_async_client = BedrockClientBoto3(aws_region=aws_region)
diff --git a/pipelex/plugin/bedrock/bedrock_llm_worker.py b/pipelex/plugins/bedrock/bedrock_llm_worker.py
similarity index 95%
rename from pipelex/plugin/bedrock/bedrock_llm_worker.py
rename to pipelex/plugins/bedrock/bedrock_llm_worker.py
index 3ce16976f..f6037759d 100644
--- a/pipelex/plugin/bedrock/bedrock_llm_worker.py
+++ b/pipelex/plugins/bedrock/bedrock_llm_worker.py
@@ -10,8 +10,8 @@ from pipelex.cogt.llm.llm_job_func import llm_job_func
from pipelex.cogt.llm.llm_models.llm_engine import LLMEngine
from pipelex.cogt.llm.llm_worker_abstract import LLMWorkerAbstract
from pipelex.cogt.llm.structured_output import StructureMethod
-from pipelex.plugin.bedrock.bedrock_client_protocol import BedrockClientProtocol
-from pipelex.plugin.bedrock.bedrock_factory import BedrockFactory
+from pipelex.plugins.bedrock.bedrock_client_protocol import BedrockClientProtocol
+from pipelex.plugins.bedrock.bedrock_factory import BedrockFactory
from pipelex.tools.typing.pydantic_utils import BaseModelTypeVar
diff --git a/pipelex/plugin/bedrock/bedrock_message.py b/pipelex/plugins/bedrock/bedrock_message.py
similarity index 100%
rename from pipelex/plugin/bedrock/bedrock_message.py
rename to pipelex/plugins/bedrock/bedrock_message.py
diff --git a/pipelex/plugin/fal/__init__.py b/pipelex/plugins/fal/__init__.py
similarity index 100%
rename from pipelex/plugin/fal/__init__.py
rename to pipelex/plugins/fal/__init__.py
diff --git a/pipelex/plugins/fal/fal_config.py b/pipelex/plugins/fal/fal_config.py
new file mode 100644
index 000000000..0de1a0b02
--- /dev/null
+++ b/pipelex/plugins/fal/fal_config.py
@@ -0,0 +1,27 @@
+from typing import Dict
+
+from pydantic import field_validator
+
+from pipelex.cogt.imgg.imgg_job_components import Quality
+from pipelex.tools.config.models import ConfigModel
+from pipelex.tools.exceptions import ConfigValidationError
+
+
+class FalConfig(ConfigModel):
+ flux_map_quality_to_steps: Dict[str, int]
+ sdxl_lightning_map_quality_to_steps: Dict[str, int]
+
+ @field_validator("flux_map_quality_to_steps", "sdxl_lightning_map_quality_to_steps")
+ @classmethod
+ def validate_quality_mapping(cls, value: Dict[str, int]) -> Dict[str, int]:
+ valid_qualities = {q.value for q in Quality}
+ missing_qualities = valid_qualities - set(value.keys())
+ invalid_qualities = set(value.keys()) - valid_qualities
+
+ if missing_qualities and invalid_qualities:
+ raise ConfigValidationError(f"Missing ({missing_qualities}) and invalid ({invalid_qualities}) quality levels in mapping")
+ elif missing_qualities:
+ raise ConfigValidationError(f"Missing quality levels in mapping: {missing_qualities}")
+ elif invalid_qualities:
+ raise ConfigValidationError(f"Invalid quality levels in mapping: {invalid_qualities}")
+ return value
diff --git a/pipelex/plugin/fal/fal_factory.py b/pipelex/plugins/fal/fal_factory.py
similarity index 95%
rename from pipelex/plugin/fal/fal_factory.py
rename to pipelex/plugins/fal/fal_factory.py
index 2778f432c..b4a2f08b3 100644
--- a/pipelex/plugin/fal/fal_factory.py
+++ b/pipelex/plugins/fal/fal_factory.py
@@ -7,6 +7,7 @@ from pipelex.cogt.imgg.imgg_engine import ImggEngine
from pipelex.cogt.imgg.imgg_handle import ImggHandle
from pipelex.cogt.imgg.imgg_job import ImggJob
from pipelex.cogt.imgg.imgg_job_components import AspectRatio, OutputFormat, Quality
+from pipelex.config import get_config
class FalFactory:
@@ -16,23 +17,13 @@ class FalFactory:
@classmethod
def make_nb_steps_from_quality_for_flux_pro(cls, quality: Quality) -> int:
- match quality:
- case Quality.LOW:
- return 14
- case Quality.MEDIUM:
- return 28
- case Quality.HIGH:
- return 56
+ flux_map_quality_to_steps = get_config().plugins.fal_config.flux_map_quality_to_steps
+ return flux_map_quality_to_steps[quality.value]
@classmethod
def make_nb_steps_from_quality_for_sdxl_lightning(cls, quality: Quality) -> int:
- match quality:
- case Quality.LOW:
- return 2
- case Quality.MEDIUM:
- return 4
- case Quality.HIGH:
- return 8
+ sdxl_lightning_map_quality_to_steps = get_config().plugins.fal_config.sdxl_lightning_map_quality_to_steps
+ return sdxl_lightning_map_quality_to_steps[quality.value]
@classmethod
def make_image_size_for_flux_1(cls, aspect_ratio: AspectRatio) -> str:
diff --git a/pipelex/plugin/fal/fal_imgg_worker.py b/pipelex/plugins/fal/fal_imgg_worker.py
similarity index 98%
rename from pipelex/plugin/fal/fal_imgg_worker.py
rename to pipelex/plugins/fal/fal_imgg_worker.py
index bf5e3fd00..edaad907b 100644
--- a/pipelex/plugin/fal/fal_imgg_worker.py
+++ b/pipelex/plugins/fal/fal_imgg_worker.py
@@ -10,7 +10,7 @@ from pipelex.cogt.imgg.imgg_engine import ImggEngine
from pipelex.cogt.imgg.imgg_job import ImggJob
from pipelex.cogt.imgg.imgg_worker_abstract import ImggWorkerAbstract, imgg_job_func
from pipelex.cogt.inference.inference_report_delegate import InferenceReportDelegate
-from pipelex.plugin.fal.fal_factory import FalFactory
+from pipelex.plugins.fal.fal_factory import FalFactory
class FalImggWorker(ImggWorkerAbstract):
diff --git a/pipelex/plugin/mistral/__init__.py b/pipelex/plugins/mistral/__init__.py
similarity index 100%
rename from pipelex/plugin/mistral/__init__.py
rename to pipelex/plugins/mistral/__init__.py
diff --git a/pipelex/plugin/mistral/mistral_config.py b/pipelex/plugins/mistral/mistral_config.py
similarity index 100%
rename from pipelex/plugin/mistral/mistral_config.py
rename to pipelex/plugins/mistral/mistral_config.py
diff --git a/pipelex/plugin/mistral/mistral_factory.py b/pipelex/plugins/mistral/mistral_factory.py
similarity index 97%
rename from pipelex/plugin/mistral/mistral_factory.py
rename to pipelex/plugins/mistral/mistral_factory.py
index 640f1fffc..a0658a5c1 100644
--- a/pipelex/plugin/mistral/mistral_factory.py
+++ b/pipelex/plugins/mistral/mistral_factory.py
@@ -26,7 +26,7 @@ from pipelex.cogt.llm.token_category import NbTokensByCategoryDict, TokenCategor
from pipelex.cogt.ocr.ocr_output import ExtractedImageFromPage, OcrOutput, Page
from pipelex.config import get_config
from pipelex.hub import get_secrets_provider
-from pipelex.plugin.openai.openai_factory import OpenAIFactory
+from pipelex.plugins.openai.openai_factory import OpenAIFactory
from pipelex.tools.misc.base_64_utils import encode_to_base64, load_binary_as_base64
@@ -38,7 +38,7 @@ class MistralFactory:
@classmethod
def make_mistral_client(cls) -> Mistral:
return Mistral(
- api_key=get_config().cogt.llm_config.mistral_config.api_key(secrets_provider=get_secrets_provider()),
+ api_key=get_config().plugins.mistral_config.api_key(secrets_provider=get_secrets_provider()),
)
#########################################################
diff --git a/pipelex/plugin/mistral/mistral_llm_worker.py b/pipelex/plugins/mistral/mistral_llm_worker.py
similarity index 98%
rename from pipelex/plugin/mistral/mistral_llm_worker.py
rename to pipelex/plugins/mistral/mistral_llm_worker.py
index d556a2f6e..6c2e64113 100644
--- a/pipelex/plugin/mistral/mistral_llm_worker.py
+++ b/pipelex/plugins/mistral/mistral_llm_worker.py
@@ -13,7 +13,7 @@ from pipelex.cogt.llm.llm_job_func import llm_job_func
from pipelex.cogt.llm.llm_models.llm_engine import LLMEngine
from pipelex.cogt.llm.llm_worker_abstract import LLMWorkerAbstract
from pipelex.cogt.llm.structured_output import StructureMethod
-from pipelex.plugin.mistral.mistral_factory import MistralFactory
+from pipelex.plugins.mistral.mistral_factory import MistralFactory
from pipelex.tools.typing.pydantic_utils import BaseModelTypeVar
diff --git a/pipelex/plugin/mistral/mistral_llms.py b/pipelex/plugins/mistral/mistral_llms.py
similarity index 89%
rename from pipelex/plugin/mistral/mistral_llms.py
rename to pipelex/plugins/mistral/mistral_llms.py
index a4dceac17..3cca53791 100644
--- a/pipelex/plugin/mistral/mistral_llms.py
+++ b/pipelex/plugins/mistral/mistral_llms.py
@@ -3,7 +3,7 @@ from typing import List
from mistralai.models import Data
from pipelex.cogt.exceptions import LLMModelProviderError
-from pipelex.plugin.mistral.mistral_factory import MistralFactory
+from pipelex.plugins.mistral.mistral_factory import MistralFactory
def list_mistral_models() -> List[Data]:
diff --git a/pipelex/plugin/mistral/mistral_ocr_worker.py b/pipelex/plugins/mistral/mistral_ocr_worker.py
similarity index 97%
rename from pipelex/plugin/mistral/mistral_ocr_worker.py
rename to pipelex/plugins/mistral/mistral_ocr_worker.py
index 0f62070b5..84a725bf7 100644
--- a/pipelex/plugin/mistral/mistral_ocr_worker.py
+++ b/pipelex/plugins/mistral/mistral_ocr_worker.py
@@ -11,8 +11,8 @@ from pipelex.cogt.ocr.ocr_input import OcrInputError
from pipelex.cogt.ocr.ocr_job import OcrJob
from pipelex.cogt.ocr.ocr_output import OcrOutput
from pipelex.cogt.ocr.ocr_worker_abstract import OcrWorkerAbstract, ocr_job_func
-from pipelex.plugin.mistral.mistral_factory import MistralFactory
-from pipelex.plugin.mistral.mistral_utils import upload_file_for_ocr
+from pipelex.plugins.mistral.mistral_factory import MistralFactory
+from pipelex.plugins.mistral.mistral_utils import upload_file_for_ocr
from pipelex.tools.misc.base_64_utils import load_binary_as_base64_async
from pipelex.tools.misc.path_utils import clarify_path_or_url
diff --git a/pipelex/plugin/mistral/mistral_utils.py b/pipelex/plugins/mistral/mistral_utils.py
similarity index 100%
rename from pipelex/plugin/mistral/mistral_utils.py
rename to pipelex/plugins/mistral/mistral_utils.py
diff --git a/pipelex/plugin/openai/__init__.py b/pipelex/plugins/openai/__init__.py
similarity index 100%
rename from pipelex/plugin/openai/__init__.py
rename to pipelex/plugins/openai/__init__.py
diff --git a/pipelex/plugin/openai/azure_openai_config.py b/pipelex/plugins/openai/azure_openai_config.py
similarity index 100%
rename from pipelex/plugin/openai/azure_openai_config.py
rename to pipelex/plugins/openai/azure_openai_config.py
diff --git a/pipelex/plugin/openai/custom_endpoint_config.py b/pipelex/plugins/openai/custom_endpoint_config.py
similarity index 100%
rename from pipelex/plugin/openai/custom_endpoint_config.py
rename to pipelex/plugins/openai/custom_endpoint_config.py
diff --git a/pipelex/plugin/openai/openai_config.py b/pipelex/plugins/openai/openai_config.py
similarity index 94%
rename from pipelex/plugin/openai/openai_config.py
rename to pipelex/plugins/openai/openai_config.py
index 443508dc9..071911b09 100644
--- a/pipelex/plugin/openai/openai_config.py
+++ b/pipelex/plugins/openai/openai_config.py
@@ -22,7 +22,8 @@ class OpenAIKeyMethod(StrEnum):
OPENAI_API_KEY_VAR_NAME = "OPENAI_API_KEY"
-class OpenAIOpenAIConfig(ConfigModel):
+class OpenAIConfig(ConfigModel):
+ image_output_compression: int = Field(ge=1, le=100)
api_key_method: OpenAIKeyMethod = Field(strict=False)
def get_api_key(self, secrets_provider: SecretsProviderAbstract) -> str:
diff --git a/pipelex/plugin/openai/openai_factory.py b/pipelex/plugins/openai/openai_factory.py
similarity index 94%
rename from pipelex/plugin/openai/openai_factory.py
rename to pipelex/plugins/openai/openai_factory.py
index 555cc4123..5fb4ae1d8 100644
--- a/pipelex/plugin/openai/openai_factory.py
+++ b/pipelex/plugins/openai/openai_factory.py
@@ -31,7 +31,7 @@ class OpenAIFactory:
api_key: Optional[str] = None
match llm_platform:
case LLMPlatform.AZURE_OPENAI:
- azure_openai_config = get_config().cogt.llm_config.azure_openai_config
+ azure_openai_config = get_config().plugins.azure_openai_config
endpoint, api_version, api_key = azure_openai_config.configure(secrets_provider=get_secrets_provider())
log.verbose(f"Making AsyncAzureOpenAI client with endpoint: {endpoint}, api_version: {api_version}")
@@ -41,7 +41,7 @@ class OpenAIFactory:
api_version=api_version,
)
case LLMPlatform.PERPLEXITY:
- perplexity_config = get_config().cogt.llm_config.perplexity_config
+ perplexity_config = get_config().plugins.perplexity_config
endpoint, api_key = perplexity_config.configure(secrets_provider=get_secrets_provider())
log.verbose(f"Making perplexity AsyncOpenAI client with endpoint: {endpoint}")
@@ -50,11 +50,11 @@ class OpenAIFactory:
base_url=endpoint,
)
case LLMPlatform.OPENAI:
- openai_openai_config = get_config().cogt.llm_config.openai_openai_config
+ openai_openai_config = get_config().plugins.openai_config
api_key = openai_openai_config.get_api_key(secrets_provider=get_secrets_provider())
the_client = openai.AsyncOpenAI(api_key=api_key)
case LLMPlatform.VERTEXAI_OPENAI:
- vertexai_config = get_config().cogt.llm_config.vertexai_config
+ vertexai_config = get_config().plugins.vertexai_config
endpoint, api_key = vertexai_config.configure(secrets_provider=get_secrets_provider())
log.verbose(f"Making vertex AsyncOpenAI client with endpoint: {endpoint}")
@@ -63,7 +63,7 @@ class OpenAIFactory:
base_url=endpoint,
)
case LLMPlatform.XAI:
- xai_config = get_config().cogt.llm_config.xai_config
+ xai_config = get_config().plugins.xai_config
endpoint, api_key = xai_config.configure(secrets_provider=get_secrets_provider())
log.verbose(f"Making Xai AsyncOpenAI client with endpoint: {endpoint}")
@@ -72,7 +72,7 @@ class OpenAIFactory:
base_url=endpoint,
)
case LLMPlatform.CUSTOM_OPENAI:
- custom_endpoint_config = get_config().cogt.llm_config.custom_endpoint_config
+ custom_endpoint_config = get_config().plugins.custom_endpoint_config
base_url, api_key = custom_endpoint_config.configure(secrets_provider=get_secrets_provider())
log.verbose(f"Making custom AsyncOpenAI client with base_url: {base_url}")
diff --git a/pipelex/plugin/openai/openai_func.py b/pipelex/plugins/openai/openai_func.py
similarity index 100%
rename from pipelex/plugin/openai/openai_func.py
rename to pipelex/plugins/openai/openai_func.py
diff --git a/pipelex/plugin/openai/openai_imgg_factory.py b/pipelex/plugins/openai/openai_imgg_factory.py
similarity index 81%
rename from pipelex/plugin/openai/openai_imgg_factory.py
rename to pipelex/plugins/openai/openai_imgg_factory.py
index 1233e53d2..9377cc256 100644
--- a/pipelex/plugin/openai/openai_imgg_factory.py
+++ b/pipelex/plugins/openai/openai_imgg_factory.py
@@ -46,20 +46,10 @@ class OpenAIImggFactory:
@classmethod
def quality_for_gpt_image_1(cls, quality: Quality) -> GptImage1QualityType:
- match quality:
- case Quality.LOW:
- return "low"
- case Quality.MEDIUM:
- return "medium"
- case Quality.HIGH:
- return "high"
+ """This method only converts the Quality string value as a Literal, as expected by the OpenAI API"""
+ return quality.value
@classmethod
def background_for_gpt_image_1(cls, background: Background) -> GptImage1BackgroundType:
- match background:
- case Background.TRANSPARENT:
- return "transparent"
- case Background.OPAQUE:
- return "opaque"
- case Background.AUTO:
- return "auto"
+ """This method only converts the Background string value as a Literal, as expected by the OpenAI API"""
+ return background.value
diff --git a/pipelex/plugin/openai/openai_imgg_worker.py b/pipelex/plugins/openai/openai_imgg_worker.py
similarity index 94%
rename from pipelex/plugin/openai/openai_imgg_worker.py
rename to pipelex/plugins/openai/openai_imgg_worker.py
index c91634b00..37cdada8c 100644
--- a/pipelex/plugin/openai/openai_imgg_worker.py
+++ b/pipelex/plugins/openai/openai_imgg_worker.py
@@ -12,7 +12,8 @@ from pipelex.cogt.imgg.imgg_job import ImggJob
from pipelex.cogt.imgg.imgg_job_components import Quality
from pipelex.cogt.imgg.imgg_worker_abstract import ImggWorkerAbstract, imgg_job_func
from pipelex.cogt.inference.inference_report_delegate import InferenceReportDelegate
-from pipelex.plugin.openai.openai_imgg_factory import OpenAIImggFactory
+from pipelex.config import get_config
+from pipelex.plugins.openai.openai_imgg_factory import OpenAIImggFactory
from pipelex.tools.misc.base_64_utils import save_base64_to_binary_file
from pipelex.tools.misc.file_utils import ensure_path
@@ -55,7 +56,7 @@ class OpenAIImggWorker(ImggWorkerAbstract):
moderation = OpenAIImggFactory.moderation_for_gpt_image_1(is_moderated=imgg_job.job_params.is_moderated)
background = OpenAIImggFactory.background_for_gpt_image_1(background=imgg_job.job_params.background)
quality = OpenAIImggFactory.quality_for_gpt_image_1(quality=imgg_job.job_params.quality or Quality.LOW)
- output_compression = 100
+ output_compression = get_config().plugins.openai_config.image_output_compression
result = await self.openai_client.images.generate(
prompt=imgg_job.imgg_prompt.positive_text,
model=self.imgg_engine.imgg_model_name,
diff --git a/pipelex/plugin/openai/openai_llm_worker.py b/pipelex/plugins/openai/openai_llm_worker.py
similarity index 99%
rename from pipelex/plugin/openai/openai_llm_worker.py
rename to pipelex/plugins/openai/openai_llm_worker.py
index 66bb1a3b3..5d6c584a6 100644
--- a/pipelex/plugin/openai/openai_llm_worker.py
+++ b/pipelex/plugins/openai/openai_llm_worker.py
@@ -15,7 +15,7 @@ from pipelex.cogt.llm.llm_models.llm_engine import LLMEngine
from pipelex.cogt.llm.llm_models.llm_family import LLMFamily
from pipelex.cogt.llm.llm_worker_abstract import LLMWorkerAbstract
from pipelex.cogt.llm.structured_output import StructureMethod
-from pipelex.plugin.openai.openai_factory import OpenAIFactory
+from pipelex.plugins.openai.openai_factory import OpenAIFactory
from pipelex.tools.typing.pydantic_utils import BaseModelTypeVar
diff --git a/pipelex/plugin/openai/openai_llms.py b/pipelex/plugins/openai/openai_llms.py
similarity index 91%
rename from pipelex/plugin/openai/openai_llms.py
rename to pipelex/plugins/openai/openai_llms.py
index 603caed75..4227709a5 100644
--- a/pipelex/plugin/openai/openai_llms.py
+++ b/pipelex/plugins/openai/openai_llms.py
@@ -4,7 +4,7 @@ from openai.types import Model
from pipelex.cogt.exceptions import LLMSDKError
from pipelex.cogt.llm.llm_models.llm_platform import LLMPlatform
-from pipelex.plugin.openai.openai_factory import OpenAIFactory
+from pipelex.plugins.openai.openai_factory import OpenAIFactory
async def openai_list_available_models(llm_platform: LLMPlatform) -> List[Model]:
diff --git a/pipelex/plugin/openai/perplexity_config.py b/pipelex/plugins/openai/perplexity_config.py
similarity index 100%
rename from pipelex/plugin/openai/perplexity_config.py
rename to pipelex/plugins/openai/perplexity_config.py
diff --git a/pipelex/plugin/openai/vertexai_config.py b/pipelex/plugins/openai/vertexai_config.py
similarity index 100%
rename from pipelex/plugin/openai/vertexai_config.py
rename to pipelex/plugins/openai/vertexai_config.py
diff --git a/pipelex/plugin/openai/xai_config.py b/pipelex/plugins/openai/xai_config.py
similarity index 100%
rename from pipelex/plugin/openai/xai_config.py
rename to pipelex/plugins/openai/xai_config.py
diff --git a/pipelex/plugins/plugins_config.py b/pipelex/plugins/plugins_config.py
new file mode 100644
index 000000000..ac5dc7d47
--- /dev/null
+++ b/pipelex/plugins/plugins_config.py
@@ -0,0 +1,24 @@
+from pipelex.plugins.anthropic.anthropic_config import AnthropicConfig
+from pipelex.plugins.bedrock.bedrock_config import BedrockConfig
+from pipelex.plugins.fal.fal_config import FalConfig
+from pipelex.plugins.mistral.mistral_config import MistralConfig
+from pipelex.plugins.openai.azure_openai_config import AzureOpenAIConfig
+from pipelex.plugins.openai.custom_endpoint_config import CustomEndpointConfig
+from pipelex.plugins.openai.openai_config import OpenAIConfig
+from pipelex.plugins.openai.perplexity_config import PerplexityConfig
+from pipelex.plugins.openai.vertexai_config import VertexAIConfig
+from pipelex.plugins.openai.xai_config import XaiConfig
+from pipelex.tools.config.models import ConfigModel
+
+
+class PluginsConfig(ConfigModel):
+ anthropic_config: AnthropicConfig
+ azure_openai_config: AzureOpenAIConfig
+ bedrock_config: BedrockConfig
+ vertexai_config: VertexAIConfig
+ mistral_config: MistralConfig
+ openai_config: OpenAIConfig
+ perplexity_config: PerplexityConfig
+ xai_config: XaiConfig
+ custom_endpoint_config: CustomEndpointConfig
+ fal_config: FalConfig
diff --git a/tests/pipelex/plugin/__init__.py b/tests/pipelex/plugins/__init__.py
similarity index 100%
rename from tests/pipelex/plugin/__init__.py
rename to tests/pipelex/plugins/__init__.py
diff --git a/tests/pipelex/plugin/conftest.py b/tests/pipelex/plugins/conftest.py
similarity index 100%
rename from tests/pipelex/plugin/conftest.py
rename to tests/pipelex/plugins/conftest.py
diff --git a/tests/pipelex/plugin/test_anthropic.py b/tests/pipelex/plugins/test_anthropic.py
similarity index 95%
rename from tests/pipelex/plugin/test_anthropic.py
rename to tests/pipelex/plugins/test_anthropic.py
index 3117040c6..904013e01 100644
--- a/tests/pipelex/plugin/test_anthropic.py
+++ b/tests/pipelex/plugins/test_anthropic.py
@@ -5,7 +5,7 @@ from rich.table import Table
from pipelex.cogt.exceptions import LLMSDKError
from pipelex.cogt.llm.llm_models.llm_platform import LLMPlatform
-from pipelex.plugin.anthropic.anthropic_llms import anthropic_list_anthropic_models
+from pipelex.plugins.anthropic.anthropic_llms import anthropic_list_anthropic_models
# make t VERBOSE=2 TEST=TestAnthropic
diff --git a/tests/pipelex/plugin/test_bedrock.py b/tests/pipelex/plugins/test_bedrock.py
similarity index 100%
rename from tests/pipelex/plugin/test_bedrock.py
rename to tests/pipelex/plugins/test_bedrock.py
diff --git a/tests/pipelex/plugin/test_mistral.py b/tests/pipelex/plugins/test_mistral.py
similarity index 96%
rename from tests/pipelex/plugin/test_mistral.py
rename to tests/pipelex/plugins/test_mistral.py
index c484d4894..b9af1c796 100644
--- a/tests/pipelex/plugin/test_mistral.py
+++ b/tests/pipelex/plugins/test_mistral.py
@@ -4,7 +4,7 @@ from rich.console import Console
from rich.table import Table
from pipelex import pretty_print
-from pipelex.plugin.mistral.mistral_llms import list_mistral_models
+from pipelex.plugins.mistral.mistral_llms import list_mistral_models
# make t VERBOSE=2 TEST=TestMistral
diff --git a/tests/pipelex/plugin/test_openai.py b/tests/pipelex/plugins/test_openai.py
similarity index 89%
rename from tests/pipelex/plugin/test_openai.py
rename to tests/pipelex/plugins/test_openai.py
index 9ece86ff8..0cf4f917c 100644
--- a/tests/pipelex/plugin/test_openai.py
+++ b/tests/pipelex/plugins/test_openai.py
@@ -5,7 +5,7 @@ from pipelex.cogt.exceptions import LLMSDKError
from pipelex.cogt.llm.llm_models.llm_platform import LLMPlatform
from pipelex.config import get_config
from pipelex.hub import get_secrets_provider
-from pipelex.plugin.openai.openai_llms import openai_list_available_models
+from pipelex.plugins.openai.openai_llms import openai_list_available_models
# make t VERBOSE=2 TEST=TestOpenAI
@@ -14,7 +14,7 @@ from pipelex.plugin.openai.openai_llms import openai_list_available_models
@pytest.mark.asyncio(loop_scope="class")
class TestOpenAI:
async def test_openai_api_key(self):
- openai_openai_config = get_config().cogt.llm_config.openai_openai_config
+ openai_openai_config = get_config().plugins.openai_config
assert openai_openai_config.get_api_key(secrets_provider=get_secrets_provider())
# pytest -k test_openai_list_available_models -s -vv
diff --git a/tests/pipelex/plugin/test_openai_image_gen.py b/tests/pipelex/plugins/test_openai_image_gen.py
similarity index 96%
rename from tests/pipelex/plugin/test_openai_image_gen.py
rename to tests/pipelex/plugins/test_openai_image_gen.py
index 67ef3e594..520eb743c 100644
--- a/tests/pipelex/plugin/test_openai_image_gen.py
+++ b/tests/pipelex/plugins/test_openai_image_gen.py
@@ -2,7 +2,7 @@ import pytest
from pipelex.cogt.exceptions import ImggGenerationError
from pipelex.cogt.llm.llm_models.llm_platform import LLMPlatform
-from pipelex.plugin.openai.openai_factory import OpenAIFactory
+from pipelex.plugins.openai.openai_factory import OpenAIFactory
from pipelex.tools.misc.base_64_utils import save_base64_to_binary_file
from pipelex.tools.misc.file_utils import ensure_path, get_incremental_file_path
from tests.conftest import TEST_OUTPUTS_DIR
diff --git a/uv.lock b/uv.lock
index 80e3d6263..f50db6d7b 100644
--- a/uv.lock
+++ b/uv.lock
@@ -160,7 +160,7 @@ wheels = [
[[package]]
name = "anthropic"
-version = "0.52.1"
+version = "0.52.2"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "anyio" },
@@ -171,9 +171,9 @@ dependencies = [
{ name = "sniffio" },
{ name = "typing-extensions" },
]
-sdist = { url = "https://files.pythonhosted.org/packages/0c/6b/7d632b680556812f36b663891ccab8f5bf71b61913a3c7332aa7aa2822be/anthropic-0.52.1.tar.gz", hash = "sha256:da4a7c3aeac0170cb45a42dc3369ca1fcf2b3238edf845cb056505d4b0c42fcf", size = 241807, upload-time = "2025-05-28T13:13:40.851Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/63/84/95126ee8df1acedd60bd03fe368d6335d65fe92e2c97581a81a82e8f576b/anthropic-0.52.2.tar.gz", hash = "sha256:9047bc960e8513950579c9cb730c16a84af3fcb56341ad7dc730772f83757050", size = 306204, upload-time = "2025-06-02T11:27:59.308Z" }
wheels = [
- { url = "https://files.pythonhosted.org/packages/9d/e7/36ad2dc22811ea004e1f511b2f5604fe71a56bdfd3b969b265892d45affa/anthropic-0.52.1-py3-none-any.whl", hash = "sha256:807cee7ebc5503753da0403a77932decf5a4c036041ddda58b4edcdb2a3da551", size = 286076, upload-time = "2025-05-28T13:13:39.052Z" },
+ { url = "https://files.pythonhosted.org/packages/77/3b/6f67f4e061d73cfaffc44dd41cbbe8b09efe1ec37b8135a2bcc043736d62/anthropic-0.52.2-py3-none-any.whl", hash = "sha256:00d52555f503e81e21aff0103db04cd93979cdf87ce8dd43c660ca6deae83ac6", size = 286262, upload-time = "2025-06-02T11:27:57.536Z" },
]
[[package]]
@@ -2009,11 +2009,11 @@ wheels = [
[[package]]
name = "types-openpyxl"
-version = "3.1.5.20250516"
+version = "3.1.5.20250602"
source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/a4/a8/d3fad3b084b477cb1eb8896de3dc7307f930e890f95099a4dc9d9f26cb8a/types_openpyxl-3.1.5.20250516.tar.gz", hash = "sha256:691339abe141a5713f115558cc39023ebdda6298bfa875d575cc5a961a3c5523", size = 100626, upload-time = "2025-05-16T03:07:16.386Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/bc/d4/33cc2f331cde82206aa4ec7d8db408beca65964785f438c6d2505d828178/types_openpyxl-3.1.5.20250602.tar.gz", hash = "sha256:d19831482022fc933780d6e9d6990464c18c2ec5f14786fea862f72c876980b5", size = 100608, upload-time = "2025-06-02T03:14:40.625Z" }
wheels = [
- { url = "https://files.pythonhosted.org/packages/62/69/bb48b9be23ce6ceae6583621d9494d47e7b1165eeee2b9feb1954bfadfee/types_openpyxl-3.1.5.20250516-py3-none-any.whl", hash = "sha256:87c6b04b30fd1cbab85dc93cbe4f57ce1eb3df5d2911f742a1a0e3bf94314dfc", size = 165927, upload-time = "2025-05-16T03:07:15.219Z" },
+ { url = "https://files.pythonhosted.org/packages/2e/69/5b924a20a4d441ec2160e94085b9fa9358dc27edde10080d71209c59101d/types_openpyxl-3.1.5.20250602-py3-none-any.whl", hash = "sha256:1f82211e086902318f6a14b5d8d865102362fda7cb82f3d63ac4dff47a1f164b", size = 165922, upload-time = "2025-06-02T03:14:39.226Z" },
]
[[package]]
@@ -2036,14 +2036,14 @@ wheels = [
[[package]]
name = "types-requests"
-version = "2.32.0.20250515"
+version = "2.32.0.20250602"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "urllib3" },
]
-sdist = { url = "https://files.pythonhosted.org/packages/06/c1/cdc4f9b8cfd9130fbe6276db574f114541f4231fcc6fb29648289e6e3390/types_requests-2.32.0.20250515.tar.gz", hash = "sha256:09c8b63c11318cb2460813871aaa48b671002e59fda67ca909e9883777787581", size = 23012, upload-time = "2025-05-15T03:04:31.817Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/48/b0/5321e6eeba5d59e4347fcf9bf06a5052f085c3aa0f4876230566d6a4dc97/types_requests-2.32.0.20250602.tar.gz", hash = "sha256:ee603aeefec42051195ae62ca7667cd909a2f8128fdf8aad9e8a5219ecfab3bf", size = 23042, upload-time = "2025-06-02T03:15:02.958Z" }
wheels = [
- { url = "https://files.pythonhosted.org/packages/fe/0f/68a997c73a129287785f418c1ebb6004f81e46b53b3caba88c0e03fcd04a/types_requests-2.32.0.20250515-py3-none-any.whl", hash = "sha256:f8eba93b3a892beee32643ff836993f15a785816acca21ea0ffa006f05ef0fb2", size = 20635, upload-time = "2025-05-15T03:04:30.5Z" },
+ { url = "https://files.pythonhosted.org/packages/da/18/9b782980e575c6581d5c0c1c99f4c6f89a1d7173dad072ee96b2756c02e6/types_requests-2.32.0.20250602-py3-none-any.whl", hash = "sha256:f4f335f87779b47ce10b8b8597b409130299f6971ead27fead4fe7ba6ea3e726", size = 20638, upload-time = "2025-06-02T03:15:01.959Z" },
]
[[package]]
@@ -2066,11 +2066,11 @@ wheels = [
[[package]]
name = "typing-extensions"
-version = "4.13.2"
+version = "4.14.0"
source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/f6/37/23083fcd6e35492953e8d2aaaa68b860eb422b34627b13f2ce3eb6106061/typing_extensions-4.13.2.tar.gz", hash = "sha256:e6c81219bd689f51865d9e372991c540bda33a0379d5573cddb9a3a23f7caaef", size = 106967, upload-time = "2025-04-10T14:19:05.416Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/d1/bc/51647cd02527e87d05cb083ccc402f93e441606ff1f01739a62c8ad09ba5/typing_extensions-4.14.0.tar.gz", hash = "sha256:8676b788e32f02ab42d9e7c61324048ae4c6d844a399eebace3d4979d75ceef4", size = 107423, upload-time = "2025-06-02T14:52:11.399Z" }
wheels = [
- { url = "https://files.pythonhosted.org/packages/8b/54/b1ae86c0973cc6f0210b53d508ca3641fb6d0c56823f288d108bc7ab3cc8/typing_extensions-4.13.2-py3-none-any.whl", hash = "sha256:a439e7c04b49fec3e5d3e2beaa21755cadbbdc391694e28ccdd36ca4a1408f8c", size = 45806, upload-time = "2025-04-10T14:19:03.967Z" },
+ { url = "https://files.pythonhosted.org/packages/69/e0/552843e0d356fbb5256d21449fa957fa4eff3bbc135a74a691ee70c7c5da/typing_extensions-4.14.0-py3-none-any.whl", hash = "sha256:a1514509136dd0b477638fc68d6a91497af5076466ad0fa6c338e44e359944af", size = 43839, upload-time = "2025-06-02T14:52:10.026Z" },
]
[[package]]