Skip to content

Build AI methods with your coding agent, then run them anywhere: as an MCP for chatbots, as a webapp for people, or via API for your software.

Feature/storage provider ingestion (#118)

Abdujabbar Mirkhalikovfe18d7fc3de1
diff --git a/docs/pages/advanced-customization/storage-provider-ingestion.md b/docs/pages/advanced-customization/storage-provider-ingestion.md
new file mode 100644
index 000000000..e380c52e1
--- /dev/null
+++ b/docs/pages/advanced-customization/storage-provider-ingestion.md
@@ -0,0 +1,3 @@
+# Storage Provider Injection
+
+⚠️ Under construction
diff --git a/pipelex/hub.py b/pipelex/hub.py
index 8a704b6ca..780c19f23 100644
--- a/pipelex/hub.py
+++ b/pipelex/hub.py
@@ -4,12 +4,16 @@ from typing import ClassVar, Dict, List, Optional, Type
from kajson.class_registry_abstract import ClassRegistryAbstract
from pipelex import log
-from pipelex.cogt.content_generation.content_generator_protocol import ContentGeneratorProtocol
+from pipelex.cogt.content_generation.content_generator_protocol import (
+ ContentGeneratorProtocol,
+)
from pipelex.cogt.imgg.imgg_worker_abstract import ImggWorkerAbstract
from pipelex.cogt.inference.inference_manager_protocol import InferenceManagerProtocol
from pipelex.cogt.llm.llm_models.llm_deck_abstract import LLMDeckAbstract
from pipelex.cogt.llm.llm_models.llm_engine_blueprint import LLMEngineBlueprint
-from pipelex.cogt.llm.llm_models.llm_model_provider_abstract import LLMModelProviderAbstract
+from pipelex.cogt.llm.llm_models.llm_model_provider_abstract import (
+ LLMModelProviderAbstract,
+)
from pipelex.cogt.llm.llm_worker_abstract import LLMWorkerAbstract
from pipelex.cogt.ocr.ocr_worker_abstract import OcrWorkerAbstract
from pipelex.cogt.plugin_manager import PluginManager
@@ -28,6 +32,7 @@ from pipelex.reporting.reporting_protocol import ReportingProtocol
from pipelex.tools.config.manager import config_manager
from pipelex.tools.config.models import ConfigRoot
from pipelex.tools.secrets.secrets_provider_abstract import SecretsProviderAbstract
+from pipelex.tools.storage.storage_provider_abstract import StorageProviderAbstract
from pipelex.tools.templating.template_provider_abstract import TemplateProviderAbstract
@@ -46,6 +51,7 @@ class PipelexHub:
self._secrets_provider: Optional[SecretsProviderAbstract] = None
self._template_provider: Optional[TemplateProviderAbstract] = None
self._class_registry: Optional[ClassRegistryAbstract] = None
+ self._storage_provider: Optional[StorageProviderAbstract] = None
# cogt
self._llm_models_provider: Optional[LLMModelProviderAbstract] = None
self._llm_deck_provider: Optional[LLMDeckAbstract] = None
@@ -112,6 +118,9 @@ class PipelexHub:
def set_secrets_provider(self, secrets_provider: SecretsProviderAbstract):
self._secrets_provider = secrets_provider
+ def set_storage_provider(self, storage_provider: StorageProviderAbstract | None):
+ self._storage_provider = storage_provider
+
def set_template_provider(self, template_provider: TemplateProviderAbstract):
self._template_provider = template_provider
@@ -198,6 +207,11 @@ class PipelexHub:
raise RuntimeError("ClassRegistry is not initialized")
return self._class_registry
+ def get_storage_provider(self) -> StorageProviderAbstract:
+ if self._storage_provider is None:
+ raise RuntimeError("StorageProvider is not initialized")
+ return self._storage_provider
+
# cogt
def get_required_llm_models_provider(self) -> LLMModelProviderAbstract:
@@ -297,6 +311,10 @@ def get_secrets_provider() -> SecretsProviderAbstract:
return get_pipelex_hub().get_required_secrets_provider()
+def get_storage_provider() -> StorageProviderAbstract:
+ return get_pipelex_hub().get_storage_provider()
+
+
def get_template_provider() -> TemplateProviderAbstract:
return get_pipelex_hub().get_required_template_provider()
@@ -391,7 +409,9 @@ def get_pipe_provider() -> PipeProviderAbstract:
return get_pipelex_hub().get_required_pipe_provider()
-def get_pipes_by_domain(excluded_domains: Optional[List[str]] = None) -> Dict[str, List[str]]:
+def get_pipes_by_domain(
+ excluded_domains: Optional[List[str]] = None,
+) -> Dict[str, List[str]]:
pipes = get_pipe_provider().get_pipes()
pipes_by_domain: Dict[str, List[str]] = defaultdict(list)
for pipe in pipes:
diff --git a/pipelex/pipelex.py b/pipelex/pipelex.py
index ea1e86e08..a6aff6f10 100644
--- a/pipelex/pipelex.py
+++ b/pipelex/pipelex.py
@@ -11,7 +11,9 @@ from typing_extensions import Self
from pipelex import log
from pipelex.cogt.content_generation.content_generator import ContentGenerator
-from pipelex.cogt.content_generation.content_generator_protocol import ContentGeneratorProtocol
+from pipelex.cogt.content_generation.content_generator_protocol import (
+ ContentGeneratorProtocol,
+)
from pipelex.cogt.inference.inference_manager import InferenceManager
from pipelex.cogt.llm.llm_models.llm_model import LATEST_VERSION_NAME
from pipelex.cogt.llm.llm_models.llm_model_library import LLMModelLibrary
@@ -24,10 +26,16 @@ from pipelex.libraries.library_manager import LibraryManager
from pipelex.pipe_works.pipe_router import PipeRouter
from pipelex.pipe_works.pipe_router_protocol import PipeRouterProtocol
from pipelex.pipeline.activity.activity_manager import ActivityManager
-from pipelex.pipeline.activity.activity_manager_protocol import ActivityManagerNoOp, ActivityManagerProtocol
+from pipelex.pipeline.activity.activity_manager_protocol import (
+ ActivityManagerNoOp,
+ ActivityManagerProtocol,
+)
from pipelex.pipeline.pipeline_manager import PipelineManager
from pipelex.pipeline.track.pipeline_tracker import PipelineTracker
-from pipelex.pipeline.track.pipeline_tracker_protocol import PipelineTrackerNoOp, PipelineTrackerProtocol
+from pipelex.pipeline.track.pipeline_tracker_protocol import (
+ PipelineTrackerNoOp,
+ PipelineTrackerProtocol,
+)
from pipelex.reporting.reporting_manager import ReportingManager
from pipelex.reporting.reporting_protocol import ReportingNoOp, ReportingProtocol
from pipelex.test_extras.registry_test_models import PipelexTestModels
@@ -36,6 +44,7 @@ from pipelex.tools.func_registry import func_registry
from pipelex.tools.runtime_manager import runtime_manager
from pipelex.tools.secrets.env_secrets_provider import EnvSecretsProvider
from pipelex.tools.secrets.secrets_provider_abstract import SecretsProviderAbstract
+from pipelex.tools.storage.storage_provider_abstract import StorageProviderAbstract
from pipelex.tools.templating.template_library import TemplateLibrary
from pipelex.tools.typing.pydantic_utils import format_pydantic_validation_error
@@ -172,10 +181,11 @@ class Pipelex:
content_generator: Optional[ContentGeneratorProtocol] = None,
pipe_router: Optional[PipeRouterProtocol] = None,
structure_classes: Optional[List[Type[Any]]] = None,
+ storage_provider: Optional[StorageProviderAbstract] = None,
):
# tools
self.pipelex_hub.set_secrets_provider(secrets_provider or EnvSecretsProvider())
-
+ self.pipelex_hub.set_storage_provider(storage_provider)
# cogt
self.pipelex_hub.set_content_generator(content_generator or ContentGenerator())
self.reporting_delegate.setup()
diff --git a/pipelex/tools/storage/__init__.py b/pipelex/tools/storage/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/pipelex/tools/storage/storage_provider_abstract.py b/pipelex/tools/storage/storage_provider_abstract.py
new file mode 100644
index 000000000..78adb784f
--- /dev/null
+++ b/pipelex/tools/storage/storage_provider_abstract.py
@@ -0,0 +1,11 @@
+from abc import ABC, abstractmethod
+
+
+class StorageProviderAbstract(ABC):
+ @abstractmethod
+ def load(self, uri: str) -> bytes:
+ pass
+
+ @abstractmethod
+ def store(self, data: bytes) -> str:
+ pass