Build AI methods with your coding agent, then run them anywhere: as an MCP for chatbots, as a webapp for people, or via API for your software.
Feature/storage provider ingestion (#118)
Abdujabbar Mirkhalikovfe18d7fc3de1
diff --git a/docs/pages/advanced-customization/storage-provider-ingestion.md b/docs/pages/advanced-customization/storage-provider-ingestion.mdnew file mode 100644index 000000000..e380c52e1--- /dev/null+++ b/docs/pages/advanced-customization/storage-provider-ingestion.md@@ -0,0 +1,3 @@+# Storage Provider Injection++⚠️ Under constructiondiff --git a/pipelex/hub.py b/pipelex/hub.pyindex 8a704b6ca..780c19f23 100644--- a/pipelex/hub.py+++ b/pipelex/hub.py@@ -4,12 +4,16 @@ from typing import ClassVar, Dict, List, Optional, Typefrom kajson.class_registry_abstract import ClassRegistryAbstractfrom pipelex import log-from pipelex.cogt.content_generation.content_generator_protocol import ContentGeneratorProtocol+from pipelex.cogt.content_generation.content_generator_protocol import (+ ContentGeneratorProtocol,+)from pipelex.cogt.imgg.imgg_worker_abstract import ImggWorkerAbstractfrom pipelex.cogt.inference.inference_manager_protocol import InferenceManagerProtocolfrom pipelex.cogt.llm.llm_models.llm_deck_abstract import LLMDeckAbstractfrom pipelex.cogt.llm.llm_models.llm_engine_blueprint import LLMEngineBlueprint-from pipelex.cogt.llm.llm_models.llm_model_provider_abstract import LLMModelProviderAbstract+from pipelex.cogt.llm.llm_models.llm_model_provider_abstract import (+ LLMModelProviderAbstract,+)from pipelex.cogt.llm.llm_worker_abstract import LLMWorkerAbstractfrom pipelex.cogt.ocr.ocr_worker_abstract import OcrWorkerAbstractfrom pipelex.cogt.plugin_manager import PluginManager@@ -28,6 +32,7 @@ from pipelex.reporting.reporting_protocol import ReportingProtocolfrom pipelex.tools.config.manager import config_managerfrom pipelex.tools.config.models import ConfigRootfrom pipelex.tools.secrets.secrets_provider_abstract import SecretsProviderAbstract+from pipelex.tools.storage.storage_provider_abstract import StorageProviderAbstractfrom pipelex.tools.templating.template_provider_abstract import TemplateProviderAbstract@@ -46,6 +51,7 @@ class PipelexHub:self._secrets_provider: Optional[SecretsProviderAbstract] = Noneself._template_provider: Optional[TemplateProviderAbstract] = Noneself._class_registry: Optional[ClassRegistryAbstract] = None+ self._storage_provider: Optional[StorageProviderAbstract] = None# cogtself._llm_models_provider: Optional[LLMModelProviderAbstract] = Noneself._llm_deck_provider: Optional[LLMDeckAbstract] = None@@ -112,6 +118,9 @@ class PipelexHub:def set_secrets_provider(self, secrets_provider: SecretsProviderAbstract):self._secrets_provider = secrets_provider+ def set_storage_provider(self, storage_provider: StorageProviderAbstract | None):+ self._storage_provider = storage_provider+def set_template_provider(self, template_provider: TemplateProviderAbstract):self._template_provider = template_provider@@ -198,6 +207,11 @@ class PipelexHub:raise RuntimeError("ClassRegistry is not initialized")return self._class_registry+ def get_storage_provider(self) -> StorageProviderAbstract:+ if self._storage_provider is None:+ raise RuntimeError("StorageProvider is not initialized")+ return self._storage_provider+# cogtdef get_required_llm_models_provider(self) -> LLMModelProviderAbstract:@@ -297,6 +311,10 @@ def get_secrets_provider() -> SecretsProviderAbstract:return get_pipelex_hub().get_required_secrets_provider()+def get_storage_provider() -> StorageProviderAbstract:+ return get_pipelex_hub().get_storage_provider()++def get_template_provider() -> TemplateProviderAbstract:return get_pipelex_hub().get_required_template_provider()@@ -391,7 +409,9 @@ def get_pipe_provider() -> PipeProviderAbstract:return get_pipelex_hub().get_required_pipe_provider()-def get_pipes_by_domain(excluded_domains: Optional[List[str]] = None) -> Dict[str, List[str]]:+def get_pipes_by_domain(+ excluded_domains: Optional[List[str]] = None,+) -> Dict[str, List[str]]:pipes = get_pipe_provider().get_pipes()pipes_by_domain: Dict[str, List[str]] = defaultdict(list)for pipe in pipes:diff --git a/pipelex/pipelex.py b/pipelex/pipelex.pyindex ea1e86e08..a6aff6f10 100644--- a/pipelex/pipelex.py+++ b/pipelex/pipelex.py@@ -11,7 +11,9 @@ from typing_extensions import Selffrom pipelex import logfrom pipelex.cogt.content_generation.content_generator import ContentGenerator-from pipelex.cogt.content_generation.content_generator_protocol import ContentGeneratorProtocol+from pipelex.cogt.content_generation.content_generator_protocol import (+ ContentGeneratorProtocol,+)from pipelex.cogt.inference.inference_manager import InferenceManagerfrom pipelex.cogt.llm.llm_models.llm_model import LATEST_VERSION_NAMEfrom pipelex.cogt.llm.llm_models.llm_model_library import LLMModelLibrary@@ -24,10 +26,16 @@ from pipelex.libraries.library_manager import LibraryManagerfrom pipelex.pipe_works.pipe_router import PipeRouterfrom pipelex.pipe_works.pipe_router_protocol import PipeRouterProtocolfrom pipelex.pipeline.activity.activity_manager import ActivityManager-from pipelex.pipeline.activity.activity_manager_protocol import ActivityManagerNoOp, ActivityManagerProtocol+from pipelex.pipeline.activity.activity_manager_protocol import (+ ActivityManagerNoOp,+ ActivityManagerProtocol,+)from pipelex.pipeline.pipeline_manager import PipelineManagerfrom pipelex.pipeline.track.pipeline_tracker import PipelineTracker-from pipelex.pipeline.track.pipeline_tracker_protocol import PipelineTrackerNoOp, PipelineTrackerProtocol+from pipelex.pipeline.track.pipeline_tracker_protocol import (+ PipelineTrackerNoOp,+ PipelineTrackerProtocol,+)from pipelex.reporting.reporting_manager import ReportingManagerfrom pipelex.reporting.reporting_protocol import ReportingNoOp, ReportingProtocolfrom pipelex.test_extras.registry_test_models import PipelexTestModels@@ -36,6 +44,7 @@ from pipelex.tools.func_registry import func_registryfrom pipelex.tools.runtime_manager import runtime_managerfrom pipelex.tools.secrets.env_secrets_provider import EnvSecretsProviderfrom pipelex.tools.secrets.secrets_provider_abstract import SecretsProviderAbstract+from pipelex.tools.storage.storage_provider_abstract import StorageProviderAbstractfrom pipelex.tools.templating.template_library import TemplateLibraryfrom pipelex.tools.typing.pydantic_utils import format_pydantic_validation_error@@ -172,10 +181,11 @@ class Pipelex:content_generator: Optional[ContentGeneratorProtocol] = None,pipe_router: Optional[PipeRouterProtocol] = None,structure_classes: Optional[List[Type[Any]]] = None,+ storage_provider: Optional[StorageProviderAbstract] = None,):# toolsself.pipelex_hub.set_secrets_provider(secrets_provider or EnvSecretsProvider())-+ self.pipelex_hub.set_storage_provider(storage_provider)# cogtself.pipelex_hub.set_content_generator(content_generator or ContentGenerator())self.reporting_delegate.setup()diff --git a/pipelex/tools/storage/__init__.py b/pipelex/tools/storage/__init__.pynew file mode 100644index 000000000..e69de29bbdiff --git a/pipelex/tools/storage/storage_provider_abstract.py b/pipelex/tools/storage/storage_provider_abstract.pynew file mode 100644index 000000000..78adb784f--- /dev/null+++ b/pipelex/tools/storage/storage_provider_abstract.py@@ -0,0 +1,11 @@+from abc import ABC, abstractmethod+++class StorageProviderAbstract(ABC):+ @abstractmethod+ def load(self, uri: str) -> bytes:+ pass++ @abstractmethod+ def store(self, data: bytes) -> str:+ pass