Skip to content

Build AI methods with your coding agent, then run them anywhere: as an MCP for chatbots, as a webapp for people, or via API for your software.

fix/Pipelex config fixed path(#151)

Thomas Hebrarda10c3d36f375
diff --git a/.github/workflows/manual-trigger-tests-check.yml b/.github/workflows/manual-trigger-tests-check.yml
new file mode 100644
index 000000000..36612c3f2
--- /dev/null
+++ b/.github/workflows/manual-trigger-tests-check.yml
@@ -0,0 +1,66 @@
+name: Manual trigger tests check
+
+on:
+ pull_request:
+ branches:
+ - dev
+ - "release/v[0-9]+.[0-9]+.[0-9]+"
+ workflow_dispatch:
+
+jobs:
+ trigger-tests:
+ runs-on: ubuntu-latest
+ environment:
+ name: manual-trigger-tests-check
+ url: ${{ github.server_url }}/${{ github.repository }}/deployments/activity_log?environment=manual-trigger-tests-check
+ steps:
+ - name: Initialize check
+ run: |
+ echo "Initializing manual trigger tests check..."
+
+ matrix-test:
+ name: Manual trigger tests check
+ needs: [trigger-tests]
+ runs-on: ubuntu-latest
+ strategy:
+ fail-fast: false
+ matrix:
+ python-version: ["3.10", "3.11", "3.12", "3.13"]
+ permissions:
+ contents: read
+ id-token: write
+ packages: read
+ actions: read
+ repository-projects: read
+ env:
+ VIRTUAL_ENV: ${{ github.workspace }}/.venv
+ ENV: dev
+
+ steps:
+ - uses: actions/checkout@v4
+
+ - name: Configure AWS Credentials
+ uses: aws-actions/configure-aws-credentials@v4
+ with:
+ role-to-assume: arn:aws:iam::975050286103:role/GitHubActionsRole
+ aws-region: eu-west-3
+
+ - name: Set up Python ${{ matrix.python-version }}
+ uses: actions/setup-python@v4
+ with:
+ python-version: ${{ matrix.python-version }}
+
+ - name: Check UV installation
+ run: make check-uv
+
+ - name: Verify UV installation
+ run: uv --version
+
+ - name: Install dependencies
+ run: PYTHON_VERSION=${{ matrix.python-version }} make install
+
+ - name: Boot test
+ run: make tp TEST=TestFundamentals
+
+ - name: Run tests
+ run: make run-manual-trigger-gha-tests
diff --git a/CHANGELOG.md b/CHANGELOG.md
index c3dd52bcb..253dde8b2 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,5 +1,14 @@
# Changelog
+## [v0.6.0] - 2025-07-12
+
+### Changed
+- The `Pipelex.make()` now takes a `relative_config_folder_path` argument, which is considered to be a relative path to the current working directory, pointing to the config folder path of pipelex. The argument `from_file` indicates if the path is relative to the caller's file path. If False, the path is considered to be absolute from where the process was started.
+
+### Added
+- Added github action for inference tests
+- `load_json_list_from_path` function in `pipelex.tools.misc.file_utils`: Loads a JSON file and ensures it contains a list.
+
## [v0.5.2] - 2025-07-11
- log a warning when dry running a `PipeFunc`
diff --git a/pipelex/cli/_cli.py b/pipelex/cli/_cli.py
index a424579a0..a304715cd 100644
--- a/pipelex/cli/_cli.py
+++ b/pipelex/cli/_cli.py
@@ -47,7 +47,7 @@ def init_libraries(
"""
try:
# TODO: Have a more proper print message regarding the overwrited files (e.g. list of files that were overwritten or not)
- LibraryConfig.export_libraries(overwrite=overwrite)
+ LibraryConfig().export_libraries(overwrite=overwrite)
if overwrite:
typer.echo("Successfully initialized pipelex libraries (all files overwritten)")
else:
@@ -76,10 +76,15 @@ def init_config(
@app.command()
-def validate() -> None:
+def validate(
+ relative_config_folder_path: Annotated[
+ str, typer.Option("--config-folder-path", "-c", help="Relative path to the config folder path")
+ ] = "pipelex_libraries",
+) -> None:
"""Run the setup sequence."""
- LibraryConfig.export_libraries()
- pipelex_instance = Pipelex.make()
+ config_folder_path = os.path.join(os.getcwd(), relative_config_folder_path)
+ LibraryConfig(config_folder_path=config_folder_path).export_libraries()
+ pipelex_instance = Pipelex.make(relative_config_folder_path=relative_config_folder_path, from_file=False)
pipelex_instance.validate_libraries()
asyncio.run(dry_run_all_pipes())
log.info("Setup sequence passed OK, config and pipelines are validated.")
@@ -96,9 +101,13 @@ def show_config() -> None:
@app.command()
-def list_pipes() -> None:
+def list_pipes(
+ relative_config_folder_path: Annotated[
+ str, typer.Option("--config-folder-path", "-c", help="Relative path to the config folder path")
+ ] = "pipelex_libraries",
+) -> None:
"""List all available pipes."""
- Pipelex.make()
+ Pipelex.make(relative_config_folder_path=relative_config_folder_path, from_file=False)
try:
get_pipe_provider().pretty_list_pipes()
diff --git a/pipelex/cogt/llm/llm_models/llm_model_library.py b/pipelex/cogt/llm/llm_models/llm_model_library.py
index cefefcc78..7b1b1ed36 100644
--- a/pipelex/cogt/llm/llm_models/llm_model_library.py
+++ b/pipelex/cogt/llm/llm_models/llm_model_library.py
@@ -1,5 +1,5 @@
import os
-from typing import Any, Dict, List, Optional
+from typing import Any, ClassVar, Dict, List, Optional
from pydantic import Field, RootModel
from typing_extensions import override
@@ -32,6 +32,12 @@ LLMModelLibraryRoot = List[LLMModel]
class LLMModelLibrary(LLMModelProviderAbstract, RootModel[LLMModelLibraryRoot]):
root: LLMModelLibraryRoot = Field(default_factory=list)
+ library_config: ClassVar[LibraryConfig]
+
+ @classmethod
+ def make_empty(cls, config_folder_path: str) -> "LLMModelLibrary":
+ cls.library_config = LibraryConfig(config_folder_path=config_folder_path)
+ return cls()
@override
def setup(self):
@@ -63,7 +69,7 @@ class LLMModelLibrary(LLMModelProviderAbstract, RootModel[LLMModelLibraryRoot]):
@classmethod
def load_llm_model_library_dict(cls) -> LLMModelLibraryDict:
- libraries_path = LibraryConfig.exported_llm_integrations_path
+ libraries_path = cls.library_config.llm_integrations_path
if not os.path.exists(libraries_path):
raise LLMModelLibraryError(f"LLM model library path `{libraries_path}` not found. Please run `pipelex init-libraries` to create it.")
llm_library: LLMModelLibraryDict = {}
diff --git a/pipelex/libraries/library_config.py b/pipelex/libraries/library_config.py
index e6c718786..4af9468e1 100644
--- a/pipelex/libraries/library_config.py
+++ b/pipelex/libraries/library_config.py
@@ -3,99 +3,117 @@ from typing import ClassVar, List
from pipelex.tools.config.models import ConfigModel
from pipelex.tools.misc.file_utils import copy_file_from_package, copy_folder_from_package, find_files_in_dir
+PIPELEX_LIBRARIES_PATH = "libraries"
+
class LibraryConfig(ConfigModel):
- # Class variables
package_name: ClassVar[str] = "pipelex"
- internal_library_root: ClassVar[str] = "libraries"
- exported_library_root: ClassVar[str] = "pipelex_libraries"
- internal_base_pipelines_path: ClassVar[str] = f"{internal_library_root}/pipelines"
- exported_pipelines_path: ClassVar[str] = f"{exported_library_root}/pipelines"
- exported_base_pipelines_path: ClassVar[str] = f"{exported_library_root}/pipelines/base_library"
- loaded_pipelines_path: ClassVar[str] = f"{exported_library_root}/pipelines"
- test_pipelines_path: ClassVar[str] = "tests/test_pipelines"
- internal_llm_integrations_path: ClassVar[str] = f"{internal_library_root}/llm_integrations"
- exported_llm_integrations_path: ClassVar[str] = f"{exported_library_root}/llm_integrations"
- internal_llm_deck_path: ClassVar[str] = f"{internal_library_root}/llm_deck"
- exported_llm_deck_path: ClassVar[str] = f"{exported_library_root}/llm_deck"
- internal_templates_path: ClassVar[str] = f"{internal_library_root}/templates"
- exported_templates_path: ClassVar[str] = f"{exported_library_root}/templates"
- internal_plugins_path: ClassVar[str] = f"{internal_library_root}/plugins"
- exported_plugins_path: ClassVar[str] = f"{exported_library_root}/plugins"
- failing_pipelines_path: ClassVar[str] = "tests/test_pipelines/failing_pipelines.toml"
-
- @classmethod
- def get_llm_deck_paths(cls) -> List[str]:
- llm_deck_paths = [str(path) for path in find_files_in_dir(dir_path=cls.exported_llm_deck_path, pattern="*.toml", is_recursive=True)]
+ config_folder_path: str = "pipelex_libraries"
+
+ @property
+ def config_file(self) -> str:
+ return f"{self.config_folder_path}/config.toml"
+
+ @property
+ def pipelines_path(self) -> str:
+ return f"{self.config_folder_path}/pipelines"
+
+ @property
+ def base_pipelines_path(self) -> str:
+ return f"{self.config_folder_path}/pipelines/base_library"
+
+ @property
+ def llm_integrations_path(self) -> str:
+ return f"{self.config_folder_path}/llm_integrations"
+
+ @property
+ def llm_deck_path(self) -> str:
+ return f"{self.config_folder_path}/llm_deck"
+
+ @property
+ def templates_path(self) -> str:
+ return f"{self.config_folder_path}/templates"
+
+ @property
+ def plugins_path(self) -> str:
+ return f"{self.config_folder_path}/plugins"
+
+ @property
+ def test_pipelines_path(self) -> str:
+ return "tests/test_pipelines"
+
+ @property
+ def failing_pipelines_path(self) -> str:
+ return "tests/test_pipelines/failing_pipelines.toml"
+
+ def get_llm_deck_paths(self) -> List[str]:
+ llm_deck_paths = [str(path) for path in find_files_in_dir(dir_path=self.llm_deck_path, pattern="*.toml", is_recursive=True)]
llm_deck_paths.sort()
return llm_deck_paths
- @classmethod
- def get_templates_paths(cls) -> List[str]:
- return [str(path) for path in find_files_in_dir(dir_path=cls.exported_templates_path, pattern="*.toml", is_recursive=True)]
+ def get_templates_paths(self) -> List[str]:
+ return [str(path) for path in find_files_in_dir(dir_path=self.templates_path, pattern="*.toml", is_recursive=True)]
- @classmethod
- def get_plugin_config_path(cls) -> str:
- return f"{cls.exported_plugins_path}/plugin_config.toml"
+ def get_default_plugin_config_path(self) -> str:
+ """Get the default plugin config path."""
+ return f"{self.plugins_path}/plugin_config.toml"
- @classmethod
- def export_libraries(cls, overwrite: bool = False) -> None:
+ def export_libraries(self, overwrite: bool = False) -> None:
"""Duplicate pipelex libraries files in the client project, preserving directory structure."""
# pipelines
copy_folder_from_package(
- package_name=cls.package_name,
- folder_path_in_package=cls.internal_base_pipelines_path,
- target_dir=cls.exported_base_pipelines_path,
+ package_name=self.package_name,
+ folder_path_in_package=f"{PIPELEX_LIBRARIES_PATH}/pipelines",
+ target_dir=self.base_pipelines_path,
overwrite=overwrite,
)
copy_file_from_package(
- package_name=cls.package_name,
- file_path_in_package=f"{cls.internal_library_root}/__init__.py",
- target_path=f"{cls.exported_library_root}/__init__.py",
+ package_name=self.package_name,
+ file_path_in_package=f"{PIPELEX_LIBRARIES_PATH}/__init__.py",
+ target_path=f"{self.config_folder_path}/__init__.py",
overwrite=overwrite,
)
copy_file_from_package(
- package_name=cls.package_name,
- file_path_in_package=f"{cls.internal_base_pipelines_path}/__init__.py",
- target_path=f"{cls.exported_pipelines_path}/__init__.py",
+ package_name=self.package_name,
+ file_path_in_package=f"{PIPELEX_LIBRARIES_PATH}/pipelines/__init__.py",
+ target_path=f"{self.pipelines_path}/__init__.py",
overwrite=overwrite,
)
copy_file_from_package(
- package_name=cls.package_name,
- file_path_in_package=f"{cls.internal_base_pipelines_path}/__init__.py",
- target_path=f"{cls.exported_base_pipelines_path}/__init__.py",
+ package_name=self.package_name,
+ file_path_in_package=f"{PIPELEX_LIBRARIES_PATH}/pipelines/__init__.py",
+ target_path=f"{self.base_pipelines_path}/__init__.py",
overwrite=overwrite,
)
-
# llm_integrations
copy_folder_from_package(
- package_name=cls.package_name,
- folder_path_in_package=cls.internal_llm_integrations_path,
- target_dir=cls.exported_llm_integrations_path,
+ package_name=self.package_name,
+ folder_path_in_package=f"{PIPELEX_LIBRARIES_PATH}/llm_integrations",
+ target_dir=self.llm_integrations_path,
overwrite=overwrite,
)
# llm_deck
copy_folder_from_package(
- package_name=cls.package_name,
- folder_path_in_package=cls.internal_llm_deck_path,
- target_dir=cls.exported_llm_deck_path,
+ package_name=self.package_name,
+ folder_path_in_package=f"{PIPELEX_LIBRARIES_PATH}/llm_deck",
+ target_dir=self.llm_deck_path,
overwrite=overwrite,
non_overwrite_files=["overrides.toml"],
)
# templates
copy_folder_from_package(
- package_name=cls.package_name,
- folder_path_in_package=cls.internal_templates_path,
- target_dir=cls.exported_templates_path,
+ package_name=self.package_name,
+ folder_path_in_package=f"{PIPELEX_LIBRARIES_PATH}/templates",
+ target_dir=self.templates_path,
overwrite=overwrite,
)
# plugins
copy_folder_from_package(
- package_name=cls.package_name,
- folder_path_in_package=cls.internal_plugins_path,
- target_dir=cls.exported_plugins_path,
+ package_name=self.package_name,
+ folder_path_in_package=f"{PIPELEX_LIBRARIES_PATH}/plugins",
+ target_dir=self.plugins_path,
overwrite=overwrite,
)
diff --git a/pipelex/libraries/library_manager.py b/pipelex/libraries/library_manager.py
index bd8d9340e..2f654e57b 100644
--- a/pipelex/libraries/library_manager.py
+++ b/pipelex/libraries/library_manager.py
@@ -61,20 +61,33 @@ class LibraryManager(LibraryManagerAbstract):
"system_prompt_to_structure",
"prompt_template_to_structure",
]
+ domain_library: DomainLibrary
+ concept_library: ConceptLibrary
+ pipe_library: PipeLibrary
+ llm_deck: Optional[LLMDeck] = None
+ library_config: ClassVar[LibraryConfig]
@classmethod
- def make_empty(cls) -> "LibraryManager":
- domain_library = DomainLibrary.make_empty()
- concept_library = ConceptLibrary.make_empty()
- pipe_library = PipeLibrary.make_empty()
- return cls(domain_library=domain_library, concept_library=concept_library, pipe_library=pipe_library)
-
- def __init__(self, domain_library: DomainLibrary, concept_library: ConceptLibrary, pipe_library: PipeLibrary) -> None:
- # TODO : avoid having an Option LLMDeck: regroup with model provider
- self.llm_deck: Optional[LLMDeck] = None
- self.domain_library = domain_library
- self.concept_library = concept_library
- self.pipe_library = pipe_library
+ def make_empty(cls, config_folder_path: str) -> "LibraryManager":
+ cls.domain_library = DomainLibrary.make_empty()
+ cls.concept_library = ConceptLibrary.make_empty()
+ cls.pipe_library = PipeLibrary.make_empty()
+ cls.library_config = LibraryConfig(config_folder_path=config_folder_path)
+ return cls()
+
+ @classmethod
+ def make(
+ cls, domain_library: DomainLibrary, concept_library: ConceptLibrary, pipe_library: PipeLibrary, config_folder_path: str
+ ) -> "LibraryManager":
+ cls.domain_library = domain_library
+ cls.concept_library = concept_library
+ cls.pipe_library = pipe_library
+ cls.library_config = LibraryConfig(config_folder_path=config_folder_path)
+ return cls()
+
+ @override
+ def get_plugin_config_path(self) -> str:
+ return self.library_config.get_default_plugin_config_path()
@override
def setup(self) -> None:
@@ -88,10 +101,10 @@ class LibraryManager(LibraryManagerAbstract):
self.domain_library.teardown()
def libraries_paths(self) -> List[str]:
- library_paths = [LibraryConfig.loaded_pipelines_path]
+ library_paths = [self.library_config.pipelines_path]
if runtime_manager.is_unit_testing:
log.debug("Registering test pipeline structures for unit testing")
- library_paths += [LibraryConfig.test_pipelines_path]
+ library_paths += [self.library_config.test_pipelines_path]
return library_paths
def load_failure_modes(self):
@@ -117,7 +130,7 @@ class LibraryManager(LibraryManagerAbstract):
self.load_combo_libraries(library_paths=toml_file_paths)
def load_deck(self) -> LLMDeck:
- llm_deck_paths = LibraryConfig.get_llm_deck_paths()
+ llm_deck_paths = self.library_config.get_llm_deck_paths()
full_llm_deck_dict: Dict[str, Any] = {}
if not llm_deck_paths:
raise LLMDeckNotFoundError("No LLM deck paths found. Please run `pipelex init-libraries` to create it.")
@@ -313,7 +326,7 @@ class LibraryManager(LibraryManagerAbstract):
"""Validate all TOML files used by the library manager for formatting issues."""
log.debug("LibraryManager validating TOML file formatting")
- llm_deck_paths = LibraryConfig.get_llm_deck_paths()
+ llm_deck_paths = self.library_config.get_llm_deck_paths()
for llm_deck_path in llm_deck_paths:
if os.path.exists(llm_deck_path):
try:
@@ -341,7 +354,7 @@ class LibraryManager(LibraryManagerAbstract):
log.error(f"TOML formatting issues in library file '{toml_path}': {exc}")
raise LibraryError(f"TOML validation failed for library file '{toml_path}': {exc}") from exc
- template_paths = LibraryConfig.get_templates_paths()
+ template_paths = self.library_config.get_templates_paths()
for template_path in template_paths:
if os.path.exists(template_path):
try:
diff --git a/pipelex/libraries/library_manager_abstract.py b/pipelex/libraries/library_manager_abstract.py
index 69d05cd00..67f1c0047 100644
--- a/pipelex/libraries/library_manager_abstract.py
+++ b/pipelex/libraries/library_manager_abstract.py
@@ -15,3 +15,7 @@ class LibraryManagerAbstract(ABC):
@abstractmethod
def load_combo_libraries(self, library_paths: List[Path]):
pass
+
+ @abstractmethod
+ def get_plugin_config_path(self) -> str:
+ pass
diff --git a/pipelex/libraries/llm_deck/base_llm_deck.toml b/pipelex/libraries/llm_deck/base_llm_deck.toml
index 64b2d2545..8a61500b0 100644
--- a/pipelex/libraries/llm_deck/base_llm_deck.toml
+++ b/pipelex/libraries/llm_deck/base_llm_deck.toml
@@ -77,3 +77,4 @@ llm_to_extract_tables = { llm_handle = "best-claude", temperature = 0.1 }
[llm_choice_defaults]
for_text = "cheap_llm_for_text"
for_object = "cheap_llm_for_object"
+
diff --git a/pipelex/libraries/llm_deck/overrides.toml b/pipelex/libraries/llm_deck/overrides.toml
index 76598a632..999cc4d1a 100644
--- a/pipelex/libraries/llm_deck/overrides.toml
+++ b/pipelex/libraries/llm_deck/overrides.toml
@@ -7,3 +7,4 @@
[llm_choice_overrides]
for_text = "disabled"
for_object = "disabled"
+
diff --git a/pipelex/libraries/llm_integrations/anthropic.toml b/pipelex/libraries/llm_integrations/anthropic.toml
index 788b72a72..510f2a212 100644
--- a/pipelex/libraries/llm_integrations/anthropic.toml
+++ b/pipelex/libraries/llm_integrations/anthropic.toml
@@ -62,3 +62,4 @@ max_prompt_images = 100
cost_per_million_tokens_usd = { input = 3.0, output = 15.0 }
platform_llm_id = { anthropic = "claude-opus-4-20250514", bedrock_anthropic = "us.anthropic.claude-opus-4-20250514-v1:0" }
default_platform = "anthropic"
+
diff --git a/pipelex/libraries/llm_integrations/bedrock.toml b/pipelex/libraries/llm_integrations/bedrock.toml
index c61e77192..bf4404370 100644
--- a/pipelex/libraries/llm_integrations/bedrock.toml
+++ b/pipelex/libraries/llm_integrations/bedrock.toml
@@ -28,3 +28,4 @@ is_gen_object_supported = false
# TODO: find out the actual cost per million tokens for nova on bedrock
cost_per_million_tokens_usd = { input = 3.0, output = 15.0 }
platform_llm_id = { bedrock = "us.amazon.nova-pro-v1:0" }
+
diff --git a/pipelex/libraries/llm_integrations/custom.toml b/pipelex/libraries/llm_integrations/custom.toml
index a551366be..ecf6d747f 100644
--- a/pipelex/libraries/llm_integrations/custom.toml
+++ b/pipelex/libraries/llm_integrations/custom.toml
@@ -27,3 +27,4 @@ is_vision_supported = false
cost_per_million_tokens_usd = { input = 0, output = 0 }
platform_llm_id = { custom_llm = "qwen3:8b" }
# TODO: support <think> tokens
+
diff --git a/pipelex/libraries/llm_integrations/mistral.toml b/pipelex/libraries/llm_integrations/mistral.toml
index bcd797b3e..c99cdd8bc 100644
--- a/pipelex/libraries/llm_integrations/mistral.toml
+++ b/pipelex/libraries/llm_integrations/mistral.toml
@@ -67,3 +67,4 @@ is_gen_object_supported = true
is_vision_supported = true
cost_per_million_tokens_usd = { input = 2.0, output = 6.0 }
platform_llm_id = { mistral = "pixtral-large-latest" }
+
diff --git a/pipelex/libraries/llm_integrations/perplexity.toml b/pipelex/libraries/llm_integrations/perplexity.toml
index a72dc5946..fb681f7bf 100644
--- a/pipelex/libraries/llm_integrations/perplexity.toml
+++ b/pipelex/libraries/llm_integrations/perplexity.toml
@@ -23,3 +23,4 @@ platform_llm_id = { perplexity = "sonar-reasoning" }
[perplexity-deepseek.perplexity-deepseek-r1.latest]
is_gen_object_supported = false
platform_llm_id = { perplexity = "r1-1776" }
+
diff --git a/pipelex/libraries/llm_integrations/vertexai.toml b/pipelex/libraries/llm_integrations/vertexai.toml
index 2b28ec505..62ca4f32f 100644
--- a/pipelex/libraries/llm_integrations/vertexai.toml
+++ b/pipelex/libraries/llm_integrations/vertexai.toml
@@ -41,3 +41,4 @@ is_vision_supported = true
max_prompt_images = 3000
cost_per_million_tokens_usd = { input = 0.15, output = 0.6 }
platform_llm_id = { vertexai = "google/gemini-2.5-flash-preview-05-20" }
+
diff --git a/pipelex/libraries/llm_integrations/xai.toml b/pipelex/libraries/llm_integrations/xai.toml
index 897b3c6a1..31393cd93 100644
--- a/pipelex/libraries/llm_integrations/xai.toml
+++ b/pipelex/libraries/llm_integrations/xai.toml
@@ -23,3 +23,4 @@ is_gen_object_supported = true
is_vision_supported = false
cost_per_million_tokens_usd = { input = 0.15, output = 4 }
platform_llm_id = { xai = "grok-3-mini-fast-latest" }
+
diff --git a/pipelex/libraries/pipelines/documents.toml b/pipelex/libraries/pipelines/documents.toml
index a3d88b110..cc52a4c66 100644
--- a/pipelex/libraries/pipelines/documents.toml
+++ b/pipelex/libraries/pipelines/documents.toml
@@ -23,3 +23,4 @@ inputs = { ocr_input = "PDF" }
output = "Page"
page_images = true
page_views = true
+
diff --git a/pipelex/libraries/pipelines/images.toml b/pipelex/libraries/pipelines/images.toml
index b2391fdbf..073bbb575 100644
--- a/pipelex/libraries/pipelines/images.toml
+++ b/pipelex/libraries/pipelines/images.toml
@@ -59,3 +59,4 @@ PipeImgGen = "Generate a photo"
inputs = { prompt = "ImgGenPrompt" }
output = "images.Photo"
nb_steps = 8
+
diff --git a/pipelex/libraries/plugins/plugin_config.toml b/pipelex/libraries/plugins/plugin_config.toml
index e3c86138b..30cde64c3 100644
--- a/pipelex/libraries/plugins/plugin_config.toml
+++ b/pipelex/libraries/plugins/plugin_config.toml
@@ -37,3 +37,4 @@ api_key_method = "env"
[fal_config]
flux_map_quality_to_steps = { "low" = 14, "medium" = 28, "high" = 56 }
sdxl_lightning_map_quality_to_steps = { "low" = 2, "medium" = 4, "high" = 8 }
+
diff --git a/pipelex/libraries/templates/base_templates.toml b/pipelex/libraries/templates/base_templates.toml
index 7a8da104b..fe3026d04 100644
--- a/pipelex/libraries/templates/base_templates.toml
+++ b/pipelex/libraries/templates/base_templates.toml
@@ -15,3 +15,4 @@ Do not create information that is not in the text.
[test_prompts]
jinja2_test_template = "I want a {{ place_holder }} t-shirt."
+
diff --git a/pipelex/pipelex.py b/pipelex/pipelex.py
index 109e5b726..5293a7c51 100644
--- a/pipelex/pipelex.py
+++ b/pipelex/pipelex.py
@@ -1,3 +1,5 @@
+import inspect
+import os
from importlib.metadata import metadata
from typing import Any, List, Optional, Type, cast
@@ -59,9 +61,10 @@ PACKAGE_VERSION = metadata(PACKAGE_NAME)["Version"]
class Pipelex(metaclass=MetaSingleton):
def __init__(
self,
+ config_folder_path: str,
+ # Dependency injection
pipelex_hub: Optional[PipelexHub] = None,
config_cls: Optional[Type[ConfigRoot]] = None,
- ready_made_config: Optional[ConfigRoot] = None,
class_registry: Optional[ClassRegistryAbstract] = None,
template_provider: Optional[TemplateLibrary] = None,
llm_model_provider: Optional[LLMModelLibrary] = None,
@@ -71,24 +74,20 @@ class Pipelex(metaclass=MetaSingleton):
activity_manager: Optional[ActivityManagerProtocol] = None,
reporting_delegate: Optional[ReportingProtocol] = None,
) -> None:
+ self.config_folder_path = config_folder_path
self.pipelex_hub = pipelex_hub or PipelexHub()
set_pipelex_hub(self.pipelex_hub)
# tools
- if ready_made_config is not None:
- if config_cls is not None:
- raise PipelexConfigError("config_cls must be None when ready_made_config is provided")
- self.pipelex_hub.set_config(ready_made_config)
- else:
- if config_cls is None:
- config_cls = PipelexConfig
- try:
- self.pipelex_hub.setup_config(config_cls=config_cls)
- except ValidationError as exc:
- error_msg = format_pydantic_validation_error(exc)
- raise PipelexConfigError(f"Error because of: {error_msg}") from exc
+ try:
+ self.pipelex_hub.setup_config(config_cls=config_cls or PipelexConfig)
+ except ValidationError as exc:
+ error_msg = format_pydantic_validation_error(exc)
+ raise PipelexConfigError(f"Error because of: {error_msg}") from exc
+
for extra_env_file in get_config().pipelex.extra_env_files:
load_dotenv(dotenv_path=extra_env_file, override=True)
+
log.configure(
project_name=get_config().project_name or "unknown_project",
log_config=get_config().pipelex.log_config,
@@ -96,8 +95,9 @@ class Pipelex(metaclass=MetaSingleton):
log.debug("Logs are configured")
# tools
- self.template_provider = template_provider or TemplateLibrary()
+ self.template_provider = template_provider or TemplateLibrary.make_empty(config_folder_path=config_folder_path)
self.pipelex_hub.set_template_provider(self.template_provider)
+
self.class_registry = class_registry or ClassRegistry()
self.pipelex_hub.set_class_registry(self.class_registry)
self.kajson_manager = KajsonManager(class_registry=self.class_registry)
@@ -105,7 +105,7 @@ class Pipelex(metaclass=MetaSingleton):
# cogt
self.plugin_manager = PluginManager()
self.pipelex_hub.set_plugin_manager(self.plugin_manager)
- self.llm_model_provider = llm_model_provider or LLMModelLibrary()
+ self.llm_model_provider = llm_model_provider or LLMModelLibrary.make_empty(config_folder_path=config_folder_path)
self.pipelex_hub.set_llm_models_provider(self.llm_model_provider)
self.inference_manager = inference_manager or InferenceManager()
self.pipelex_hub.set_inference_manager(self.inference_manager)
@@ -124,10 +124,11 @@ class Pipelex(metaclass=MetaSingleton):
self.pipelex_hub.set_domain_provider(domain_provider=domain_library)
self.pipelex_hub.set_concept_provider(concept_provider=concept_library)
self.pipelex_hub.set_pipe_provider(pipe_provider=pipe_library)
- self.library_manager = LibraryManager(
+ self.library_manager = LibraryManager.make(
domain_library=domain_library,
concept_library=concept_library,
pipe_library=pipe_library,
+ config_folder_path=config_folder_path,
)
self.library_manager.setup()
self.pipelex_hub.set_library_manager(library_manager=self.library_manager)
@@ -167,7 +168,7 @@ class Pipelex(metaclass=MetaSingleton):
self.pipelex_hub.set_secrets_provider(secrets_provider or EnvSecretsProvider())
self.pipelex_hub.set_storage_provider(storage_provider)
# cogt
- self.plugin_manager.setup()
+ self.plugin_manager.setup(library_config=self.library_manager.library_config)
self.pipelex_hub.set_content_generator(content_generator or ContentGenerator())
self.reporting_delegate.setup()
self.class_registry.register_classes(PipelexRegistryModels.get_all_models())
@@ -187,7 +188,7 @@ class Pipelex(metaclass=MetaSingleton):
log.debug(f"{PACKAGE_NAME} version {PACKAGE_VERSION} setup done for {get_config().project_name}")
- def setup_libraries(self):
+ def setup_libraries(self, relative_config_folder_path: str):
try:
self.template_provider.setup()
self.llm_model_provider.setup()
@@ -241,10 +242,22 @@ class Pipelex(metaclass=MetaSingleton):
# TODO: add kwargs to make() so that subclasses can employ specific parameters
@classmethod
- def make(cls, structure_classes: Optional[List[Type[Any]]] = None) -> Self:
- pipelex_instance = cls()
- pipelex_instance.setup(structure_classes=structure_classes)
- pipelex_instance.setup_libraries()
+ def make(cls, relative_config_folder_path: str, from_file: Optional[bool] = True) -> Self:
+ if from_file: # from_file means it's going to merge the relative_config_folder_path with the caller's file path
+ current_frame = inspect.currentframe()
+ if current_frame is None:
+ raise RuntimeError("Failed to get current frame")
+ if current_frame.f_back is None:
+ raise RuntimeError("Failed to get caller frame")
+ caller_file = current_frame.f_back.f_code.co_filename
+ caller_dir = os.path.dirname(os.path.abspath(caller_file))
+ config_folder_path = os.path.abspath(os.path.join(caller_dir, relative_config_folder_path))
+ else:
+ config_folder_path = os.path.abspath(os.path.join(os.getcwd(), relative_config_folder_path))
+
+ pipelex_instance = cls(config_folder_path=config_folder_path)
+ pipelex_instance.setup()
+ pipelex_instance.setup_libraries(relative_config_folder_path=relative_config_folder_path)
log.info(f"Pipelex {PACKAGE_VERSION} initialized.")
return pipelex_instance
diff --git a/pipelex/plugins/plugin_manager.py b/pipelex/plugins/plugin_manager.py
index 9c14cb73d..90b0adfdf 100644
--- a/pipelex/plugins/plugin_manager.py
+++ b/pipelex/plugins/plugin_manager.py
@@ -17,8 +17,8 @@ class PluginManager:
raise RuntimeError("Plugin configs not loaded")
return self._plugin_configs
- def setup(self):
- plugin_config_path = LibraryConfig.get_plugin_config_path()
+ def setup(self, library_config: LibraryConfig):
+ plugin_config_path = library_config.get_default_plugin_config_path()
plugin_config_dict = load_toml_from_path(path=plugin_config_path)
self._plugin_configs = PluginConfig.model_validate(plugin_config_dict)
diff --git a/pipelex/test_extras/shared_pytest_plugins.py b/pipelex/test_extras/shared_pytest_plugins.py
index db30a233d..113654024 100644
--- a/pipelex/test_extras/shared_pytest_plugins.py
+++ b/pipelex/test_extras/shared_pytest_plugins.py
@@ -13,7 +13,7 @@ def set_run_mode():
@pytest.fixture(scope="session")
def manage_pipelex_libraries():
- LibraryConfig.export_libraries(overwrite=False)
+ LibraryConfig(config_folder_path="./pipelex_libraries").export_libraries(overwrite=False)
yield
# TODO: make it safe to erase/replace standard libraries in client projects without touching custom stuff
# LibraryConfig.remove_libraries()
@@ -21,7 +21,7 @@ def manage_pipelex_libraries():
@pytest.fixture(scope="session")
def manage_pipelex_libraries_with_overwrite():
- LibraryConfig.export_libraries(overwrite=True)
+ LibraryConfig(config_folder_path="./pipelex_libraries").export_libraries(overwrite=True)
yield
# TODO: make it safe to erase/replace standard libraries in client projects without touching custom stuff
# LibraryConfig.remove_libraries()
diff --git a/pipelex/tools/misc/json_utils.py b/pipelex/tools/misc/json_utils.py
index 6b4247bac..2c3fb5fd6 100644
--- a/pipelex/tools/misc/json_utils.py
+++ b/pipelex/tools/misc/json_utils.py
@@ -128,6 +128,31 @@ def load_json_dict_from_path(path: str) -> Dict[Any, Any]:
raise JsonTypeError(f"{path} is not a dict")
+def load_json_list_from_path(path: str) -> List[Any]:
+ """
+ Loads a JSON file and ensures it contains a list.
+
+ This function reads a JSON file and verifies that its content is a list.
+ It uses load_json_from_path internally and adds type checking.
+
+ Args:
+ path (str): The file path to the JSON file to be loaded.
+
+ Returns:
+ List[Any]: The parsed JSON content as a Python list.
+
+ Raises:
+ JsonTypeError: If the JSON content is not a list.
+ FileNotFoundError: If the file does not exist.
+ json.JSONDecodeError: If the file contains invalid JSON.
+ """
+ json_content: JsonContent = load_json_from_path(path)
+ if isinstance(json_content, list):
+ return json_content
+ else:
+ raise JsonTypeError(f"{path} is not a list")
+
+
def deep_update(target_dict: Dict[str, Any], updates: Dict[str, Any]):
"""
Recursively updates a dictionary with values from another dictionary.
diff --git a/pipelex/tools/templating/template_library.py b/pipelex/tools/templating/template_library.py
index 8f64a73c3..e94006862 100644
--- a/pipelex/tools/templating/template_library.py
+++ b/pipelex/tools/templating/template_library.py
@@ -1,4 +1,4 @@
-from typing import Any, Dict
+from typing import Any, ClassVar, Dict
from jinja2 import TemplateSyntaxError
from pydantic import Field, RootModel, ValidationError
@@ -23,10 +23,16 @@ class TemplateLibraryError(ToolException):
class TemplateLibrary(TemplateProviderAbstract, RootModel[TemplateLibraryRoot]):
root: TemplateLibraryRoot = Field(default_factory=dict)
+ library_config: ClassVar[LibraryConfig]
+
+ @classmethod
+ def make_empty(cls, config_folder_path: str) -> "TemplateLibrary":
+ cls.library_config = LibraryConfig(config_folder_path=config_folder_path)
+ return cls()
@override
def setup(self) -> None:
- template_toml_paths = LibraryConfig.get_templates_paths()
+ template_toml_paths = self.library_config.get_templates_paths()
for template_toml_path in template_toml_paths:
self._load_from_toml(toml_path=template_toml_path)
self.validate_templates(template_category=Jinja2TemplateCategory.LLM_PROMPT)
diff --git a/pipelex/tools/typing/module_inspector.py b/pipelex/tools/typing/module_inspector.py
index 7805bb98c..bbe6dac70 100644
--- a/pipelex/tools/typing/module_inspector.py
+++ b/pipelex/tools/typing/module_inspector.py
@@ -27,14 +27,44 @@ def import_module_from_file(file_path: str) -> Any:
if not file_path.endswith(".py"):
raise ModuleFileError(f"File {file_path} is not a Python file (must end with .py)")
- module_name = file_path[:-3].replace(os.sep, ".")
- module = importlib.import_module(module_name)
+ # Convert file path to module-style path to use as the actual module name
+ module_name = _convert_file_path_to_module_path(file_path)
+
+ # Check if module is already loaded to avoid duplicate loading
+ if module_name in sys.modules:
+ return sys.modules[module_name]
+
+ # Use importlib.util to load the module from file path
+ spec = importlib.util.spec_from_file_location(module_name, file_path)
+ if spec is None or spec.loader is None:
+ raise ModuleFileError(f"Could not load module from {file_path}")
+
+ module = importlib.util.module_from_spec(spec)
# Add the module to sys.modules to ensure proper imports within the module
sys.modules[module_name] = module
+
+ # Execute the module
+ spec.loader.exec_module(module)
+
return module
+def _convert_file_path_to_module_path(file_path: str) -> str:
+ """Convert a file path to a module-style path."""
+ # Remove .py extension
+ module_path = file_path[:-3] if file_path.endswith(".py") else file_path
+
+ # Replace path separators with dots
+ module_path = module_path.replace(os.sep, ".")
+
+ # Handle __init__.py files by removing the __init__ part
+ if module_path.endswith(".__init__"):
+ module_path = module_path[:-9]
+
+ return module_path
+
+
def find_classes_in_module(
module: Any,
base_class: Optional[Type[Any]],
diff --git a/pyproject.toml b/pyproject.toml
index fceefc715..7b157939b 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -1,6 +1,6 @@
[project]
name = "pipelex"
-version = "0.5.2"
+version = "0.6.0"
description = "Pipelex is an open-source dev tool based on a simple declarative language that lets you define replicable, structured, composable LLM pipelines."
authors = [{ name = "Evotis S.A.S.", email = "[email protected]" }]
maintainers = [{ name = "Pipelex staff", email = "[email protected]" }]
diff --git a/tests/conftest.py b/tests/conftest.py
index 35d6afc56..56656e634 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -25,7 +25,7 @@ def reset_pipelex_config_fixture():
# Code to run before each test
print("[magenta]pipelex setup[/magenta]")
try:
- pipelex_instance = pipelex.pipelex.Pipelex.make()
+ pipelex_instance = pipelex.pipelex.Pipelex.make(relative_config_folder_path="../pipelex/libraries")
config = get_config()
log.verbose(config, title="Test config")
assert isinstance(config, pipelex.config.PipelexConfig)
diff --git a/tests/integration/pipelex/test_libraries.py b/tests/integration/pipelex/test_libraries.py
index 2be8c9c29..428383e0c 100644
--- a/tests/integration/pipelex/test_libraries.py
+++ b/tests/integration/pipelex/test_libraries.py
@@ -90,7 +90,7 @@ class TestLibraries:
known_concept: str,
known_pipe: str,
):
- library_manager = LibraryManager.make_empty()
+ library_manager = LibraryManager.make_empty(config_folder_path="pipelex_libraries")
test_pipelines_path = get_config().pipelex.library_config.test_pipelines_path
test_library_paths = library_manager.list_toml_files_from_path(library_paths=[test_pipelines_path])
library_manager.load_combo_libraries(library_paths=test_library_paths)
diff --git a/tests/unit/pipelex/tools/typing/test_module_inspector.py b/tests/unit/pipelex/tools/typing/test_module_inspector.py
index 346fed801..686affe16 100644
--- a/tests/unit/pipelex/tools/typing/test_module_inspector.py
+++ b/tests/unit/pipelex/tools/typing/test_module_inspector.py
@@ -69,7 +69,7 @@ class TestClass:
def test_import_nonexistent_file_raises_error(self, tmp_path: Path):
"""Test that importing a nonexistent file raises FileNotFoundError."""
nonexistent_file_path = "nonexistent.py"
- with pytest.raises(ModuleNotFoundError):
+ with pytest.raises(FileNotFoundError):
import_module_from_file(nonexistent_file_path)
def test_import_file_with_syntax_error_raises_error(self, tmp_path: Path):
@@ -102,7 +102,9 @@ def test_function():
return "test_value"
""")
module = import_module_from_file(test_file_path)
+ # The module name should be the actual module path derived from the file path
expected_module_name = "test_module"
+ assert module.__name__ == expected_module_name
assert expected_module_name in sys.modules
assert sys.modules[expected_module_name] is module
diff --git a/uv.lock b/uv.lock
index cdeb9cf39..5f7317061 100644
--- a/uv.lock
+++ b/uv.lock
@@ -1619,7 +1619,7 @@ wheels = [
[[package]]
name = "pipelex"
-version = "0.5.0"
+version = "0.6.0"
source = { editable = "." }
dependencies = [
{ name = "aiofiles" },