Build AI methods with your coding agent, then run them anywhere: as an MCP for chatbots, as a webapp for people, or via API for your software.
fix/Pipelex config fixed path(#151)
Thomas Hebrarda10c3d36f375
diff --git a/.github/workflows/manual-trigger-tests-check.yml b/.github/workflows/manual-trigger-tests-check.ymlnew file mode 100644index 000000000..36612c3f2--- /dev/null+++ b/.github/workflows/manual-trigger-tests-check.yml@@ -0,0 +1,66 @@+name: Manual trigger tests check++on:+ pull_request:+ branches:+ - dev+ - "release/v[0-9]+.[0-9]+.[0-9]+"+ workflow_dispatch:++jobs:+ trigger-tests:+ runs-on: ubuntu-latest+ environment:+ name: manual-trigger-tests-check+ url: ${{ github.server_url }}/${{ github.repository }}/deployments/activity_log?environment=manual-trigger-tests-check+ steps:+ - name: Initialize check+ run: |+ echo "Initializing manual trigger tests check..."++ matrix-test:+ name: Manual trigger tests check+ needs: [trigger-tests]+ runs-on: ubuntu-latest+ strategy:+ fail-fast: false+ matrix:+ python-version: ["3.10", "3.11", "3.12", "3.13"]+ permissions:+ contents: read+ id-token: write+ packages: read+ actions: read+ repository-projects: read+ env:+ VIRTUAL_ENV: ${{ github.workspace }}/.venv+ ENV: dev++ steps:+ - uses: actions/checkout@v4++ - name: Configure AWS Credentials+ uses: aws-actions/configure-aws-credentials@v4+ with:+ role-to-assume: arn:aws:iam::975050286103:role/GitHubActionsRole+ aws-region: eu-west-3++ - name: Set up Python ${{ matrix.python-version }}+ uses: actions/setup-python@v4+ with:+ python-version: ${{ matrix.python-version }}++ - name: Check UV installation+ run: make check-uv++ - name: Verify UV installation+ run: uv --version++ - name: Install dependencies+ run: PYTHON_VERSION=${{ matrix.python-version }} make install++ - name: Boot test+ run: make tp TEST=TestFundamentals++ - name: Run tests+ run: make run-manual-trigger-gha-testsdiff --git a/CHANGELOG.md b/CHANGELOG.mdindex c3dd52bcb..253dde8b2 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -1,5 +1,14 @@# Changelog+## [v0.6.0] - 2025-07-12++### Changed+- The `Pipelex.make()` now takes a `relative_config_folder_path` argument, which is considered to be a relative path to the current working directory, pointing to the config folder path of pipelex. The argument `from_file` indicates if the path is relative to the caller's file path. If False, the path is considered to be absolute from where the process was started.++### Added+- Added github action for inference tests+- `load_json_list_from_path` function in `pipelex.tools.misc.file_utils`: Loads a JSON file and ensures it contains a list.+## [v0.5.2] - 2025-07-11- log a warning when dry running a `PipeFunc`diff --git a/pipelex/cli/_cli.py b/pipelex/cli/_cli.pyindex a424579a0..a304715cd 100644--- a/pipelex/cli/_cli.py+++ b/pipelex/cli/_cli.py@@ -47,7 +47,7 @@ def init_libraries("""try:# TODO: Have a more proper print message regarding the overwrited files (e.g. list of files that were overwritten or not)- LibraryConfig.export_libraries(overwrite=overwrite)+ LibraryConfig().export_libraries(overwrite=overwrite)if overwrite:typer.echo("Successfully initialized pipelex libraries (all files overwritten)")else:@@ -76,10 +76,15 @@ def init_config(@app.command()-def validate() -> None:+def validate(+ relative_config_folder_path: Annotated[+ str, typer.Option("--config-folder-path", "-c", help="Relative path to the config folder path")+ ] = "pipelex_libraries",+) -> None:"""Run the setup sequence."""- LibraryConfig.export_libraries()- pipelex_instance = Pipelex.make()+ config_folder_path = os.path.join(os.getcwd(), relative_config_folder_path)+ LibraryConfig(config_folder_path=config_folder_path).export_libraries()+ pipelex_instance = Pipelex.make(relative_config_folder_path=relative_config_folder_path, from_file=False)pipelex_instance.validate_libraries()asyncio.run(dry_run_all_pipes())log.info("Setup sequence passed OK, config and pipelines are validated.")@@ -96,9 +101,13 @@ def show_config() -> None:@app.command()-def list_pipes() -> None:+def list_pipes(+ relative_config_folder_path: Annotated[+ str, typer.Option("--config-folder-path", "-c", help="Relative path to the config folder path")+ ] = "pipelex_libraries",+) -> None:"""List all available pipes."""- Pipelex.make()+ Pipelex.make(relative_config_folder_path=relative_config_folder_path, from_file=False)try:get_pipe_provider().pretty_list_pipes()diff --git a/pipelex/cogt/llm/llm_models/llm_model_library.py b/pipelex/cogt/llm/llm_models/llm_model_library.pyindex cefefcc78..7b1b1ed36 100644--- a/pipelex/cogt/llm/llm_models/llm_model_library.py+++ b/pipelex/cogt/llm/llm_models/llm_model_library.py@@ -1,5 +1,5 @@import os-from typing import Any, Dict, List, Optional+from typing import Any, ClassVar, Dict, List, Optionalfrom pydantic import Field, RootModelfrom typing_extensions import override@@ -32,6 +32,12 @@ LLMModelLibraryRoot = List[LLMModel]class LLMModelLibrary(LLMModelProviderAbstract, RootModel[LLMModelLibraryRoot]):root: LLMModelLibraryRoot = Field(default_factory=list)+ library_config: ClassVar[LibraryConfig]++ @classmethod+ def make_empty(cls, config_folder_path: str) -> "LLMModelLibrary":+ cls.library_config = LibraryConfig(config_folder_path=config_folder_path)+ return cls()@overridedef setup(self):@@ -63,7 +69,7 @@ class LLMModelLibrary(LLMModelProviderAbstract, RootModel[LLMModelLibraryRoot]):@classmethoddef load_llm_model_library_dict(cls) -> LLMModelLibraryDict:- libraries_path = LibraryConfig.exported_llm_integrations_path+ libraries_path = cls.library_config.llm_integrations_pathif not os.path.exists(libraries_path):raise LLMModelLibraryError(f"LLM model library path `{libraries_path}` not found. Please run `pipelex init-libraries` to create it.")llm_library: LLMModelLibraryDict = {}diff --git a/pipelex/libraries/library_config.py b/pipelex/libraries/library_config.pyindex e6c718786..4af9468e1 100644--- a/pipelex/libraries/library_config.py+++ b/pipelex/libraries/library_config.py@@ -3,99 +3,117 @@ from typing import ClassVar, Listfrom pipelex.tools.config.models import ConfigModelfrom pipelex.tools.misc.file_utils import copy_file_from_package, copy_folder_from_package, find_files_in_dir+PIPELEX_LIBRARIES_PATH = "libraries"+class LibraryConfig(ConfigModel):- # Class variablespackage_name: ClassVar[str] = "pipelex"- internal_library_root: ClassVar[str] = "libraries"- exported_library_root: ClassVar[str] = "pipelex_libraries"- internal_base_pipelines_path: ClassVar[str] = f"{internal_library_root}/pipelines"- exported_pipelines_path: ClassVar[str] = f"{exported_library_root}/pipelines"- exported_base_pipelines_path: ClassVar[str] = f"{exported_library_root}/pipelines/base_library"- loaded_pipelines_path: ClassVar[str] = f"{exported_library_root}/pipelines"- test_pipelines_path: ClassVar[str] = "tests/test_pipelines"- internal_llm_integrations_path: ClassVar[str] = f"{internal_library_root}/llm_integrations"- exported_llm_integrations_path: ClassVar[str] = f"{exported_library_root}/llm_integrations"- internal_llm_deck_path: ClassVar[str] = f"{internal_library_root}/llm_deck"- exported_llm_deck_path: ClassVar[str] = f"{exported_library_root}/llm_deck"- internal_templates_path: ClassVar[str] = f"{internal_library_root}/templates"- exported_templates_path: ClassVar[str] = f"{exported_library_root}/templates"- internal_plugins_path: ClassVar[str] = f"{internal_library_root}/plugins"- exported_plugins_path: ClassVar[str] = f"{exported_library_root}/plugins"- failing_pipelines_path: ClassVar[str] = "tests/test_pipelines/failing_pipelines.toml"-- @classmethod- def get_llm_deck_paths(cls) -> List[str]:- llm_deck_paths = [str(path) for path in find_files_in_dir(dir_path=cls.exported_llm_deck_path, pattern="*.toml", is_recursive=True)]+ config_folder_path: str = "pipelex_libraries"++ @property+ def config_file(self) -> str:+ return f"{self.config_folder_path}/config.toml"++ @property+ def pipelines_path(self) -> str:+ return f"{self.config_folder_path}/pipelines"++ @property+ def base_pipelines_path(self) -> str:+ return f"{self.config_folder_path}/pipelines/base_library"++ @property+ def llm_integrations_path(self) -> str:+ return f"{self.config_folder_path}/llm_integrations"++ @property+ def llm_deck_path(self) -> str:+ return f"{self.config_folder_path}/llm_deck"++ @property+ def templates_path(self) -> str:+ return f"{self.config_folder_path}/templates"++ @property+ def plugins_path(self) -> str:+ return f"{self.config_folder_path}/plugins"++ @property+ def test_pipelines_path(self) -> str:+ return "tests/test_pipelines"++ @property+ def failing_pipelines_path(self) -> str:+ return "tests/test_pipelines/failing_pipelines.toml"++ def get_llm_deck_paths(self) -> List[str]:+ llm_deck_paths = [str(path) for path in find_files_in_dir(dir_path=self.llm_deck_path, pattern="*.toml", is_recursive=True)]llm_deck_paths.sort()return llm_deck_paths- @classmethod- def get_templates_paths(cls) -> List[str]:- return [str(path) for path in find_files_in_dir(dir_path=cls.exported_templates_path, pattern="*.toml", is_recursive=True)]+ def get_templates_paths(self) -> List[str]:+ return [str(path) for path in find_files_in_dir(dir_path=self.templates_path, pattern="*.toml", is_recursive=True)]- @classmethod- def get_plugin_config_path(cls) -> str:- return f"{cls.exported_plugins_path}/plugin_config.toml"+ def get_default_plugin_config_path(self) -> str:+ """Get the default plugin config path."""+ return f"{self.plugins_path}/plugin_config.toml"- @classmethod- def export_libraries(cls, overwrite: bool = False) -> None:+ def export_libraries(self, overwrite: bool = False) -> None:"""Duplicate pipelex libraries files in the client project, preserving directory structure."""# pipelinescopy_folder_from_package(- package_name=cls.package_name,- folder_path_in_package=cls.internal_base_pipelines_path,- target_dir=cls.exported_base_pipelines_path,+ package_name=self.package_name,+ folder_path_in_package=f"{PIPELEX_LIBRARIES_PATH}/pipelines",+ target_dir=self.base_pipelines_path,overwrite=overwrite,)copy_file_from_package(- package_name=cls.package_name,- file_path_in_package=f"{cls.internal_library_root}/__init__.py",- target_path=f"{cls.exported_library_root}/__init__.py",+ package_name=self.package_name,+ file_path_in_package=f"{PIPELEX_LIBRARIES_PATH}/__init__.py",+ target_path=f"{self.config_folder_path}/__init__.py",overwrite=overwrite,)copy_file_from_package(- package_name=cls.package_name,- file_path_in_package=f"{cls.internal_base_pipelines_path}/__init__.py",- target_path=f"{cls.exported_pipelines_path}/__init__.py",+ package_name=self.package_name,+ file_path_in_package=f"{PIPELEX_LIBRARIES_PATH}/pipelines/__init__.py",+ target_path=f"{self.pipelines_path}/__init__.py",overwrite=overwrite,)copy_file_from_package(- package_name=cls.package_name,- file_path_in_package=f"{cls.internal_base_pipelines_path}/__init__.py",- target_path=f"{cls.exported_base_pipelines_path}/__init__.py",+ package_name=self.package_name,+ file_path_in_package=f"{PIPELEX_LIBRARIES_PATH}/pipelines/__init__.py",+ target_path=f"{self.base_pipelines_path}/__init__.py",overwrite=overwrite,)-# llm_integrationscopy_folder_from_package(- package_name=cls.package_name,- folder_path_in_package=cls.internal_llm_integrations_path,- target_dir=cls.exported_llm_integrations_path,+ package_name=self.package_name,+ folder_path_in_package=f"{PIPELEX_LIBRARIES_PATH}/llm_integrations",+ target_dir=self.llm_integrations_path,overwrite=overwrite,)# llm_deckcopy_folder_from_package(- package_name=cls.package_name,- folder_path_in_package=cls.internal_llm_deck_path,- target_dir=cls.exported_llm_deck_path,+ package_name=self.package_name,+ folder_path_in_package=f"{PIPELEX_LIBRARIES_PATH}/llm_deck",+ target_dir=self.llm_deck_path,overwrite=overwrite,non_overwrite_files=["overrides.toml"],)# templatescopy_folder_from_package(- package_name=cls.package_name,- folder_path_in_package=cls.internal_templates_path,- target_dir=cls.exported_templates_path,+ package_name=self.package_name,+ folder_path_in_package=f"{PIPELEX_LIBRARIES_PATH}/templates",+ target_dir=self.templates_path,overwrite=overwrite,)# pluginscopy_folder_from_package(- package_name=cls.package_name,- folder_path_in_package=cls.internal_plugins_path,- target_dir=cls.exported_plugins_path,+ package_name=self.package_name,+ folder_path_in_package=f"{PIPELEX_LIBRARIES_PATH}/plugins",+ target_dir=self.plugins_path,overwrite=overwrite,)diff --git a/pipelex/libraries/library_manager.py b/pipelex/libraries/library_manager.pyindex bd8d9340e..2f654e57b 100644--- a/pipelex/libraries/library_manager.py+++ b/pipelex/libraries/library_manager.py@@ -61,20 +61,33 @@ class LibraryManager(LibraryManagerAbstract):"system_prompt_to_structure","prompt_template_to_structure",]+ domain_library: DomainLibrary+ concept_library: ConceptLibrary+ pipe_library: PipeLibrary+ llm_deck: Optional[LLMDeck] = None+ library_config: ClassVar[LibraryConfig]@classmethod- def make_empty(cls) -> "LibraryManager":- domain_library = DomainLibrary.make_empty()- concept_library = ConceptLibrary.make_empty()- pipe_library = PipeLibrary.make_empty()- return cls(domain_library=domain_library, concept_library=concept_library, pipe_library=pipe_library)-- def __init__(self, domain_library: DomainLibrary, concept_library: ConceptLibrary, pipe_library: PipeLibrary) -> None:- # TODO : avoid having an Option LLMDeck: regroup with model provider- self.llm_deck: Optional[LLMDeck] = None- self.domain_library = domain_library- self.concept_library = concept_library- self.pipe_library = pipe_library+ def make_empty(cls, config_folder_path: str) -> "LibraryManager":+ cls.domain_library = DomainLibrary.make_empty()+ cls.concept_library = ConceptLibrary.make_empty()+ cls.pipe_library = PipeLibrary.make_empty()+ cls.library_config = LibraryConfig(config_folder_path=config_folder_path)+ return cls()++ @classmethod+ def make(+ cls, domain_library: DomainLibrary, concept_library: ConceptLibrary, pipe_library: PipeLibrary, config_folder_path: str+ ) -> "LibraryManager":+ cls.domain_library = domain_library+ cls.concept_library = concept_library+ cls.pipe_library = pipe_library+ cls.library_config = LibraryConfig(config_folder_path=config_folder_path)+ return cls()++ @override+ def get_plugin_config_path(self) -> str:+ return self.library_config.get_default_plugin_config_path()@overridedef setup(self) -> None:@@ -88,10 +101,10 @@ class LibraryManager(LibraryManagerAbstract):self.domain_library.teardown()def libraries_paths(self) -> List[str]:- library_paths = [LibraryConfig.loaded_pipelines_path]+ library_paths = [self.library_config.pipelines_path]if runtime_manager.is_unit_testing:log.debug("Registering test pipeline structures for unit testing")- library_paths += [LibraryConfig.test_pipelines_path]+ library_paths += [self.library_config.test_pipelines_path]return library_pathsdef load_failure_modes(self):@@ -117,7 +130,7 @@ class LibraryManager(LibraryManagerAbstract):self.load_combo_libraries(library_paths=toml_file_paths)def load_deck(self) -> LLMDeck:- llm_deck_paths = LibraryConfig.get_llm_deck_paths()+ llm_deck_paths = self.library_config.get_llm_deck_paths()full_llm_deck_dict: Dict[str, Any] = {}if not llm_deck_paths:raise LLMDeckNotFoundError("No LLM deck paths found. Please run `pipelex init-libraries` to create it.")@@ -313,7 +326,7 @@ class LibraryManager(LibraryManagerAbstract):"""Validate all TOML files used by the library manager for formatting issues."""log.debug("LibraryManager validating TOML file formatting")- llm_deck_paths = LibraryConfig.get_llm_deck_paths()+ llm_deck_paths = self.library_config.get_llm_deck_paths()for llm_deck_path in llm_deck_paths:if os.path.exists(llm_deck_path):try:@@ -341,7 +354,7 @@ class LibraryManager(LibraryManagerAbstract):log.error(f"TOML formatting issues in library file '{toml_path}': {exc}")raise LibraryError(f"TOML validation failed for library file '{toml_path}': {exc}") from exc- template_paths = LibraryConfig.get_templates_paths()+ template_paths = self.library_config.get_templates_paths()for template_path in template_paths:if os.path.exists(template_path):try:diff --git a/pipelex/libraries/library_manager_abstract.py b/pipelex/libraries/library_manager_abstract.pyindex 69d05cd00..67f1c0047 100644--- a/pipelex/libraries/library_manager_abstract.py+++ b/pipelex/libraries/library_manager_abstract.py@@ -15,3 +15,7 @@ class LibraryManagerAbstract(ABC):@abstractmethoddef load_combo_libraries(self, library_paths: List[Path]):pass++ @abstractmethod+ def get_plugin_config_path(self) -> str:+ passdiff --git a/pipelex/libraries/llm_deck/base_llm_deck.toml b/pipelex/libraries/llm_deck/base_llm_deck.tomlindex 64b2d2545..8a61500b0 100644--- a/pipelex/libraries/llm_deck/base_llm_deck.toml+++ b/pipelex/libraries/llm_deck/base_llm_deck.toml@@ -77,3 +77,4 @@ llm_to_extract_tables = { llm_handle = "best-claude", temperature = 0.1 }[llm_choice_defaults]for_text = "cheap_llm_for_text"for_object = "cheap_llm_for_object"+diff --git a/pipelex/libraries/llm_deck/overrides.toml b/pipelex/libraries/llm_deck/overrides.tomlindex 76598a632..999cc4d1a 100644--- a/pipelex/libraries/llm_deck/overrides.toml+++ b/pipelex/libraries/llm_deck/overrides.toml@@ -7,3 +7,4 @@[llm_choice_overrides]for_text = "disabled"for_object = "disabled"+diff --git a/pipelex/libraries/llm_integrations/anthropic.toml b/pipelex/libraries/llm_integrations/anthropic.tomlindex 788b72a72..510f2a212 100644--- a/pipelex/libraries/llm_integrations/anthropic.toml+++ b/pipelex/libraries/llm_integrations/anthropic.toml@@ -62,3 +62,4 @@ max_prompt_images = 100cost_per_million_tokens_usd = { input = 3.0, output = 15.0 }platform_llm_id = { anthropic = "claude-opus-4-20250514", bedrock_anthropic = "us.anthropic.claude-opus-4-20250514-v1:0" }default_platform = "anthropic"+diff --git a/pipelex/libraries/llm_integrations/bedrock.toml b/pipelex/libraries/llm_integrations/bedrock.tomlindex c61e77192..bf4404370 100644--- a/pipelex/libraries/llm_integrations/bedrock.toml+++ b/pipelex/libraries/llm_integrations/bedrock.toml@@ -28,3 +28,4 @@ is_gen_object_supported = false# TODO: find out the actual cost per million tokens for nova on bedrockcost_per_million_tokens_usd = { input = 3.0, output = 15.0 }platform_llm_id = { bedrock = "us.amazon.nova-pro-v1:0" }+diff --git a/pipelex/libraries/llm_integrations/custom.toml b/pipelex/libraries/llm_integrations/custom.tomlindex a551366be..ecf6d747f 100644--- a/pipelex/libraries/llm_integrations/custom.toml+++ b/pipelex/libraries/llm_integrations/custom.toml@@ -27,3 +27,4 @@ is_vision_supported = falsecost_per_million_tokens_usd = { input = 0, output = 0 }platform_llm_id = { custom_llm = "qwen3:8b" }# TODO: support <think> tokens+diff --git a/pipelex/libraries/llm_integrations/mistral.toml b/pipelex/libraries/llm_integrations/mistral.tomlindex bcd797b3e..c99cdd8bc 100644--- a/pipelex/libraries/llm_integrations/mistral.toml+++ b/pipelex/libraries/llm_integrations/mistral.toml@@ -67,3 +67,4 @@ is_gen_object_supported = trueis_vision_supported = truecost_per_million_tokens_usd = { input = 2.0, output = 6.0 }platform_llm_id = { mistral = "pixtral-large-latest" }+diff --git a/pipelex/libraries/llm_integrations/perplexity.toml b/pipelex/libraries/llm_integrations/perplexity.tomlindex a72dc5946..fb681f7bf 100644--- a/pipelex/libraries/llm_integrations/perplexity.toml+++ b/pipelex/libraries/llm_integrations/perplexity.toml@@ -23,3 +23,4 @@ platform_llm_id = { perplexity = "sonar-reasoning" }[perplexity-deepseek.perplexity-deepseek-r1.latest]is_gen_object_supported = falseplatform_llm_id = { perplexity = "r1-1776" }+diff --git a/pipelex/libraries/llm_integrations/vertexai.toml b/pipelex/libraries/llm_integrations/vertexai.tomlindex 2b28ec505..62ca4f32f 100644--- a/pipelex/libraries/llm_integrations/vertexai.toml+++ b/pipelex/libraries/llm_integrations/vertexai.toml@@ -41,3 +41,4 @@ is_vision_supported = truemax_prompt_images = 3000cost_per_million_tokens_usd = { input = 0.15, output = 0.6 }platform_llm_id = { vertexai = "google/gemini-2.5-flash-preview-05-20" }+diff --git a/pipelex/libraries/llm_integrations/xai.toml b/pipelex/libraries/llm_integrations/xai.tomlindex 897b3c6a1..31393cd93 100644--- a/pipelex/libraries/llm_integrations/xai.toml+++ b/pipelex/libraries/llm_integrations/xai.toml@@ -23,3 +23,4 @@ is_gen_object_supported = trueis_vision_supported = falsecost_per_million_tokens_usd = { input = 0.15, output = 4 }platform_llm_id = { xai = "grok-3-mini-fast-latest" }+diff --git a/pipelex/libraries/pipelines/documents.toml b/pipelex/libraries/pipelines/documents.tomlindex a3d88b110..cc52a4c66 100644--- a/pipelex/libraries/pipelines/documents.toml+++ b/pipelex/libraries/pipelines/documents.toml@@ -23,3 +23,4 @@ inputs = { ocr_input = "PDF" }output = "Page"page_images = truepage_views = true+diff --git a/pipelex/libraries/pipelines/images.toml b/pipelex/libraries/pipelines/images.tomlindex b2391fdbf..073bbb575 100644--- a/pipelex/libraries/pipelines/images.toml+++ b/pipelex/libraries/pipelines/images.toml@@ -59,3 +59,4 @@ PipeImgGen = "Generate a photo"inputs = { prompt = "ImgGenPrompt" }output = "images.Photo"nb_steps = 8+diff --git a/pipelex/libraries/plugins/plugin_config.toml b/pipelex/libraries/plugins/plugin_config.tomlindex e3c86138b..30cde64c3 100644--- a/pipelex/libraries/plugins/plugin_config.toml+++ b/pipelex/libraries/plugins/plugin_config.toml@@ -37,3 +37,4 @@ api_key_method = "env"[fal_config]flux_map_quality_to_steps = { "low" = 14, "medium" = 28, "high" = 56 }sdxl_lightning_map_quality_to_steps = { "low" = 2, "medium" = 4, "high" = 8 }+diff --git a/pipelex/libraries/templates/base_templates.toml b/pipelex/libraries/templates/base_templates.tomlindex 7a8da104b..fe3026d04 100644--- a/pipelex/libraries/templates/base_templates.toml+++ b/pipelex/libraries/templates/base_templates.toml@@ -15,3 +15,4 @@ Do not create information that is not in the text.[test_prompts]jinja2_test_template = "I want a {{ place_holder }} t-shirt."+diff --git a/pipelex/pipelex.py b/pipelex/pipelex.pyindex 109e5b726..5293a7c51 100644--- a/pipelex/pipelex.py+++ b/pipelex/pipelex.py@@ -1,3 +1,5 @@+import inspect+import osfrom importlib.metadata import metadatafrom typing import Any, List, Optional, Type, cast@@ -59,9 +61,10 @@ PACKAGE_VERSION = metadata(PACKAGE_NAME)["Version"]class Pipelex(metaclass=MetaSingleton):def __init__(self,+ config_folder_path: str,+ # Dependency injectionpipelex_hub: Optional[PipelexHub] = None,config_cls: Optional[Type[ConfigRoot]] = None,- ready_made_config: Optional[ConfigRoot] = None,class_registry: Optional[ClassRegistryAbstract] = None,template_provider: Optional[TemplateLibrary] = None,llm_model_provider: Optional[LLMModelLibrary] = None,@@ -71,24 +74,20 @@ class Pipelex(metaclass=MetaSingleton):activity_manager: Optional[ActivityManagerProtocol] = None,reporting_delegate: Optional[ReportingProtocol] = None,) -> None:+ self.config_folder_path = config_folder_pathself.pipelex_hub = pipelex_hub or PipelexHub()set_pipelex_hub(self.pipelex_hub)# tools- if ready_made_config is not None:- if config_cls is not None:- raise PipelexConfigError("config_cls must be None when ready_made_config is provided")- self.pipelex_hub.set_config(ready_made_config)- else:- if config_cls is None:- config_cls = PipelexConfig- try:- self.pipelex_hub.setup_config(config_cls=config_cls)- except ValidationError as exc:- error_msg = format_pydantic_validation_error(exc)- raise PipelexConfigError(f"Error because of: {error_msg}") from exc+ try:+ self.pipelex_hub.setup_config(config_cls=config_cls or PipelexConfig)+ except ValidationError as exc:+ error_msg = format_pydantic_validation_error(exc)+ raise PipelexConfigError(f"Error because of: {error_msg}") from exc+for extra_env_file in get_config().pipelex.extra_env_files:load_dotenv(dotenv_path=extra_env_file, override=True)+log.configure(project_name=get_config().project_name or "unknown_project",log_config=get_config().pipelex.log_config,@@ -96,8 +95,9 @@ class Pipelex(metaclass=MetaSingleton):log.debug("Logs are configured")# tools- self.template_provider = template_provider or TemplateLibrary()+ self.template_provider = template_provider or TemplateLibrary.make_empty(config_folder_path=config_folder_path)self.pipelex_hub.set_template_provider(self.template_provider)+self.class_registry = class_registry or ClassRegistry()self.pipelex_hub.set_class_registry(self.class_registry)self.kajson_manager = KajsonManager(class_registry=self.class_registry)@@ -105,7 +105,7 @@ class Pipelex(metaclass=MetaSingleton):# cogtself.plugin_manager = PluginManager()self.pipelex_hub.set_plugin_manager(self.plugin_manager)- self.llm_model_provider = llm_model_provider or LLMModelLibrary()+ self.llm_model_provider = llm_model_provider or LLMModelLibrary.make_empty(config_folder_path=config_folder_path)self.pipelex_hub.set_llm_models_provider(self.llm_model_provider)self.inference_manager = inference_manager or InferenceManager()self.pipelex_hub.set_inference_manager(self.inference_manager)@@ -124,10 +124,11 @@ class Pipelex(metaclass=MetaSingleton):self.pipelex_hub.set_domain_provider(domain_provider=domain_library)self.pipelex_hub.set_concept_provider(concept_provider=concept_library)self.pipelex_hub.set_pipe_provider(pipe_provider=pipe_library)- self.library_manager = LibraryManager(+ self.library_manager = LibraryManager.make(domain_library=domain_library,concept_library=concept_library,pipe_library=pipe_library,+ config_folder_path=config_folder_path,)self.library_manager.setup()self.pipelex_hub.set_library_manager(library_manager=self.library_manager)@@ -167,7 +168,7 @@ class Pipelex(metaclass=MetaSingleton):self.pipelex_hub.set_secrets_provider(secrets_provider or EnvSecretsProvider())self.pipelex_hub.set_storage_provider(storage_provider)# cogt- self.plugin_manager.setup()+ self.plugin_manager.setup(library_config=self.library_manager.library_config)self.pipelex_hub.set_content_generator(content_generator or ContentGenerator())self.reporting_delegate.setup()self.class_registry.register_classes(PipelexRegistryModels.get_all_models())@@ -187,7 +188,7 @@ class Pipelex(metaclass=MetaSingleton):log.debug(f"{PACKAGE_NAME} version {PACKAGE_VERSION} setup done for {get_config().project_name}")- def setup_libraries(self):+ def setup_libraries(self, relative_config_folder_path: str):try:self.template_provider.setup()self.llm_model_provider.setup()@@ -241,10 +242,22 @@ class Pipelex(metaclass=MetaSingleton):# TODO: add kwargs to make() so that subclasses can employ specific parameters@classmethod- def make(cls, structure_classes: Optional[List[Type[Any]]] = None) -> Self:- pipelex_instance = cls()- pipelex_instance.setup(structure_classes=structure_classes)- pipelex_instance.setup_libraries()+ def make(cls, relative_config_folder_path: str, from_file: Optional[bool] = True) -> Self:+ if from_file: # from_file means it's going to merge the relative_config_folder_path with the caller's file path+ current_frame = inspect.currentframe()+ if current_frame is None:+ raise RuntimeError("Failed to get current frame")+ if current_frame.f_back is None:+ raise RuntimeError("Failed to get caller frame")+ caller_file = current_frame.f_back.f_code.co_filename+ caller_dir = os.path.dirname(os.path.abspath(caller_file))+ config_folder_path = os.path.abspath(os.path.join(caller_dir, relative_config_folder_path))+ else:+ config_folder_path = os.path.abspath(os.path.join(os.getcwd(), relative_config_folder_path))++ pipelex_instance = cls(config_folder_path=config_folder_path)+ pipelex_instance.setup()+ pipelex_instance.setup_libraries(relative_config_folder_path=relative_config_folder_path)log.info(f"Pipelex {PACKAGE_VERSION} initialized.")return pipelex_instancediff --git a/pipelex/plugins/plugin_manager.py b/pipelex/plugins/plugin_manager.pyindex 9c14cb73d..90b0adfdf 100644--- a/pipelex/plugins/plugin_manager.py+++ b/pipelex/plugins/plugin_manager.py@@ -17,8 +17,8 @@ class PluginManager:raise RuntimeError("Plugin configs not loaded")return self._plugin_configs- def setup(self):- plugin_config_path = LibraryConfig.get_plugin_config_path()+ def setup(self, library_config: LibraryConfig):+ plugin_config_path = library_config.get_default_plugin_config_path()plugin_config_dict = load_toml_from_path(path=plugin_config_path)self._plugin_configs = PluginConfig.model_validate(plugin_config_dict)diff --git a/pipelex/test_extras/shared_pytest_plugins.py b/pipelex/test_extras/shared_pytest_plugins.pyindex db30a233d..113654024 100644--- a/pipelex/test_extras/shared_pytest_plugins.py+++ b/pipelex/test_extras/shared_pytest_plugins.py@@ -13,7 +13,7 @@ def set_run_mode():@pytest.fixture(scope="session")def manage_pipelex_libraries():- LibraryConfig.export_libraries(overwrite=False)+ LibraryConfig(config_folder_path="./pipelex_libraries").export_libraries(overwrite=False)yield# TODO: make it safe to erase/replace standard libraries in client projects without touching custom stuff# LibraryConfig.remove_libraries()@@ -21,7 +21,7 @@ def manage_pipelex_libraries():@pytest.fixture(scope="session")def manage_pipelex_libraries_with_overwrite():- LibraryConfig.export_libraries(overwrite=True)+ LibraryConfig(config_folder_path="./pipelex_libraries").export_libraries(overwrite=True)yield# TODO: make it safe to erase/replace standard libraries in client projects without touching custom stuff# LibraryConfig.remove_libraries()diff --git a/pipelex/tools/misc/json_utils.py b/pipelex/tools/misc/json_utils.pyindex 6b4247bac..2c3fb5fd6 100644--- a/pipelex/tools/misc/json_utils.py+++ b/pipelex/tools/misc/json_utils.py@@ -128,6 +128,31 @@ def load_json_dict_from_path(path: str) -> Dict[Any, Any]:raise JsonTypeError(f"{path} is not a dict")+def load_json_list_from_path(path: str) -> List[Any]:+ """+ Loads a JSON file and ensures it contains a list.++ This function reads a JSON file and verifies that its content is a list.+ It uses load_json_from_path internally and adds type checking.++ Args:+ path (str): The file path to the JSON file to be loaded.++ Returns:+ List[Any]: The parsed JSON content as a Python list.++ Raises:+ JsonTypeError: If the JSON content is not a list.+ FileNotFoundError: If the file does not exist.+ json.JSONDecodeError: If the file contains invalid JSON.+ """+ json_content: JsonContent = load_json_from_path(path)+ if isinstance(json_content, list):+ return json_content+ else:+ raise JsonTypeError(f"{path} is not a list")++def deep_update(target_dict: Dict[str, Any], updates: Dict[str, Any]):"""Recursively updates a dictionary with values from another dictionary.diff --git a/pipelex/tools/templating/template_library.py b/pipelex/tools/templating/template_library.pyindex 8f64a73c3..e94006862 100644--- a/pipelex/tools/templating/template_library.py+++ b/pipelex/tools/templating/template_library.py@@ -1,4 +1,4 @@-from typing import Any, Dict+from typing import Any, ClassVar, Dictfrom jinja2 import TemplateSyntaxErrorfrom pydantic import Field, RootModel, ValidationError@@ -23,10 +23,16 @@ class TemplateLibraryError(ToolException):class TemplateLibrary(TemplateProviderAbstract, RootModel[TemplateLibraryRoot]):root: TemplateLibraryRoot = Field(default_factory=dict)+ library_config: ClassVar[LibraryConfig]++ @classmethod+ def make_empty(cls, config_folder_path: str) -> "TemplateLibrary":+ cls.library_config = LibraryConfig(config_folder_path=config_folder_path)+ return cls()@overridedef setup(self) -> None:- template_toml_paths = LibraryConfig.get_templates_paths()+ template_toml_paths = self.library_config.get_templates_paths()for template_toml_path in template_toml_paths:self._load_from_toml(toml_path=template_toml_path)self.validate_templates(template_category=Jinja2TemplateCategory.LLM_PROMPT)diff --git a/pipelex/tools/typing/module_inspector.py b/pipelex/tools/typing/module_inspector.pyindex 7805bb98c..bbe6dac70 100644--- a/pipelex/tools/typing/module_inspector.py+++ b/pipelex/tools/typing/module_inspector.py@@ -27,14 +27,44 @@ def import_module_from_file(file_path: str) -> Any:if not file_path.endswith(".py"):raise ModuleFileError(f"File {file_path} is not a Python file (must end with .py)")- module_name = file_path[:-3].replace(os.sep, ".")- module = importlib.import_module(module_name)+ # Convert file path to module-style path to use as the actual module name+ module_name = _convert_file_path_to_module_path(file_path)++ # Check if module is already loaded to avoid duplicate loading+ if module_name in sys.modules:+ return sys.modules[module_name]++ # Use importlib.util to load the module from file path+ spec = importlib.util.spec_from_file_location(module_name, file_path)+ if spec is None or spec.loader is None:+ raise ModuleFileError(f"Could not load module from {file_path}")++ module = importlib.util.module_from_spec(spec)# Add the module to sys.modules to ensure proper imports within the modulesys.modules[module_name] = module++ # Execute the module+ spec.loader.exec_module(module)+return module+def _convert_file_path_to_module_path(file_path: str) -> str:+ """Convert a file path to a module-style path."""+ # Remove .py extension+ module_path = file_path[:-3] if file_path.endswith(".py") else file_path++ # Replace path separators with dots+ module_path = module_path.replace(os.sep, ".")++ # Handle __init__.py files by removing the __init__ part+ if module_path.endswith(".__init__"):+ module_path = module_path[:-9]++ return module_path++def find_classes_in_module(module: Any,base_class: Optional[Type[Any]],diff --git a/pyproject.toml b/pyproject.tomlindex fceefc715..7b157939b 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -1,6 +1,6 @@[project]name = "pipelex"-version = "0.5.2"+version = "0.6.0"description = "Pipelex is an open-source dev tool based on a simple declarative language that lets you define replicable, structured, composable LLM pipelines."authors = [{ name = "Evotis S.A.S.", email = "[email protected]" }]maintainers = [{ name = "Pipelex staff", email = "[email protected]" }]diff --git a/tests/conftest.py b/tests/conftest.pyindex 35d6afc56..56656e634 100644--- a/tests/conftest.py+++ b/tests/conftest.py@@ -25,7 +25,7 @@ def reset_pipelex_config_fixture():# Code to run before each testprint("[magenta]pipelex setup[/magenta]")try:- pipelex_instance = pipelex.pipelex.Pipelex.make()+ pipelex_instance = pipelex.pipelex.Pipelex.make(relative_config_folder_path="../pipelex/libraries")config = get_config()log.verbose(config, title="Test config")assert isinstance(config, pipelex.config.PipelexConfig)diff --git a/tests/integration/pipelex/test_libraries.py b/tests/integration/pipelex/test_libraries.pyindex 2be8c9c29..428383e0c 100644--- a/tests/integration/pipelex/test_libraries.py+++ b/tests/integration/pipelex/test_libraries.py@@ -90,7 +90,7 @@ class TestLibraries:known_concept: str,known_pipe: str,):- library_manager = LibraryManager.make_empty()+ library_manager = LibraryManager.make_empty(config_folder_path="pipelex_libraries")test_pipelines_path = get_config().pipelex.library_config.test_pipelines_pathtest_library_paths = library_manager.list_toml_files_from_path(library_paths=[test_pipelines_path])library_manager.load_combo_libraries(library_paths=test_library_paths)diff --git a/tests/unit/pipelex/tools/typing/test_module_inspector.py b/tests/unit/pipelex/tools/typing/test_module_inspector.pyindex 346fed801..686affe16 100644--- a/tests/unit/pipelex/tools/typing/test_module_inspector.py+++ b/tests/unit/pipelex/tools/typing/test_module_inspector.py@@ -69,7 +69,7 @@ class TestClass:def test_import_nonexistent_file_raises_error(self, tmp_path: Path):"""Test that importing a nonexistent file raises FileNotFoundError."""nonexistent_file_path = "nonexistent.py"- with pytest.raises(ModuleNotFoundError):+ with pytest.raises(FileNotFoundError):import_module_from_file(nonexistent_file_path)def test_import_file_with_syntax_error_raises_error(self, tmp_path: Path):@@ -102,7 +102,9 @@ def test_function():return "test_value"""")module = import_module_from_file(test_file_path)+ # The module name should be the actual module path derived from the file pathexpected_module_name = "test_module"+ assert module.__name__ == expected_module_nameassert expected_module_name in sys.modulesassert sys.modules[expected_module_name] is modulediff --git a/uv.lock b/uv.lockindex cdeb9cf39..5f7317061 100644--- a/uv.lock+++ b/uv.lock@@ -1619,7 +1619,7 @@ wheels = [[[package]]name = "pipelex"-version = "0.5.0"+version = "0.6.0"source = { editable = "." }dependencies = [{ name = "aiofiles" },