Skip to content

Build AI methods with your coding agent, then run them anywhere: as an MCP for chatbots, as a webapp for people, or via API for your software.

fix/cli (#177)

Thomas Hebrard9110adce4530
diff --git a/.github/workflows/manual-trigger-tests-check.yml b/.github/workflows/manual-trigger-tests-check.yml
index b09443edc..03acd8636 100644
--- a/.github/workflows/manual-trigger-tests-check.yml
+++ b/.github/workflows/manual-trigger-tests-check.yml
@@ -3,10 +3,11 @@ name: Manual trigger tests check
on:
pull_request:
branches:
- - main
- - dev
- - "release/v[0-9]+.[0-9]+.[0-9]+"
- workflow_dispatch:
+ - manual-trigger-tests-check
+ # - main
+ # - dev
+ # - "release/v[0-9]+.[0-9]+.[0-9]+"
+ # workflow_dispatch:
jobs:
trigger-tests:
diff --git a/.gitignore b/.gitignore
index fb58afe0d..7b442388d 100644
--- a/.gitignore
+++ b/.gitignore
@@ -44,7 +44,7 @@ reports/
results/
# Pipelex libraries, duplicated from pipelex/libraries/ by `pipelex init-libraries`
-pipelex_libraries/
+/pipelex_libraries
# personnal pipelex config file that overrides the default one
pipelex_super.toml
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 7cd580bd3..3d4fb2aa0 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,5 +1,16 @@
# Changelog
+## [v0.6.2] - 2025-07-18
+
+### Added
+- New `dry-run-pipe` cli command to dry run a single pipe by its code
+- New `show-pipe` cli command to display pipe definitions from the pipe library
+- New `dry_run_single_pipe()` function for running individual pipe dry runs
+
+### Changed
+- Updated `init-libraries` command to accept a directory argument and create `pipelex_libraries` folder in specified location
+- Updated `validate` command to use `-c` flag for the config folder path
+
## [v0.6.1] - 2025-07-16
- Can execute pipelines with `input_memory`: It is a `CompactMemory: Dict[str, Dict[str, Any]]`
diff --git a/Makefile b/Makefile
index d90a8d64a..97c7ebcb2 100644
--- a/Makefile
+++ b/Makefile
@@ -137,18 +137,11 @@ env: check-uv
fi
@echo "Using Python: $$($(VENV_PYTHON) --version) from $$(which $$(readlink -f $(VENV_PYTHON)))"
-init: env
- $(call PRINT_TITLE,"Running pipelex init-libraries and init-config")
- $(VENV_PIPELEX) init-libraries
- $(VENV_PIPELEX) init-config
-
install: env
$(call PRINT_TITLE,"Installing dependencies")
@. $(VIRTUAL_ENV)/bin/activate && \
uv sync --all-extras && \
- $(VENV_PIPELEX) init-libraries && \
- $(VENV_PIPELEX) init-config && \
- echo "Installed Pipelex dependencies in ${VIRTUAL_ENV} with all extras and initialized Pipelex";
+ echo "Installed Pipelex dependencies in ${VIRTUAL_ENV} with all extras.";
lock: env
$(call PRINT_TITLE,"Resolving dependencies without update")
@@ -161,9 +154,14 @@ update: env
uv sync --all-extras && \
echo "Updated dependencies in ${VIRTUAL_ENV}";
+init: env
+ $(call PRINT_TITLE,"Running pipelex init-libraries and init-config")
+ $(VENV_PIPELEX) init-libraries
+ $(VENV_PIPELEX) init-config
+
validate: env
$(call PRINT_TITLE,"Running setup sequence")
- $(VENV_PIPELEX) validate
+ $(VENV_PIPELEX) validate -c pipelex/libraries
build: env
$(call PRINT_TITLE,"Building the wheels")
@@ -439,16 +437,16 @@ check-unused-imports: env
$(call PRINT_TITLE,"Checking for unused imports without fixing")
$(VENV_RUFF) check --select=F401 --no-fix .
-c: init format lint pyright mypy
+c: format lint pyright mypy
@echo "> done: c = check"
-cc: init cleanderived c
- @echo "> done: cc = init cleanderived init format lint pyright mypy"
+cc: cleanderived c
+ @echo "> done: cc = cleanderived format lint pyright mypy"
check: cc check-unused-imports
@echo "> done: check"
-v: init validate
+v: validate
@echo "> done: v = validate"
li: lock install
diff --git a/pipelex/cli/_cli.py b/pipelex/cli/_cli.py
index a304715cd..93d6a83ca 100644
--- a/pipelex/cli/_cli.py
+++ b/pipelex/cli/_cli.py
@@ -10,13 +10,42 @@ from typing_extensions import override
from pipelex import log, pretty_print
from pipelex.exceptions import PipelexCLIError, PipelexConfigError
-from pipelex.hub import get_pipe_provider
+from pipelex.hub import get_pipe_provider, get_pipeline_tracker, get_required_pipe
from pipelex.libraries.library_config import LibraryConfig
-from pipelex.pipe_works.pipe_dry import dry_run_all_pipes
+from pipelex.pipe_works.pipe_dry import dry_run_all_pipes, dry_run_single_pipe
from pipelex.pipelex import Pipelex
from pipelex.tools.config.manager import config_manager
+def is_pipelex_libraries_folder(folder_path: str) -> bool:
+ """Check if the given folder path contains a valid pipelex libraries structure.
+
+ A valid pipelex libraries folder should contain the following subdirectories:
+ - pipelines
+ - llm_deck
+ - llm_integrations
+ - plugins
+ - templates
+
+ Args:
+ folder_path: Path to the folder to check
+
+ Returns:
+ True if the folder contains all required subdirectories, False otherwise
+ """
+ if not os.path.exists(folder_path) or not os.path.isdir(folder_path):
+ return False
+
+ required_subdirs = ["pipelines", "llm_deck", "llm_integrations", "plugins", "templates"]
+
+ for subdir in required_subdirs:
+ subdir_path = os.path.join(folder_path, subdir)
+ if not os.path.exists(subdir_path) or not os.path.isdir(subdir_path):
+ return False
+
+ return True
+
+
class PipelexCLI(TyperGroup):
@override
def get_command(self, ctx: Context, cmd_name: str) -> Optional[Command]:
@@ -38,22 +67,31 @@ app = typer.Typer(
@app.command("init-libraries")
def init_libraries(
+ directory: Annotated[str, typer.Argument(help="Directory where to create the pipelex_libraries folder")] = ".",
overwrite: Annotated[bool, typer.Option("--overwrite", "-o", help="Warning: If set, existing files will be overwritten.")] = False,
) -> None:
- """Initialize pipelex libraries in the current directory.
+ """Initialize pipelex libraries in a pipelex_libraries folder in the specified directory.
If overwrite is False, only create files that don't exist yet.
If overwrite is True, all files will be overwritten even if they exist.
"""
try:
- # TODO: Have a more proper print message regarding the overwrited files (e.g. list of files that were overwritten or not)
- LibraryConfig().export_libraries(overwrite=overwrite)
+ # Always create a pipelex_libraries folder in the specified directory
+ target_path = os.path.join(directory, "pipelex_libraries")
+
+ # Create the target directory if it doesn't exist
+ os.makedirs(directory, exist_ok=True)
+
+ # Create a LibraryConfig instance with the target path
+ library_config = LibraryConfig(config_folder_path=target_path)
+ library_config.export_libraries(overwrite=overwrite)
+
if overwrite:
- typer.echo("Successfully initialized pipelex libraries (all files overwritten)")
+ typer.echo(f"✅ Successfully initialized pipelex libraries at '{target_path}' (all files overwritten)")
else:
- typer.echo("Successfully initialized pipelex libraries (only created non-existing files)")
+ typer.echo(f"✅ Successfully initialized pipelex libraries at '{target_path}' (only created non-existing files)")
except Exception as e:
- raise PipelexCLIError(f"Failed to initialize libraries: {e}")
+ raise PipelexCLIError(f"Failed to initialize libraries at '{directory}': {e}")
@app.command("init-config")
@@ -79,17 +117,49 @@ def init_config(
def validate(
relative_config_folder_path: Annotated[
str, typer.Option("--config-folder-path", "-c", help="Relative path to the config folder path")
- ] = "pipelex_libraries",
+ ] = "./pipelex_libraries",
) -> None:
"""Run the setup sequence."""
- config_folder_path = os.path.join(os.getcwd(), relative_config_folder_path)
- LibraryConfig(config_folder_path=config_folder_path).export_libraries()
+ # Check if pipelex libraries folder exists
+ if not is_pipelex_libraries_folder(relative_config_folder_path):
+ typer.echo(f"❌ No pipelex libraries folder found at '{relative_config_folder_path}'")
+ typer.echo("To create a pipelex libraries folder, run: pipelex init-libraries")
+ raise typer.Exit(1)
+
pipelex_instance = Pipelex.make(relative_config_folder_path=relative_config_folder_path, from_file=False)
pipelex_instance.validate_libraries()
asyncio.run(dry_run_all_pipes())
log.info("Setup sequence passed OK, config and pipelines are validated.")
+def dry_run_pipe(
+ pipe_code: Annotated[str, typer.Argument(help="The pipe code to dry run")],
+ relative_config_folder_path: Annotated[
+ str, typer.Option("--config-folder-path", "-c", help="Relative path to the config folder path")
+ ] = "./pipelex_libraries",
+) -> None:
+ """Dry run a single pipe."""
+ # Check if pipelex libraries folder exists
+ if not is_pipelex_libraries_folder(relative_config_folder_path):
+ typer.echo(f"❌ No pipelex libraries folder found at '{relative_config_folder_path}'")
+ typer.echo("To create a pipelex libraries folder, run: pipelex init-libraries")
+ raise typer.Exit(1)
+
+ try:
+ # Initialize Pipelex
+ pipelex_instance = Pipelex.make(relative_config_folder_path=relative_config_folder_path, from_file=False)
+ pipelex_instance.validate_libraries()
+
+ # Run the single pipe dry run
+ asyncio.run(dry_run_single_pipe(pipe_code))
+ get_pipeline_tracker().output_flowchart()
+
+ except Exception as e:
+ typer.echo(f"❌ Error running dry run for pipe '{pipe_code}': {e}")
+ raise typer.Exit(1)
+
+
@app.command()
def show_config() -> None:
"""Show the pipelex configuration."""
@@ -116,6 +186,22 @@ def list_pipes(
raise PipelexCLIError(f"Failed to list pipes: {e}")
[email protected]("show-pipe")
+def show_pipe(
+ pipe_code: Annotated[
+ str,
+ typer.Argument(help="Pipeline code to show definition for"),
+ ],
+ relative_config_folder_path: Annotated[
+ str, typer.Option("--config-folder-path", "-c", help="Relative path to the config folder path")
+ ] = "./pipelex_libraries",
+) -> None:
+ """Show pipe from the pipe library."""
+ Pipelex.make(relative_config_folder_path=relative_config_folder_path, from_file=False)
+ pipe = get_required_pipe(pipe_code=pipe_code)
+ pretty_print(pipe, title=f"Pipe '{pipe_code}'")
+
+
def main() -> None:
"""Entry point for the pipelex CLI."""
app()
diff --git a/pipelex/pipe_works/pipe_dry.py b/pipelex/pipe_works/pipe_dry.py
index 2b29b58fa..8b7109b5f 100644
--- a/pipelex/pipe_works/pipe_dry.py
+++ b/pipelex/pipe_works/pipe_dry.py
@@ -21,6 +21,29 @@ async def dry_run_all_pipes():
await dry_run_pipes(pipes=all_pipes)
+async def dry_run_single_pipe(pipe_code: str) -> str:
+ """
+ Dry run a single pipe by its code.
+
+ Args:
+ pipe_code: The code of the pipe to dry run
+
+ Returns:
+ Status string: "SUCCESS" or error message
+ """
+ try:
+ # Get the pipe using the hub function
+ pipe = get_pipe_provider().get_optional_pipe(pipe_code=pipe_code)
+ if not pipe:
+ return f"FAILED: Pipe '{pipe_code}' not found"
+
+ # Run the single pipe
+ result = await dry_run_pipes(pipes=[pipe])
+ return result.get(pipe_code, f"FAILED: No result for pipe '{pipe_code}'")
+ except Exception as e:
+ return f"FAILED: {str(e)}"
+
+
# TODO: add a function to dry run a single pipe, make it callable as a param of `pipelex validate`
async def dry_run_pipes(pipes: List[PipeAbstract]) -> Dict[str, str]:
"""
diff --git a/pipelex/pipelex_template.toml b/pipelex/pipelex_template.toml
index 55e1cc4a5..030f8bc14 100644
--- a/pipelex/pipelex_template.toml
+++ b/pipelex/pipelex_template.toml
@@ -1,3 +1,10 @@
+[pipelex]
+[pipelex.feature_config]
+# WIP/Experimental feature flags
+is_pipeline_tracking_enabled = true
+is_activity_tracking_enabled = true
+is_reporting_enabled = true
+
[pipelex.aws_config]
api_key_method = "env"
# The possible values are "env" and "secret_provider".
diff --git a/pyproject.toml b/pyproject.toml
index 431f6ac36..96b7eeac4 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -1,6 +1,6 @@
[project]
name = "pipelex"
-version = "0.6.1"
+version = "0.6.2"
description = "Pipelex is an open-source dev tool based on a simple declarative language that lets you define replicable, structured, composable LLM pipelines."
authors = [{ name = "Evotis S.A.S.", email = "[email protected]" }]
maintainers = [{ name = "Pipelex staff", email = "[email protected]" }]
diff --git a/uv.lock b/uv.lock
index 401aa9e90..433957950 100644
--- a/uv.lock
+++ b/uv.lock
@@ -1619,7 +1619,7 @@ wheels = [
[[package]]
name = "pipelex"
-version = "0.6.1"
+version = "0.6.2"
source = { editable = "." }
dependencies = [
{ name = "aiofiles" },