Skip to content

Build AI methods with your coding agent, then run them anywhere: as an MCP for chatbots, as a webapp for people, or via API for your software.

Fix/cli init and bugs with pipe controllers (#400)

Louis Choquelc18deea7e176
diff --git a/.vscode/launch.json b/.vscode/launch.json
index 5b57e620c..1827cff67 100644
--- a/.vscode/launch.json
+++ b/.vscode/launch.json
@@ -23,7 +23,7 @@
"args": [
"-s",
"-vv",
- "-k test_boot"
+ "-k test_discord_newsletter_dry_run_working_memory"
],
"console": "integratedTerminal"
},
@@ -41,19 +41,32 @@
"console": "integratedTerminal"
},
{
- "name": "Debug Validate Pipe",
+ "name": "Debug validate pipe",
"type": "debugpy",
"request": "launch",
"program": "${workspaceFolder}/.venv/bin/pipelex",
"args": [
"validate",
- "pipe",
"describe_image",
"-c",
"pipelex/libraries",
],
"console": "integratedTerminal",
"justMyCode": false
+ },
+ {
+ "name": "Debug run pipe",
+ "type": "debugpy",
+ "request": "launch",
+ "program": "${workspaceFolder}/.venv/bin/pipelex",
+ "args": [
+ "run",
+ "results/test_1.plx",
+ "--inputs",
+ "results/inputs_2.json",
+ ],
+ "console": "integratedTerminal",
+ "justMyCode": false
}
]
}
\ No newline at end of file
diff --git a/pipelex/cli/_cli.py b/pipelex/cli/_cli.py
index 997229975..6ce815e04 100644
--- a/pipelex/cli/_cli.py
+++ b/pipelex/cli/_cli.py
@@ -68,7 +68,7 @@ def app_callback(ctx: typer.Context) -> None:
if ctx.invoked_subcommand is None or ctx.invoked_subcommand in ("init", "doctor"):
return
- init_cmd()
+ init_cmd(silent=True)
@app.command(name="init", help="Initialize Pipelex configuration in a `.pipelex` directory")
diff --git a/pipelex/cli/commands/build_cmd.py b/pipelex/cli/commands/build_cmd.py
index e0de0dee6..673aab60e 100644
--- a/pipelex/cli/commands/build_cmd.py
+++ b/pipelex/cli/commands/build_cmd.py
@@ -35,6 +35,7 @@ pipelex build pipe "Imagine a cute animal mascot for a startup based on its elev
pipelex build pipe "Given an expense report, apply company rules"
pipelex build pipe "Take a CV in a PDF file, a Job offer text, and analyze if they match"
pipelex build pipe "Take a CV in a PDF file and a Job offer text, analyze if they match and generate 5 questions for the interview"
+pipelex build pipe "Take a CV and Job offer in PDF, analyze if they match and generate 5 questions for the interview"
pipelex build partial "Given an expense report, apply company rules" -o results/generated.json
pipelex build flow "Given an expense report, apply company rules" -o results/flow.json
@@ -74,9 +75,7 @@ def build_pipe_cmd(
] = False,
) -> None:
Pipelex.make(integration_mode=IntegrationMode.CLI)
- typer.echo("=" * 70)
- typer.secho("πŸ”₯ Starting pipe builder... πŸš€", fg=typer.colors.GREEN)
- typer.echo("")
+ typer.secho("πŸ”₯ Starting pipe builder... πŸš€\n", fg=typer.colors.GREEN)
async def run_pipeline():
if no_output:
@@ -293,9 +292,7 @@ def build_one_shot_cmd(
] = False,
) -> None:
Pipelex.make(integration_mode=IntegrationMode.CLI)
- typer.echo("=" * 70)
- typer.secho("πŸ”₯ Starting pipe builder... πŸš€", fg=typer.colors.GREEN)
- typer.echo("")
+ typer.secho("πŸ”₯ Starting pipe builder... πŸš€\n", fg=typer.colors.GREEN)
async def run_pipeline():
if no_output:
@@ -370,9 +367,7 @@ def build_partial_cmd(
] = False,
) -> None:
Pipelex.make(integration_mode=IntegrationMode.CLI)
- typer.echo("=" * 70)
- typer.secho("πŸ”₯ Starting pipe builder... πŸš€", fg=typer.colors.GREEN)
- typer.echo("")
+ typer.secho("πŸ”₯ Starting pipe builder... πŸš€\n", fg=typer.colors.GREEN)
async def run_pipeline():
output_path: str | None = None
diff --git a/pipelex/cli/commands/init_cmd.py b/pipelex/cli/commands/init_cmd.py
index 63312092c..7a7e198dc 100644
--- a/pipelex/cli/commands/init_cmd.py
+++ b/pipelex/cli/commands/init_cmd.py
@@ -349,6 +349,7 @@ def init_cmd(
focus: InitFocus = InitFocus.ALL,
reset: bool = False,
skip_confirmation: bool = False,
+ silent: bool = False,
):
"""Initialize Pipelex configuration, inference backends, and telemetry if needed, in a unified flow.
@@ -356,6 +357,7 @@ def init_cmd(
focus: What to initialize - 'config', 'inference', 'telemetry', or 'all' (default)
reset: Whether to reset/overwrite existing files
skip_confirmation: If True, skip the confirmation prompt (used when called from doctor --fix)
+ silent: If True, suppress all output when everything is already configured
"""
console = Console()
pipelex_config_dir = config_manager.pipelex_config_dir
@@ -382,6 +384,10 @@ def init_cmd(
# If nothing needs to be done, handle based on focus
if not needs_config and not needs_inference and not needs_telemetry:
+ # In silent mode, just return without any output
+ if silent:
+ return
+
if handle_already_configured(focus, console, backends_toml_path, telemetry_config_path):
# User wants to reconfigure
needs_config, needs_inference, needs_telemetry = update_needs_for_reconfigure(focus)
diff --git a/pipelex/cli/commands/run_cmd.py b/pipelex/cli/commands/run_cmd.py
index 5c85e3e28..8572f9050 100644
--- a/pipelex/cli/commands/run_cmd.py
+++ b/pipelex/cli/commands/run_cmd.py
@@ -7,6 +7,7 @@ from typing import Annotated
import click
import typer
from posthog import new_context, tag
+from rich.console import Console
from pipelex import log, pretty_print_md
from pipelex.builder.builder import load_and_validate_bundle
@@ -190,7 +191,9 @@ def run_cmd(
except Exception as exc:
log.error(f"Error executing pipeline: {exc}")
- typer.secho(f"Failed to execute pipeline: {exc}", fg=typer.colors.RED, err=True)
+ console = Console(stderr=True)
+ console.print("\n[bold red]Failed to execute pipeline[/bold red]\n")
+ console.print_exception(show_locals=True)
raise typer.Exit(1) from exc
with new_context():
diff --git a/pipelex/core/concepts/concept.py b/pipelex/core/concepts/concept.py
index b3e968752..832ee04df 100644
--- a/pipelex/core/concepts/concept.py
+++ b/pipelex/core/concepts/concept.py
@@ -65,6 +65,8 @@ class Concept(BaseModel):
@classmethod
def are_concept_compatible(cls, concept_1: "Concept", concept_2: "Concept", strict: bool = False) -> bool:
+ if NativeConceptCode.is_dynamic_concept(concept_code=concept_1.code):
+ return True
if NativeConceptCode.is_dynamic_concept(concept_code=concept_2.code):
return True
if concept_1.concept_string == concept_2.concept_string:
diff --git a/pipelex/core/memory/working_memory.py b/pipelex/core/memory/working_memory.py
index 5889785b2..32673afe6 100644
--- a/pipelex/core/memory/working_memory.py
+++ b/pipelex/core/memory/working_memory.py
@@ -81,7 +81,6 @@ class WorkingMemory(BaseModel, ContextProviderAbstract):
def get_main_stuff(self) -> Stuff:
return self.get_stuff(name=MAIN_STUFF_NAME)
- # TODO: all calls to get_stuff should catch WorkingMemoryStuffNotFoundError in order to indicate what pipe is missing a required stuff
def get_stuff(self, name: str) -> Stuff:
if named_stuff := self.root.get(name):
return named_stuff
diff --git a/pipelex/core/pipes/pipe_abstract.py b/pipelex/core/pipes/pipe_abstract.py
index bf7736bf8..551d87746 100644
--- a/pipelex/core/pipes/pipe_abstract.py
+++ b/pipelex/core/pipes/pipe_abstract.py
@@ -61,7 +61,7 @@ class PipeAbstract(ABC, BaseModel):
If None, starts recursion detection with an empty set.
Returns:
- PipeInput containing all needed inputs for this pipe
+ InputRequirements containing all needed inputs for this pipe
"""
diff --git a/pipelex/exceptions.py b/pipelex/exceptions.py
index f198563a5..ef83c534d 100644
--- a/pipelex/exceptions.py
+++ b/pipelex/exceptions.py
@@ -97,7 +97,10 @@ class WorkingMemoryStuffAttributeNotFoundError(WorkingMemoryVariableError):
class WorkingMemoryStuffNotFoundError(WorkingMemoryVariableError):
- pass
+ def __init__(self, message: str, variable_name: str, pipe_code: str | None = None, concept_code: str | None = None):
+ super().__init__(message, variable_name)
+ self.pipe_code = pipe_code
+ self.concept_code = concept_code
class PipelexCLIError(PipelexException, ClickException):
@@ -227,6 +230,13 @@ class PipeInputError(PipelexException):
super().__init__(message)
+class PipeRunInputsError(PipelexException):
+ def __init__(self, message: str, pipe_code: str, missing_inputs: dict[str, str]):
+ self.pipe_code = pipe_code
+ self.missing_inputs = missing_inputs
+ super().__init__(message)
+
+
class DomainLoadingError(LibraryLoadingError):
def __init__(self, message: str, domain_code: str, description: str, source: str | None = None):
self.domain_code = domain_code
diff --git a/pipelex/pipe_controllers/batch/pipe_batch.py b/pipelex/pipe_controllers/batch/pipe_batch.py
index cedd97ea3..5ad58495c 100644
--- a/pipelex/pipe_controllers/batch/pipe_batch.py
+++ b/pipelex/pipe_controllers/batch/pipe_batch.py
@@ -7,6 +7,7 @@ from typing_extensions import override
from pipelex.config import get_config
from pipelex.core.memory.working_memory import MAIN_STUFF_NAME, WorkingMemory
+from pipelex.core.pipe_errors import PipeDefinitionError
from pipelex.core.pipes.input_requirements import InputRequirements
from pipelex.core.pipes.pipe_output import PipeOutput
from pipelex.core.stuffs.list_content import ListContent
@@ -14,6 +15,7 @@ from pipelex.core.stuffs.stuff_factory import StuffFactory
from pipelex.exceptions import (
PipeInputError,
PipeInputNotFoundError,
+ PipeRunInputsError,
WorkingMemoryStuffNotFoundError,
)
from pipelex.hub import get_pipeline_tracker, get_required_pipe
@@ -77,6 +79,20 @@ class PipeBatch(PipeController):
def needed_inputs(self, visited_pipes: set[str] | None = None) -> InputRequirements:
return self.inputs
+ @override
+ def _validate_inputs_in_memory(self, working_memory: WorkingMemory) -> None:
+ if not self.batch_params:
+ raise PipeDefinitionError(message=f"PipeBatch '{self.code}' must have a batch_params", pipe_code=self.code)
+ required_concept_code = self.inputs.get_required_input_requirement(variable_name=self.batch_params.input_item_stuff_name).concept.code
+ required_stuff_name = self.batch_params.input_list_stuff_name
+ try:
+ working_memory.get_stuff(required_stuff_name)
+ except WorkingMemoryStuffNotFoundError as exc:
+ variable_name: str = exc.variable_name or required_stuff_name
+ missing_inputs: dict[str, str] = {variable_name: exc.concept_code or required_concept_code}
+ msg = f"Missing required inputs for pipe '{self.code}': {missing_inputs}"
+ raise PipeRunInputsError(message=msg, pipe_code=self.code, missing_inputs=missing_inputs) from exc
+
async def _run_batch_pipe(
self,
job_metadata: JobMetadata,
diff --git a/pipelex/pipe_controllers/pipe_controller.py b/pipelex/pipe_controllers/pipe_controller.py
index 71b87c11c..0fe00ca94 100644
--- a/pipelex/pipe_controllers/pipe_controller.py
+++ b/pipelex/pipe_controllers/pipe_controller.py
@@ -7,6 +7,7 @@ from pipelex import log
from pipelex.core.memory.working_memory import WorkingMemory
from pipelex.core.pipes.pipe_abstract import PipeAbstract
from pipelex.core.pipes.pipe_output import PipeOutput
+from pipelex.exceptions import PipeRunInputsError, WorkingMemoryStuffNotFoundError
from pipelex.pipe_run.pipe_run_mode import PipeRunMode
from pipelex.pipe_run.pipe_run_params import PipeRunParams
from pipelex.pipeline.job_metadata import JobMetadata
@@ -19,6 +20,19 @@ class PipeController(PipeAbstract):
def class_name(self) -> str:
return self.__class__.__name__
+ def _validate_inputs_in_memory(self, working_memory: WorkingMemory) -> None:
+ missing_inputs: dict[str, str] = {}
+ for required_stuff_name, requirement in self.needed_inputs().items:
+ try:
+ working_memory.get_stuff(required_stuff_name)
+ except WorkingMemoryStuffNotFoundError as exc:
+ variable_name: str = exc.variable_name or required_stuff_name
+ missing_inputs[variable_name] = exc.concept_code or requirement.concept.code
+ if missing_inputs:
+ raise PipeRunInputsError(
+ message=f"Missing required inputs for pipe '{self.code}': {missing_inputs}", pipe_code=self.code, missing_inputs=missing_inputs
+ )
+
@override
async def run_pipe(
self,
@@ -36,6 +50,9 @@ class PipeController(PipeAbstract):
)
job_metadata.update(updated_metadata=updated_metadata)
+ # check we have the required inputs in the working memory
+ self._validate_inputs_in_memory(working_memory=working_memory)
+
match pipe_run_params.run_mode:
case PipeRunMode.LIVE:
indent_level = len(pipe_run_params.pipe_stack) - 1
diff --git a/pipelex/pipe_controllers/sub_pipe.py b/pipelex/pipe_controllers/sub_pipe.py
index 4e85b9fac..42666bb07 100644
--- a/pipelex/pipe_controllers/sub_pipe.py
+++ b/pipelex/pipe_controllers/sub_pipe.py
@@ -87,7 +87,7 @@ class SubPipe(BaseModel):
try:
required_stuffs = working_memory.get_stuffs(names=required_stuff_names)
except WorkingMemoryStuffNotFoundError as exc:
- sub_pipe_path = [*sub_pipe_run_params.pipe_layers, self.pipe_code]
+ sub_pipe_path = [*sub_pipe_run_params.pipe_stack, self.pipe_code]
sub_pipe_path_str = ".".join(sub_pipe_path)
error_details = f"SubPipe '{sub_pipe_path_str}', required_variables: {required_variables}, missing: '{exc.variable_name}'"
msg = f"Some required stuff(s) not found: {error_details}"
diff --git a/pipelex/pipelex.py b/pipelex/pipelex.py
index 13da1eebb..705e04cd2 100644
--- a/pipelex/pipelex.py
+++ b/pipelex/pipelex.py
@@ -399,7 +399,7 @@ If you need help, drop by our Discord: we're happy to assist: {URLs.discord}.
**kwargs,
)
pipelex_instance.setup_libraries()
- log.info(f"{PACKAGE_NAME} version {PACKAGE_VERSION} ready")
+ log.verbose(f"{PACKAGE_NAME} version {PACKAGE_VERSION} ready")
return pipelex_instance
@classmethod
diff --git a/pipelex/tools/misc/context_provider_abstract.py b/pipelex/tools/misc/context_provider_abstract.py
index 259e66567..ca8ff8f37 100644
--- a/pipelex/tools/misc/context_provider_abstract.py
+++ b/pipelex/tools/misc/context_provider_abstract.py
@@ -5,9 +5,9 @@ from pipelex.system.exceptions import ToolException
class ContextProviderException(ToolException):
- def __init__(self, variable_name: str, message: str, *args: object, **kwargs: object) -> None:
+ def __init__(self, message: str, variable_name: str):
+ super().__init__(message=message)
self.variable_name = variable_name
- super().__init__(message, *args, **kwargs)
class ContextProviderAbstract(ABC):
diff --git a/tests/e2e/pipelex/pipes/pipe_controller/pipe_condition/test_pipe_condition_simple.py b/tests/e2e/pipelex/pipes/pipe_controller/pipe_condition/test_pipe_condition_simple.py
index adada5154..c13df7201 100644
--- a/tests/e2e/pipelex/pipes/pipe_controller/pipe_condition/test_pipe_condition_simple.py
+++ b/tests/e2e/pipelex/pipes/pipe_controller/pipe_condition/test_pipe_condition_simple.py
@@ -8,7 +8,7 @@ from pipelex.core.concepts.concept_native import NativeConceptCode
from pipelex.core.domains.domain import SpecialDomain
from pipelex.core.memory.working_memory_factory import WorkingMemoryFactory
from pipelex.core.pipes.input_requirements import TypedNamedInputRequirement
-from pipelex.exceptions import DryRunMissingInputsError
+from pipelex.exceptions import DryRunMissingInputsError, PipeRunInputsError
from pipelex.pipe_controllers.condition.pipe_condition_blueprint import PipeConditionBlueprint
from pipelex.pipe_controllers.condition.pipe_condition_factory import PipeConditionFactory
from pipelex.pipe_run.pipe_run_params import PipeRunMode
@@ -42,7 +42,7 @@ class TestPipeConditionSimple:
# Test with empty working memory - should FAIL
empty_working_memory = WorkingMemoryFactory.make_empty()
- with pytest.raises(DryRunMissingInputsError) as exc_info:
+ with pytest.raises(PipeRunInputsError) as exc_info:
await pipe_condition.run_pipe(
job_metadata=JobMetadata(job_name="test_direct_condition_fail"),
working_memory=empty_working_memory,
@@ -53,7 +53,7 @@ class TestPipeConditionSimple:
error = exc_info.value
assert error.pipe_code == "test_condition_fail"
assert "user_category" in error.missing_inputs
- assert "missing required inputs" in str(error)
+ assert "Missing required inputs" in str(error)
async def test_direct_pipe_condition_should_succeed(self):
"""Test a PipeCondition created directly in code that should SUCCEED dry run."""
diff --git a/tests/integration/pipelex/pipes/controller/pipe_condition/test_pipe_condition_complex.py b/tests/integration/pipelex/pipes/controller/pipe_condition/test_pipe_condition_complex.py
index d400470dc..93eef19e7 100644
--- a/tests/integration/pipelex/pipes/controller/pipe_condition/test_pipe_condition_complex.py
+++ b/tests/integration/pipelex/pipes/controller/pipe_condition/test_pipe_condition_complex.py
@@ -10,7 +10,7 @@ from pipelex.core.concepts.concept_factory import ConceptFactory
from pipelex.core.memory.working_memory_factory import WorkingMemoryFactory
from pipelex.core.stuffs.stuff_factory import StuffFactory
from pipelex.core.stuffs.text_content import TextContent
-from pipelex.exceptions import PipeRouterError
+from pipelex.exceptions import PipeRunInputsError
from pipelex.hub import get_pipe_router, get_required_pipe
from pipelex.pipe_run.pipe_job_factory import PipeJobFactory
from pipelex.pipe_run.pipe_run_params import PipeRunMode
@@ -280,7 +280,7 @@ class TestPipeConditionComplex:
working_memory = WorkingMemoryFactory.make_from_single_stuff(doc_stuff)
- with pytest.raises(PipeRouterError) as exc_info:
+ with pytest.raises(PipeRunInputsError) as exc_info:
await get_pipe_router().run(
pipe_job=PipeJobFactory.make_pipe_job(
pipe=get_required_pipe(pipe_code="complex_document_processor"),
@@ -294,7 +294,7 @@ class TestPipeConditionComplex:
assert error.pipe_code == "complex_document_processor"
assert error.missing_inputs is not None
assert "user_profile" in error.missing_inputs
- assert "missing required inputs" in str(error)
+ assert "Missing required inputs" in str(error)
@pytest.mark.parametrize(
("doc_type", "priority", "user_level", "department", "complexity", "language", "expected_output_contains"),
diff --git a/tests/integration/pipelex/pipes/controller/pipe_condition/test_pipe_condition_simple.py b/tests/integration/pipelex/pipes/controller/pipe_condition/test_pipe_condition_simple.py
index d3a5180a2..7aa3da4d0 100644
--- a/tests/integration/pipelex/pipes/controller/pipe_condition/test_pipe_condition_simple.py
+++ b/tests/integration/pipelex/pipes/controller/pipe_condition/test_pipe_condition_simple.py
@@ -12,7 +12,7 @@ from pipelex.core.domains.domain import SpecialDomain
from pipelex.core.memory.working_memory_factory import WorkingMemoryFactory
from pipelex.core.stuffs.stuff_factory import StuffFactory
from pipelex.core.stuffs.text_content import TextContent
-from pipelex.exceptions import PipeRouterError
+from pipelex.exceptions import PipeRunInputsError
from pipelex.hub import get_pipe_router, get_required_pipe
from pipelex.pipe_controllers.condition.pipe_condition_blueprint import PipeConditionBlueprint
from pipelex.pipe_controllers.condition.pipe_condition_factory import PipeConditionFactory
@@ -219,7 +219,7 @@ class TestPipeConditionSimple:
empty_working_memory = WorkingMemoryFactory.make_empty()
# Run dry run using the real pipe - this should fail with PipeRouterError
- with pytest.raises(PipeRouterError) as exc_info:
+ with pytest.raises(PipeRunInputsError) as exc_info:
await get_pipe_router().run(
pipe_job=PipeJobFactory.make_pipe_job(
pipe=get_required_pipe(pipe_code="basic_condition_by_category_2"),
@@ -234,8 +234,7 @@ class TestPipeConditionSimple:
assert error.pipe_code == "basic_condition_by_category_2"
assert error.missing_inputs is not None
assert "input_data" in error.missing_inputs
- assert "missing required inputs" in str(error)
- assert "input_data" in str(error)
+ assert "Missing required inputs" in str(error)
async def test_condition_dry_run_medium_category_validation(self, request: FixtureRequest):
"""Test PipeCondition dry run with medium category - should validate the 'medium' branch."""
diff --git a/tests/test_pipelines/discord_newsletter.plx b/tests/test_pipelines/discord_newsletter.plx
index 885ab08f6..f912d9308 100644
--- a/tests/test_pipelines/discord_newsletter.plx
+++ b/tests/test_pipelines/discord_newsletter.plx
@@ -9,7 +9,7 @@ HtmlNewsletter = "The final newsletter content in html format with organized cha
[pipe.write_discord_newsletter]
type = "PipeSequence"
description = "Create a newsletter from Discord articles by summarizing channels and organizing content"
-inputs = { discord_channel_updates = "DiscordChannelUpdate" }
+inputs = { discord_channel_updates = "DiscordChannelUpdate[]" }
output = "HtmlNewsletter"
steps = [
{ pipe = "summarize_discord_channel_update", batch_over = "discord_channel_updates", batch_as = "discord_channel_update", result = "channel_summaries" },