Build AI methods with your coding agent, then run them anywhere: as an MCP for chatbots, as a webapp for people, or via API for your software.
Fix/cli init and bugs with pipe controllers (#400)
Louis Choquelc18deea7e176
diff --git a/.vscode/launch.json b/.vscode/launch.jsonindex 5b57e620c..1827cff67 100644--- a/.vscode/launch.json+++ b/.vscode/launch.json@@ -23,7 +23,7 @@"args": ["-s","-vv",- "-k test_boot"+ "-k test_discord_newsletter_dry_run_working_memory"],"console": "integratedTerminal"},@@ -41,19 +41,32 @@"console": "integratedTerminal"},{- "name": "Debug Validate Pipe",+ "name": "Debug validate pipe","type": "debugpy","request": "launch","program": "${workspaceFolder}/.venv/bin/pipelex","args": ["validate",- "pipe","describe_image","-c","pipelex/libraries",],"console": "integratedTerminal","justMyCode": false+ },+ {+ "name": "Debug run pipe",+ "type": "debugpy",+ "request": "launch",+ "program": "${workspaceFolder}/.venv/bin/pipelex",+ "args": [+ "run",+ "results/test_1.plx",+ "--inputs",+ "results/inputs_2.json",+ ],+ "console": "integratedTerminal",+ "justMyCode": false}]}\ No newline at end of filediff --git a/pipelex/cli/_cli.py b/pipelex/cli/_cli.pyindex 997229975..6ce815e04 100644--- a/pipelex/cli/_cli.py+++ b/pipelex/cli/_cli.py@@ -68,7 +68,7 @@ def app_callback(ctx: typer.Context) -> None:if ctx.invoked_subcommand is None or ctx.invoked_subcommand in ("init", "doctor"):return- init_cmd()+ init_cmd(silent=True)@app.command(name="init", help="Initialize Pipelex configuration in a `.pipelex` directory")diff --git a/pipelex/cli/commands/build_cmd.py b/pipelex/cli/commands/build_cmd.pyindex e0de0dee6..673aab60e 100644--- a/pipelex/cli/commands/build_cmd.py+++ b/pipelex/cli/commands/build_cmd.py@@ -35,6 +35,7 @@ pipelex build pipe "Imagine a cute animal mascot for a startup based on its elevpipelex build pipe "Given an expense report, apply company rules"pipelex build pipe "Take a CV in a PDF file, a Job offer text, and analyze if they match"pipelex build pipe "Take a CV in a PDF file and a Job offer text, analyze if they match and generate 5 questions for the interview"+pipelex build pipe "Take a CV and Job offer in PDF, analyze if they match and generate 5 questions for the interview"pipelex build partial "Given an expense report, apply company rules" -o results/generated.jsonpipelex build flow "Given an expense report, apply company rules" -o results/flow.json@@ -74,9 +75,7 @@ def build_pipe_cmd(] = False,) -> None:Pipelex.make(integration_mode=IntegrationMode.CLI)- typer.echo("=" * 70)- typer.secho("π₯ Starting pipe builder... π", fg=typer.colors.GREEN)- typer.echo("")+ typer.secho("π₯ Starting pipe builder... π\n", fg=typer.colors.GREEN)async def run_pipeline():if no_output:@@ -293,9 +292,7 @@ def build_one_shot_cmd(] = False,) -> None:Pipelex.make(integration_mode=IntegrationMode.CLI)- typer.echo("=" * 70)- typer.secho("π₯ Starting pipe builder... π", fg=typer.colors.GREEN)- typer.echo("")+ typer.secho("π₯ Starting pipe builder... π\n", fg=typer.colors.GREEN)async def run_pipeline():if no_output:@@ -370,9 +367,7 @@ def build_partial_cmd(] = False,) -> None:Pipelex.make(integration_mode=IntegrationMode.CLI)- typer.echo("=" * 70)- typer.secho("π₯ Starting pipe builder... π", fg=typer.colors.GREEN)- typer.echo("")+ typer.secho("π₯ Starting pipe builder... π\n", fg=typer.colors.GREEN)async def run_pipeline():output_path: str | None = Nonediff --git a/pipelex/cli/commands/init_cmd.py b/pipelex/cli/commands/init_cmd.pyindex 63312092c..7a7e198dc 100644--- a/pipelex/cli/commands/init_cmd.py+++ b/pipelex/cli/commands/init_cmd.py@@ -349,6 +349,7 @@ def init_cmd(focus: InitFocus = InitFocus.ALL,reset: bool = False,skip_confirmation: bool = False,+ silent: bool = False,):"""Initialize Pipelex configuration, inference backends, and telemetry if needed, in a unified flow.@@ -356,6 +357,7 @@ def init_cmd(focus: What to initialize - 'config', 'inference', 'telemetry', or 'all' (default)reset: Whether to reset/overwrite existing filesskip_confirmation: If True, skip the confirmation prompt (used when called from doctor --fix)+ silent: If True, suppress all output when everything is already configured"""console = Console()pipelex_config_dir = config_manager.pipelex_config_dir@@ -382,6 +384,10 @@ def init_cmd(# If nothing needs to be done, handle based on focusif not needs_config and not needs_inference and not needs_telemetry:+ # In silent mode, just return without any output+ if silent:+ return+if handle_already_configured(focus, console, backends_toml_path, telemetry_config_path):# User wants to reconfigureneeds_config, needs_inference, needs_telemetry = update_needs_for_reconfigure(focus)diff --git a/pipelex/cli/commands/run_cmd.py b/pipelex/cli/commands/run_cmd.pyindex 5c85e3e28..8572f9050 100644--- a/pipelex/cli/commands/run_cmd.py+++ b/pipelex/cli/commands/run_cmd.py@@ -7,6 +7,7 @@ from typing import Annotatedimport clickimport typerfrom posthog import new_context, tag+from rich.console import Consolefrom pipelex import log, pretty_print_mdfrom pipelex.builder.builder import load_and_validate_bundle@@ -190,7 +191,9 @@ def run_cmd(except Exception as exc:log.error(f"Error executing pipeline: {exc}")- typer.secho(f"Failed to execute pipeline: {exc}", fg=typer.colors.RED, err=True)+ console = Console(stderr=True)+ console.print("\n[bold red]Failed to execute pipeline[/bold red]\n")+ console.print_exception(show_locals=True)raise typer.Exit(1) from excwith new_context():diff --git a/pipelex/core/concepts/concept.py b/pipelex/core/concepts/concept.pyindex b3e968752..832ee04df 100644--- a/pipelex/core/concepts/concept.py+++ b/pipelex/core/concepts/concept.py@@ -65,6 +65,8 @@ class Concept(BaseModel):@classmethoddef are_concept_compatible(cls, concept_1: "Concept", concept_2: "Concept", strict: bool = False) -> bool:+ if NativeConceptCode.is_dynamic_concept(concept_code=concept_1.code):+ return Trueif NativeConceptCode.is_dynamic_concept(concept_code=concept_2.code):return Trueif concept_1.concept_string == concept_2.concept_string:diff --git a/pipelex/core/memory/working_memory.py b/pipelex/core/memory/working_memory.pyindex 5889785b2..32673afe6 100644--- a/pipelex/core/memory/working_memory.py+++ b/pipelex/core/memory/working_memory.py@@ -81,7 +81,6 @@ class WorkingMemory(BaseModel, ContextProviderAbstract):def get_main_stuff(self) -> Stuff:return self.get_stuff(name=MAIN_STUFF_NAME)- # TODO: all calls to get_stuff should catch WorkingMemoryStuffNotFoundError in order to indicate what pipe is missing a required stuffdef get_stuff(self, name: str) -> Stuff:if named_stuff := self.root.get(name):return named_stuffdiff --git a/pipelex/core/pipes/pipe_abstract.py b/pipelex/core/pipes/pipe_abstract.pyindex bf7736bf8..551d87746 100644--- a/pipelex/core/pipes/pipe_abstract.py+++ b/pipelex/core/pipes/pipe_abstract.py@@ -61,7 +61,7 @@ class PipeAbstract(ABC, BaseModel):If None, starts recursion detection with an empty set.Returns:- PipeInput containing all needed inputs for this pipe+ InputRequirements containing all needed inputs for this pipe"""diff --git a/pipelex/exceptions.py b/pipelex/exceptions.pyindex f198563a5..ef83c534d 100644--- a/pipelex/exceptions.py+++ b/pipelex/exceptions.py@@ -97,7 +97,10 @@ class WorkingMemoryStuffAttributeNotFoundError(WorkingMemoryVariableError):class WorkingMemoryStuffNotFoundError(WorkingMemoryVariableError):- pass+ def __init__(self, message: str, variable_name: str, pipe_code: str | None = None, concept_code: str | None = None):+ super().__init__(message, variable_name)+ self.pipe_code = pipe_code+ self.concept_code = concept_codeclass PipelexCLIError(PipelexException, ClickException):@@ -227,6 +230,13 @@ class PipeInputError(PipelexException):super().__init__(message)+class PipeRunInputsError(PipelexException):+ def __init__(self, message: str, pipe_code: str, missing_inputs: dict[str, str]):+ self.pipe_code = pipe_code+ self.missing_inputs = missing_inputs+ super().__init__(message)++class DomainLoadingError(LibraryLoadingError):def __init__(self, message: str, domain_code: str, description: str, source: str | None = None):self.domain_code = domain_codediff --git a/pipelex/pipe_controllers/batch/pipe_batch.py b/pipelex/pipe_controllers/batch/pipe_batch.pyindex cedd97ea3..5ad58495c 100644--- a/pipelex/pipe_controllers/batch/pipe_batch.py+++ b/pipelex/pipe_controllers/batch/pipe_batch.py@@ -7,6 +7,7 @@ from typing_extensions import overridefrom pipelex.config import get_configfrom pipelex.core.memory.working_memory import MAIN_STUFF_NAME, WorkingMemory+from pipelex.core.pipe_errors import PipeDefinitionErrorfrom pipelex.core.pipes.input_requirements import InputRequirementsfrom pipelex.core.pipes.pipe_output import PipeOutputfrom pipelex.core.stuffs.list_content import ListContent@@ -14,6 +15,7 @@ from pipelex.core.stuffs.stuff_factory import StuffFactoryfrom pipelex.exceptions import (PipeInputError,PipeInputNotFoundError,+ PipeRunInputsError,WorkingMemoryStuffNotFoundError,)from pipelex.hub import get_pipeline_tracker, get_required_pipe@@ -77,6 +79,20 @@ class PipeBatch(PipeController):def needed_inputs(self, visited_pipes: set[str] | None = None) -> InputRequirements:return self.inputs+ @override+ def _validate_inputs_in_memory(self, working_memory: WorkingMemory) -> None:+ if not self.batch_params:+ raise PipeDefinitionError(message=f"PipeBatch '{self.code}' must have a batch_params", pipe_code=self.code)+ required_concept_code = self.inputs.get_required_input_requirement(variable_name=self.batch_params.input_item_stuff_name).concept.code+ required_stuff_name = self.batch_params.input_list_stuff_name+ try:+ working_memory.get_stuff(required_stuff_name)+ except WorkingMemoryStuffNotFoundError as exc:+ variable_name: str = exc.variable_name or required_stuff_name+ missing_inputs: dict[str, str] = {variable_name: exc.concept_code or required_concept_code}+ msg = f"Missing required inputs for pipe '{self.code}': {missing_inputs}"+ raise PipeRunInputsError(message=msg, pipe_code=self.code, missing_inputs=missing_inputs) from exc+async def _run_batch_pipe(self,job_metadata: JobMetadata,diff --git a/pipelex/pipe_controllers/pipe_controller.py b/pipelex/pipe_controllers/pipe_controller.pyindex 71b87c11c..0fe00ca94 100644--- a/pipelex/pipe_controllers/pipe_controller.py+++ b/pipelex/pipe_controllers/pipe_controller.py@@ -7,6 +7,7 @@ from pipelex import logfrom pipelex.core.memory.working_memory import WorkingMemoryfrom pipelex.core.pipes.pipe_abstract import PipeAbstractfrom pipelex.core.pipes.pipe_output import PipeOutput+from pipelex.exceptions import PipeRunInputsError, WorkingMemoryStuffNotFoundErrorfrom pipelex.pipe_run.pipe_run_mode import PipeRunModefrom pipelex.pipe_run.pipe_run_params import PipeRunParamsfrom pipelex.pipeline.job_metadata import JobMetadata@@ -19,6 +20,19 @@ class PipeController(PipeAbstract):def class_name(self) -> str:return self.__class__.__name__+ def _validate_inputs_in_memory(self, working_memory: WorkingMemory) -> None:+ missing_inputs: dict[str, str] = {}+ for required_stuff_name, requirement in self.needed_inputs().items:+ try:+ working_memory.get_stuff(required_stuff_name)+ except WorkingMemoryStuffNotFoundError as exc:+ variable_name: str = exc.variable_name or required_stuff_name+ missing_inputs[variable_name] = exc.concept_code or requirement.concept.code+ if missing_inputs:+ raise PipeRunInputsError(+ message=f"Missing required inputs for pipe '{self.code}': {missing_inputs}", pipe_code=self.code, missing_inputs=missing_inputs+ )+@overrideasync def run_pipe(self,@@ -36,6 +50,9 @@ class PipeController(PipeAbstract):)job_metadata.update(updated_metadata=updated_metadata)+ # check we have the required inputs in the working memory+ self._validate_inputs_in_memory(working_memory=working_memory)+match pipe_run_params.run_mode:case PipeRunMode.LIVE:indent_level = len(pipe_run_params.pipe_stack) - 1diff --git a/pipelex/pipe_controllers/sub_pipe.py b/pipelex/pipe_controllers/sub_pipe.pyindex 4e85b9fac..42666bb07 100644--- a/pipelex/pipe_controllers/sub_pipe.py+++ b/pipelex/pipe_controllers/sub_pipe.py@@ -87,7 +87,7 @@ class SubPipe(BaseModel):try:required_stuffs = working_memory.get_stuffs(names=required_stuff_names)except WorkingMemoryStuffNotFoundError as exc:- sub_pipe_path = [*sub_pipe_run_params.pipe_layers, self.pipe_code]+ sub_pipe_path = [*sub_pipe_run_params.pipe_stack, self.pipe_code]sub_pipe_path_str = ".".join(sub_pipe_path)error_details = f"SubPipe '{sub_pipe_path_str}', required_variables: {required_variables}, missing: '{exc.variable_name}'"msg = f"Some required stuff(s) not found: {error_details}"diff --git a/pipelex/pipelex.py b/pipelex/pipelex.pyindex 13da1eebb..705e04cd2 100644--- a/pipelex/pipelex.py+++ b/pipelex/pipelex.py@@ -399,7 +399,7 @@ If you need help, drop by our Discord: we're happy to assist: {URLs.discord}.**kwargs,)pipelex_instance.setup_libraries()- log.info(f"{PACKAGE_NAME} version {PACKAGE_VERSION} ready")+ log.verbose(f"{PACKAGE_NAME} version {PACKAGE_VERSION} ready")return pipelex_instance@classmethoddiff --git a/pipelex/tools/misc/context_provider_abstract.py b/pipelex/tools/misc/context_provider_abstract.pyindex 259e66567..ca8ff8f37 100644--- a/pipelex/tools/misc/context_provider_abstract.py+++ b/pipelex/tools/misc/context_provider_abstract.py@@ -5,9 +5,9 @@ from pipelex.system.exceptions import ToolExceptionclass ContextProviderException(ToolException):- def __init__(self, variable_name: str, message: str, *args: object, **kwargs: object) -> None:+ def __init__(self, message: str, variable_name: str):+ super().__init__(message=message)self.variable_name = variable_name- super().__init__(message, *args, **kwargs)class ContextProviderAbstract(ABC):diff --git a/tests/e2e/pipelex/pipes/pipe_controller/pipe_condition/test_pipe_condition_simple.py b/tests/e2e/pipelex/pipes/pipe_controller/pipe_condition/test_pipe_condition_simple.pyindex adada5154..c13df7201 100644--- a/tests/e2e/pipelex/pipes/pipe_controller/pipe_condition/test_pipe_condition_simple.py+++ b/tests/e2e/pipelex/pipes/pipe_controller/pipe_condition/test_pipe_condition_simple.py@@ -8,7 +8,7 @@ from pipelex.core.concepts.concept_native import NativeConceptCodefrom pipelex.core.domains.domain import SpecialDomainfrom pipelex.core.memory.working_memory_factory import WorkingMemoryFactoryfrom pipelex.core.pipes.input_requirements import TypedNamedInputRequirement-from pipelex.exceptions import DryRunMissingInputsError+from pipelex.exceptions import DryRunMissingInputsError, PipeRunInputsErrorfrom pipelex.pipe_controllers.condition.pipe_condition_blueprint import PipeConditionBlueprintfrom pipelex.pipe_controllers.condition.pipe_condition_factory import PipeConditionFactoryfrom pipelex.pipe_run.pipe_run_params import PipeRunMode@@ -42,7 +42,7 @@ class TestPipeConditionSimple:# Test with empty working memory - should FAILempty_working_memory = WorkingMemoryFactory.make_empty()- with pytest.raises(DryRunMissingInputsError) as exc_info:+ with pytest.raises(PipeRunInputsError) as exc_info:await pipe_condition.run_pipe(job_metadata=JobMetadata(job_name="test_direct_condition_fail"),working_memory=empty_working_memory,@@ -53,7 +53,7 @@ class TestPipeConditionSimple:error = exc_info.valueassert error.pipe_code == "test_condition_fail"assert "user_category" in error.missing_inputs- assert "missing required inputs" in str(error)+ assert "Missing required inputs" in str(error)async def test_direct_pipe_condition_should_succeed(self):"""Test a PipeCondition created directly in code that should SUCCEED dry run."""diff --git a/tests/integration/pipelex/pipes/controller/pipe_condition/test_pipe_condition_complex.py b/tests/integration/pipelex/pipes/controller/pipe_condition/test_pipe_condition_complex.pyindex d400470dc..93eef19e7 100644--- a/tests/integration/pipelex/pipes/controller/pipe_condition/test_pipe_condition_complex.py+++ b/tests/integration/pipelex/pipes/controller/pipe_condition/test_pipe_condition_complex.py@@ -10,7 +10,7 @@ from pipelex.core.concepts.concept_factory import ConceptFactoryfrom pipelex.core.memory.working_memory_factory import WorkingMemoryFactoryfrom pipelex.core.stuffs.stuff_factory import StuffFactoryfrom pipelex.core.stuffs.text_content import TextContent-from pipelex.exceptions import PipeRouterError+from pipelex.exceptions import PipeRunInputsErrorfrom pipelex.hub import get_pipe_router, get_required_pipefrom pipelex.pipe_run.pipe_job_factory import PipeJobFactoryfrom pipelex.pipe_run.pipe_run_params import PipeRunMode@@ -280,7 +280,7 @@ class TestPipeConditionComplex:working_memory = WorkingMemoryFactory.make_from_single_stuff(doc_stuff)- with pytest.raises(PipeRouterError) as exc_info:+ with pytest.raises(PipeRunInputsError) as exc_info:await get_pipe_router().run(pipe_job=PipeJobFactory.make_pipe_job(pipe=get_required_pipe(pipe_code="complex_document_processor"),@@ -294,7 +294,7 @@ class TestPipeConditionComplex:assert error.pipe_code == "complex_document_processor"assert error.missing_inputs is not Noneassert "user_profile" in error.missing_inputs- assert "missing required inputs" in str(error)+ assert "Missing required inputs" in str(error)@pytest.mark.parametrize(("doc_type", "priority", "user_level", "department", "complexity", "language", "expected_output_contains"),diff --git a/tests/integration/pipelex/pipes/controller/pipe_condition/test_pipe_condition_simple.py b/tests/integration/pipelex/pipes/controller/pipe_condition/test_pipe_condition_simple.pyindex d3a5180a2..7aa3da4d0 100644--- a/tests/integration/pipelex/pipes/controller/pipe_condition/test_pipe_condition_simple.py+++ b/tests/integration/pipelex/pipes/controller/pipe_condition/test_pipe_condition_simple.py@@ -12,7 +12,7 @@ from pipelex.core.domains.domain import SpecialDomainfrom pipelex.core.memory.working_memory_factory import WorkingMemoryFactoryfrom pipelex.core.stuffs.stuff_factory import StuffFactoryfrom pipelex.core.stuffs.text_content import TextContent-from pipelex.exceptions import PipeRouterError+from pipelex.exceptions import PipeRunInputsErrorfrom pipelex.hub import get_pipe_router, get_required_pipefrom pipelex.pipe_controllers.condition.pipe_condition_blueprint import PipeConditionBlueprintfrom pipelex.pipe_controllers.condition.pipe_condition_factory import PipeConditionFactory@@ -219,7 +219,7 @@ class TestPipeConditionSimple:empty_working_memory = WorkingMemoryFactory.make_empty()# Run dry run using the real pipe - this should fail with PipeRouterError- with pytest.raises(PipeRouterError) as exc_info:+ with pytest.raises(PipeRunInputsError) as exc_info:await get_pipe_router().run(pipe_job=PipeJobFactory.make_pipe_job(pipe=get_required_pipe(pipe_code="basic_condition_by_category_2"),@@ -234,8 +234,7 @@ class TestPipeConditionSimple:assert error.pipe_code == "basic_condition_by_category_2"assert error.missing_inputs is not Noneassert "input_data" in error.missing_inputs- assert "missing required inputs" in str(error)- assert "input_data" in str(error)+ assert "Missing required inputs" in str(error)async def test_condition_dry_run_medium_category_validation(self, request: FixtureRequest):"""Test PipeCondition dry run with medium category - should validate the 'medium' branch."""diff --git a/tests/test_pipelines/discord_newsletter.plx b/tests/test_pipelines/discord_newsletter.plxindex 885ab08f6..f912d9308 100644--- a/tests/test_pipelines/discord_newsletter.plx+++ b/tests/test_pipelines/discord_newsletter.plx@@ -9,7 +9,7 @@ HtmlNewsletter = "The final newsletter content in html format with organized cha[pipe.write_discord_newsletter]type = "PipeSequence"description = "Create a newsletter from Discord articles by summarizing channels and organizing content"-inputs = { discord_channel_updates = "DiscordChannelUpdate" }+inputs = { discord_channel_updates = "DiscordChannelUpdate[]" }output = "HtmlNewsletter"steps = [{ pipe = "summarize_discord_channel_update", batch_over = "discord_channel_updates", batch_as = "discord_channel_update", result = "channel_summaries" },