Build AI methods with your coding agent, then run them anywhere: as an MCP for chatbots, as a webapp for people, or via API for your software.
Rename PipelineBlueprint, improve prompts
Louis Choquel12acf596f4aa
diff --git a/.pipelex/create/build_pipelines.md b/.pipelex/create/build_pipelines.mdindex deab04c86..1dc666313 100644--- a/.pipelex/create/build_pipelines.md+++ b/.pipelex/create/build_pipelines.md@@ -1,6 +1,6 @@# Building Pipelex Pipelines from Drafts-This guide explains how to translate a markdown pipeline draft into a formal `PipelineLibraryBlueprint` structure that can be validated and executed by Pipelex.+This guide explains how to translate a markdown pipeline draft into a formal `PipelineBlueprint` structure that can be validated and executed by Pipelex.## Overviewdiff --git a/.pipelex/create/design_pipelines.md b/.pipelex/create/design_pipelines.mdindex 5a13bbf5e..2a3fcfb80 100644--- a/.pipelex/create/design_pipelines.md+++ b/.pipelex/create/design_pipelines.md@@ -6,7 +6,7 @@ This section explains how to design pipelines for Pipelex.- Plan the pipeline in natural language (goal, inputs, steps, outputs).-- Translate the plan into a `PipelineLibraryBlueprint` object graph: domain, concepts, pipes.+- Translate the plan into a `PipelineBlueprint` object graph: domain, concepts, pipes.- Define controllers first (`PipeSequence`, `PipeParallel`, `PipeCondition`), then operators (`PipeLLM`, `PipeOcr`, etc.).@@ -16,9 +16,9 @@ This section explains how to design pipelines for Pipelex.- Choose appropriate native or structured output concepts.-### Root blueprint object: PipelineLibraryBlueprint+### Root blueprint object: PipelineBlueprint-`PipelineLibraryBlueprint` fields:+`PipelineBlueprint` fields:- `domain: str` — required. Single word or snake_case code for the domain.diff --git a/pipelex/core/registry_models.py b/pipelex/core/registry_models.pyindex 498dd3c93..c9b2bfd95 100644--- a/pipelex/core/registry_models.py+++ b/pipelex/core/registry_models.py@@ -17,7 +17,7 @@ from pipelex.core.stuff_content import (TextAndImagesContent,TextContent,)-from pipelex.libraries.pipeline_blueprint import PipelineLibraryBlueprint+from pipelex.libraries.pipeline_blueprint import PipelineBlueprintfrom pipelex.libraries.pipelines.meta.pipeline_draft import PipelineDraftfrom pipelex.pipe_controllers.pipe_batch import PipeBatchfrom pipelex.pipe_controllers.pipe_batch_factory import PipeBatchFactory@@ -88,7 +88,7 @@ class PipelexRegistryModels(RegistryModels):PDFContent,TextAndImagesContent,PageContent,- PipelineLibraryBlueprint,+ PipelineBlueprint,PipelineDraft,]diff --git a/pipelex/create/build_blueprint.py b/pipelex/create/build_blueprint.pyindex 8f594d160..affce8844 100644--- a/pipelex/create/build_blueprint.py+++ b/pipelex/create/build_blueprint.py@@ -13,7 +13,7 @@ from pipelex.create.pipeline_toml import save_pipeline_blueprint_toml_to_pathfrom pipelex.exceptions import PipeDefinitionError, PipelexCLIErrorfrom pipelex.hub import get_library_managerfrom pipelex.libraries.library_manager import LibraryManager-from pipelex.libraries.pipeline_blueprint import PipelineLibraryBlueprint+from pipelex.libraries.pipeline_blueprint import PipelineBlueprintfrom pipelex.pipe_works.pipe_dry import dry_run_pipe_codesfrom pipelex.pipeline.execute import execute_pipelinefrom pipelex.tools.misc.file_utils import save_text_to_path@@ -38,7 +38,7 @@ async def do_build_blueprint("build_pipeline_rules": get_support_file(subpath="create/build_pipelines.md"),},)- blueprint = pipe_output.main_stuff_as(content_type=PipelineLibraryBlueprint)+ blueprint = pipe_output.main_stuff_as(content_type=PipelineBlueprint)pretty_print(blueprint, title="Pipeline Blueprint")pipeline_draft = pipe_output.working_memory.get_stuff_as_str(name="pipeline_draft")pretty_print(pipeline_draft, title="Pipeline Draft")@@ -66,8 +66,8 @@ async def do_build_blueprint(asyncio.run(dry_run_pipe_codes(pipe_codes=generated_pipe_codes))-def _load_pipes_from_generated_blueprint(blueprint: PipelineLibraryBlueprint) -> None:- """Instantiate and register all pipes from a generated PipelineLibraryBlueprint.+def _load_pipes_from_generated_blueprint(blueprint: PipelineBlueprint) -> None:+ """Instantiate and register all pipes from a generated PipelineBlueprint.This constructs the appropriate typed PipeBlueprint for each pipe entry anduses LibraryManager.load_pipe_from_blueprint to create the Pipe, then adds itdiff --git a/pipelex/create/pipeline_toml.py b/pipelex/create/pipeline_toml.pyindex 947b7690f..9e6bd8a5c 100644--- a/pipelex/create/pipeline_toml.py+++ b/pipelex/create/pipeline_toml.py@@ -6,7 +6,7 @@ import tomlkitfrom tomlkit import array, document, inline_table, tablefrom tomlkit import string as tk_string-from pipelex.libraries.pipeline_blueprint import PipelineLibraryBlueprint+from pipelex.libraries.pipeline_blueprint import PipelineBlueprintfrom pipelex.tools.misc.file_utils import save_text_to_pathfrom pipelex.tools.misc.json_utils import remove_none_values_from_dictfrom pipelex.tools.misc.toml_utils import clean_trailing_whitespace@@ -113,7 +113,7 @@ def dict_to_toml(data: Mapping[str, Any]) -> str:return dumped_content-def pipeline_blueprint_to_toml(blueprint: PipelineLibraryBlueprint) -> str:+def pipeline_blueprint_to_toml(blueprint: PipelineBlueprint) -> str:blueprint_dict = blueprint.smart_dump()return dict_to_toml(data=blueprint_dict)@@ -133,6 +133,6 @@ def save_toml_to_path(data: Dict[str, Any], path: str) -> None:file.write(cleaned_content)-def save_pipeline_blueprint_toml_to_path(blueprint: PipelineLibraryBlueprint, path: str) -> None:+def save_pipeline_blueprint_toml_to_path(blueprint: PipelineBlueprint, path: str) -> None:pipeline_blueprint_toml = pipeline_blueprint_to_toml(blueprint=blueprint)save_text_to_path(text=pipeline_blueprint_toml, path=path)diff --git a/pipelex/libraries/library_manager.py b/pipelex/libraries/library_manager.pyindex 1730c9ed2..b6e0a56e1 100644--- a/pipelex/libraries/library_manager.py+++ b/pipelex/libraries/library_manager.py@@ -30,8 +30,8 @@ from pipelex.libraries.library_manager_abstract import LibraryManagerAbstractfrom pipelex.libraries.pipeline_blueprint import (ConceptBlueprintError,PipeBlueprintError,+ PipelineBlueprint,PipelineBlueprintValidationError,- PipelineLibraryBlueprint,)from pipelex.tools.class_registry_utils import ClassRegistryUtilsfrom pipelex.tools.misc.file_utils import find_files_in_dir@@ -212,11 +212,11 @@ class LibraryManager(LibraryManagerAbstract):nb_pipes_loaded = len(self.pipe_library.root) - nb_pipes_beforelog.verbose(f"Loaded {nb_pipes_loaded} pipes from '{toml_path.name}'")- def _load_blueprint_from_file(self, toml_path: Path) -> PipelineLibraryBlueprint:+ def _load_blueprint_from_file(self, toml_path: Path) -> PipelineBlueprint:"""Load and validate a pipeline blueprint from a TOML file."""try:toml_data = load_toml_from_path(path=str(toml_path))- blueprint = PipelineLibraryBlueprint.model_validate(toml_data)+ blueprint = PipelineBlueprint.model_validate(toml_data)return blueprintexcept ValidationError as exc:error_msg = format_pydantic_validation_error(exc)@@ -227,7 +227,7 @@ class LibraryManager(LibraryManagerAbstract):except Exception as exc:raise LibraryError(f"Failed to load TOML file '{toml_path}': {exc}") from exc- def _load_concepts_from_blueprint(self, blueprint: PipelineLibraryBlueprint, file_path: str):+ def _load_concepts_from_blueprint(self, blueprint: PipelineBlueprint, file_path: str):"""Load concepts from a validated blueprint."""for concept_name, concept_data in blueprint.concept.items():try:@@ -255,7 +255,7 @@ class LibraryManager(LibraryManagerAbstract):error_msg=error_msg,) from exc- def _load_pipes_from_blueprint(self, blueprint: PipelineLibraryBlueprint, file_path: str):+ def _load_pipes_from_blueprint(self, blueprint: PipelineBlueprint, file_path: str):"""Load pipes from a validated blueprint."""for pipe_name, pipe_data in blueprint.pipe.items():try:diff --git a/pipelex/libraries/pipeline_blueprint.py b/pipelex/libraries/pipeline_blueprint.pyindex 85e3c8ea7..54ebb0331 100644--- a/pipelex/libraries/pipeline_blueprint.py+++ b/pipelex/libraries/pipeline_blueprint.py@@ -34,8 +34,8 @@ class PipeBlueprintError(LibraryError):super().__init__(f"Error loading pipe '{pipe_name}' from '{file_path}': {error_msg}")-class PipelineLibraryBlueprint(StructuredContent):- """Complete blueprint of a pipeline library TOML file."""+class PipelineBlueprint(StructuredContent):+ """Complete blueprint of a pipeline TOML definition."""# Domain information (required)domain: strdiff --git a/pipelex/libraries/pipelines/meta/meta_pipeline.toml b/pipelex/libraries/pipelines/meta/meta_pipeline.tomlindex 7b29af98b..21d6d99d0 100644--- a/pipelex/libraries/pipelines/meta/meta_pipeline.toml+++ b/pipelex/libraries/pipelines/meta/meta_pipeline.toml@@ -5,7 +5,7 @@ definition = "Meta domain is the domain of pipelines about pipelines"PipelineDraft = "Rough solution to create a pipeline"PipelineDraftText = "Rough solution to create a pipeline in text format"PipeDraft = "Rough solution to create a pipe"-PipelineLibraryBlueprint = "Details enabling to create a pipeline"+PipelineBlueprint = "Details enabling to create a pipeline"ConceptBlueprint = "Details enabling to create a concept"PipeBlueprint = "Details enabling to create a pipe"@@ -15,7 +15,7 @@ PipeBlueprint = "Details enabling to create a pipe"type = "PipeSequence"definition = "Build a pipeline blueprint from natural language requirements"inputs = { draft_pipeline_rules = "Text", build_pipeline_rules = "Text", requirements = "Text", pipeline_name = "Text", domain = "Text" }-output = "PipelineLibraryBlueprint"+output = "PipelineBlueprint"steps = [{ pipe = "draft_pipeline_text", result = "pipeline_draft" },{ pipe = "structure_blueprint", result = "pipeline_blueprint" },@@ -32,17 +32,19 @@ You are a Pipelex pipeline architect. Build a pipeline draft text showing how we@draft_pipeline_rules-@requirements+---+Name of the pipeline: $pipeline_name.Domain: $domain.-Name of the pipeline: $pipeline_name.++Requirements: $requirements"""[pipe.structure_blueprint]type = "PipeLLM"definition = "Generate a pipeline blueprint from natural language requirements"-inputs = { build_pipeline_rules = "Text", pipeline_draft = "Text", pipeline_name = "Text", domain = "Text" }-output = "PipelineLibraryBlueprint"+inputs = { build_pipeline_rules = "Text", pipeline_draft = "Text", pipeline_name = "Text", domain = "Text", requirements = "Text" }+output = "PipelineBlueprint"llm = "llm_to_pipelex"# structuring_method = "preliminary_text"prompt_template = """@@ -52,7 +54,11 @@ You are a Pipelex pipeline architect. Build a structured pipeline blueprint from@pipeline_draft-Domain: $domain.+---Name of the pipeline: $pipeline_name.++Domain: $domain.++Requirements: $requirements."""diff --git a/tests/integration/pipelex/pipes/pipe_operator/pipe_llm/test_generate_pipeline_blueprint.py b/tests/integration/pipelex/pipes/pipe_operator/pipe_llm/test_generate_pipeline_blueprint.pyindex 51e28d102..1e1805bba 100644--- a/tests/integration/pipelex/pipes/pipe_operator/pipe_llm/test_generate_pipeline_blueprint.py+++ b/tests/integration/pipelex/pipes/pipe_operator/pipe_llm/test_generate_pipeline_blueprint.py@@ -3,7 +3,7 @@ import pytestfrom pipelex import pretty_printfrom pipelex.core.pipe_run_params import PipeRunModefrom pipelex.create.helpers import get_support_file-from pipelex.libraries.pipeline_blueprint import PipelineLibraryBlueprint+from pipelex.libraries.pipeline_blueprint import PipelineBlueprintfrom pipelex.pipeline.execute import execute_pipeline@@ -32,13 +32,14 @@ async def test_generate_pipeline_blueprint(pipe_run_mode: PipeRunMode, requireme"domain": "test_domain","pipeline_name": "test_pipeline","requirements": requirements,- "rules": get_support_file("create/design_pipelines.md"),+ "draft_pipeline_rules": get_support_file(subpath="create/draft_pipelines.md"),+ "build_pipeline_rules": get_support_file(subpath="create/build_pipelines.md"),},pipe_run_mode=pipe_run_mode,)- blueprint = pipe_output.main_stuff_as(content_type=PipelineLibraryBlueprint)- assert isinstance(blueprint, PipelineLibraryBlueprint)+ blueprint = pipe_output.main_stuff_as(content_type=PipelineBlueprint)+ assert isinstance(blueprint, PipelineBlueprint)# Basic sanity checksassert isinstance(blueprint.domain, str) and blueprint.domain != ""