Build AI methods with your coding agent, then run them anywhere: as an MCP for chatbots, as a webapp for people, or via API for your software.
Pipeline draft, as text or structured
Louis Choquel59b165ef42f3
diff --git a/pipelex/cli/commands/build_cmd.py b/pipelex/cli/commands/build_cmd.pyindex b0aca2346..061799c30 100644--- a/pipelex/cli/commands/build_cmd.py+++ b/pipelex/cli/commands/build_cmd.py@@ -1,113 +1,76 @@from __future__ import annotationsimport asyncio-from typing import Annotated, Any, Dict, Optional, cast+from typing import Annotated, Optionalimport typer-from pydantic import ValidationError-from pipelex import pretty_print-from pipelex.exceptions import PipeDefinitionError, PipelexCLIError-from pipelex.hub import get_library_manager-from pipelex.libraries.library_manager import LibraryManager-from pipelex.libraries.pipeline_blueprint import PipelineLibraryBlueprint-from pipelex.pipe_works.pipe_dry import dry_run_pipe_codes+from pipelex.create.build_blueprint import do_build_blueprint+from pipelex.create.draft_pipeline import do_draft_pipeline, do_draft_pipeline_text+from pipelex.exceptions import PipelexCLIErrorfrom pipelex.pipelex import Pipelex-from pipelex.pipeline.execute import execute_pipelinefrom pipelex.tools.misc.file_utils import load_text_from_path-from pipelex.tools.typing.pydantic_utils import format_pydantic_validation_error--def _load_requirements_text(requirements: Optional[str], requirements_file: Optional[str]) -> str:- if requirements_file:- return load_text_from_path(requirements_file)- if not requirements:- raise PipelexCLIError("You must provide requirements text via --requirements or a file via --file")- return requirements---async def _generate_blueprint_async(requirements_text: str) -> PipelineLibraryBlueprint:- pipe_output = await execute_pipeline(- pipe_code="generate_pipeline_blueprint",- input_memory={- "requirements": requirements_text,- },- )- return pipe_output.main_stuff_as(content_type=PipelineLibraryBlueprint)+# Typer group for build commands+build_app = typer.Typer(help="Build artifacts like pipeline blueprints")-def do_build_blueprint(- relative_config_folder_path: str,- requirements: Optional[str],- requirements_file: Optional[str],- output_path: Optional[str],- validate: bool,+@build_app.command("draft")+def build_draft_pipeline_cmd(+ requirements: Annotated[+ Optional[str],+ typer.Option("--requirements", "-r", help="Requirements text to generate the pipeline blueprint from"),+ ] = None,+ requirements_file: Annotated[+ Optional[str],+ typer.Option("--file", "-f", help="Path to a file containing the requirements text"),+ ] = None,+ output_path: Annotated[+ Optional[str],+ typer.Option("--output", "-o", help="Path to save the generated TOML blueprint (optional)"),+ ] = None,+ raw: Annotated[+ bool,+ typer.Option("--raw", help="Raw text draft"),+ ] = False,+ relative_config_folder_path: Annotated[+ str,+ typer.Option(+ "--config-folder-path",+ "-c",+ help="Relative path to the config folder path (libraries)",+ ),+ ] = "./pipelex_libraries",) -> None:# Initialize Pipelex (loads libraries and pipes)Pipelex.make(relative_config_folder_path=relative_config_folder_path, from_file=False)# Get requirements texttry:- requirements_text = _load_requirements_text(requirements=requirements, requirements_file=requirements_file)+ requirements_text: str+ if requirements_file:+ requirements_text = load_text_from_path(requirements_file)+ else:+ if not requirements:+ raise PipelexCLIError("You must provide requirements text via --requirements or a file via --file")+ requirements_text = requirementsexcept Exception as exc:raise PipelexCLIError(f"Failed to load requirements: {exc}") from exc- # Generate blueprint- blueprint = asyncio.run(_generate_blueprint_async(requirements_text=requirements_text))-- pretty_print(blueprint, title="Pipeline Blueprint")- pretty_print(blueprint.to_toml_dict(), title="Pipeline Blueprint (TOML)")-- # Save or display result- output_path = output_path or "pipelex/libraries/pipelines/temp/generated_blueprint.toml"- blueprint.save_to_file(output_path)- typer.echo(f"✅ Blueprint saved to '{output_path}'")-- if validate:- try:- _load_pipes_from_generated_blueprint(blueprint=blueprint)- except PipeDefinitionError as exc:- raise PipelexCLIError(f"Failed to load pipes from generated blueprint at '{output_path}': {exc}") from exc-- generated_pipe_codes = list(blueprint.pipe.keys())- if not generated_pipe_codes:- raise PipelexCLIError("No pipe found in generated blueprint to validate")- asyncio.run(dry_run_pipe_codes(pipe_codes=generated_pipe_codes))---def _load_pipes_from_generated_blueprint(blueprint: PipelineLibraryBlueprint) -> None:- """Instantiate and register all pipes from a generated PipelineLibraryBlueprint.-- This constructs the appropriate typed PipeBlueprint for each pipe entry and- uses LibraryManager.load_pipe_from_blueprint to create the Pipe, then adds it- to the in-memory PipeLibrary so that validation can run against them.- """- library_manager = cast(LibraryManager, get_library_manager())- domain_code = blueprint.domain-- for pipe_code, details in blueprint.pipe.items():- details_dict: Dict[str, Any] = details.copy()-- # Build the concrete PipeBlueprint via LibraryManager helper- try:- pipe_blueprint = LibraryManager.make_pipe_blueprint_from_details(- domain_code=domain_code,- details_dict=details_dict,+ if raw:+ asyncio.run(+ do_draft_pipeline_text(+ requirements=requirements_text,+ output_path=output_path,+ )+ )+ else:+ asyncio.run(+ do_draft_pipeline(+ requirements=requirements_text,+ output_path=output_path,)- except ValidationError as exc:- error_msg = format_pydantic_validation_error(exc=exc)- raise PipeDefinitionError(f"Failed to build pipe blueprint for pipe '{pipe_code}': {error_msg}") from exc-- # Create pipe and register in library- pipe = LibraryManager.load_pipe_from_blueprint(- pipe_code=pipe_code,- pipe_blueprint=pipe_blueprint,)- library_manager.pipe_library.add_new_pipe(pipe=pipe)---# Typer group for build commands-build_app = typer.Typer(help="Build artifacts like pipeline blueprints")@build_app.command("blueprint")@@ -137,22 +100,25 @@ def build_blueprint_cmd(),] = "./pipelex_libraries",) -> None:- # try:- # do_build_blueprint(- # relative_config_folder_path=relative_config_folder_path,- # requirements=requirements,- # requirements_file=requirements_file,- # output_path=output_path,- # validate=validate,- # )- # except Exception as exc:- # typer.echo(f"❌ Error: {exc}")- # raise typer.Exit(1)+ # Initialize Pipelex (loads libraries and pipes)+ Pipelex.make(relative_config_folder_path=relative_config_folder_path, from_file=False)- do_build_blueprint(- relative_config_folder_path=relative_config_folder_path,- requirements=requirements,- requirements_file=requirements_file,- output_path=output_path,- validate=validate,+ # Get requirements text+ try:+ requirements_text: str+ if requirements_file:+ requirements_text = load_text_from_path(requirements_file)+ else:+ if not requirements:+ raise PipelexCLIError("You must provide requirements text via --requirements or a file via --file")+ requirements_text = requirements+ except Exception as exc:+ raise PipelexCLIError(f"Failed to load requirements: {exc}") from exc++ asyncio.run(+ do_build_blueprint(+ requirements=requirements_text,+ output_path=output_path,+ validate=validate,+ ))diff --git a/pipelex/core/registry_models.py b/pipelex/core/registry_models.pyindex 0d9c0faad..498dd3c93 100644--- a/pipelex/core/registry_models.py+++ b/pipelex/core/registry_models.py@@ -18,6 +18,7 @@ from pipelex.core.stuff_content import (TextContent,)from pipelex.libraries.pipeline_blueprint import PipelineLibraryBlueprint+from pipelex.libraries.pipelines.meta.pipeline_draft import PipelineDraftfrom pipelex.pipe_controllers.pipe_batch import PipeBatchfrom pipelex.pipe_controllers.pipe_batch_factory import PipeBatchFactoryfrom pipelex.pipe_controllers.pipe_condition import PipeCondition@@ -88,6 +89,7 @@ class PipelexRegistryModels(RegistryModels):TextAndImagesContent,PageContent,PipelineLibraryBlueprint,+ PipelineDraft,]EXPERIMENTAL: ClassVar[List[ModelType]] = [diff --git a/pipelex/create/build_blueprint.py b/pipelex/create/build_blueprint.pynew file mode 100644index 000000000..d73460547--- /dev/null+++ b/pipelex/create/build_blueprint.py@@ -0,0 +1,82 @@+from __future__ import annotations++import asyncio+from typing import Annotated, Any, Dict, Optional, cast++import typer+from pydantic import ValidationError++from pipelex import pretty_print+from pipelex.exceptions import PipeDefinitionError, PipelexCLIError+from pipelex.hub import get_library_manager+from pipelex.libraries.library_manager import LibraryManager+from pipelex.libraries.pipeline_blueprint import PipelineLibraryBlueprint+from pipelex.libraries.pipelines.meta.pipeline_draft import PipelineDraft+from pipelex.pipe_works.pipe_dry import dry_run_pipe_codes+from pipelex.pipelex import Pipelex+from pipelex.pipeline.execute import execute_pipeline+from pipelex.tools.misc.file_utils import load_text_from_path+from pipelex.tools.typing.pydantic_utils import format_pydantic_validation_error+++async def do_build_blueprint(+ requirements: str,+ output_path: Optional[str],+ validate: bool,+) -> None:+ pipe_output = await execute_pipeline(+ pipe_code="draft_pipeline",+ input_memory={+ "requirements": requirements,+ },+ )+ blueprint = pipe_output.main_stuff_as(content_type=PipelineLibraryBlueprint)+ pretty_print(blueprint, title="Pipeline Blueprint")+ pretty_print(blueprint.to_toml_dict(), title="Pipeline Blueprint (TOML)")++ # Save or display result+ output_path = output_path or "pipelex/libraries/pipelines/temp/generated_blueprint.toml"+ blueprint.save_to_file(output_path)+ typer.echo(f"✅ Blueprint saved to '{output_path}'")++ if validate:+ try:+ _load_pipes_from_generated_blueprint(blueprint=blueprint)+ except PipeDefinitionError as exc:+ raise PipelexCLIError(f"Failed to load pipes from generated blueprint at '{output_path}': {exc}") from exc++ generated_pipe_codes = list(blueprint.pipe.keys())+ if not generated_pipe_codes:+ raise PipelexCLIError("No pipe found in generated blueprint to validate")+ asyncio.run(dry_run_pipe_codes(pipe_codes=generated_pipe_codes))+++def _load_pipes_from_generated_blueprint(blueprint: PipelineLibraryBlueprint) -> None:+ """Instantiate and register all pipes from a generated PipelineLibraryBlueprint.++ This constructs the appropriate typed PipeBlueprint for each pipe entry and+ uses LibraryManager.load_pipe_from_blueprint to create the Pipe, then adds it+ to the in-memory PipeLibrary so that validation can run against them.+ """+ library_manager = cast(LibraryManager, get_library_manager())+ domain_code = blueprint.domain++ for pipe_code, details in blueprint.pipe.items():+ details_dict: Dict[str, Any] = details.copy()++ # Build the concrete PipeBlueprint via LibraryManager helper+ try:+ pipe_blueprint = LibraryManager.make_pipe_blueprint_from_details(+ domain_code=domain_code,+ details_dict=details_dict,+ )+ except ValidationError as exc:+ error_msg = format_pydantic_validation_error(exc=exc)+ raise PipeDefinitionError(f"Failed to build pipe blueprint for pipe '{pipe_code}': {error_msg}") from exc++ # Create pipe and register in library+ pipe = LibraryManager.load_pipe_from_blueprint(+ pipe_code=pipe_code,+ pipe_blueprint=pipe_blueprint,+ )+ library_manager.pipe_library.add_new_pipe(pipe=pipe)diff --git a/pipelex/create/draft_pipeline.py b/pipelex/create/draft_pipeline.pynew file mode 100644index 000000000..26fcf1948--- /dev/null+++ b/pipelex/create/draft_pipeline.py@@ -0,0 +1,58 @@+from __future__ import annotations++import asyncio+from typing import Annotated, Any, Dict, Optional, cast++import typer+from pydantic import ValidationError++from pipelex import pretty_print+from pipelex.exceptions import PipeDefinitionError, PipelexCLIError+from pipelex.hub import get_library_manager+from pipelex.libraries.library_manager import LibraryManager+from pipelex.libraries.pipeline_blueprint import PipelineLibraryBlueprint+from pipelex.libraries.pipelines.meta.pipeline_draft import PipelineDraft+from pipelex.pipe_works.pipe_dry import dry_run_pipe_codes+from pipelex.pipelex import Pipelex+from pipelex.pipeline.execute import execute_pipeline+from pipelex.tools.misc.file_utils import load_text_from_path, save_text_to_path+from pipelex.tools.typing.pydantic_utils import format_pydantic_validation_error+++async def do_draft_pipeline_text(+ requirements: str,+ output_path: Optional[str],+) -> None:+ pipe_output = await execute_pipeline(+ pipe_code="draft_pipeline_text",+ input_memory={+ "requirements": requirements,+ },+ )+ draft_text = pipe_output.main_stuff_as_str+ pretty_print(draft_text, title="Pipeline Draft Text")++ # Save or display result+ output_path = output_path or "pipelex/libraries/pipelines/temp/pipeline_draft.text"+ save_text_to_path(text=draft_text, path=output_path)+ typer.echo(f"✅ Pipeline draft text saved to '{output_path}'")+++async def do_draft_pipeline(+ requirements: str,+ output_path: Optional[str],+) -> None:+ pipe_output = await execute_pipeline(+ pipe_code="draft_pipeline",+ input_memory={+ "requirements": requirements,+ },+ )+ draft = pipe_output.main_stuff_as(content_type=PipelineDraft)+ pretty_print(draft, title="Pipeline Draft")+ pretty_print(draft.to_toml_dict(), title="Pipeline Draft (TOML)")++ # Save or display result+ output_path = output_path or "pipelex/libraries/pipelines/temp/generated_blueprint.toml"+ draft.save_to_file(output_path)+ typer.echo(f"✅ Blueprint saved to '{output_path}'")diff --git a/pipelex/libraries/pipeline_blueprint.py b/pipelex/libraries/pipeline_blueprint.pyindex 7d5d4177d..6a5149baa 100644--- a/pipelex/libraries/pipeline_blueprint.py+++ b/pipelex/libraries/pipeline_blueprint.py@@ -1,7 +1,7 @@from pathlib import Pathfrom typing import Any, Dict, Optional, Union-from pydantic import BaseModel, Field+from pydantic import Fieldfrom pipelex.core.concept_factory import ConceptBlueprintfrom pipelex.core.stuff_content import StructuredContent@@ -36,13 +36,6 @@ class PipeBlueprintError(LibraryError):super().__init__(f"Error loading pipe '{pipe_name}' from '{file_path}': {error_msg}")-class PipeStepBlueprint(BaseModel):- """Blueprint of a step in a pipe sequence."""-- pipe: str- result: str--class PipelineLibraryBlueprint(StructuredContent):"""Complete blueprint of a pipeline library TOML file."""diff --git a/pipelex/libraries/pipelines/meta/meta_pipeline.toml b/pipelex/libraries/pipelines/meta/meta_pipeline.tomlindex 3d09717a3..ba0b14c5a 100644--- a/pipelex/libraries/pipelines/meta/meta_pipeline.toml+++ b/pipelex/libraries/pipelines/meta/meta_pipeline.toml@@ -2,30 +2,60 @@ domain = "meta"definition = "Meta domain is the domain of pipelines about pipelines"[concept]+PipelineDraft = "Rough solution to create a pipeline"+PipelineDraftText = "Rough solution to create a pipeline in text format"+PipeDraft = "Rough solution to create a pipe"PipelineLibraryBlueprint = "Details enabling to create a pipeline"ConceptBlueprint = "Details enabling to create a concept"PipeBlueprint = "Details enabling to create a pipe"[pipe]-[pipe.generate_pipeline_blueprint]+[pipe.draft_pipeline_text]type = "PipeLLM"-definition = "Generate a PipelineLibraryBlueprint from natural language requirements"+definition = "Generate a rough pipeline draft from natural language requirements"inputs = { requirements = "Text" }-output = "PipelineLibraryBlueprint"+output = "PipelineDraftText"+prompt_template = """+You are a Pipelex pipeline architect. Build a pipeline draft showing how we solve the problem.++Think on the pipeline requirements: what are the pipeline's inputs and what is the expected output?++- Keep domain and pipe codes short, lowercase, and meaningful.+- In concept, map ConceptName (PascalCase, singular) to its natural language definition string. Do not redefine native concepts (Text, Image, PDF, TextAndImages, Number, Page).+- In pipe, define at least one pipe that accomplishes the main goal implied by the requirements. Use operators/controllers supported by Pipelex (e.g., PipeLLM, PipeOcr, PipeSequence...).+- Ensure all input variables referenced in prompts are listed in the pipe `inputs`.++@requirements+"""+++[pipe.draft_pipeline]+type = "PipeLLM"+definition = "Generate a rough pipeline draft from natural language requirements"+inputs = { requirements = "Text" }+output = "PipelineDraft"structuring_method = "preliminary_text"prompt_template = """-You are a Pipelex pipeline architect. Convert the following requirements into a minimal, correct PipelineLibraryBlueprint object.+You are a Pipelex pipeline architect. Build a pipeline draft showing how we solve the problem.-Rules:-- Only fill fields that exist on the PipelineLibraryBlueprint schema: domain, definition (optional), concept, pipe.+Think on the pipeline requirements: what are the pipeline's inputs and what is the expected output?++- Keep domain and pipe codes short, lowercase, and meaningful.- In concept, map ConceptName (PascalCase, singular) to its natural language definition string. Do not redefine native concepts (Text, Image, PDF, TextAndImages, Number, Page).- In pipe, define at least one pipe that accomplishes the main goal implied by the requirements. Use operators/controllers supported by Pipelex (e.g., PipeLLM, PipeOcr, PipeSequence...).-- For pipe definitions, use the concise form (e.g., `PipeLLM = "..."`, then `inputs = { ... }`, `output = "..."`).- Ensure all input variables referenced in prompts are listed in the pipe `inputs`.-- Keep domain code short, lowercase, and meaningful.-Requirements:+@requirements+"""++[pipe.build_blueprint]+type = "PipeLLM"+definition = "Generate a pipeline blueprint from natural language requirements"+inputs = { requirements = "Text" }+output = "PipelineLibraryBlueprint"+prompt_template = """+You are a Pipelex pipeline architect. Build a pipeline blueprint showing how we solve the problem.@requirements"""diff --git a/pipelex/libraries/pipelines/meta/pipeline_draft.py b/pipelex/libraries/pipelines/meta/pipeline_draft.pynew file mode 100644index 000000000..a1769dfe0--- /dev/null+++ b/pipelex/libraries/pipelines/meta/pipeline_draft.py@@ -0,0 +1,57 @@+from pathlib import Path+from typing import Any, Dict, Union++from pydantic import Field++from pipelex.core.stuff_content import StructuredContent+from pipelex.tools.misc.toml_utils import dict_to_toml_string, save_toml_to_path+++class PipeDraft(StructuredContent):+ code: str+ type: str+ definition: str+ inputs: Dict[str, str]+ output: str+++class PipelineDraft(StructuredContent):+ """Complete blueprint of a pipeline library TOML file."""++ # Domain information (required)+ domain: str+ definition: str++ # Concepts section - concept_name -> definition (string) or blueprint (dict)+ concept: Dict[str, str] = Field(default_factory=dict)++ # Pipes section - pipe_name -> blueprint dict+ pipe: Dict[str, PipeDraft] = Field(default_factory=dict)++ def to_toml_dict(self) -> Dict[str, Any]:+ """Convert blueprint back to TOML-compatible dictionary structure."""+ result: Dict[str, Any] = {"domain": self.domain}++ # Add optional domain-level fields+ if self.definition:+ result["definition"] = self.definition++ # Add concepts section if not empty+ if self.concept:+ result["concept"] = self.concept++ # Add pipes section if not empty+ if self.pipe:+ result["pipe"] = self.pipe++ return result++ def to_toml_string(self) -> str:+ """Convert blueprint to TOML string format."""+ return dict_to_toml_string(self.to_toml_dict())++ def save_to_file(self, file_path: Union[str, Path]) -> None:+ """Save blueprint to a TOML file."""+ path = Path(file_path)+ path.parent.mkdir(parents=True, exist_ok=True)+ save_toml_to_path(data=self.to_toml_dict(), path=str(path))diff --git a/pipelex/pipe_operators/pipe_llm.py b/pipelex/pipe_operators/pipe_llm.pyindex f92f8ac05..36739516a 100644--- a/pipelex/pipe_operators/pipe_llm.py+++ b/pipelex/pipe_operators/pipe_llm.py@@ -11,6 +11,7 @@ from pipelex.cogt.llm.llm_models.llm_setting import LLMSetting, LLMSettingChoicefrom pipelex.cogt.llm.llm_prompt import LLMPromptfrom pipelex.cogt.llm.llm_prompt_factory_abstract import LLMPromptFactoryAbstractfrom pipelex.config import StaticValidationReaction, get_config+from pipelex.core.concept import Conceptfrom pipelex.core.concept_code_factory import ConceptCodeFactoryfrom pipelex.core.concept_native import NativeConcept, NativeConceptClassfrom pipelex.core.domain import Domain, SpecialDomain@@ -79,7 +80,11 @@ class PipeLLM(PipeOperator):if self.structuring_method is not None:output_concept = get_required_concept(concept_code=self.output_concept_code)if output_concept.structure_class_name == NativeConceptClass.TEXT:- raise PipeDefinitionError(f"Output concept '{self.output_concept_code}' is a Text concept, so it cannot be structured")+ concept_name = Concept.extract_concept_name_from_str(concept_str=self.output_concept_code)+ raise PipeDefinitionError(+ f"Output concept '{self.output_concept_code}' is considered a Text concept, "+ f"so it cannot be structured. Maybe you forgot to add '{concept_name}' to the class registry?"+ )return self@overridediff --git a/tests/integration/pipelex/pipes/pipe_operator/pipe_llm/test_generate_pipeline_blueprint.py b/tests/integration/pipelex/pipes/pipe_operator/pipe_llm/test_generate_pipeline_blueprint.pyindex 80162e6df..6f643b1d1 100644--- a/tests/integration/pipelex/pipes/pipe_operator/pipe_llm/test_generate_pipeline_blueprint.py+++ b/tests/integration/pipelex/pipes/pipe_operator/pipe_llm/test_generate_pipeline_blueprint.py@@ -24,9 +24,9 @@ from pipelex.pipeline.execute import execute_pipeline),],)-async def test_generate_pipeline_blueprint_dry_run(pipe_run_mode: PipeRunMode, requirements: str):+async def test_generate_pipeline_blueprint(pipe_run_mode: PipeRunMode, requirements: str):pipe_output = await execute_pipeline(- pipe_code="generate_pipeline_blueprint",+ pipe_code="build_blueprint",input_memory={"requirements": requirements,},