Skip to content

Build AI methods with your coding agent, then run them anywhere: as an MCP for chatbots, as a webapp for people, or via API for your software.

Pipeline draft, as text or structured

Louis Choquel59b165ef42f3
diff --git a/pipelex/cli/commands/build_cmd.py b/pipelex/cli/commands/build_cmd.py
index b0aca2346..061799c30 100644
--- a/pipelex/cli/commands/build_cmd.py
+++ b/pipelex/cli/commands/build_cmd.py
@@ -1,113 +1,76 @@
from __future__ import annotations
import asyncio
-from typing import Annotated, Any, Dict, Optional, cast
+from typing import Annotated, Optional
import typer
-from pydantic import ValidationError
-from pipelex import pretty_print
-from pipelex.exceptions import PipeDefinitionError, PipelexCLIError
-from pipelex.hub import get_library_manager
-from pipelex.libraries.library_manager import LibraryManager
-from pipelex.libraries.pipeline_blueprint import PipelineLibraryBlueprint
-from pipelex.pipe_works.pipe_dry import dry_run_pipe_codes
+from pipelex.create.build_blueprint import do_build_blueprint
+from pipelex.create.draft_pipeline import do_draft_pipeline, do_draft_pipeline_text
+from pipelex.exceptions import PipelexCLIError
from pipelex.pipelex import Pipelex
-from pipelex.pipeline.execute import execute_pipeline
from pipelex.tools.misc.file_utils import load_text_from_path
-from pipelex.tools.typing.pydantic_utils import format_pydantic_validation_error
-
-def _load_requirements_text(requirements: Optional[str], requirements_file: Optional[str]) -> str:
- if requirements_file:
- return load_text_from_path(requirements_file)
- if not requirements:
- raise PipelexCLIError("You must provide requirements text via --requirements or a file via --file")
- return requirements
-
-
-async def _generate_blueprint_async(requirements_text: str) -> PipelineLibraryBlueprint:
- pipe_output = await execute_pipeline(
- pipe_code="generate_pipeline_blueprint",
- input_memory={
- "requirements": requirements_text,
- },
- )
- return pipe_output.main_stuff_as(content_type=PipelineLibraryBlueprint)
+# Typer group for build commands
+build_app = typer.Typer(help="Build artifacts like pipeline blueprints")
-def do_build_blueprint(
- relative_config_folder_path: str,
- requirements: Optional[str],
- requirements_file: Optional[str],
- output_path: Optional[str],
- validate: bool,
+@build_app.command("draft")
+def build_draft_pipeline_cmd(
+ requirements: Annotated[
+ Optional[str],
+ typer.Option("--requirements", "-r", help="Requirements text to generate the pipeline blueprint from"),
+ ] = None,
+ requirements_file: Annotated[
+ Optional[str],
+ typer.Option("--file", "-f", help="Path to a file containing the requirements text"),
+ ] = None,
+ output_path: Annotated[
+ Optional[str],
+ typer.Option("--output", "-o", help="Path to save the generated TOML blueprint (optional)"),
+ ] = None,
+ raw: Annotated[
+ bool,
+ typer.Option("--raw", help="Raw text draft"),
+ ] = False,
+ relative_config_folder_path: Annotated[
+ str,
+ typer.Option(
+ "--config-folder-path",
+ "-c",
+ help="Relative path to the config folder path (libraries)",
+ ),
+ ] = "./pipelex_libraries",
) -> None:
# Initialize Pipelex (loads libraries and pipes)
Pipelex.make(relative_config_folder_path=relative_config_folder_path, from_file=False)
# Get requirements text
try:
- requirements_text = _load_requirements_text(requirements=requirements, requirements_file=requirements_file)
+ requirements_text: str
+ if requirements_file:
+ requirements_text = load_text_from_path(requirements_file)
+ else:
+ if not requirements:
+ raise PipelexCLIError("You must provide requirements text via --requirements or a file via --file")
+ requirements_text = requirements
except Exception as exc:
raise PipelexCLIError(f"Failed to load requirements: {exc}") from exc
- # Generate blueprint
- blueprint = asyncio.run(_generate_blueprint_async(requirements_text=requirements_text))
-
- pretty_print(blueprint, title="Pipeline Blueprint")
- pretty_print(blueprint.to_toml_dict(), title="Pipeline Blueprint (TOML)")
-
- # Save or display result
- output_path = output_path or "pipelex/libraries/pipelines/temp/generated_blueprint.toml"
- blueprint.save_to_file(output_path)
- typer.echo(f"✅ Blueprint saved to '{output_path}'")
-
- if validate:
- try:
- _load_pipes_from_generated_blueprint(blueprint=blueprint)
- except PipeDefinitionError as exc:
- raise PipelexCLIError(f"Failed to load pipes from generated blueprint at '{output_path}': {exc}") from exc
-
- generated_pipe_codes = list(blueprint.pipe.keys())
- if not generated_pipe_codes:
- raise PipelexCLIError("No pipe found in generated blueprint to validate")
- asyncio.run(dry_run_pipe_codes(pipe_codes=generated_pipe_codes))
-
-
-def _load_pipes_from_generated_blueprint(blueprint: PipelineLibraryBlueprint) -> None:
- """Instantiate and register all pipes from a generated PipelineLibraryBlueprint.
-
- This constructs the appropriate typed PipeBlueprint for each pipe entry and
- uses LibraryManager.load_pipe_from_blueprint to create the Pipe, then adds it
- to the in-memory PipeLibrary so that validation can run against them.
- """
- library_manager = cast(LibraryManager, get_library_manager())
- domain_code = blueprint.domain
-
- for pipe_code, details in blueprint.pipe.items():
- details_dict: Dict[str, Any] = details.copy()
-
- # Build the concrete PipeBlueprint via LibraryManager helper
- try:
- pipe_blueprint = LibraryManager.make_pipe_blueprint_from_details(
- domain_code=domain_code,
- details_dict=details_dict,
+ if raw:
+ asyncio.run(
+ do_draft_pipeline_text(
+ requirements=requirements_text,
+ output_path=output_path,
+ )
+ )
+ else:
+ asyncio.run(
+ do_draft_pipeline(
+ requirements=requirements_text,
+ output_path=output_path,
)
- except ValidationError as exc:
- error_msg = format_pydantic_validation_error(exc=exc)
- raise PipeDefinitionError(f"Failed to build pipe blueprint for pipe '{pipe_code}': {error_msg}") from exc
-
- # Create pipe and register in library
- pipe = LibraryManager.load_pipe_from_blueprint(
- pipe_code=pipe_code,
- pipe_blueprint=pipe_blueprint,
)
- library_manager.pipe_library.add_new_pipe(pipe=pipe)
-
-
-# Typer group for build commands
-build_app = typer.Typer(help="Build artifacts like pipeline blueprints")
@build_app.command("blueprint")
@@ -137,22 +100,25 @@ def build_blueprint_cmd(
),
] = "./pipelex_libraries",
) -> None:
- # try:
- # do_build_blueprint(
- # relative_config_folder_path=relative_config_folder_path,
- # requirements=requirements,
- # requirements_file=requirements_file,
- # output_path=output_path,
- # validate=validate,
- # )
- # except Exception as exc:
- # typer.echo(f"❌ Error: {exc}")
- # raise typer.Exit(1)
+ # Initialize Pipelex (loads libraries and pipes)
+ Pipelex.make(relative_config_folder_path=relative_config_folder_path, from_file=False)
- do_build_blueprint(
- relative_config_folder_path=relative_config_folder_path,
- requirements=requirements,
- requirements_file=requirements_file,
- output_path=output_path,
- validate=validate,
+ # Get requirements text
+ try:
+ requirements_text: str
+ if requirements_file:
+ requirements_text = load_text_from_path(requirements_file)
+ else:
+ if not requirements:
+ raise PipelexCLIError("You must provide requirements text via --requirements or a file via --file")
+ requirements_text = requirements
+ except Exception as exc:
+ raise PipelexCLIError(f"Failed to load requirements: {exc}") from exc
+
+ asyncio.run(
+ do_build_blueprint(
+ requirements=requirements_text,
+ output_path=output_path,
+ validate=validate,
+ )
)
diff --git a/pipelex/core/registry_models.py b/pipelex/core/registry_models.py
index 0d9c0faad..498dd3c93 100644
--- a/pipelex/core/registry_models.py
+++ b/pipelex/core/registry_models.py
@@ -18,6 +18,7 @@ from pipelex.core.stuff_content import (
TextContent,
)
from pipelex.libraries.pipeline_blueprint import PipelineLibraryBlueprint
+from pipelex.libraries.pipelines.meta.pipeline_draft import PipelineDraft
from pipelex.pipe_controllers.pipe_batch import PipeBatch
from pipelex.pipe_controllers.pipe_batch_factory import PipeBatchFactory
from pipelex.pipe_controllers.pipe_condition import PipeCondition
@@ -88,6 +89,7 @@ class PipelexRegistryModels(RegistryModels):
TextAndImagesContent,
PageContent,
PipelineLibraryBlueprint,
+ PipelineDraft,
]
EXPERIMENTAL: ClassVar[List[ModelType]] = [
diff --git a/pipelex/create/build_blueprint.py b/pipelex/create/build_blueprint.py
new file mode 100644
index 000000000..d73460547
--- /dev/null
+++ b/pipelex/create/build_blueprint.py
@@ -0,0 +1,82 @@
+from __future__ import annotations
+
+import asyncio
+from typing import Annotated, Any, Dict, Optional, cast
+
+import typer
+from pydantic import ValidationError
+
+from pipelex import pretty_print
+from pipelex.exceptions import PipeDefinitionError, PipelexCLIError
+from pipelex.hub import get_library_manager
+from pipelex.libraries.library_manager import LibraryManager
+from pipelex.libraries.pipeline_blueprint import PipelineLibraryBlueprint
+from pipelex.libraries.pipelines.meta.pipeline_draft import PipelineDraft
+from pipelex.pipe_works.pipe_dry import dry_run_pipe_codes
+from pipelex.pipelex import Pipelex
+from pipelex.pipeline.execute import execute_pipeline
+from pipelex.tools.misc.file_utils import load_text_from_path
+from pipelex.tools.typing.pydantic_utils import format_pydantic_validation_error
+
+
+async def do_build_blueprint(
+ requirements: str,
+ output_path: Optional[str],
+ validate: bool,
+) -> None:
+ pipe_output = await execute_pipeline(
+ pipe_code="draft_pipeline",
+ input_memory={
+ "requirements": requirements,
+ },
+ )
+ blueprint = pipe_output.main_stuff_as(content_type=PipelineLibraryBlueprint)
+ pretty_print(blueprint, title="Pipeline Blueprint")
+ pretty_print(blueprint.to_toml_dict(), title="Pipeline Blueprint (TOML)")
+
+ # Save or display result
+ output_path = output_path or "pipelex/libraries/pipelines/temp/generated_blueprint.toml"
+ blueprint.save_to_file(output_path)
+ typer.echo(f"✅ Blueprint saved to '{output_path}'")
+
+ if validate:
+ try:
+ _load_pipes_from_generated_blueprint(blueprint=blueprint)
+ except PipeDefinitionError as exc:
+ raise PipelexCLIError(f"Failed to load pipes from generated blueprint at '{output_path}': {exc}") from exc
+
+ generated_pipe_codes = list(blueprint.pipe.keys())
+ if not generated_pipe_codes:
+ raise PipelexCLIError("No pipe found in generated blueprint to validate")
+ asyncio.run(dry_run_pipe_codes(pipe_codes=generated_pipe_codes))
+
+
+def _load_pipes_from_generated_blueprint(blueprint: PipelineLibraryBlueprint) -> None:
+ """Instantiate and register all pipes from a generated PipelineLibraryBlueprint.
+
+ This constructs the appropriate typed PipeBlueprint for each pipe entry and
+ uses LibraryManager.load_pipe_from_blueprint to create the Pipe, then adds it
+ to the in-memory PipeLibrary so that validation can run against them.
+ """
+ library_manager = cast(LibraryManager, get_library_manager())
+ domain_code = blueprint.domain
+
+ for pipe_code, details in blueprint.pipe.items():
+ details_dict: Dict[str, Any] = details.copy()
+
+ # Build the concrete PipeBlueprint via LibraryManager helper
+ try:
+ pipe_blueprint = LibraryManager.make_pipe_blueprint_from_details(
+ domain_code=domain_code,
+ details_dict=details_dict,
+ )
+ except ValidationError as exc:
+ error_msg = format_pydantic_validation_error(exc=exc)
+ raise PipeDefinitionError(f"Failed to build pipe blueprint for pipe '{pipe_code}': {error_msg}") from exc
+
+ # Create pipe and register in library
+ pipe = LibraryManager.load_pipe_from_blueprint(
+ pipe_code=pipe_code,
+ pipe_blueprint=pipe_blueprint,
+ )
+ library_manager.pipe_library.add_new_pipe(pipe=pipe)
diff --git a/pipelex/create/draft_pipeline.py b/pipelex/create/draft_pipeline.py
new file mode 100644
index 000000000..26fcf1948
--- /dev/null
+++ b/pipelex/create/draft_pipeline.py
@@ -0,0 +1,58 @@
+from __future__ import annotations
+
+import asyncio
+from typing import Annotated, Any, Dict, Optional, cast
+
+import typer
+from pydantic import ValidationError
+
+from pipelex import pretty_print
+from pipelex.exceptions import PipeDefinitionError, PipelexCLIError
+from pipelex.hub import get_library_manager
+from pipelex.libraries.library_manager import LibraryManager
+from pipelex.libraries.pipeline_blueprint import PipelineLibraryBlueprint
+from pipelex.libraries.pipelines.meta.pipeline_draft import PipelineDraft
+from pipelex.pipe_works.pipe_dry import dry_run_pipe_codes
+from pipelex.pipelex import Pipelex
+from pipelex.pipeline.execute import execute_pipeline
+from pipelex.tools.misc.file_utils import load_text_from_path, save_text_to_path
+from pipelex.tools.typing.pydantic_utils import format_pydantic_validation_error
+
+
+async def do_draft_pipeline_text(
+ requirements: str,
+ output_path: Optional[str],
+) -> None:
+ pipe_output = await execute_pipeline(
+ pipe_code="draft_pipeline_text",
+ input_memory={
+ "requirements": requirements,
+ },
+ )
+ draft_text = pipe_output.main_stuff_as_str
+ pretty_print(draft_text, title="Pipeline Draft Text")
+
+ # Save or display result
+ output_path = output_path or "pipelex/libraries/pipelines/temp/pipeline_draft.text"
+ save_text_to_path(text=draft_text, path=output_path)
+ typer.echo(f"✅ Pipeline draft text saved to '{output_path}'")
+
+
+async def do_draft_pipeline(
+ requirements: str,
+ output_path: Optional[str],
+) -> None:
+ pipe_output = await execute_pipeline(
+ pipe_code="draft_pipeline",
+ input_memory={
+ "requirements": requirements,
+ },
+ )
+ draft = pipe_output.main_stuff_as(content_type=PipelineDraft)
+ pretty_print(draft, title="Pipeline Draft")
+ pretty_print(draft.to_toml_dict(), title="Pipeline Draft (TOML)")
+
+ # Save or display result
+ output_path = output_path or "pipelex/libraries/pipelines/temp/generated_blueprint.toml"
+ draft.save_to_file(output_path)
+ typer.echo(f"✅ Blueprint saved to '{output_path}'")
diff --git a/pipelex/libraries/pipeline_blueprint.py b/pipelex/libraries/pipeline_blueprint.py
index 7d5d4177d..6a5149baa 100644
--- a/pipelex/libraries/pipeline_blueprint.py
+++ b/pipelex/libraries/pipeline_blueprint.py
@@ -1,7 +1,7 @@
from pathlib import Path
from typing import Any, Dict, Optional, Union
-from pydantic import BaseModel, Field
+from pydantic import Field
from pipelex.core.concept_factory import ConceptBlueprint
from pipelex.core.stuff_content import StructuredContent
@@ -36,13 +36,6 @@ class PipeBlueprintError(LibraryError):
super().__init__(f"Error loading pipe '{pipe_name}' from '{file_path}': {error_msg}")
-class PipeStepBlueprint(BaseModel):
- """Blueprint of a step in a pipe sequence."""
-
- pipe: str
- result: str
-
-
class PipelineLibraryBlueprint(StructuredContent):
"""Complete blueprint of a pipeline library TOML file."""
diff --git a/pipelex/libraries/pipelines/meta/meta_pipeline.toml b/pipelex/libraries/pipelines/meta/meta_pipeline.toml
index 3d09717a3..ba0b14c5a 100644
--- a/pipelex/libraries/pipelines/meta/meta_pipeline.toml
+++ b/pipelex/libraries/pipelines/meta/meta_pipeline.toml
@@ -2,30 +2,60 @@ domain = "meta"
definition = "Meta domain is the domain of pipelines about pipelines"
[concept]
+PipelineDraft = "Rough solution to create a pipeline"
+PipelineDraftText = "Rough solution to create a pipeline in text format"
+PipeDraft = "Rough solution to create a pipe"
PipelineLibraryBlueprint = "Details enabling to create a pipeline"
ConceptBlueprint = "Details enabling to create a concept"
PipeBlueprint = "Details enabling to create a pipe"
[pipe]
-[pipe.generate_pipeline_blueprint]
+[pipe.draft_pipeline_text]
type = "PipeLLM"
-definition = "Generate a PipelineLibraryBlueprint from natural language requirements"
+definition = "Generate a rough pipeline draft from natural language requirements"
inputs = { requirements = "Text" }
-output = "PipelineLibraryBlueprint"
+output = "PipelineDraftText"
+prompt_template = """
+You are a Pipelex pipeline architect. Build a pipeline draft showing how we solve the problem.
+
+Think on the pipeline requirements: what are the pipeline's inputs and what is the expected output?
+
+- Keep domain and pipe codes short, lowercase, and meaningful.
+- In concept, map ConceptName (PascalCase, singular) to its natural language definition string. Do not redefine native concepts (Text, Image, PDF, TextAndImages, Number, Page).
+- In pipe, define at least one pipe that accomplishes the main goal implied by the requirements. Use operators/controllers supported by Pipelex (e.g., PipeLLM, PipeOcr, PipeSequence...).
+- Ensure all input variables referenced in prompts are listed in the pipe `inputs`.
+
+@requirements
+"""
+
+
+[pipe.draft_pipeline]
+type = "PipeLLM"
+definition = "Generate a rough pipeline draft from natural language requirements"
+inputs = { requirements = "Text" }
+output = "PipelineDraft"
structuring_method = "preliminary_text"
prompt_template = """
-You are a Pipelex pipeline architect. Convert the following requirements into a minimal, correct PipelineLibraryBlueprint object.
+You are a Pipelex pipeline architect. Build a pipeline draft showing how we solve the problem.
-Rules:
-- Only fill fields that exist on the PipelineLibraryBlueprint schema: domain, definition (optional), concept, pipe.
+Think on the pipeline requirements: what are the pipeline's inputs and what is the expected output?
+
+- Keep domain and pipe codes short, lowercase, and meaningful.
- In concept, map ConceptName (PascalCase, singular) to its natural language definition string. Do not redefine native concepts (Text, Image, PDF, TextAndImages, Number, Page).
- In pipe, define at least one pipe that accomplishes the main goal implied by the requirements. Use operators/controllers supported by Pipelex (e.g., PipeLLM, PipeOcr, PipeSequence...).
-- For pipe definitions, use the concise form (e.g., `PipeLLM = "..."`, then `inputs = { ... }`, `output = "..."`).
- Ensure all input variables referenced in prompts are listed in the pipe `inputs`.
-- Keep domain code short, lowercase, and meaningful.
-Requirements:
+@requirements
+"""
+
+[pipe.build_blueprint]
+type = "PipeLLM"
+definition = "Generate a pipeline blueprint from natural language requirements"
+inputs = { requirements = "Text" }
+output = "PipelineLibraryBlueprint"
+prompt_template = """
+You are a Pipelex pipeline architect. Build a pipeline blueprint showing how we solve the problem.
@requirements
"""
diff --git a/pipelex/libraries/pipelines/meta/pipeline_draft.py b/pipelex/libraries/pipelines/meta/pipeline_draft.py
new file mode 100644
index 000000000..a1769dfe0
--- /dev/null
+++ b/pipelex/libraries/pipelines/meta/pipeline_draft.py
@@ -0,0 +1,57 @@
+from pathlib import Path
+from typing import Any, Dict, Union
+
+from pydantic import Field
+
+from pipelex.core.stuff_content import StructuredContent
+from pipelex.tools.misc.toml_utils import dict_to_toml_string, save_toml_to_path
+
+
+class PipeDraft(StructuredContent):
+ code: str
+ type: str
+ definition: str
+ inputs: Dict[str, str]
+ output: str
+
+
+class PipelineDraft(StructuredContent):
+ """Complete blueprint of a pipeline library TOML file."""
+
+ # Domain information (required)
+ domain: str
+ definition: str
+
+ # Concepts section - concept_name -> definition (string) or blueprint (dict)
+ concept: Dict[str, str] = Field(default_factory=dict)
+
+ # Pipes section - pipe_name -> blueprint dict
+ pipe: Dict[str, PipeDraft] = Field(default_factory=dict)
+
+ def to_toml_dict(self) -> Dict[str, Any]:
+ """Convert blueprint back to TOML-compatible dictionary structure."""
+ result: Dict[str, Any] = {"domain": self.domain}
+
+ # Add optional domain-level fields
+ if self.definition:
+ result["definition"] = self.definition
+
+ # Add concepts section if not empty
+ if self.concept:
+ result["concept"] = self.concept
+
+ # Add pipes section if not empty
+ if self.pipe:
+ result["pipe"] = self.pipe
+
+ return result
+
+ def to_toml_string(self) -> str:
+ """Convert blueprint to TOML string format."""
+ return dict_to_toml_string(self.to_toml_dict())
+
+ def save_to_file(self, file_path: Union[str, Path]) -> None:
+ """Save blueprint to a TOML file."""
+ path = Path(file_path)
+ path.parent.mkdir(parents=True, exist_ok=True)
+ save_toml_to_path(data=self.to_toml_dict(), path=str(path))
diff --git a/pipelex/pipe_operators/pipe_llm.py b/pipelex/pipe_operators/pipe_llm.py
index f92f8ac05..36739516a 100644
--- a/pipelex/pipe_operators/pipe_llm.py
+++ b/pipelex/pipe_operators/pipe_llm.py
@@ -11,6 +11,7 @@ from pipelex.cogt.llm.llm_models.llm_setting import LLMSetting, LLMSettingChoice
from pipelex.cogt.llm.llm_prompt import LLMPrompt
from pipelex.cogt.llm.llm_prompt_factory_abstract import LLMPromptFactoryAbstract
from pipelex.config import StaticValidationReaction, get_config
+from pipelex.core.concept import Concept
from pipelex.core.concept_code_factory import ConceptCodeFactory
from pipelex.core.concept_native import NativeConcept, NativeConceptClass
from pipelex.core.domain import Domain, SpecialDomain
@@ -79,7 +80,11 @@ class PipeLLM(PipeOperator):
if self.structuring_method is not None:
output_concept = get_required_concept(concept_code=self.output_concept_code)
if output_concept.structure_class_name == NativeConceptClass.TEXT:
- raise PipeDefinitionError(f"Output concept '{self.output_concept_code}' is a Text concept, so it cannot be structured")
+ concept_name = Concept.extract_concept_name_from_str(concept_str=self.output_concept_code)
+ raise PipeDefinitionError(
+ f"Output concept '{self.output_concept_code}' is considered a Text concept, "
+ f"so it cannot be structured. Maybe you forgot to add '{concept_name}' to the class registry?"
+ )
return self
@override
diff --git a/tests/integration/pipelex/pipes/pipe_operator/pipe_llm/test_generate_pipeline_blueprint.py b/tests/integration/pipelex/pipes/pipe_operator/pipe_llm/test_generate_pipeline_blueprint.py
index 80162e6df..6f643b1d1 100644
--- a/tests/integration/pipelex/pipes/pipe_operator/pipe_llm/test_generate_pipeline_blueprint.py
+++ b/tests/integration/pipelex/pipes/pipe_operator/pipe_llm/test_generate_pipeline_blueprint.py
@@ -24,9 +24,9 @@ from pipelex.pipeline.execute import execute_pipeline
),
],
)
-async def test_generate_pipeline_blueprint_dry_run(pipe_run_mode: PipeRunMode, requirements: str):
+async def test_generate_pipeline_blueprint(pipe_run_mode: PipeRunMode, requirements: str):
pipe_output = await execute_pipeline(
- pipe_code="generate_pipeline_blueprint",
+ pipe_code="build_blueprint",
input_memory={
"requirements": requirements,
},