Skip to content

Build AI methods with your coding agent, then run them anywhere: as an MCP for chatbots, as a webapp for people, or via API for your software.

WIP pipeline drafting

Louis Choquelba6a7f5b3342
diff --git a/pipelex/cli/commands/build_cmd.py b/pipelex/cli/commands/build_cmd.py
index c08d402bf..bf0601ee0 100644
--- a/pipelex/cli/commands/build_cmd.py
+++ b/pipelex/cli/commands/build_cmd.py
@@ -19,8 +19,12 @@ build_app = typer.Typer(help="Build artifacts like pipeline blueprints")
def build_draft_pipeline_cmd(
pipeline_name: Annotated[
str,
- typer.Argument(help="Name/code of the pipeline to generate (used inside the draft)"),
+ typer.Argument(help="Name/code of the pipeline to generate"),
],
+ domain: Annotated[
+ str,
+ typer.Option("--domain", "-d", help="Domain of the pipeline to generate"),
+ ] = "wip_domain",
requirements: Annotated[
Optional[str],
typer.Option("--requirements", "-r", help="Requirements text to generate the pipeline blueprint from"),
@@ -64,6 +68,7 @@ def build_draft_pipeline_cmd(
if raw:
asyncio.run(
do_draft_pipeline_text(
+ domain=domain,
pipeline_name=pipeline_name,
requirements=requirements_text,
output_path=output_path,
diff --git a/pipelex/create/build_blueprint.py b/pipelex/create/build_blueprint.py
index 6e9ab90da..e5b08c68c 100644
--- a/pipelex/create/build_blueprint.py
+++ b/pipelex/create/build_blueprint.py
@@ -8,7 +8,7 @@ import typer
from pydantic import ValidationError
from pipelex import pretty_print
-from pipelex.create.helpers import get_pipeline_creation_rules
+from pipelex.create.helpers import get_support_file
from pipelex.create.pipeline_toml import save_pipeline_blueprint_toml_to_path
from pipelex.exceptions import PipeDefinitionError, PipelexCLIError
from pipelex.hub import get_library_manager
@@ -31,7 +31,7 @@ async def do_build_blueprint(
input_memory={
"pipeline_name": pipeline_name,
"requirements": requirements,
- "rules": get_pipeline_creation_rules(),
+ "rules": get_support_file(subpath="design_pipelines.md"),
},
)
blueprint = pipe_output.main_stuff_as(content_type=PipelineLibraryBlueprint)
diff --git a/pipelex/create/draft_pipeline.py b/pipelex/create/draft_pipeline.py
index bccf24e74..3b739cfd2 100644
--- a/pipelex/create/draft_pipeline.py
+++ b/pipelex/create/draft_pipeline.py
@@ -5,7 +5,7 @@ from typing import Optional
import typer
from pipelex import pretty_print
-from pipelex.create.helpers import get_pipeline_creation_rules
+from pipelex.create.helpers import get_support_file
from pipelex.create.pipeline_toml import dict_to_toml, save_toml_to_path
from pipelex.libraries.pipelines.meta.pipeline_draft import PipelineDraft
from pipelex.pipeline.execute import execute_pipeline
@@ -13,6 +13,7 @@ from pipelex.tools.misc.file_utils import save_text_to_path
async def do_draft_pipeline_text(
+ domain: str,
pipeline_name: str,
requirements: str,
output_path: Optional[str],
@@ -20,16 +21,17 @@ async def do_draft_pipeline_text(
pipe_output = await execute_pipeline(
pipe_code="draft_pipeline_text",
input_memory={
+ "domain": domain,
"pipeline_name": pipeline_name,
"requirements": requirements,
- "rules": get_pipeline_creation_rules(),
+ "rules": get_support_file(subpath="draft_pipelines.md"),
},
)
draft_text = pipe_output.main_stuff_as_str
pretty_print(draft_text, title="Pipeline Draft Text")
# Save or display result
- output_path = output_path or "pipelex/libraries/pipelines/temp/pipeline_draft.text"
+ output_path = output_path or "pipelex/libraries/pipelines/temp/pipeline_draft.md"
save_text_to_path(text=draft_text, path=output_path)
typer.echo(f"✅ Pipeline draft text saved to '{output_path}'")
@@ -44,7 +46,7 @@ async def do_draft_pipeline(
input_memory={
"pipeline_name": pipeline_name,
"requirements": requirements,
- "rules": get_pipeline_creation_rules(),
+ "rules": get_support_file(subpath="draft_pipelines.md"),
},
)
draft = pipe_output.main_stuff_as(content_type=PipelineDraft)
diff --git a/pipelex/create/helpers.py b/pipelex/create/helpers.py
index ecb61e057..664f4df53 100644
--- a/pipelex/create/helpers.py
+++ b/pipelex/create/helpers.py
@@ -1,5 +1,55 @@
+from pathlib import Path
+from typing import Dict, List, Optional
+
from pipelex.tools.misc.file_utils import load_text_from_path
-def get_pipeline_creation_rules() -> str:
- return load_text_from_path(".pipelex/design_pipelines.md")
+class PipelexSupportFiles:
+ """Dictionary-like interface to access all files in .pipelex by their subpath."""
+
+ def __init__(self, base_path: str = ".pipelex"):
+ self.base_path = Path(base_path)
+ self._file_cache: Dict[str, str] = {}
+
+ def __getitem__(self, subpath: str) -> str:
+ """Get file content by subpath (e.g., 'design_pipelines.md')."""
+ if subpath not in self._file_cache:
+ file_path = self.base_path / subpath
+ if not file_path.exists():
+ raise KeyError(f"File '{subpath}' not found in {self.base_path}")
+ self._file_cache[subpath] = load_text_from_path(str(file_path))
+ return self._file_cache[subpath]
+
+ def __contains__(self, subpath: str) -> bool:
+ """Check if a file exists by subpath."""
+ file_path = self.base_path / subpath
+ return file_path.exists() and file_path.is_file()
+
+ def get(self, subpath: str, default: Optional[str] = None) -> Optional[str]:
+ """Get file content by subpath with optional default."""
+ try:
+ return self[subpath]
+ except KeyError:
+ return default
+
+ def keys(self) -> List[str]:
+ """Get all available file subpaths."""
+ if not self.base_path.exists():
+ return []
+
+ files: List[str] = []
+ for file_path in self.base_path.rglob("*"):
+ if file_path.is_file():
+ subpath = str(file_path.relative_to(self.base_path))
+ files.append(subpath)
+ return sorted(files)
+
+ def clear_cache(self) -> None:
+ """Clear the internal file cache."""
+ self._file_cache.clear()
+
+
+def get_support_file(subpath: str) -> str:
+ """Get pipeline creation rules from design_pipelines.md."""
+ support_files = PipelexSupportFiles()
+ return support_files[subpath]
diff --git a/pipelex/libraries/pipelines/meta/meta_pipeline.toml b/pipelex/libraries/pipelines/meta/meta_pipeline.toml
index d8396250e..bf9565f6c 100644
--- a/pipelex/libraries/pipelines/meta/meta_pipeline.toml
+++ b/pipelex/libraries/pipelines/meta/meta_pipeline.toml
@@ -14,7 +14,7 @@ PipeBlueprint = "Details enabling to create a pipe"
[pipe.draft_pipeline_text]
type = "PipeLLM"
definition = "Generate a rough pipeline draft from natural language requirements"
-inputs = { rules = "Text", requirements = "Text", pipeline_name = "Text" }
+inputs = { rules = "Text", requirements = "Text", pipeline_name = "Text", domain = "Text" }
output = "PipelineDraftText"
llm = "llm_to_pipelex"
prompt_template = """
@@ -24,7 +24,8 @@ You are a Pipelex pipeline architect. Build a pipeline draft text showing how we
@requirements
-Name of the pipeline, i.e. name of the main pipe: $pipeline_name.
+Domain: $domain.
+Name of the pipeline: $pipeline_name.
"""
diff --git a/tests/integration/pipelex/pipes/pipe_operator/pipe_llm/test_generate_pipeline_blueprint.py b/tests/integration/pipelex/pipes/pipe_operator/pipe_llm/test_generate_pipeline_blueprint.py
index 61014da91..043ca1b6a 100644
--- a/tests/integration/pipelex/pipes/pipe_operator/pipe_llm/test_generate_pipeline_blueprint.py
+++ b/tests/integration/pipelex/pipes/pipe_operator/pipe_llm/test_generate_pipeline_blueprint.py
@@ -2,7 +2,7 @@ import pytest
from pipelex import pretty_print
from pipelex.core.pipe_run_params import PipeRunMode
-from pipelex.create.helpers import get_pipeline_creation_rules
+from pipelex.create.helpers import get_support_file
from pipelex.libraries.pipeline_blueprint import PipelineLibraryBlueprint
from pipelex.pipeline.execute import execute_pipeline
@@ -31,7 +31,7 @@ async def test_generate_pipeline_blueprint(pipe_run_mode: PipeRunMode, requireme
input_memory={
"pipeline_name": "test_pipeline",
"requirements": requirements,
- "rules": get_pipeline_creation_rules(),
+ "rules": get_support_file(),
},
pipe_run_mode=pipe_run_mode,
)