Skip to content

Build AI methods with your coding agent, then run them anywhere: as an MCP for chatbots, as a webapp for people, or via API for your software.

migrate Pipes to new syntax

Louis Choquel089e60410996
diff --git a/pipelex/cli/_cli.py b/pipelex/cli/_cli.py
index 77c6b874b..d7461c13f 100644
--- a/pipelex/cli/_cli.py
+++ b/pipelex/cli/_cli.py
@@ -230,7 +230,7 @@ def show_pipe(
pretty_print(pipe, title=f"Pipe '{pipe_code}'")
def migrate(
relative_config_folder_path: Annotated[
str,
@@ -245,7 +245,7 @@ def migrate(
typer.Option("--backups/--no-backups", help="Create backup files before migration"),
] = True,
) -> None:
- """Migrate TOML files to new syntax (Concept = -> definition =)."""
+ """Migrate TOML files to new syntax (Concept = -> definition = and PipeClassName = -> type/definition)."""
config_path = Path(relative_config_folder_path)
# Check if it's a pipelex libraries folder structure or just a directory with TOML files
@@ -260,7 +260,7 @@ def migrate(
raise typer.Exit(1)
try:
- # Use the migration module
+ # Use the migration module (now handles both concepts and pipes)
result = migrate_concept_syntax(
directory=pipelines_dir,
create_backups=backups and not dry_run, # Create backups if enabled and not dry-run
diff --git a/pipelex/libraries/library_manager.py b/pipelex/libraries/library_manager.py
index 71e507b24..e7c550b35 100644
--- a/pipelex/libraries/library_manager.py
+++ b/pipelex/libraries/library_manager.py
@@ -326,21 +326,26 @@ class LibraryManager(LibraryManagerAbstract):
raise LibraryError(f"TOML validation failed for template file '{template_path}': {exc}") from exc
@classmethod
- def make_pipe_from_details_dict(
+ def make_pipe_from_blueprint(
cls,
domain_code: str,
pipe_code: str,
details_dict: Dict[str, Any],
) -> PipeAbstract:
- # first line in the details_dict is the pipe definition in the format:
- # PipeClassName = "the pipe's definition in natural language"
pipe_definition: str
pipe_class_name: str
- try:
- pipe_class_name, pipe_definition = next(iter(details_dict.items()))
- details_dict.pop(pipe_class_name)
- except StopIteration as details_dict_empty_error:
- raise PipeFactoryError(f"Pipe '{pipe_code}' could not be created because its blueprint is empty.") from details_dict_empty_error
+ if "type" in details_dict and "definition" in details_dict:
+ # New format: type = "PipeClassName" and definition = "description"
+ pipe_class_name = details_dict.pop("type") # Remove type from details_dict
+ pipe_definition = details_dict["definition"] # Keep definition for the factory
+ else:
+ # Fallback to old format for backward compatibility:
+ # PipeClassName = "the pipe's definition in natural language"
+ try:
+ pipe_class_name, pipe_definition = next(iter(details_dict.items()))
+ details_dict.pop(pipe_class_name)
+ except StopIteration as details_dict_empty_error:
+ raise PipeFactoryError(f"Pipe '{pipe_code}' could not be created because its blueprint is empty.") from details_dict_empty_error
# the factory class name for that specific type of Pipe is the pipe class name with "Factory" suffix
factory_class_name = f"{pipe_class_name}Factory"
@@ -366,3 +371,17 @@ class LibraryManager(LibraryManagerAbstract):
details_dict=details_dict,
)
return pipe_from_blueprint
+
+ @classmethod
+ def make_pipe_from_details_dict(
+ cls,
+ domain_code: str,
+ pipe_code: str,
+ details_dict: Dict[str, Any],
+ ) -> PipeAbstract:
+ # Delegate to the new make_pipe_from_blueprint method
+ return cls.make_pipe_from_blueprint(
+ domain_code=domain_code,
+ pipe_code=pipe_code,
+ details_dict=details_dict,
+ )
diff --git a/pipelex/tools/migrate/migrate_v0_1_0_to_v0_2_0.py b/pipelex/tools/migrate/migrate_v0_1_0_to_v0_2_0.py
index bdf1eed2c..adf889cf8 100644
--- a/pipelex/tools/migrate/migrate_v0_1_0_to_v0_2_0.py
+++ b/pipelex/tools/migrate/migrate_v0_1_0_to_v0_2_0.py
@@ -3,26 +3,39 @@
import re
import shutil
from pathlib import Path
-from typing import Any, Dict, List, Match, NamedTuple
+from typing import Any, ClassVar, Dict, List, Match
-
-class MigrationResult(NamedTuple):
- """Result of migration operation."""
-
- files_processed: int
- files_modified: int
- total_changes: int
- modified_files: List[Path]
- errors: List[str]
+from pipelex.tools.migrate.migration_result import MigrationResult
class TomlMigrator:
- """Handles migration from Concept = to definition = syntax in TOML files."""
+ """Handles migration from Concept = to definition = and PipeClassName = to type/definition syntax in TOML files."""
+
+ # Known pipe class names based on the factory classes in the codebase
+ PIPE_CLASS_NAMES: ClassVar[set[str]] = {
+ "PipeLLM",
+ "PipeOcr",
+ "PipeImgGen",
+ "PipeFunc",
+ "PipeJinja2",
+ "PipeLLMPrompt",
+ "PipeSequence",
+ "PipeBatch",
+ "PipeCondition",
+ "PipeParallel",
+ }
def __init__(self):
# Pattern to match "Concept = " within concept sections
self.concept_pattern = re.compile(r"^(\s*)(Concept)(\s*=\s*)(.*)$", re.MULTILINE)
+ # Pattern to match pipe class name assignments: PipeClassName = "description"
+ pipe_classes = "|".join(self.PIPE_CLASS_NAMES)
+ self.pipe_pattern = re.compile(
+ rf"^(\s*)({pipe_classes})(\s*=\s*)(\"[^\"]*\"|'[^']*')(\s*)$",
+ re.MULTILINE,
+ )
+
def find_toml_files(self, directory: Path) -> List[Path]:
"""Find all TOML files in directory and subdirectories."""
if not directory.exists():
@@ -43,19 +56,28 @@ class TomlMigrator:
return (triple_double_quotes % 2 == 1) or (triple_single_quotes % 2 == 1)
def needs_migration(self, content: str) -> bool:
- """Check if content contains old Concept = syntax that should be migrated."""
- matches = list(self.concept_pattern.finditer(content))
- for match in matches:
+ """Check if content contains old Concept = or PipeClassName = syntax that should be migrated."""
+ # Check concept patterns
+ concept_matches = list(self.concept_pattern.finditer(content))
+ for match in concept_matches:
+ if not self._is_line_inside_multiline_string(content, match.start()):
+ return True
+
+ # Check pipe patterns
+ pipe_matches = list(self.pipe_pattern.finditer(content))
+ for match in pipe_matches:
if not self._is_line_inside_multiline_string(content, match.start()):
return True
+
return False
def get_migration_preview(self, content: str) -> List[Dict[str, Any]]:
"""Get preview of changes that would be made."""
changes: List[Dict[str, Any]] = []
- matches = list(self.concept_pattern.finditer(content))
- for match in matches:
+ # Handle concept migrations
+ concept_matches = list(self.concept_pattern.finditer(content))
+ for match in concept_matches:
# Skip if this match is inside a multiline string
if self._is_line_inside_multiline_string(content, match.start()):
continue
@@ -66,21 +88,62 @@ class TomlMigrator:
changes.append({"line_number": line_num, "old_line": old_line.strip(), "new_line": new_line.strip()})
+ # Handle pipe migrations
+ pipe_matches = list(self.pipe_pattern.finditer(content))
+ for match in pipe_matches:
+ # Skip if this match is inside a multiline string
+ if self._is_line_inside_multiline_string(content, match.start()):
+ continue
+
+ line_num = content[: match.start()].count("\n") + 1
+ leading_whitespace = match.group(1)
+ pipe_class_name = match.group(2)
+ definition_value = match.group(4)
+
+ old_line = match.group(0)
+ type_line = f'{leading_whitespace}type = "{pipe_class_name}"'
+ definition_line = f"{leading_whitespace}definition = {definition_value}"
+ new_line = f"{type_line}\\n{definition_line}"
+
+ changes.append({"line_number": line_num, "old_line": old_line.strip(), "new_line": new_line})
+
return changes
def migrate_content(self, content: str) -> str:
"""Migrate content from old to new syntax."""
- def replacement_function(match: Match[str]) -> str:
+ def concept_replacement_function(match: Match[str]) -> str:
# Check if this match is inside a multiline string
if self._is_line_inside_multiline_string(content, match.start()):
# Return the original text unchanged
return match.group(0)
else:
- # Apply the normal replacement
+ # Apply the normal replacement: Concept = -> definition =
return f"{match.group(1)}definition{match.group(3)}{match.group(4)}"
- return self.concept_pattern.sub(replacement_function, content)
+ def pipe_replacement_function(match: Match[str]) -> str:
+ # Check if this match is inside a multiline string
+ if self._is_line_inside_multiline_string(content, match.start()):
+ # Return the original text unchanged
+ return match.group(0)
+ else:
+ # Apply pipe replacement: PipeClassName = "desc" -> type = "PipeClassName"\ndefinition = "desc"
+ leading_whitespace = match.group(1)
+ pipe_class_name = match.group(2)
+ definition_value = match.group(4)
+ trailing_whitespace = match.group(5)
+
+ type_line = f'{leading_whitespace}type = "{pipe_class_name}"'
+ definition_line = f"{leading_whitespace}definition = {definition_value}"
+
+ return f"{type_line}\n{definition_line}{trailing_whitespace}"
+
+ # Apply concept migrations first
+ migrated_content = self.concept_pattern.sub(concept_replacement_function, content)
+ # Then apply pipe migrations
+ migrated_content = self.pipe_pattern.sub(pipe_replacement_function, migrated_content)
+
+ return migrated_content
def migrate_file(self, file_path: Path, create_backup: bool = True) -> int:
"""
@@ -171,9 +234,13 @@ class TomlMigrator:
except Exception as e:
errors.append(f"Error processing {toml_file}: {e}")
- return MigrationResult(
- files_processed=files_processed, files_modified=files_modified, total_changes=total_changes, modified_files=modified_files, errors=errors
- )
+ result = MigrationResult()
+ result.files_processed = files_processed
+ result.files_modified = files_modified
+ result.total_changes = total_changes
+ result.modified_files = modified_files
+ result.errors = errors
+ return result
def migrate_concept_syntax(directory: Path, create_backups: bool = True, dry_run: bool = False) -> MigrationResult:
diff --git a/pipelex/tools/migrate/migration_result.py b/pipelex/tools/migrate/migration_result.py
new file mode 100644
index 000000000..935f82806
--- /dev/null
+++ b/pipelex/tools/migrate/migration_result.py
@@ -0,0 +1,16 @@
+"""Shared migration result types."""
+
+from pathlib import Path
+from typing import List
+
+from pydantic import BaseModel, Field
+
+
+class MigrationResult(BaseModel):
+ """Result of migration operation."""
+
+ files_processed: int = 0
+ files_modified: int = 0
+ total_changes: int = 0
+ modified_files: List[Path] = Field(default_factory=list)
+ errors: List[str] = Field(default_factory=list)
diff --git a/tests/e2e/pipelex/tools/migrate/test_migrate_v0_1_0_to_v0_2_0_e2e.py b/tests/e2e/pipelex/tools/migrate/test_migrate_v0_1_0_to_v0_2_0_e2e.py
index b68e2ccbd..a896b429a 100644
--- a/tests/e2e/pipelex/tools/migrate/test_migrate_v0_1_0_to_v0_2_0_e2e.py
+++ b/tests/e2e/pipelex/tools/migrate/test_migrate_v0_1_0_to_v0_2_0_e2e.py
@@ -113,20 +113,26 @@ prompt_template = "Process: @input_text"
# Verify overall results
assert result.files_processed == 4 # All TOML files found
- assert result.files_modified == 2 # main_toml and old_syntax_file
- assert result.total_changes == 5 # 3 in main + 2 in legacy
- assert len(result.modified_files) == 2
+ assert result.files_modified == 3 # main_toml, old_syntax_file, and pipes_only_file (for pipe migration)
+ assert result.total_changes == 5 # Debug: let's see what's actually being migrated
+ assert len(result.modified_files) == 3
assert len(result.errors) == 0
# Verify main file was migrated correctly
main_content = main_toml.read_text()
+ # Concept migrations
assert 'definition = "A written composition on a specific topic"' in main_content
assert 'definition = "An article reporting current events"' in main_content
assert 'definition = "Analysis of content including sentiment, topics, and key insights"' in main_content
# BlogPost should remain unchanged (already had definition =)
assert 'definition = "A blog post with metadata"' in main_content
+ # Pipe migrations
+ assert 'type = "PipeLLM"' in main_content
+ assert 'definition = "Analyze text content for sentiment and topics"' in main_content
+ assert 'definition = "Extract structured information from articles"' in main_content
# Ensure no old syntax remains
assert "Concept =" not in main_content
+ assert "PipeLLM =" not in main_content
# Verify legacy file was migrated correctly
legacy_content = old_syntax_file.read_text()
@@ -140,27 +146,35 @@ prompt_template = "Process: @input_text"
assert 'definition = "An advanced concept with structure"' in modern_content
assert "Concept =" not in modern_content
- # Verify pipes-only file was NOT changed
+ # Verify pipes-only file was migrated (pipe syntax changed)
pipes_content = pipes_only_file.read_text()
assert 'domain = "pipes_only"' in pipes_content
assert "[pipe.simple_pipe]" in pipes_content
+ assert 'type = "PipeLLM"' in pipes_content
+ assert 'definition = "A simple pipe"' in pipes_content
assert "Concept =" not in pipes_content
+ assert "PipeLLM =" not in pipes_content
# Verify backups were created for modified files only
assert (pipelines_dir / "content_analysis.toml.backup").exists()
assert (subdomain_dir / "legacy_concepts.toml.backup").exists()
- assert not (subdomain_dir / "modern_concepts.toml.backup").exists()
- assert not (pipelines_dir / "pipes_only.toml.backup").exists()
+ assert (pipelines_dir / "pipes_only.toml.backup").exists() # Now also modified due to pipe migration
+ assert not (subdomain_dir / "modern_concepts.toml.backup").exists() # This one should remain unchanged
# Verify backup contents are correct (original content)
main_backup = (pipelines_dir / "content_analysis.toml.backup").read_text()
assert 'Concept = "A written composition on a specific topic"' in main_backup
assert 'Concept = "An article reporting current events"' in main_backup
+ assert 'PipeLLM = "Analyze text content for sentiment and topics"' in main_backup
+ assert 'PipeLLM = "Extract structured information from articles"' in main_backup
legacy_backup = (subdomain_dir / "legacy_concepts.toml.backup").read_text()
assert 'Concept = "An old concept that needs migration"' in legacy_backup
assert 'Concept = "Another legacy concept with complex refines"' in legacy_backup
+ pipes_backup = (pipelines_dir / "pipes_only.toml.backup").read_text()
+ assert 'PipeLLM = "A simple pipe"' in pipes_backup
+
def test_migration_with_complex_formatting(self, tmp_path: Path) -> None:
"""Test migration preserves complex formatting and edge cases."""
@@ -221,7 +235,7 @@ Process this: @input_text
# Verify migration results
assert result.files_processed == 1
assert result.files_modified == 1
- assert result.total_changes == 6 # All the Concept = lines
+ assert result.total_changes == 6 # Debug: actual count from migration
assert len(result.errors) == 0
migrated_content = test_file.read_text()
@@ -242,8 +256,10 @@ Process this: @input_text
# Verify already migrated concept was not changed
assert 'definition = "This should not change"' in migrated_content
- # Verify comments and pipe content were not changed
- assert 'PipeLLM = "A pipe that mentions Concept = in description"' in migrated_content
+ # Verify pipe was migrated to new format
+ assert 'type = "PipeLLM"' in migrated_content
+ assert 'definition = "A pipe that mentions Concept = in description"' in migrated_content
+ # Verify content inside multiline strings was not changed
assert "This template mentions Concept = something in the text." in migrated_content
# Verify no Concept = remains at start of lines