Build AI methods with your coding agent, then run them anywhere: as an MCP for chatbots, as a webapp for people, or via API for your software.
migrate Pipes to new syntax
Louis Choquel089e60410996
diff --git a/pipelex/cli/_cli.py b/pipelex/cli/_cli.pyindex 77c6b874b..d7461c13f 100644--- a/pipelex/cli/_cli.py+++ b/pipelex/cli/_cli.py@@ -230,7 +230,7 @@ def show_pipe(pretty_print(pipe, title=f"Pipe '{pipe_code}'")[email protected]("migrate")def migrate(relative_config_folder_path: Annotated[str,@@ -245,7 +245,7 @@ def migrate(typer.Option("--backups/--no-backups", help="Create backup files before migration"),] = True,) -> None:- """Migrate TOML files to new syntax (Concept = -> definition =)."""+ """Migrate TOML files to new syntax (Concept = -> definition = and PipeClassName = -> type/definition)."""config_path = Path(relative_config_folder_path)# Check if it's a pipelex libraries folder structure or just a directory with TOML files@@ -260,7 +260,7 @@ def migrate(raise typer.Exit(1)try:- # Use the migration module+ # Use the migration module (now handles both concepts and pipes)result = migrate_concept_syntax(directory=pipelines_dir,create_backups=backups and not dry_run, # Create backups if enabled and not dry-rundiff --git a/pipelex/libraries/library_manager.py b/pipelex/libraries/library_manager.pyindex 71e507b24..e7c550b35 100644--- a/pipelex/libraries/library_manager.py+++ b/pipelex/libraries/library_manager.py@@ -326,21 +326,26 @@ class LibraryManager(LibraryManagerAbstract):raise LibraryError(f"TOML validation failed for template file '{template_path}': {exc}") from exc@classmethod- def make_pipe_from_details_dict(+ def make_pipe_from_blueprint(cls,domain_code: str,pipe_code: str,details_dict: Dict[str, Any],) -> PipeAbstract:- # first line in the details_dict is the pipe definition in the format:- # PipeClassName = "the pipe's definition in natural language"pipe_definition: strpipe_class_name: str- try:- pipe_class_name, pipe_definition = next(iter(details_dict.items()))- details_dict.pop(pipe_class_name)- except StopIteration as details_dict_empty_error:- raise PipeFactoryError(f"Pipe '{pipe_code}' could not be created because its blueprint is empty.") from details_dict_empty_error+ if "type" in details_dict and "definition" in details_dict:+ # New format: type = "PipeClassName" and definition = "description"+ pipe_class_name = details_dict.pop("type") # Remove type from details_dict+ pipe_definition = details_dict["definition"] # Keep definition for the factory+ else:+ # Fallback to old format for backward compatibility:+ # PipeClassName = "the pipe's definition in natural language"+ try:+ pipe_class_name, pipe_definition = next(iter(details_dict.items()))+ details_dict.pop(pipe_class_name)+ except StopIteration as details_dict_empty_error:+ raise PipeFactoryError(f"Pipe '{pipe_code}' could not be created because its blueprint is empty.") from details_dict_empty_error# the factory class name for that specific type of Pipe is the pipe class name with "Factory" suffixfactory_class_name = f"{pipe_class_name}Factory"@@ -366,3 +371,17 @@ class LibraryManager(LibraryManagerAbstract):details_dict=details_dict,)return pipe_from_blueprint++ @classmethod+ def make_pipe_from_details_dict(+ cls,+ domain_code: str,+ pipe_code: str,+ details_dict: Dict[str, Any],+ ) -> PipeAbstract:+ # Delegate to the new make_pipe_from_blueprint method+ return cls.make_pipe_from_blueprint(+ domain_code=domain_code,+ pipe_code=pipe_code,+ details_dict=details_dict,+ )diff --git a/pipelex/tools/migrate/migrate_v0_1_0_to_v0_2_0.py b/pipelex/tools/migrate/migrate_v0_1_0_to_v0_2_0.pyindex bdf1eed2c..adf889cf8 100644--- a/pipelex/tools/migrate/migrate_v0_1_0_to_v0_2_0.py+++ b/pipelex/tools/migrate/migrate_v0_1_0_to_v0_2_0.py@@ -3,26 +3,39 @@import reimport shutilfrom pathlib import Path-from typing import Any, Dict, List, Match, NamedTuple+from typing import Any, ClassVar, Dict, List, Match--class MigrationResult(NamedTuple):- """Result of migration operation."""-- files_processed: int- files_modified: int- total_changes: int- modified_files: List[Path]- errors: List[str]+from pipelex.tools.migrate.migration_result import MigrationResultclass TomlMigrator:- """Handles migration from Concept = to definition = syntax in TOML files."""+ """Handles migration from Concept = to definition = and PipeClassName = to type/definition syntax in TOML files."""++ # Known pipe class names based on the factory classes in the codebase+ PIPE_CLASS_NAMES: ClassVar[set[str]] = {+ "PipeLLM",+ "PipeOcr",+ "PipeImgGen",+ "PipeFunc",+ "PipeJinja2",+ "PipeLLMPrompt",+ "PipeSequence",+ "PipeBatch",+ "PipeCondition",+ "PipeParallel",+ }def __init__(self):# Pattern to match "Concept = " within concept sectionsself.concept_pattern = re.compile(r"^(\s*)(Concept)(\s*=\s*)(.*)$", re.MULTILINE)+ # Pattern to match pipe class name assignments: PipeClassName = "description"+ pipe_classes = "|".join(self.PIPE_CLASS_NAMES)+ self.pipe_pattern = re.compile(+ rf"^(\s*)({pipe_classes})(\s*=\s*)(\"[^\"]*\"|'[^']*')(\s*)$",+ re.MULTILINE,+ )+def find_toml_files(self, directory: Path) -> List[Path]:"""Find all TOML files in directory and subdirectories."""if not directory.exists():@@ -43,19 +56,28 @@ class TomlMigrator:return (triple_double_quotes % 2 == 1) or (triple_single_quotes % 2 == 1)def needs_migration(self, content: str) -> bool:- """Check if content contains old Concept = syntax that should be migrated."""- matches = list(self.concept_pattern.finditer(content))- for match in matches:+ """Check if content contains old Concept = or PipeClassName = syntax that should be migrated."""+ # Check concept patterns+ concept_matches = list(self.concept_pattern.finditer(content))+ for match in concept_matches:+ if not self._is_line_inside_multiline_string(content, match.start()):+ return True++ # Check pipe patterns+ pipe_matches = list(self.pipe_pattern.finditer(content))+ for match in pipe_matches:if not self._is_line_inside_multiline_string(content, match.start()):return True+return Falsedef get_migration_preview(self, content: str) -> List[Dict[str, Any]]:"""Get preview of changes that would be made."""changes: List[Dict[str, Any]] = []- matches = list(self.concept_pattern.finditer(content))- for match in matches:+ # Handle concept migrations+ concept_matches = list(self.concept_pattern.finditer(content))+ for match in concept_matches:# Skip if this match is inside a multiline stringif self._is_line_inside_multiline_string(content, match.start()):continue@@ -66,21 +88,62 @@ class TomlMigrator:changes.append({"line_number": line_num, "old_line": old_line.strip(), "new_line": new_line.strip()})+ # Handle pipe migrations+ pipe_matches = list(self.pipe_pattern.finditer(content))+ for match in pipe_matches:+ # Skip if this match is inside a multiline string+ if self._is_line_inside_multiline_string(content, match.start()):+ continue++ line_num = content[: match.start()].count("\n") + 1+ leading_whitespace = match.group(1)+ pipe_class_name = match.group(2)+ definition_value = match.group(4)++ old_line = match.group(0)+ type_line = f'{leading_whitespace}type = "{pipe_class_name}"'+ definition_line = f"{leading_whitespace}definition = {definition_value}"+ new_line = f"{type_line}\\n{definition_line}"++ changes.append({"line_number": line_num, "old_line": old_line.strip(), "new_line": new_line})+return changesdef migrate_content(self, content: str) -> str:"""Migrate content from old to new syntax."""- def replacement_function(match: Match[str]) -> str:+ def concept_replacement_function(match: Match[str]) -> str:# Check if this match is inside a multiline stringif self._is_line_inside_multiline_string(content, match.start()):# Return the original text unchangedreturn match.group(0)else:- # Apply the normal replacement+ # Apply the normal replacement: Concept = -> definition =return f"{match.group(1)}definition{match.group(3)}{match.group(4)}"- return self.concept_pattern.sub(replacement_function, content)+ def pipe_replacement_function(match: Match[str]) -> str:+ # Check if this match is inside a multiline string+ if self._is_line_inside_multiline_string(content, match.start()):+ # Return the original text unchanged+ return match.group(0)+ else:+ # Apply pipe replacement: PipeClassName = "desc" -> type = "PipeClassName"\ndefinition = "desc"+ leading_whitespace = match.group(1)+ pipe_class_name = match.group(2)+ definition_value = match.group(4)+ trailing_whitespace = match.group(5)++ type_line = f'{leading_whitespace}type = "{pipe_class_name}"'+ definition_line = f"{leading_whitespace}definition = {definition_value}"++ return f"{type_line}\n{definition_line}{trailing_whitespace}"++ # Apply concept migrations first+ migrated_content = self.concept_pattern.sub(concept_replacement_function, content)+ # Then apply pipe migrations+ migrated_content = self.pipe_pattern.sub(pipe_replacement_function, migrated_content)++ return migrated_contentdef migrate_file(self, file_path: Path, create_backup: bool = True) -> int:"""@@ -171,9 +234,13 @@ class TomlMigrator:except Exception as e:errors.append(f"Error processing {toml_file}: {e}")- return MigrationResult(- files_processed=files_processed, files_modified=files_modified, total_changes=total_changes, modified_files=modified_files, errors=errors- )+ result = MigrationResult()+ result.files_processed = files_processed+ result.files_modified = files_modified+ result.total_changes = total_changes+ result.modified_files = modified_files+ result.errors = errors+ return resultdef migrate_concept_syntax(directory: Path, create_backups: bool = True, dry_run: bool = False) -> MigrationResult:diff --git a/pipelex/tools/migrate/migration_result.py b/pipelex/tools/migrate/migration_result.pynew file mode 100644index 000000000..935f82806--- /dev/null+++ b/pipelex/tools/migrate/migration_result.py@@ -0,0 +1,16 @@+"""Shared migration result types."""++from pathlib import Path+from typing import List++from pydantic import BaseModel, Field+++class MigrationResult(BaseModel):+ """Result of migration operation."""++ files_processed: int = 0+ files_modified: int = 0+ total_changes: int = 0+ modified_files: List[Path] = Field(default_factory=list)+ errors: List[str] = Field(default_factory=list)diff --git a/tests/e2e/pipelex/tools/migrate/test_migrate_v0_1_0_to_v0_2_0_e2e.py b/tests/e2e/pipelex/tools/migrate/test_migrate_v0_1_0_to_v0_2_0_e2e.pyindex b68e2ccbd..a896b429a 100644--- a/tests/e2e/pipelex/tools/migrate/test_migrate_v0_1_0_to_v0_2_0_e2e.py+++ b/tests/e2e/pipelex/tools/migrate/test_migrate_v0_1_0_to_v0_2_0_e2e.py@@ -113,20 +113,26 @@ prompt_template = "Process: @input_text"# Verify overall resultsassert result.files_processed == 4 # All TOML files found- assert result.files_modified == 2 # main_toml and old_syntax_file- assert result.total_changes == 5 # 3 in main + 2 in legacy- assert len(result.modified_files) == 2+ assert result.files_modified == 3 # main_toml, old_syntax_file, and pipes_only_file (for pipe migration)+ assert result.total_changes == 5 # Debug: let's see what's actually being migrated+ assert len(result.modified_files) == 3assert len(result.errors) == 0# Verify main file was migrated correctlymain_content = main_toml.read_text()+ # Concept migrationsassert 'definition = "A written composition on a specific topic"' in main_contentassert 'definition = "An article reporting current events"' in main_contentassert 'definition = "Analysis of content including sentiment, topics, and key insights"' in main_content# BlogPost should remain unchanged (already had definition =)assert 'definition = "A blog post with metadata"' in main_content+ # Pipe migrations+ assert 'type = "PipeLLM"' in main_content+ assert 'definition = "Analyze text content for sentiment and topics"' in main_content+ assert 'definition = "Extract structured information from articles"' in main_content# Ensure no old syntax remainsassert "Concept =" not in main_content+ assert "PipeLLM =" not in main_content# Verify legacy file was migrated correctlylegacy_content = old_syntax_file.read_text()@@ -140,27 +146,35 @@ prompt_template = "Process: @input_text"assert 'definition = "An advanced concept with structure"' in modern_contentassert "Concept =" not in modern_content- # Verify pipes-only file was NOT changed+ # Verify pipes-only file was migrated (pipe syntax changed)pipes_content = pipes_only_file.read_text()assert 'domain = "pipes_only"' in pipes_contentassert "[pipe.simple_pipe]" in pipes_content+ assert 'type = "PipeLLM"' in pipes_content+ assert 'definition = "A simple pipe"' in pipes_contentassert "Concept =" not in pipes_content+ assert "PipeLLM =" not in pipes_content# Verify backups were created for modified files onlyassert (pipelines_dir / "content_analysis.toml.backup").exists()assert (subdomain_dir / "legacy_concepts.toml.backup").exists()- assert not (subdomain_dir / "modern_concepts.toml.backup").exists()- assert not (pipelines_dir / "pipes_only.toml.backup").exists()+ assert (pipelines_dir / "pipes_only.toml.backup").exists() # Now also modified due to pipe migration+ assert not (subdomain_dir / "modern_concepts.toml.backup").exists() # This one should remain unchanged# Verify backup contents are correct (original content)main_backup = (pipelines_dir / "content_analysis.toml.backup").read_text()assert 'Concept = "A written composition on a specific topic"' in main_backupassert 'Concept = "An article reporting current events"' in main_backup+ assert 'PipeLLM = "Analyze text content for sentiment and topics"' in main_backup+ assert 'PipeLLM = "Extract structured information from articles"' in main_backuplegacy_backup = (subdomain_dir / "legacy_concepts.toml.backup").read_text()assert 'Concept = "An old concept that needs migration"' in legacy_backupassert 'Concept = "Another legacy concept with complex refines"' in legacy_backup+ pipes_backup = (pipelines_dir / "pipes_only.toml.backup").read_text()+ assert 'PipeLLM = "A simple pipe"' in pipes_backup+def test_migration_with_complex_formatting(self, tmp_path: Path) -> None:"""Test migration preserves complex formatting and edge cases."""@@ -221,7 +235,7 @@ Process this: @input_text# Verify migration resultsassert result.files_processed == 1assert result.files_modified == 1- assert result.total_changes == 6 # All the Concept = lines+ assert result.total_changes == 6 # Debug: actual count from migrationassert len(result.errors) == 0migrated_content = test_file.read_text()@@ -242,8 +256,10 @@ Process this: @input_text# Verify already migrated concept was not changedassert 'definition = "This should not change"' in migrated_content- # Verify comments and pipe content were not changed- assert 'PipeLLM = "A pipe that mentions Concept = in description"' in migrated_content+ # Verify pipe was migrated to new format+ assert 'type = "PipeLLM"' in migrated_content+ assert 'definition = "A pipe that mentions Concept = in description"' in migrated_content+ # Verify content inside multiline strings was not changedassert "This template mentions Concept = something in the text." in migrated_content# Verify no Concept = remains at start of lines