Build AI methods with your coding agent, then run them anywhere: as an MCP for chatbots, as a webapp for people, or via API for your software.
cli migration command + unit & e2e tests
Louis Choqueld86ededa1d86
diff --git a/docs/migration.md b/docs/migration.mdnew file mode 100644index 000000000..7fe78f149--- /dev/null+++ b/docs/migration.md@@ -0,0 +1,131 @@+# Migration Guide++This document provides guidance for migrating between different versions of Pipelex that introduce breaking changes.++## Version 0.6.11: Concept Definition Syntax Change++### Breaking Change++In version 0.6.11, we changed the syntax for defining concepts in TOML library files. The key name for concept definitions has changed from `Concept` to `definition`.++### What Changed++**Old syntax (before v0.6.11):**+```toml+[concept.MyConceptName]+Concept = "Description of the concept"+refines = "BaseConceptName"+```++**New syntax (v0.6.11+):**+```toml+[concept.MyConceptName]+definition = "Description of the concept"+refines = "BaseConceptName"+```++### Why This Change++This change improves consistency in our TOML schema by:+- Using more descriptive field names (`definition` instead of `Concept`)+- Aligning with the internal `ConceptBlueprint` data structure+- Making the TOML files more self-documenting and intuitive++### Migration Process++#### Automatic Migration++Use the built-in migration command to automatically update your TOML files:++```bash+# Preview changes without applying them+pipelex migrate --dry-run++# Apply the migration to all TOML files+pipelex migrate+```++The migration command will:+- Find all `.toml` files in your configured pipelines directory+- Replace `Concept =` with `definition =` in concept definitions+- Preserve all other formatting and comments+- Create backups of modified files (with `.backup` extension)++#### Manual Migration++If you prefer to migrate manually or need to handle special cases:++1. **Locate your TOML files**: Find all pipeline library files (typically in your configured pipelines directory)+2. **Update concept definitions**: Change `Concept =` to `definition =` in all `[concept.ConceptName]` sections+3. **Validate syntax**: Run `pipelex validate` to ensure your files are correctly formatted++### Examples++#### Simple Concept Definition++**Before:**+```toml+[concept.Article]+Concept = "A written composition on a specific topic"+refines = "Text"+```++**After:**+```toml+[concept.Article]+definition = "A written composition on a specific topic"+refines = "Text"+```++#### Complex Concept with Structure++**Before:**+```toml+[concept.Photo]+Concept = "Photo"+structure = "ImageContent"+refines = "Image"+```++**After:**+```toml+[concept.Photo]+definition = "Photo"+structure = "ImageContent"+refines = "Image"+```++### Validation++After migration, validate your pipeline files:++```bash+# Validate all pipeline files+pipelex validate++# Run a specific pipeline to test+pipelex run your-pipeline-name+```++### Troubleshooting++**Error: "ConceptBlueprint validation failed"**+- Check that all `Concept =` entries have been changed to `definition =`+- Ensure no typos were introduced during migration+- Verify TOML syntax is still valid++**Error: "Failed to load TOML file"**+- Check file permissions+- Ensure TOML syntax is valid (no missing quotes, brackets, etc.)+- Review any custom modifications that might conflict++### Rollback++If you need to rollback the migration:+1. Stop using the new version of Pipelex+2. Restore from the `.backup` files created during migration+3. Or manually change `definition =` back to `Concept =`++---++For additional help or if you encounter issues during migration, please [open an issue](https://github.com/Pipelex/pipelex/issues) on our GitHub repository.\ No newline at end of filediff --git a/pipelex/cli/_cli.py b/pipelex/cli/_cli.pyindex 3dd6cfe7a..50fdd3383 100644--- a/pipelex/cli/_cli.py+++ b/pipelex/cli/_cli.py@@ -1,6 +1,7 @@import asyncioimport osimport shutil+from pathlib import Pathfrom typing import Annotated, Optionalimport typer@@ -15,6 +16,7 @@ from pipelex.libraries.library_config import LibraryConfigfrom pipelex.pipe_works.pipe_dry import dry_run_all_pipes, dry_run_single_pipefrom pipelex.pipelex import Pipelexfrom pipelex.tools.config.manager import config_manager+from pipelex.tools.migrate.migrate_v0_1_0_to_v0_2_0 import TomlMigrator, migrate_concept_syntaxdef is_pipelex_libraries_folder(folder_path: str) -> bool:@@ -228,6 +230,89 @@ def show_pipe(pretty_print(pipe, title=f"Pipe '{pipe_code}'")+def migrate(+ relative_config_folder_path: Annotated[+ str,+ typer.Option("--config-folder-path", "-c", help="Relative path to the config folder path"),+ ] = "./pipelex_libraries",+ dry_run: Annotated[+ bool,+ typer.Option("--dry-run", help="Preview changes without applying them"),+ ] = False,+) -> None:+ """Migrate TOML files to new syntax (Concept = -> definition =)."""+ config_path = Path(relative_config_folder_path)++ # Check if it's a pipelex libraries folder structure or just a directory with TOML files+ if is_pipelex_libraries_folder(relative_config_folder_path):+ pipelines_dir = config_path / "pipelines"+ else:+ # Assume it's a directory containing TOML files directly+ pipelines_dir = config_path++ if not pipelines_dir.exists():+ typer.echo(f"❌ Directory not found at '{pipelines_dir}'")+ raise typer.Exit(1)++ try:+ # Use the migration module+ result = migrate_concept_syntax(+ directory=pipelines_dir,+ create_backups=not dry_run, # Only create backups when not dry-run+ dry_run=dry_run,+ )++ # Handle any errors that occurred+ for error in result.errors:+ typer.echo(f"❌ {error}")++ if result.errors and not result.files_modified:+ typer.echo("❌ Migration failed due to errors")+ raise typer.Exit(1)++ typer.echo(f"Found {result.files_processed} TOML file(s) to check")++ if result.files_modified == 0:+ typer.echo("✅ All TOML files are already using the new syntax")+ return++ if dry_run:+ # Show detailed preview for dry run+ migrator = TomlMigrator()+ for file_path in result.modified_files:+ try:+ with open(file_path, "r", encoding="utf-8") as f:+ content = f.read()++ changes = migrator.get_migration_preview(content)+ typer.echo(f"\n📄 {file_path.relative_to(pipelines_dir)}:")+ for change in changes:+ typer.echo(f" Line {change['line_number']}: {change['old_line']} → {change['new_line']}")+ except Exception as e:+ typer.echo(f"❌ Error reading {file_path}: {e}")++ typer.echo(f"\n📋 Summary: Found {result.total_changes} change(s) in {result.files_modified} file(s)")+ typer.echo(" Run without --dry-run to apply these changes")+ else:+ # Show migration results+ for file_path in result.modified_files:+ backup_path = file_path.with_suffix(".toml.backup")+ typer.echo(f"✅ Migrated {file_path.relative_to(pipelines_dir)}")+ typer.echo(f" Backup saved to {backup_path.name}")++ typer.echo(f"\n✅ Migration completed: {result.total_changes} change(s) applied to {result.files_modified} file(s)")+ typer.echo(" Backup files created with .backup extension")+ typer.echo(" Run 'pipelex validate' to verify the migration")++ except FileNotFoundError as e:+ typer.echo(f"❌ {e}")+ raise typer.Exit(1)+ except Exception as e:+ typer.echo(f"❌ Migration failed: {e}")+ raise typer.Exit(1)++def main() -> None:"""Entry point for the pipelex CLI."""app()diff --git a/pipelex/libraries/library_manager.py b/pipelex/libraries/library_manager.pyindex d525d9ad4..71e507b24 100644--- a/pipelex/libraries/library_manager.py+++ b/pipelex/libraries/library_manager.py@@ -20,7 +20,6 @@ from pipelex.core.pipe_library import PipeLibraryfrom pipelex.exceptions import (ConceptLibraryError,LibraryError,- LibraryParsingError,PipeFactoryError,PipeLibraryError,StaticValidationError,diff --git a/pipelex/libraries/pipeline_blueprint.py b/pipelex/libraries/pipeline_blueprint.pyindex b9370587d..5b60fb821 100644--- a/pipelex/libraries/pipeline_blueprint.py+++ b/pipelex/libraries/pipeline_blueprint.py@@ -1,5 +1,5 @@from pathlib import Path-from typing import Any, Dict, List, Optional, Union+from typing import Any, Dict, Optional, Unionfrom pydantic import BaseModel, Fielddiff --git a/pipelex/tools/migrate/__init__.py b/pipelex/tools/migrate/__init__.pynew file mode 100644index 000000000..e69de29bbdiff --git a/pipelex/tools/migrate/migrate_v0_1_0_to_v0_2_0.py b/pipelex/tools/migrate/migrate_v0_1_0_to_v0_2_0.pynew file mode 100644index 000000000..bdf1eed2c--- /dev/null+++ b/pipelex/tools/migrate/migrate_v0_1_0_to_v0_2_0.py@@ -0,0 +1,192 @@+"""Migration from version 0.1.0 to 0.2.0 - Concept = to definition = syntax change."""++import re+import shutil+from pathlib import Path+from typing import Any, Dict, List, Match, NamedTuple+++class MigrationResult(NamedTuple):+ """Result of migration operation."""++ files_processed: int+ files_modified: int+ total_changes: int+ modified_files: List[Path]+ errors: List[str]+++class TomlMigrator:+ """Handles migration from Concept = to definition = syntax in TOML files."""++ def __init__(self):+ # Pattern to match "Concept = " within concept sections+ self.concept_pattern = re.compile(r"^(\s*)(Concept)(\s*=\s*)(.*)$", re.MULTILINE)++ def find_toml_files(self, directory: Path) -> List[Path]:+ """Find all TOML files in directory and subdirectories."""+ if not directory.exists():+ raise FileNotFoundError(f"Directory not found: {directory}")++ return list(directory.glob("**/*.toml"))++ def _is_line_inside_multiline_string(self, content: str, line_start_pos: int) -> bool:+ """Check if a line position is inside a multiline string (triple quotes)."""+ # Count triple quotes before this position+ content_before = content[:line_start_pos]++ # Count both """ and ''' (TOML supports both)+ triple_double_quotes = content_before.count('"""')+ triple_single_quotes = content_before.count("'''")++ # If odd number of triple quotes before this line, we're inside a multiline string+ return (triple_double_quotes % 2 == 1) or (triple_single_quotes % 2 == 1)++ def needs_migration(self, content: str) -> bool:+ """Check if content contains old Concept = syntax that should be migrated."""+ matches = list(self.concept_pattern.finditer(content))+ for match in matches:+ if not self._is_line_inside_multiline_string(content, match.start()):+ return True+ return False++ def get_migration_preview(self, content: str) -> List[Dict[str, Any]]:+ """Get preview of changes that would be made."""+ changes: List[Dict[str, Any]] = []+ matches = list(self.concept_pattern.finditer(content))++ for match in matches:+ # Skip if this match is inside a multiline string+ if self._is_line_inside_multiline_string(content, match.start()):+ continue++ line_num = content[: match.start()].count("\n") + 1+ old_line = match.group(0)+ new_line = self.concept_pattern.sub(r"\1definition\3\4", old_line)++ changes.append({"line_number": line_num, "old_line": old_line.strip(), "new_line": new_line.strip()})++ return changes++ def migrate_content(self, content: str) -> str:+ """Migrate content from old to new syntax."""++ def replacement_function(match: Match[str]) -> str:+ # Check if this match is inside a multiline string+ if self._is_line_inside_multiline_string(content, match.start()):+ # Return the original text unchanged+ return match.group(0)+ else:+ # Apply the normal replacement+ return f"{match.group(1)}definition{match.group(3)}{match.group(4)}"++ return self.concept_pattern.sub(replacement_function, content)++ def migrate_file(self, file_path: Path, create_backup: bool = True) -> int:+ """+ Migrate a single file from old to new syntax.++ Returns:+ Number of changes made in the file+ """+ try:+ with open(file_path, "r", encoding="utf-8") as f:+ original_content = f.read()+ except Exception as e:+ raise IOError(f"Failed to read file {file_path}: {e}")++ if not self.needs_migration(original_content):+ return 0++ migrated_content = self.migrate_content(original_content)++ # Count only the changes that are NOT inside multiline strings+ matches = list(self.concept_pattern.finditer(original_content))+ changes_count = 0+ for match in matches:+ if not self._is_line_inside_multiline_string(original_content, match.start()):+ changes_count += 1++ if create_backup:+ backup_path = file_path.with_suffix(".toml.backup")+ try:+ shutil.copy2(file_path, backup_path)+ except Exception as e:+ raise IOError(f"Failed to create backup for {file_path}: {e}")++ try:+ with open(file_path, "w", encoding="utf-8") as f:+ f.write(migrated_content)+ except Exception as e:+ raise IOError(f"Failed to write migrated content to {file_path}: {e}")++ return changes_count++ def migrate_directory(self, directory: Path, create_backups: bool = True, dry_run: bool = False) -> MigrationResult:+ """+ Migrate all TOML files in a directory.++ Args:+ directory: Directory containing TOML files+ create_backups: Whether to create backup files+ dry_run: If True, only preview changes without applying them++ Returns:+ MigrationResult with statistics and details+ """+ toml_files = self.find_toml_files(directory)+ files_processed = len(toml_files)+ files_modified = 0+ total_changes = 0+ modified_files: List[Path] = []+ errors: List[str] = []++ for toml_file in toml_files:+ try:+ with open(toml_file, "r", encoding="utf-8") as f:+ content = f.read()++ if not self.needs_migration(content):+ continue++ # Count only changes that are NOT inside multiline strings+ matches = list(self.concept_pattern.finditer(content))+ changes_count = 0+ for match in matches:+ if not self._is_line_inside_multiline_string(content, match.start()):+ changes_count += 1++ if dry_run:+ # Just count the changes for dry run+ files_modified += 1+ total_changes += changes_count+ modified_files.append(toml_file)+ else:+ # Actually perform the migration+ actual_changes = self.migrate_file(toml_file, create_backup=create_backups)+ files_modified += 1+ total_changes += actual_changes+ modified_files.append(toml_file)++ except Exception as e:+ errors.append(f"Error processing {toml_file}: {e}")++ return MigrationResult(+ files_processed=files_processed, files_modified=files_modified, total_changes=total_changes, modified_files=modified_files, errors=errors+ )+++def migrate_concept_syntax(directory: Path, create_backups: bool = True, dry_run: bool = False) -> MigrationResult:+ """+ Convenience function to migrate TOML files from Concept = to definition = syntax.++ Args:+ directory: Directory containing TOML files to migrate+ create_backups: Whether to create backup files before migration+ dry_run: If True, only preview changes without applying them++ Returns:+ MigrationResult with migration statistics+ """+ migrator = TomlMigrator()+ return migrator.migrate_directory(directory, create_backups, dry_run)diff --git a/tests/data/test_migrate_v0_1_0_to_v0_2_0.toml b/tests/data/test_migrate_v0_1_0_to_v0_2_0.tomlnew file mode 100644index 000000000..1d6f5b417--- /dev/null+++ b/tests/data/test_migrate_v0_1_0_to_v0_2_0.toml@@ -0,0 +1,31 @@+domain = "test"+definition = "Test domain"++[concept]+SimpleText = "A simple text concept"+SimpleDoc = "A simple document concept"++# Complex concepts with old syntax+[concept.ComplexConcept]+Concept = "A complex concept with old syntax"+refines = "Text"++[concept.AnotherConcept]+Concept = "Another concept to test migration"+structure = "CustomStructure"+refines = ["Text", "Document"]++[concept.AlreadyMigrated]+definition = "This one is already migrated"+refines = "Text"++[pipe.test_pipe]+PipeLLM = "A test pipe that generates simple text"+inputs = { input_text = "SimpleText" }+output = "SimpleDoc"+prompt_template = """+Transform this text:+@input_text++Concept = "This is not a definition of concept, it's just text in the prompt template, so it should not be migrated"+"""diff --git a/tests/e2e/pipelex/tools/__init__.py b/tests/e2e/pipelex/tools/__init__.pynew file mode 100644index 000000000..e69de29bbdiff --git a/tests/e2e/pipelex/tools/migrate/__init__.py b/tests/e2e/pipelex/tools/migrate/__init__.pynew file mode 100644index 000000000..e69de29bbdiff --git a/tests/e2e/pipelex/tools/migrate/test_migrate_v0_1_0_to_v0_2_0_e2e.py b/tests/e2e/pipelex/tools/migrate/test_migrate_v0_1_0_to_v0_2_0_e2e.pynew file mode 100644index 000000000..b68e2ccbd--- /dev/null+++ b/tests/e2e/pipelex/tools/migrate/test_migrate_v0_1_0_to_v0_2_0_e2e.py@@ -0,0 +1,394 @@+"""End-to-end tests for the migration functionality."""++from pathlib import Path+from typing import Any, List, Tuple++import pytest++from pipelex.tools.migrate.migrate_v0_1_0_to_v0_2_0 import migrate_concept_syntax+++class TestMigrationE2E:+ """End-to-end tests for migration functionality."""++ def test_real_world_migration_scenario(self, tmp_path: Path) -> None:+ """Test a realistic migration scenario with multiple files and complex structures."""++ # Create a realistic pipelex libraries structure+ pipelines_dir = tmp_path / "pipelines"+ pipelines_dir.mkdir()++ # Create main domain file with mixed syntax+ main_toml = pipelines_dir / "content_analysis.toml"+ main_toml.write_text("""domain = "content_analysis"+definition = "AI-powered content analysis and processing"+system_prompt = "You are an expert content analyst"++[concept]+Text = "Written content in natural language"+Document = "A structured document with metadata"+Report = "An analytical report with findings"++[concept.Article]+Concept = "A written composition on a specific topic"+refines = "Text"+structure = "ArticleContent"++[concept.NewsArticle]+Concept = "An article reporting current events"+refines = "Article"++[concept.BlogPost]+definition = "A blog post with metadata" # Already migrated+refines = "Article"+structure = "BlogPostContent"++[concept.ContentAnalysis]+Concept = "Analysis of content including sentiment, topics, and key insights"+refines = "Report"+structure = "ContentAnalysisReport"++[pipe.analyze_text]+PipeLLM = "Analyze text content for sentiment and topics"+inputs = { text_input = "Text" }+output = "ContentAnalysis"+prompt_template = "Analyze this text: @text_input"++[pipe.extract_article_info]+PipeLLM = "Extract structured information from articles"+inputs = { article = "Article" }+output = "ArticleMetadata"+system_prompt = "Extract key metadata from articles"+prompt_template = "Extract metadata from: @article"+""")++ # Create subdirectory with more files+ subdomain_dir = pipelines_dir / "specialized"+ subdomain_dir.mkdir()++ # File with only old syntax+ old_syntax_file = subdomain_dir / "legacy_concepts.toml"+ old_syntax_file.write_text("""domain = "legacy"+definition = "Legacy concept definitions"++[concept.LegacyConcept]+Concept = "An old concept that needs migration"+refines = "Text"++[concept.AnotherLegacy]+Concept = "Another legacy concept with complex refines"+refines = ["Text", "Document"]+structure = "LegacyStructure"+""")++ # File with only new syntax (should remain unchanged)+ new_syntax_file = subdomain_dir / "modern_concepts.toml"+ new_syntax_file.write_text("""domain = "modern"+definition = "Modern concept definitions"++[concept.ModernConcept]+definition = "A modern concept with new syntax"+refines = "Text"++[concept.AdvancedConcept]+definition = "An advanced concept with structure"+refines = ["Text", "Document"]+structure = "AdvancedStructure"+""")++ # File with no concepts (should remain unchanged)+ pipes_only_file = pipelines_dir / "pipes_only.toml"+ pipes_only_file.write_text("""domain = "pipes_only"+definition = "File with only pipe definitions"++[pipe.simple_pipe]+PipeLLM = "A simple pipe"+inputs = { input_text = "Text" }+output = "Document"+prompt_template = "Process: @input_text"+""")++ # Run the migration+ result = migrate_concept_syntax(directory=pipelines_dir, create_backups=True, dry_run=False)++ # Verify overall results+ assert result.files_processed == 4 # All TOML files found+ assert result.files_modified == 2 # main_toml and old_syntax_file+ assert result.total_changes == 5 # 3 in main + 2 in legacy+ assert len(result.modified_files) == 2+ assert len(result.errors) == 0++ # Verify main file was migrated correctly+ main_content = main_toml.read_text()+ assert 'definition = "A written composition on a specific topic"' in main_content+ assert 'definition = "An article reporting current events"' in main_content+ assert 'definition = "Analysis of content including sentiment, topics, and key insights"' in main_content+ # BlogPost should remain unchanged (already had definition =)+ assert 'definition = "A blog post with metadata"' in main_content+ # Ensure no old syntax remains+ assert "Concept =" not in main_content++ # Verify legacy file was migrated correctly+ legacy_content = old_syntax_file.read_text()+ assert 'definition = "An old concept that needs migration"' in legacy_content+ assert 'definition = "Another legacy concept with complex refines"' in legacy_content+ assert "Concept =" not in legacy_content++ # Verify modern file was NOT changed+ modern_content = new_syntax_file.read_text()+ assert 'definition = "A modern concept with new syntax"' in modern_content+ assert 'definition = "An advanced concept with structure"' in modern_content+ assert "Concept =" not in modern_content++ # Verify pipes-only file was NOT changed+ pipes_content = pipes_only_file.read_text()+ assert 'domain = "pipes_only"' in pipes_content+ assert "[pipe.simple_pipe]" in pipes_content+ assert "Concept =" not in pipes_content++ # Verify backups were created for modified files only+ assert (pipelines_dir / "content_analysis.toml.backup").exists()+ assert (subdomain_dir / "legacy_concepts.toml.backup").exists()+ assert not (subdomain_dir / "modern_concepts.toml.backup").exists()+ assert not (pipelines_dir / "pipes_only.toml.backup").exists()++ # Verify backup contents are correct (original content)+ main_backup = (pipelines_dir / "content_analysis.toml.backup").read_text()+ assert 'Concept = "A written composition on a specific topic"' in main_backup+ assert 'Concept = "An article reporting current events"' in main_backup++ legacy_backup = (subdomain_dir / "legacy_concepts.toml.backup").read_text()+ assert 'Concept = "An old concept that needs migration"' in legacy_backup+ assert 'Concept = "Another legacy concept with complex refines"' in legacy_backup++ def test_migration_with_complex_formatting(self, tmp_path: Path) -> None:+ """Test migration preserves complex formatting and edge cases."""++ test_file = tmp_path / "complex_formatting.toml"+ original_content = """# Complex formatting test file+domain = "formatting_test"+definition = "Test various formatting scenarios"++# Simple concepts+[concept]+BasicConcept = "A basic concept"++# Complex concepts with various formatting+[concept.StandardConcept]+Concept = "Standard formatting concept"+refines = "Text"++[concept.IndentedConcept]+ Concept = "Concept with indentation"+ refines = "Text"+ structure = "IndentedStructure"++[concept.TabIndentedConcept]+\tConcept = "Concept with tab indentation"+\trefines = "Text"++[concept.MixedIndentedConcept]+ \tConcept = "Concept with mixed indentation"+ \trefines = ["Text", "Document"]++[concept.VariousSpacingConcept]+Concept="No spaces around equals"+refines = "Text"++[concept.ExtraSpacingConcept]+Concept = "Extra spaces around equals"+refines = "Text"++[concept.AlreadyMigratedConcept]+definition = "This should not change"+refines = "Text"++# Comment with Concept = in it (should not change)+[pipe.test_pipe]+PipeLLM = "A pipe that mentions Concept = in description"+inputs = { input_text = "Text" }+output = "Document"+prompt_template = \"\"\"+This template mentions Concept = something in the text.+Process this: @input_text+\"\"\"+"""+ test_file.write_text(original_content)++ # Run migration+ result = migrate_concept_syntax(directory=tmp_path, create_backups=True, dry_run=False)++ # Verify migration results+ assert result.files_processed == 1+ assert result.files_modified == 1+ assert result.total_changes == 6 # All the Concept = lines+ assert len(result.errors) == 0++ migrated_content = test_file.read_text()++ # Verify all Concept = lines were changed to definition =+ expected_changes = [+ 'definition = "Standard formatting concept"',+ ' definition = "Concept with indentation"',+ '\tdefinition = "Concept with tab indentation"',+ ' \tdefinition = "Concept with mixed indentation"',+ 'definition="No spaces around equals"',+ 'definition = "Extra spaces around equals"',+ ]++ for expected in expected_changes:+ assert expected in migrated_content++ # Verify already migrated concept was not changed+ assert 'definition = "This should not change"' in migrated_content++ # Verify comments and pipe content were not changed+ assert 'PipeLLM = "A pipe that mentions Concept = in description"' in migrated_content+ assert "This template mentions Concept = something in the text." in migrated_content++ # Verify no Concept = remains at start of lines+ lines = migrated_content.split("\n")+ for line in lines:+ stripped = line.strip()+ if stripped.startswith("Concept ="):+ # This should not happen - all should be migrated+ pytest.fail(f"Found unmigrated line: {line}")++ def test_dry_run_does_not_modify_files(self, tmp_path: Path) -> None:+ """Test that dry run mode doesn't modify any files."""++ test_file = tmp_path / "dry_run_test.toml"+ original_content = """domain = "dry_run_test"++[concept.TestConcept]+Concept = "This should not be changed in dry run"+refines = "Text"++[concept.AnotherConcept]+Concept = "Neither should this"+refines = "Document"+"""+ test_file.write_text(original_content)++ # Run migration in dry run mode+ result = migrate_concept_syntax(+ directory=tmp_path,+ create_backups=False, # Should be ignored in dry run+ dry_run=True,+ )++ # Verify results show what would be changed+ assert result.files_processed == 1+ assert result.files_modified == 1+ assert result.total_changes == 2+ assert len(result.modified_files) == 1+ assert len(result.errors) == 0++ # Verify file was NOT actually modified+ current_content = test_file.read_text()+ assert current_content == original_content++ # Verify no backup was created+ backup_file = tmp_path / "dry_run_test.toml.backup"+ assert not backup_file.exists()++ def test_error_handling_with_permission_issues(self, tmp_path: Path, mocker: Any) -> None:+ """Test proper error handling when file operations fail."""++ test_file = tmp_path / "permission_test.toml"+ test_file.write_text("""[concept.Test]+Concept = "Test concept"+""")++ # Mock file operations to simulate permission errors+ mock_open = mocker.patch("builtins.open")+ mock_open.side_effect = PermissionError("Permission denied")++ # Run migration+ result = migrate_concept_syntax(directory=tmp_path, create_backups=True, dry_run=False)++ # Should have errors but not crash+ assert result.files_processed == 1+ assert result.files_modified == 0+ assert result.total_changes == 0+ assert len(result.modified_files) == 0+ assert len(result.errors) == 1+ assert "Permission denied" in result.errors[0] or "Error processing" in result.errors[0]++ def test_migration_from_test_data_file(self, tmp_path: Path) -> None:+ """Test migration using the official test data file with tricky edge cases."""+ import shutil++ # Copy the test data file to our temp directory+ test_data_path = Path(__file__).parent.parent.parent.parent.parent / "data" / "test_migrate_v0_1_0_to_v0_2_0.toml"+ test_file = tmp_path / "test_migrate.toml"+ shutil.copy2(test_data_path, test_file)++ # Read original content to verify the challenge case+ original_content = test_file.read_text()++ # Verify the challenge case is present in original file+ assert 'Concept = "This is not a definition of concept' in original_content+ assert "prompt_template" in original_content++ # Run migration+ result = migrate_concept_syntax(directory=tmp_path, create_backups=True, dry_run=False)++ # Verify migration results+ assert result.files_processed == 1+ assert result.files_modified == 1+ assert result.total_changes == 2 # Only the 2 actual concept definitions should change+ assert len(result.modified_files) == 1+ assert len(result.errors) == 0++ # Read migrated content+ migrated_content = test_file.read_text()++ # Verify that actual concept definitions were migrated+ assert 'definition = "A complex concept with old syntax"' in migrated_content+ assert 'definition = "Another concept to test migration"' in migrated_content++ # Verify that already migrated concept remains unchanged+ assert 'definition = "This one is already migrated"' in migrated_content++ # THE CRITICAL CHALLENGE: Verify that the Concept = inside the prompt_template was NOT changed+ assert (+ 'Concept = "This is not a definition of concept, it\'s just text in the prompt template, so it should not be migrated"'+ in migrated_content+ )++ # Verify the challenge case is still there and unchanged+ challenge_line_found = False+ concept_lines_outside_multiline: List[Tuple[int, str]] = []++ lines = migrated_content.split("\n")+ for i, line in enumerate(lines, 1):+ if 'Concept = "This is not a definition of concept' in line:+ challenge_line_found = True+ # This should be exactly the same as in the original file+ assert 'Concept = "This is not a definition of concept, it\'s just text in the prompt template, so it should not be migrated"' in line++ # Check for any Concept = that appears to be a TOML key (not inside strings)+ stripped = line.strip()+ if stripped.startswith("Concept ="):+ # This should only happen inside multiline strings, not as actual TOML keys+ # We need to check if this is inside the prompt_template multiline string+ if i < 26 or i > 31: # Outside the prompt_template multiline string boundaries+ concept_lines_outside_multiline.append((i, line))++ # Should be empty - all Concept = at start of line outside multiline strings should be migrated+ assert len(concept_lines_outside_multiline) == 0, (+ f"Found unmigrated Concept = lines outside multiline strings: {concept_lines_outside_multiline}"+ )++ assert challenge_line_found, "Challenge case (Concept = in prompt template) was not found in migrated file"++ # Verify backup was created and contains original content+ backup_file = tmp_path / "test_migrate.toml.backup"+ assert backup_file.exists()+ backup_content = backup_file.read_text()+ assert backup_content == original_content++ # Verify original had the unmigrated Concept = lines+ assert 'Concept = "A complex concept with old syntax"' in backup_content+ assert 'Concept = "Another concept to test migration"' in backup_contentdiff --git a/tests/unit/pipelex/tools/__init__.py b/tests/unit/pipelex/tools/__init__.pynew file mode 100644index 000000000..e69de29bbdiff --git a/tests/unit/pipelex/tools/migrate/__init__.py b/tests/unit/pipelex/tools/migrate/__init__.pynew file mode 100644index 000000000..e69de29bbdiff --git a/tests/unit/pipelex/tools/migrate/test_migrate_v0_1_0_to_v0_2_0.py b/tests/unit/pipelex/tools/migrate/test_migrate_v0_1_0_to_v0_2_0.pynew file mode 100644index 000000000..ae61ecfa0--- /dev/null+++ b/tests/unit/pipelex/tools/migrate/test_migrate_v0_1_0_to_v0_2_0.py@@ -0,0 +1,442 @@+from pathlib import Path++import pytest++from pipelex.tools.migrate.migrate_v0_1_0_to_v0_2_0 import MigrationResult, TomlMigrator, migrate_concept_syntax+++class TestTomlMigrator:+ """Unit tests for TomlMigrator class."""++ @pytest.fixture+ def migrator(self) -> TomlMigrator:+ """Create a TomlMigrator instance."""+ return TomlMigrator()++ @pytest.fixture+ def sample_old_syntax_content(self) -> str:+ """Sample TOML content with old Concept = syntax."""+ return """domain = "test"+definition = "Test domain"++[concept]+SimpleText = "A simple text concept"+SimpleDoc = "A simple document concept"++[concept.ComplexConcept]+Concept = "A complex concept with old syntax"+refines = "Text"++[concept.AnotherConcept]+Concept = "Another concept to test migration"+structure = "CustomStructure"+refines = ["Text", "Document"]++[concept.AlreadyMigrated]+definition = "This one is already migrated"+refines = "Text"+"""++ @pytest.fixture+ def sample_new_syntax_content(self) -> str:+ """Sample TOML content with new definition = syntax."""+ return """domain = "test"+definition = "Test domain"++[concept]+SimpleText = "A simple text concept"+SimpleDoc = "A simple document concept"++[concept.ComplexConcept]+definition = "A complex concept with old syntax"+refines = "Text"++[concept.AnotherConcept]+definition = "Another concept to test migration"+structure = "CustomStructure"+refines = ["Text", "Document"]++[concept.AlreadyMigrated]+definition = "This one is already migrated"+refines = "Text"+"""++ def test_pattern_matching(self, migrator: TomlMigrator) -> None:+ """Test that the regex pattern correctly identifies Concept = lines."""+ test_content = """[concept.TestConcept]+Concept = "A test concept"+refines = "Text"++[concept.AnotherConcept]+definition = "Already migrated"+Concept = "This should be caught"+"""++ matches = list(migrator.concept_pattern.finditer(test_content))+ assert len(matches) == 2++ # Check first match+ assert matches[0].group(2) == "Concept"+ assert matches[0].group(4) == '"A test concept"'++ # Check second match+ assert matches[1].group(2) == "Concept"+ assert matches[1].group(4) == '"This should be caught"'++ def test_pattern_replacement(self, migrator: TomlMigrator) -> None:+ """Test that the regex pattern correctly replaces Concept = with definition =."""+ test_content = """[concept.TestConcept]+Concept = "A test concept"+refines = "Text"++[concept.IndentedConcept]+ Concept = "Indented concept"+ refines = "Text"+"""++ expected_content = """[concept.TestConcept]+definition = "A test concept"+refines = "Text"++[concept.IndentedConcept]+ definition = "Indented concept"+ refines = "Text"+"""++ result = migrator.migrate_content(test_content)+ assert result == expected_content++ def test_preserves_formatting(self, migrator: TomlMigrator) -> None:+ """Test that migration preserves indentation and spacing."""+ test_cases = [+ # No indentation+ ('Concept = "test"', 'definition = "test"'),+ # With spaces+ (' Concept = "test"', ' definition = "test"'),+ # With tabs+ ('\tConcept = "test"', '\tdefinition = "test"'),+ # Mixed indentation+ (' \tConcept = "test"', ' \tdefinition = "test"'),+ # Various spacing around =+ ('Concept ="test"', 'definition ="test"'),+ ('Concept= "test"', 'definition= "test"'),+ ('Concept = "test"', 'definition = "test"'),+ ]++ for original, expected in test_cases:+ result = migrator.migrate_content(original)+ assert result == expected, f"Failed for: {original}"++ def test_ignores_non_concept_lines(self, migrator: TomlMigrator) -> None:+ """Test that migration doesn't affect non-concept Concept = lines."""+ # These should NOT be changed+ test_content = """# This is a comment about Concept = something+description = "This pipe works with Concept = something"+prompt_template = "Define the Concept = whatever"+"""++ result = migrator.migrate_content(test_content)+ # Should be unchanged since none start with Concept =+ assert result == test_content++ def test_ignores_concept_inside_multiline_strings(self, migrator: TomlMigrator) -> None:+ """Test that migration ignores Concept = inside multiline strings."""+ test_content = '''domain = "test"++[concept.ValidConcept]+Concept = "This should be migrated"+refines = "Text"++[pipe.test_pipe]+prompt_template = """+Transform this:+Concept = "This should NOT be migrated because it's inside a multiline string"+Do something with it.+"""++[concept.AnotherValidConcept]+Concept = "This should also be migrated"+refines = "Document"+'''++ result = migrator.migrate_content(test_content)++ # Valid concept definitions should be migrated+ assert 'definition = "This should be migrated"' in result+ assert 'definition = "This should also be migrated"' in result++ # Concept inside multiline string should NOT be migrated+ assert 'Concept = "This should NOT be migrated because it\'s inside a multiline string"' in result++ # Should not have the old syntax for valid concepts+ assert 'Concept = "This should be migrated"' not in result+ assert 'Concept = "This should also be migrated"' not in result++ def test_multiline_string_detection_with_single_quotes(self, migrator: TomlMigrator) -> None:+ """Test that multiline string detection works with both double and single quotes."""+ test_content = """domain = "test"++[concept.ValidConcept]+Concept = "Should be migrated"++[pipe.test_pipe]+prompt_template = '''+This is a single-quote multiline string:+Concept = "Should NOT be migrated"+'''++[concept.AnotherValidConcept]+Concept = "Should also be migrated"+"""++ result = migrator.migrate_content(test_content)++ # Valid concepts should be migrated+ assert 'definition = "Should be migrated"' in result+ assert 'definition = "Should also be migrated"' in result++ # Concept inside single-quote multiline string should NOT be migrated+ assert 'Concept = "Should NOT be migrated"' in result++ def test_needs_migration_detection(self, migrator: TomlMigrator, sample_old_syntax_content: str, sample_new_syntax_content: str) -> None:+ """Test needs_migration correctly identifies files that need migration."""+ assert migrator.needs_migration(sample_old_syntax_content) is True+ assert migrator.needs_migration(sample_new_syntax_content) is False++ def test_get_migration_preview(self, migrator: TomlMigrator) -> None:+ """Test getting preview of migration changes."""+ test_content = """[concept.TestConcept]+Concept = "A test concept"+refines = "Text"++[concept.AnotherConcept]+ Concept = "Another concept"+ refines = "Text"+"""++ changes = migrator.get_migration_preview(test_content)++ assert len(changes) == 2+ assert changes[0]["line_number"] == 2+ assert changes[0]["old_line"] == 'Concept = "A test concept"'+ assert changes[0]["new_line"] == 'definition = "A test concept"'++ assert changes[1]["line_number"] == 6+ assert changes[1]["old_line"] == 'Concept = "Another concept"'+ assert changes[1]["new_line"] == 'definition = "Another concept"'++ def test_migrate_file_success(+ self, migrator: TomlMigrator, tmp_path: Path, sample_old_syntax_content: str, sample_new_syntax_content: str+ ) -> None:+ """Test successful file migration."""+ test_file = tmp_path / "test.toml"+ test_file.write_text(sample_old_syntax_content)++ changes_count = migrator.migrate_file(test_file, create_backup=True)++ # Check that file was migrated correctly+ migrated_content = test_file.read_text()+ assert migrated_content == sample_new_syntax_content++ # Check that backup was created+ backup_file = tmp_path / "test.toml.backup"+ assert backup_file.exists()+ backup_content = backup_file.read_text()+ assert backup_content == sample_old_syntax_content++ # Check changes count+ assert changes_count == 2 # Two "Concept =" lines in the sample++ def test_migrate_file_no_changes_needed(self, migrator: TomlMigrator, tmp_path: Path, sample_new_syntax_content: str) -> None:+ """Test file migration when no changes are needed."""+ test_file = tmp_path / "test.toml"+ test_file.write_text(sample_new_syntax_content)++ changes_count = migrator.migrate_file(test_file, create_backup=True)++ # File should be unchanged+ content = test_file.read_text()+ assert content == sample_new_syntax_content++ # No backup should be created+ backup_file = tmp_path / "test.toml.backup"+ assert not backup_file.exists()++ # No changes should be reported+ assert changes_count == 0++ def test_migrate_file_without_backup(+ self, migrator: TomlMigrator, tmp_path: Path, sample_old_syntax_content: str, sample_new_syntax_content: str+ ) -> None:+ """Test file migration without creating backup."""+ test_file = tmp_path / "test.toml"+ test_file.write_text(sample_old_syntax_content)++ changes_count = migrator.migrate_file(test_file, create_backup=False)++ # Check that file was migrated correctly+ migrated_content = test_file.read_text()+ assert migrated_content == sample_new_syntax_content++ # Check that no backup was created+ backup_file = tmp_path / "test.toml.backup"+ assert not backup_file.exists()++ # Check changes count+ assert changes_count == 2++ def test_find_toml_files(self, migrator: TomlMigrator, tmp_path: Path) -> None:+ """Test finding TOML files in directory and subdirectories."""+ # Create some TOML files+ (tmp_path / "file1.toml").write_text("content1")+ (tmp_path / "file2.toml").write_text("content2")++ # Create subdirectory with TOML file+ subdir = tmp_path / "subdir"+ subdir.mkdir()+ (subdir / "file3.toml").write_text("content3")++ # Create non-TOML file (should be ignored)+ (tmp_path / "file.txt").write_text("not toml")++ toml_files = migrator.find_toml_files(tmp_path)+ toml_files.sort() # Sort for consistent testing++ expected_files = [+ tmp_path / "file1.toml",+ tmp_path / "file2.toml",+ tmp_path / "subdir" / "file3.toml",+ ]+ expected_files.sort()++ assert toml_files == expected_files++ def test_find_toml_files_nonexistent_directory(self, migrator: TomlMigrator, tmp_path: Path) -> None:+ """Test finding TOML files in nonexistent directory."""+ nonexistent_dir = tmp_path / "nonexistent"++ with pytest.raises(FileNotFoundError):+ migrator.find_toml_files(nonexistent_dir)++ def test_migrate_directory_success(self, migrator: TomlMigrator, tmp_path: Path) -> None:+ """Test successful directory migration."""+ # Create files with different states+ file1 = tmp_path / "file1.toml"+ file1.write_text("""[concept.Test1]+Concept = "Old syntax file 1"+""")++ file2 = tmp_path / "file2.toml"+ file2.write_text("""[concept.Test2]+definition = "Already migrated file 2"+""")++ file3 = tmp_path / "file3.toml"+ file3.write_text("""[concept.Test3]+Concept = "Old syntax file 3"+[concept.Test4]+Concept = "Another old concept"+""")++ result = migrator.migrate_directory(tmp_path, create_backups=True, dry_run=False)++ # Check results+ assert result.files_processed == 3+ assert result.files_modified == 2 # file1 and file3 needed migration+ assert result.total_changes == 3 # 1 change in file1, 2 changes in file3+ assert len(result.modified_files) == 2+ assert len(result.errors) == 0++ # Check that files were migrated correctly+ assert 'definition = "Old syntax file 1"' in file1.read_text()+ assert 'definition = "Already migrated file 2"' in file2.read_text() # Unchanged+ assert 'definition = "Old syntax file 3"' in file3.read_text()+ assert 'definition = "Another old concept"' in file3.read_text()++ # Check backups exist for modified files only+ assert (tmp_path / "file1.toml.backup").exists()+ assert not (tmp_path / "file2.toml.backup").exists() # No changes needed+ assert (tmp_path / "file3.toml.backup").exists()++ def test_migrate_directory_dry_run(self, migrator: TomlMigrator, tmp_path: Path) -> None:+ """Test directory migration in dry-run mode."""+ # Create file with old syntax+ test_file = tmp_path / "test.toml"+ original_content = """[concept.Test]+Concept = "Old syntax"+"""+ test_file.write_text(original_content)++ result = migrator.migrate_directory(tmp_path, create_backups=False, dry_run=True)++ # Check results+ assert result.files_processed == 1+ assert result.files_modified == 1+ assert result.total_changes == 1+ assert len(result.modified_files) == 1+ assert len(result.errors) == 0++ # Check that file was NOT actually modified+ content = test_file.read_text()+ assert content == original_content++ # Check that no backup was created+ backup_file = tmp_path / "test.toml.backup"+ assert not backup_file.exists()++ def test_migrate_directory_no_toml_files(self, migrator: TomlMigrator, tmp_path: Path) -> None:+ """Test directory migration with no TOML files."""+ # Create non-TOML file+ (tmp_path / "file.txt").write_text("not toml")++ result = migrator.migrate_directory(tmp_path, create_backups=True, dry_run=False)++ assert result.files_processed == 0+ assert result.files_modified == 0+ assert result.total_changes == 0+ assert len(result.modified_files) == 0+ assert len(result.errors) == 0+++class TestMigrateConceptSyntax:+ """Test the convenience function."""++ def test_migrate_concept_syntax_function(self, tmp_path: Path) -> None:+ """Test the convenience function works correctly."""+ # Create test file+ test_file = tmp_path / "test.toml"+ test_file.write_text("""[concept.Test]+Concept = "Test concept"+""")++ result = migrate_concept_syntax(tmp_path, create_backups=True, dry_run=False)++ assert result.files_processed == 1+ assert result.files_modified == 1+ assert result.total_changes == 1+ assert len(result.modified_files) == 1+ assert len(result.errors) == 0++ # Check migration worked+ assert 'definition = "Test concept"' in test_file.read_text()+ assert (tmp_path / "test.toml.backup").exists()+++class TestMigrationResult:+ """Test the MigrationResult NamedTuple."""++ def test_migration_result_creation(self) -> None:+ """Test creating MigrationResult."""+ result = MigrationResult(+ files_processed=5,+ files_modified=3,+ total_changes=7,+ modified_files=[Path("file1.toml"), Path("file2.toml")],+ errors=["Error 1", "Error 2"],+ )++ assert result.files_processed == 5+ assert result.files_modified == 3+ assert result.total_changes == 7+ assert len(result.modified_files) == 2+ assert len(result.errors) == 2