Skip to content

Build AI methods with your coding agent, then run them anywhere: as an MCP for chatbots, as a webapp for people, or via API for your software.

cli migration command + unit & e2e tests

Louis Choqueld86ededa1d86
diff --git a/docs/migration.md b/docs/migration.md
new file mode 100644
index 000000000..7fe78f149
--- /dev/null
+++ b/docs/migration.md
@@ -0,0 +1,131 @@
+# Migration Guide
+
+This document provides guidance for migrating between different versions of Pipelex that introduce breaking changes.
+
+## Version 0.6.11: Concept Definition Syntax Change
+
+### Breaking Change
+
+In version 0.6.11, we changed the syntax for defining concepts in TOML library files. The key name for concept definitions has changed from `Concept` to `definition`.
+
+### What Changed
+
+**Old syntax (before v0.6.11):**
+```toml
+[concept.MyConceptName]
+Concept = "Description of the concept"
+refines = "BaseConceptName"
+```
+
+**New syntax (v0.6.11+):**
+```toml
+[concept.MyConceptName]
+definition = "Description of the concept"
+refines = "BaseConceptName"
+```
+
+### Why This Change
+
+This change improves consistency in our TOML schema by:
+- Using more descriptive field names (`definition` instead of `Concept`)
+- Aligning with the internal `ConceptBlueprint` data structure
+- Making the TOML files more self-documenting and intuitive
+
+### Migration Process
+
+#### Automatic Migration
+
+Use the built-in migration command to automatically update your TOML files:
+
+```bash
+# Preview changes without applying them
+pipelex migrate --dry-run
+
+# Apply the migration to all TOML files
+pipelex migrate
+```
+
+The migration command will:
+- Find all `.toml` files in your configured pipelines directory
+- Replace `Concept =` with `definition =` in concept definitions
+- Preserve all other formatting and comments
+- Create backups of modified files (with `.backup` extension)
+
+#### Manual Migration
+
+If you prefer to migrate manually or need to handle special cases:
+
+1. **Locate your TOML files**: Find all pipeline library files (typically in your configured pipelines directory)
+2. **Update concept definitions**: Change `Concept =` to `definition =` in all `[concept.ConceptName]` sections
+3. **Validate syntax**: Run `pipelex validate` to ensure your files are correctly formatted
+
+### Examples
+
+#### Simple Concept Definition
+
+**Before:**
+```toml
+[concept.Article]
+Concept = "A written composition on a specific topic"
+refines = "Text"
+```
+
+**After:**
+```toml
+[concept.Article]
+definition = "A written composition on a specific topic"
+refines = "Text"
+```
+
+#### Complex Concept with Structure
+
+**Before:**
+```toml
+[concept.Photo]
+Concept = "Photo"
+structure = "ImageContent"
+refines = "Image"
+```
+
+**After:**
+```toml
+[concept.Photo]
+definition = "Photo"
+structure = "ImageContent"
+refines = "Image"
+```
+
+### Validation
+
+After migration, validate your pipeline files:
+
+```bash
+# Validate all pipeline files
+pipelex validate
+
+# Run a specific pipeline to test
+pipelex run your-pipeline-name
+```
+
+### Troubleshooting
+
+**Error: "ConceptBlueprint validation failed"**
+- Check that all `Concept =` entries have been changed to `definition =`
+- Ensure no typos were introduced during migration
+- Verify TOML syntax is still valid
+
+**Error: "Failed to load TOML file"**
+- Check file permissions
+- Ensure TOML syntax is valid (no missing quotes, brackets, etc.)
+- Review any custom modifications that might conflict
+
+### Rollback
+
+If you need to rollback the migration:
+1. Stop using the new version of Pipelex
+2. Restore from the `.backup` files created during migration
+3. Or manually change `definition =` back to `Concept =`
+
+---
+
+For additional help or if you encounter issues during migration, please [open an issue](https://github.com/Pipelex/pipelex/issues) on our GitHub repository.
\ No newline at end of file
diff --git a/pipelex/cli/_cli.py b/pipelex/cli/_cli.py
index 3dd6cfe7a..50fdd3383 100644
--- a/pipelex/cli/_cli.py
+++ b/pipelex/cli/_cli.py
@@ -1,6 +1,7 @@
import asyncio
import os
import shutil
+from pathlib import Path
from typing import Annotated, Optional
import typer
@@ -15,6 +16,7 @@ from pipelex.libraries.library_config import LibraryConfig
from pipelex.pipe_works.pipe_dry import dry_run_all_pipes, dry_run_single_pipe
from pipelex.pipelex import Pipelex
from pipelex.tools.config.manager import config_manager
+from pipelex.tools.migrate.migrate_v0_1_0_to_v0_2_0 import TomlMigrator, migrate_concept_syntax
def is_pipelex_libraries_folder(folder_path: str) -> bool:
@@ -228,6 +230,89 @@ def show_pipe(
pretty_print(pipe, title=f"Pipe '{pipe_code}'")
+def migrate(
+ relative_config_folder_path: Annotated[
+ str,
+ typer.Option("--config-folder-path", "-c", help="Relative path to the config folder path"),
+ ] = "./pipelex_libraries",
+ dry_run: Annotated[
+ bool,
+ typer.Option("--dry-run", help="Preview changes without applying them"),
+ ] = False,
+) -> None:
+ """Migrate TOML files to new syntax (Concept = -> definition =)."""
+ config_path = Path(relative_config_folder_path)
+
+ # Check if it's a pipelex libraries folder structure or just a directory with TOML files
+ if is_pipelex_libraries_folder(relative_config_folder_path):
+ pipelines_dir = config_path / "pipelines"
+ else:
+ # Assume it's a directory containing TOML files directly
+ pipelines_dir = config_path
+
+ if not pipelines_dir.exists():
+ typer.echo(f"❌ Directory not found at '{pipelines_dir}'")
+ raise typer.Exit(1)
+
+ try:
+ # Use the migration module
+ result = migrate_concept_syntax(
+ directory=pipelines_dir,
+ create_backups=not dry_run, # Only create backups when not dry-run
+ dry_run=dry_run,
+ )
+
+ # Handle any errors that occurred
+ for error in result.errors:
+ typer.echo(f"❌ {error}")
+
+ if result.errors and not result.files_modified:
+ typer.echo("❌ Migration failed due to errors")
+ raise typer.Exit(1)
+
+ typer.echo(f"Found {result.files_processed} TOML file(s) to check")
+
+ if result.files_modified == 0:
+ typer.echo("✅ All TOML files are already using the new syntax")
+ return
+
+ if dry_run:
+ # Show detailed preview for dry run
+ migrator = TomlMigrator()
+ for file_path in result.modified_files:
+ try:
+ with open(file_path, "r", encoding="utf-8") as f:
+ content = f.read()
+
+ changes = migrator.get_migration_preview(content)
+ typer.echo(f"\n📄 {file_path.relative_to(pipelines_dir)}:")
+ for change in changes:
+ typer.echo(f" Line {change['line_number']}: {change['old_line']} → {change['new_line']}")
+ except Exception as e:
+ typer.echo(f"❌ Error reading {file_path}: {e}")
+
+ typer.echo(f"\n📋 Summary: Found {result.total_changes} change(s) in {result.files_modified} file(s)")
+ typer.echo(" Run without --dry-run to apply these changes")
+ else:
+ # Show migration results
+ for file_path in result.modified_files:
+ backup_path = file_path.with_suffix(".toml.backup")
+ typer.echo(f"✅ Migrated {file_path.relative_to(pipelines_dir)}")
+ typer.echo(f" Backup saved to {backup_path.name}")
+
+ typer.echo(f"\n✅ Migration completed: {result.total_changes} change(s) applied to {result.files_modified} file(s)")
+ typer.echo(" Backup files created with .backup extension")
+ typer.echo(" Run 'pipelex validate' to verify the migration")
+
+ except FileNotFoundError as e:
+ typer.echo(f"❌ {e}")
+ raise typer.Exit(1)
+ except Exception as e:
+ typer.echo(f"❌ Migration failed: {e}")
+ raise typer.Exit(1)
+
+
def main() -> None:
"""Entry point for the pipelex CLI."""
app()
diff --git a/pipelex/libraries/library_manager.py b/pipelex/libraries/library_manager.py
index d525d9ad4..71e507b24 100644
--- a/pipelex/libraries/library_manager.py
+++ b/pipelex/libraries/library_manager.py
@@ -20,7 +20,6 @@ from pipelex.core.pipe_library import PipeLibrary
from pipelex.exceptions import (
ConceptLibraryError,
LibraryError,
- LibraryParsingError,
PipeFactoryError,
PipeLibraryError,
StaticValidationError,
diff --git a/pipelex/libraries/pipeline_blueprint.py b/pipelex/libraries/pipeline_blueprint.py
index b9370587d..5b60fb821 100644
--- a/pipelex/libraries/pipeline_blueprint.py
+++ b/pipelex/libraries/pipeline_blueprint.py
@@ -1,5 +1,5 @@
from pathlib import Path
-from typing import Any, Dict, List, Optional, Union
+from typing import Any, Dict, Optional, Union
from pydantic import BaseModel, Field
diff --git a/pipelex/tools/migrate/__init__.py b/pipelex/tools/migrate/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/pipelex/tools/migrate/migrate_v0_1_0_to_v0_2_0.py b/pipelex/tools/migrate/migrate_v0_1_0_to_v0_2_0.py
new file mode 100644
index 000000000..bdf1eed2c
--- /dev/null
+++ b/pipelex/tools/migrate/migrate_v0_1_0_to_v0_2_0.py
@@ -0,0 +1,192 @@
+"""Migration from version 0.1.0 to 0.2.0 - Concept = to definition = syntax change."""
+
+import re
+import shutil
+from pathlib import Path
+from typing import Any, Dict, List, Match, NamedTuple
+
+
+class MigrationResult(NamedTuple):
+ """Result of migration operation."""
+
+ files_processed: int
+ files_modified: int
+ total_changes: int
+ modified_files: List[Path]
+ errors: List[str]
+
+
+class TomlMigrator:
+ """Handles migration from Concept = to definition = syntax in TOML files."""
+
+ def __init__(self):
+ # Pattern to match "Concept = " within concept sections
+ self.concept_pattern = re.compile(r"^(\s*)(Concept)(\s*=\s*)(.*)$", re.MULTILINE)
+
+ def find_toml_files(self, directory: Path) -> List[Path]:
+ """Find all TOML files in directory and subdirectories."""
+ if not directory.exists():
+ raise FileNotFoundError(f"Directory not found: {directory}")
+
+ return list(directory.glob("**/*.toml"))
+
+ def _is_line_inside_multiline_string(self, content: str, line_start_pos: int) -> bool:
+ """Check if a line position is inside a multiline string (triple quotes)."""
+ # Count triple quotes before this position
+ content_before = content[:line_start_pos]
+
+ # Count both """ and ''' (TOML supports both)
+ triple_double_quotes = content_before.count('"""')
+ triple_single_quotes = content_before.count("'''")
+
+ # If odd number of triple quotes before this line, we're inside a multiline string
+ return (triple_double_quotes % 2 == 1) or (triple_single_quotes % 2 == 1)
+
+ def needs_migration(self, content: str) -> bool:
+ """Check if content contains old Concept = syntax that should be migrated."""
+ matches = list(self.concept_pattern.finditer(content))
+ for match in matches:
+ if not self._is_line_inside_multiline_string(content, match.start()):
+ return True
+ return False
+
+ def get_migration_preview(self, content: str) -> List[Dict[str, Any]]:
+ """Get preview of changes that would be made."""
+ changes: List[Dict[str, Any]] = []
+ matches = list(self.concept_pattern.finditer(content))
+
+ for match in matches:
+ # Skip if this match is inside a multiline string
+ if self._is_line_inside_multiline_string(content, match.start()):
+ continue
+
+ line_num = content[: match.start()].count("\n") + 1
+ old_line = match.group(0)
+ new_line = self.concept_pattern.sub(r"\1definition\3\4", old_line)
+
+ changes.append({"line_number": line_num, "old_line": old_line.strip(), "new_line": new_line.strip()})
+
+ return changes
+
+ def migrate_content(self, content: str) -> str:
+ """Migrate content from old to new syntax."""
+
+ def replacement_function(match: Match[str]) -> str:
+ # Check if this match is inside a multiline string
+ if self._is_line_inside_multiline_string(content, match.start()):
+ # Return the original text unchanged
+ return match.group(0)
+ else:
+ # Apply the normal replacement
+ return f"{match.group(1)}definition{match.group(3)}{match.group(4)}"
+
+ return self.concept_pattern.sub(replacement_function, content)
+
+ def migrate_file(self, file_path: Path, create_backup: bool = True) -> int:
+ """
+ Migrate a single file from old to new syntax.
+
+ Returns:
+ Number of changes made in the file
+ """
+ try:
+ with open(file_path, "r", encoding="utf-8") as f:
+ original_content = f.read()
+ except Exception as e:
+ raise IOError(f"Failed to read file {file_path}: {e}")
+
+ if not self.needs_migration(original_content):
+ return 0
+
+ migrated_content = self.migrate_content(original_content)
+
+ # Count only the changes that are NOT inside multiline strings
+ matches = list(self.concept_pattern.finditer(original_content))
+ changes_count = 0
+ for match in matches:
+ if not self._is_line_inside_multiline_string(original_content, match.start()):
+ changes_count += 1
+
+ if create_backup:
+ backup_path = file_path.with_suffix(".toml.backup")
+ try:
+ shutil.copy2(file_path, backup_path)
+ except Exception as e:
+ raise IOError(f"Failed to create backup for {file_path}: {e}")
+
+ try:
+ with open(file_path, "w", encoding="utf-8") as f:
+ f.write(migrated_content)
+ except Exception as e:
+ raise IOError(f"Failed to write migrated content to {file_path}: {e}")
+
+ return changes_count
+
+ def migrate_directory(self, directory: Path, create_backups: bool = True, dry_run: bool = False) -> MigrationResult:
+ """
+ Migrate all TOML files in a directory.
+
+ Args:
+ directory: Directory containing TOML files
+ create_backups: Whether to create backup files
+ dry_run: If True, only preview changes without applying them
+
+ Returns:
+ MigrationResult with statistics and details
+ """
+ toml_files = self.find_toml_files(directory)
+ files_processed = len(toml_files)
+ files_modified = 0
+ total_changes = 0
+ modified_files: List[Path] = []
+ errors: List[str] = []
+
+ for toml_file in toml_files:
+ try:
+ with open(toml_file, "r", encoding="utf-8") as f:
+ content = f.read()
+
+ if not self.needs_migration(content):
+ continue
+
+ # Count only changes that are NOT inside multiline strings
+ matches = list(self.concept_pattern.finditer(content))
+ changes_count = 0
+ for match in matches:
+ if not self._is_line_inside_multiline_string(content, match.start()):
+ changes_count += 1
+
+ if dry_run:
+ # Just count the changes for dry run
+ files_modified += 1
+ total_changes += changes_count
+ modified_files.append(toml_file)
+ else:
+ # Actually perform the migration
+ actual_changes = self.migrate_file(toml_file, create_backup=create_backups)
+ files_modified += 1
+ total_changes += actual_changes
+ modified_files.append(toml_file)
+
+ except Exception as e:
+ errors.append(f"Error processing {toml_file}: {e}")
+
+ return MigrationResult(
+ files_processed=files_processed, files_modified=files_modified, total_changes=total_changes, modified_files=modified_files, errors=errors
+ )
+
+
+def migrate_concept_syntax(directory: Path, create_backups: bool = True, dry_run: bool = False) -> MigrationResult:
+ """
+ Convenience function to migrate TOML files from Concept = to definition = syntax.
+
+ Args:
+ directory: Directory containing TOML files to migrate
+ create_backups: Whether to create backup files before migration
+ dry_run: If True, only preview changes without applying them
+
+ Returns:
+ MigrationResult with migration statistics
+ """
+ migrator = TomlMigrator()
+ return migrator.migrate_directory(directory, create_backups, dry_run)
diff --git a/tests/data/test_migrate_v0_1_0_to_v0_2_0.toml b/tests/data/test_migrate_v0_1_0_to_v0_2_0.toml
new file mode 100644
index 000000000..1d6f5b417
--- /dev/null
+++ b/tests/data/test_migrate_v0_1_0_to_v0_2_0.toml
@@ -0,0 +1,31 @@
+domain = "test"
+definition = "Test domain"
+
+[concept]
+SimpleText = "A simple text concept"
+SimpleDoc = "A simple document concept"
+
+# Complex concepts with old syntax
+[concept.ComplexConcept]
+Concept = "A complex concept with old syntax"
+refines = "Text"
+
+[concept.AnotherConcept]
+Concept = "Another concept to test migration"
+structure = "CustomStructure"
+refines = ["Text", "Document"]
+
+[concept.AlreadyMigrated]
+definition = "This one is already migrated"
+refines = "Text"
+
+[pipe.test_pipe]
+PipeLLM = "A test pipe that generates simple text"
+inputs = { input_text = "SimpleText" }
+output = "SimpleDoc"
+prompt_template = """
+Transform this text:
+@input_text
+
+Concept = "This is not a definition of concept, it's just text in the prompt template, so it should not be migrated"
+"""
diff --git a/tests/e2e/pipelex/tools/__init__.py b/tests/e2e/pipelex/tools/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/tests/e2e/pipelex/tools/migrate/__init__.py b/tests/e2e/pipelex/tools/migrate/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/tests/e2e/pipelex/tools/migrate/test_migrate_v0_1_0_to_v0_2_0_e2e.py b/tests/e2e/pipelex/tools/migrate/test_migrate_v0_1_0_to_v0_2_0_e2e.py
new file mode 100644
index 000000000..b68e2ccbd
--- /dev/null
+++ b/tests/e2e/pipelex/tools/migrate/test_migrate_v0_1_0_to_v0_2_0_e2e.py
@@ -0,0 +1,394 @@
+"""End-to-end tests for the migration functionality."""
+
+from pathlib import Path
+from typing import Any, List, Tuple
+
+import pytest
+
+from pipelex.tools.migrate.migrate_v0_1_0_to_v0_2_0 import migrate_concept_syntax
+
+
+class TestMigrationE2E:
+ """End-to-end tests for migration functionality."""
+
+ def test_real_world_migration_scenario(self, tmp_path: Path) -> None:
+ """Test a realistic migration scenario with multiple files and complex structures."""
+
+ # Create a realistic pipelex libraries structure
+ pipelines_dir = tmp_path / "pipelines"
+ pipelines_dir.mkdir()
+
+ # Create main domain file with mixed syntax
+ main_toml = pipelines_dir / "content_analysis.toml"
+ main_toml.write_text("""domain = "content_analysis"
+definition = "AI-powered content analysis and processing"
+system_prompt = "You are an expert content analyst"
+
+[concept]
+Text = "Written content in natural language"
+Document = "A structured document with metadata"
+Report = "An analytical report with findings"
+
+[concept.Article]
+Concept = "A written composition on a specific topic"
+refines = "Text"
+structure = "ArticleContent"
+
+[concept.NewsArticle]
+Concept = "An article reporting current events"
+refines = "Article"
+
+[concept.BlogPost]
+definition = "A blog post with metadata" # Already migrated
+refines = "Article"
+structure = "BlogPostContent"
+
+[concept.ContentAnalysis]
+Concept = "Analysis of content including sentiment, topics, and key insights"
+refines = "Report"
+structure = "ContentAnalysisReport"
+
+[pipe.analyze_text]
+PipeLLM = "Analyze text content for sentiment and topics"
+inputs = { text_input = "Text" }
+output = "ContentAnalysis"
+prompt_template = "Analyze this text: @text_input"
+
+[pipe.extract_article_info]
+PipeLLM = "Extract structured information from articles"
+inputs = { article = "Article" }
+output = "ArticleMetadata"
+system_prompt = "Extract key metadata from articles"
+prompt_template = "Extract metadata from: @article"
+""")
+
+ # Create subdirectory with more files
+ subdomain_dir = pipelines_dir / "specialized"
+ subdomain_dir.mkdir()
+
+ # File with only old syntax
+ old_syntax_file = subdomain_dir / "legacy_concepts.toml"
+ old_syntax_file.write_text("""domain = "legacy"
+definition = "Legacy concept definitions"
+
+[concept.LegacyConcept]
+Concept = "An old concept that needs migration"
+refines = "Text"
+
+[concept.AnotherLegacy]
+Concept = "Another legacy concept with complex refines"
+refines = ["Text", "Document"]
+structure = "LegacyStructure"
+""")
+
+ # File with only new syntax (should remain unchanged)
+ new_syntax_file = subdomain_dir / "modern_concepts.toml"
+ new_syntax_file.write_text("""domain = "modern"
+definition = "Modern concept definitions"
+
+[concept.ModernConcept]
+definition = "A modern concept with new syntax"
+refines = "Text"
+
+[concept.AdvancedConcept]
+definition = "An advanced concept with structure"
+refines = ["Text", "Document"]
+structure = "AdvancedStructure"
+""")
+
+ # File with no concepts (should remain unchanged)
+ pipes_only_file = pipelines_dir / "pipes_only.toml"
+ pipes_only_file.write_text("""domain = "pipes_only"
+definition = "File with only pipe definitions"
+
+[pipe.simple_pipe]
+PipeLLM = "A simple pipe"
+inputs = { input_text = "Text" }
+output = "Document"
+prompt_template = "Process: @input_text"
+""")
+
+ # Run the migration
+ result = migrate_concept_syntax(directory=pipelines_dir, create_backups=True, dry_run=False)
+
+ # Verify overall results
+ assert result.files_processed == 4 # All TOML files found
+ assert result.files_modified == 2 # main_toml and old_syntax_file
+ assert result.total_changes == 5 # 3 in main + 2 in legacy
+ assert len(result.modified_files) == 2
+ assert len(result.errors) == 0
+
+ # Verify main file was migrated correctly
+ main_content = main_toml.read_text()
+ assert 'definition = "A written composition on a specific topic"' in main_content
+ assert 'definition = "An article reporting current events"' in main_content
+ assert 'definition = "Analysis of content including sentiment, topics, and key insights"' in main_content
+ # BlogPost should remain unchanged (already had definition =)
+ assert 'definition = "A blog post with metadata"' in main_content
+ # Ensure no old syntax remains
+ assert "Concept =" not in main_content
+
+ # Verify legacy file was migrated correctly
+ legacy_content = old_syntax_file.read_text()
+ assert 'definition = "An old concept that needs migration"' in legacy_content
+ assert 'definition = "Another legacy concept with complex refines"' in legacy_content
+ assert "Concept =" not in legacy_content
+
+ # Verify modern file was NOT changed
+ modern_content = new_syntax_file.read_text()
+ assert 'definition = "A modern concept with new syntax"' in modern_content
+ assert 'definition = "An advanced concept with structure"' in modern_content
+ assert "Concept =" not in modern_content
+
+ # Verify pipes-only file was NOT changed
+ pipes_content = pipes_only_file.read_text()
+ assert 'domain = "pipes_only"' in pipes_content
+ assert "[pipe.simple_pipe]" in pipes_content
+ assert "Concept =" not in pipes_content
+
+ # Verify backups were created for modified files only
+ assert (pipelines_dir / "content_analysis.toml.backup").exists()
+ assert (subdomain_dir / "legacy_concepts.toml.backup").exists()
+ assert not (subdomain_dir / "modern_concepts.toml.backup").exists()
+ assert not (pipelines_dir / "pipes_only.toml.backup").exists()
+
+ # Verify backup contents are correct (original content)
+ main_backup = (pipelines_dir / "content_analysis.toml.backup").read_text()
+ assert 'Concept = "A written composition on a specific topic"' in main_backup
+ assert 'Concept = "An article reporting current events"' in main_backup
+
+ legacy_backup = (subdomain_dir / "legacy_concepts.toml.backup").read_text()
+ assert 'Concept = "An old concept that needs migration"' in legacy_backup
+ assert 'Concept = "Another legacy concept with complex refines"' in legacy_backup
+
+ def test_migration_with_complex_formatting(self, tmp_path: Path) -> None:
+ """Test migration preserves complex formatting and edge cases."""
+
+ test_file = tmp_path / "complex_formatting.toml"
+ original_content = """# Complex formatting test file
+domain = "formatting_test"
+definition = "Test various formatting scenarios"
+
+# Simple concepts
+[concept]
+BasicConcept = "A basic concept"
+
+# Complex concepts with various formatting
+[concept.StandardConcept]
+Concept = "Standard formatting concept"
+refines = "Text"
+
+[concept.IndentedConcept]
+ Concept = "Concept with indentation"
+ refines = "Text"
+ structure = "IndentedStructure"
+
+[concept.TabIndentedConcept]
+\tConcept = "Concept with tab indentation"
+\trefines = "Text"
+
+[concept.MixedIndentedConcept]
+ \tConcept = "Concept with mixed indentation"
+ \trefines = ["Text", "Document"]
+
+[concept.VariousSpacingConcept]
+Concept="No spaces around equals"
+refines = "Text"
+
+[concept.ExtraSpacingConcept]
+Concept = "Extra spaces around equals"
+refines = "Text"
+
+[concept.AlreadyMigratedConcept]
+definition = "This should not change"
+refines = "Text"
+
+# Comment with Concept = in it (should not change)
+[pipe.test_pipe]
+PipeLLM = "A pipe that mentions Concept = in description"
+inputs = { input_text = "Text" }
+output = "Document"
+prompt_template = \"\"\"
+This template mentions Concept = something in the text.
+Process this: @input_text
+\"\"\"
+"""
+ test_file.write_text(original_content)
+
+ # Run migration
+ result = migrate_concept_syntax(directory=tmp_path, create_backups=True, dry_run=False)
+
+ # Verify migration results
+ assert result.files_processed == 1
+ assert result.files_modified == 1
+ assert result.total_changes == 6 # All the Concept = lines
+ assert len(result.errors) == 0
+
+ migrated_content = test_file.read_text()
+
+ # Verify all Concept = lines were changed to definition =
+ expected_changes = [
+ 'definition = "Standard formatting concept"',
+ ' definition = "Concept with indentation"',
+ '\tdefinition = "Concept with tab indentation"',
+ ' \tdefinition = "Concept with mixed indentation"',
+ 'definition="No spaces around equals"',
+ 'definition = "Extra spaces around equals"',
+ ]
+
+ for expected in expected_changes:
+ assert expected in migrated_content
+
+ # Verify already migrated concept was not changed
+ assert 'definition = "This should not change"' in migrated_content
+
+ # Verify comments and pipe content were not changed
+ assert 'PipeLLM = "A pipe that mentions Concept = in description"' in migrated_content
+ assert "This template mentions Concept = something in the text." in migrated_content
+
+ # Verify no Concept = remains at start of lines
+ lines = migrated_content.split("\n")
+ for line in lines:
+ stripped = line.strip()
+ if stripped.startswith("Concept ="):
+ # This should not happen - all should be migrated
+ pytest.fail(f"Found unmigrated line: {line}")
+
+ def test_dry_run_does_not_modify_files(self, tmp_path: Path) -> None:
+ """Test that dry run mode doesn't modify any files."""
+
+ test_file = tmp_path / "dry_run_test.toml"
+ original_content = """domain = "dry_run_test"
+
+[concept.TestConcept]
+Concept = "This should not be changed in dry run"
+refines = "Text"
+
+[concept.AnotherConcept]
+Concept = "Neither should this"
+refines = "Document"
+"""
+ test_file.write_text(original_content)
+
+ # Run migration in dry run mode
+ result = migrate_concept_syntax(
+ directory=tmp_path,
+ create_backups=False, # Should be ignored in dry run
+ dry_run=True,
+ )
+
+ # Verify results show what would be changed
+ assert result.files_processed == 1
+ assert result.files_modified == 1
+ assert result.total_changes == 2
+ assert len(result.modified_files) == 1
+ assert len(result.errors) == 0
+
+ # Verify file was NOT actually modified
+ current_content = test_file.read_text()
+ assert current_content == original_content
+
+ # Verify no backup was created
+ backup_file = tmp_path / "dry_run_test.toml.backup"
+ assert not backup_file.exists()
+
+ def test_error_handling_with_permission_issues(self, tmp_path: Path, mocker: Any) -> None:
+ """Test proper error handling when file operations fail."""
+
+ test_file = tmp_path / "permission_test.toml"
+ test_file.write_text("""[concept.Test]
+Concept = "Test concept"
+""")
+
+ # Mock file operations to simulate permission errors
+ mock_open = mocker.patch("builtins.open")
+ mock_open.side_effect = PermissionError("Permission denied")
+
+ # Run migration
+ result = migrate_concept_syntax(directory=tmp_path, create_backups=True, dry_run=False)
+
+ # Should have errors but not crash
+ assert result.files_processed == 1
+ assert result.files_modified == 0
+ assert result.total_changes == 0
+ assert len(result.modified_files) == 0
+ assert len(result.errors) == 1
+ assert "Permission denied" in result.errors[0] or "Error processing" in result.errors[0]
+
+ def test_migration_from_test_data_file(self, tmp_path: Path) -> None:
+ """Test migration using the official test data file with tricky edge cases."""
+ import shutil
+
+ # Copy the test data file to our temp directory
+ test_data_path = Path(__file__).parent.parent.parent.parent.parent / "data" / "test_migrate_v0_1_0_to_v0_2_0.toml"
+ test_file = tmp_path / "test_migrate.toml"
+ shutil.copy2(test_data_path, test_file)
+
+ # Read original content to verify the challenge case
+ original_content = test_file.read_text()
+
+ # Verify the challenge case is present in original file
+ assert 'Concept = "This is not a definition of concept' in original_content
+ assert "prompt_template" in original_content
+
+ # Run migration
+ result = migrate_concept_syntax(directory=tmp_path, create_backups=True, dry_run=False)
+
+ # Verify migration results
+ assert result.files_processed == 1
+ assert result.files_modified == 1
+ assert result.total_changes == 2 # Only the 2 actual concept definitions should change
+ assert len(result.modified_files) == 1
+ assert len(result.errors) == 0
+
+ # Read migrated content
+ migrated_content = test_file.read_text()
+
+ # Verify that actual concept definitions were migrated
+ assert 'definition = "A complex concept with old syntax"' in migrated_content
+ assert 'definition = "Another concept to test migration"' in migrated_content
+
+ # Verify that already migrated concept remains unchanged
+ assert 'definition = "This one is already migrated"' in migrated_content
+
+ # THE CRITICAL CHALLENGE: Verify that the Concept = inside the prompt_template was NOT changed
+ assert (
+ 'Concept = "This is not a definition of concept, it\'s just text in the prompt template, so it should not be migrated"'
+ in migrated_content
+ )
+
+ # Verify the challenge case is still there and unchanged
+ challenge_line_found = False
+ concept_lines_outside_multiline: List[Tuple[int, str]] = []
+
+ lines = migrated_content.split("\n")
+ for i, line in enumerate(lines, 1):
+ if 'Concept = "This is not a definition of concept' in line:
+ challenge_line_found = True
+ # This should be exactly the same as in the original file
+ assert 'Concept = "This is not a definition of concept, it\'s just text in the prompt template, so it should not be migrated"' in line
+
+ # Check for any Concept = that appears to be a TOML key (not inside strings)
+ stripped = line.strip()
+ if stripped.startswith("Concept ="):
+ # This should only happen inside multiline strings, not as actual TOML keys
+ # We need to check if this is inside the prompt_template multiline string
+ if i < 26 or i > 31: # Outside the prompt_template multiline string boundaries
+ concept_lines_outside_multiline.append((i, line))
+
+ # Should be empty - all Concept = at start of line outside multiline strings should be migrated
+ assert len(concept_lines_outside_multiline) == 0, (
+ f"Found unmigrated Concept = lines outside multiline strings: {concept_lines_outside_multiline}"
+ )
+
+ assert challenge_line_found, "Challenge case (Concept = in prompt template) was not found in migrated file"
+
+ # Verify backup was created and contains original content
+ backup_file = tmp_path / "test_migrate.toml.backup"
+ assert backup_file.exists()
+ backup_content = backup_file.read_text()
+ assert backup_content == original_content
+
+ # Verify original had the unmigrated Concept = lines
+ assert 'Concept = "A complex concept with old syntax"' in backup_content
+ assert 'Concept = "Another concept to test migration"' in backup_content
diff --git a/tests/unit/pipelex/tools/__init__.py b/tests/unit/pipelex/tools/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/tests/unit/pipelex/tools/migrate/__init__.py b/tests/unit/pipelex/tools/migrate/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/tests/unit/pipelex/tools/migrate/test_migrate_v0_1_0_to_v0_2_0.py b/tests/unit/pipelex/tools/migrate/test_migrate_v0_1_0_to_v0_2_0.py
new file mode 100644
index 000000000..ae61ecfa0
--- /dev/null
+++ b/tests/unit/pipelex/tools/migrate/test_migrate_v0_1_0_to_v0_2_0.py
@@ -0,0 +1,442 @@
+from pathlib import Path
+
+import pytest
+
+from pipelex.tools.migrate.migrate_v0_1_0_to_v0_2_0 import MigrationResult, TomlMigrator, migrate_concept_syntax
+
+
+class TestTomlMigrator:
+ """Unit tests for TomlMigrator class."""
+
+ @pytest.fixture
+ def migrator(self) -> TomlMigrator:
+ """Create a TomlMigrator instance."""
+ return TomlMigrator()
+
+ @pytest.fixture
+ def sample_old_syntax_content(self) -> str:
+ """Sample TOML content with old Concept = syntax."""
+ return """domain = "test"
+definition = "Test domain"
+
+[concept]
+SimpleText = "A simple text concept"
+SimpleDoc = "A simple document concept"
+
+[concept.ComplexConcept]
+Concept = "A complex concept with old syntax"
+refines = "Text"
+
+[concept.AnotherConcept]
+Concept = "Another concept to test migration"
+structure = "CustomStructure"
+refines = ["Text", "Document"]
+
+[concept.AlreadyMigrated]
+definition = "This one is already migrated"
+refines = "Text"
+"""
+
+ @pytest.fixture
+ def sample_new_syntax_content(self) -> str:
+ """Sample TOML content with new definition = syntax."""
+ return """domain = "test"
+definition = "Test domain"
+
+[concept]
+SimpleText = "A simple text concept"
+SimpleDoc = "A simple document concept"
+
+[concept.ComplexConcept]
+definition = "A complex concept with old syntax"
+refines = "Text"
+
+[concept.AnotherConcept]
+definition = "Another concept to test migration"
+structure = "CustomStructure"
+refines = ["Text", "Document"]
+
+[concept.AlreadyMigrated]
+definition = "This one is already migrated"
+refines = "Text"
+"""
+
+ def test_pattern_matching(self, migrator: TomlMigrator) -> None:
+ """Test that the regex pattern correctly identifies Concept = lines."""
+ test_content = """[concept.TestConcept]
+Concept = "A test concept"
+refines = "Text"
+
+[concept.AnotherConcept]
+definition = "Already migrated"
+Concept = "This should be caught"
+"""
+
+ matches = list(migrator.concept_pattern.finditer(test_content))
+ assert len(matches) == 2
+
+ # Check first match
+ assert matches[0].group(2) == "Concept"
+ assert matches[0].group(4) == '"A test concept"'
+
+ # Check second match
+ assert matches[1].group(2) == "Concept"
+ assert matches[1].group(4) == '"This should be caught"'
+
+ def test_pattern_replacement(self, migrator: TomlMigrator) -> None:
+ """Test that the regex pattern correctly replaces Concept = with definition =."""
+ test_content = """[concept.TestConcept]
+Concept = "A test concept"
+refines = "Text"
+
+[concept.IndentedConcept]
+ Concept = "Indented concept"
+ refines = "Text"
+"""
+
+ expected_content = """[concept.TestConcept]
+definition = "A test concept"
+refines = "Text"
+
+[concept.IndentedConcept]
+ definition = "Indented concept"
+ refines = "Text"
+"""
+
+ result = migrator.migrate_content(test_content)
+ assert result == expected_content
+
+ def test_preserves_formatting(self, migrator: TomlMigrator) -> None:
+ """Test that migration preserves indentation and spacing."""
+ test_cases = [
+ # No indentation
+ ('Concept = "test"', 'definition = "test"'),
+ # With spaces
+ (' Concept = "test"', ' definition = "test"'),
+ # With tabs
+ ('\tConcept = "test"', '\tdefinition = "test"'),
+ # Mixed indentation
+ (' \tConcept = "test"', ' \tdefinition = "test"'),
+ # Various spacing around =
+ ('Concept ="test"', 'definition ="test"'),
+ ('Concept= "test"', 'definition= "test"'),
+ ('Concept = "test"', 'definition = "test"'),
+ ]
+
+ for original, expected in test_cases:
+ result = migrator.migrate_content(original)
+ assert result == expected, f"Failed for: {original}"
+
+ def test_ignores_non_concept_lines(self, migrator: TomlMigrator) -> None:
+ """Test that migration doesn't affect non-concept Concept = lines."""
+ # These should NOT be changed
+ test_content = """# This is a comment about Concept = something
+description = "This pipe works with Concept = something"
+prompt_template = "Define the Concept = whatever"
+"""
+
+ result = migrator.migrate_content(test_content)
+ # Should be unchanged since none start with Concept =
+ assert result == test_content
+
+ def test_ignores_concept_inside_multiline_strings(self, migrator: TomlMigrator) -> None:
+ """Test that migration ignores Concept = inside multiline strings."""
+ test_content = '''domain = "test"
+
+[concept.ValidConcept]
+Concept = "This should be migrated"
+refines = "Text"
+
+[pipe.test_pipe]
+prompt_template = """
+Transform this:
+Concept = "This should NOT be migrated because it's inside a multiline string"
+Do something with it.
+"""
+
+[concept.AnotherValidConcept]
+Concept = "This should also be migrated"
+refines = "Document"
+'''
+
+ result = migrator.migrate_content(test_content)
+
+ # Valid concept definitions should be migrated
+ assert 'definition = "This should be migrated"' in result
+ assert 'definition = "This should also be migrated"' in result
+
+ # Concept inside multiline string should NOT be migrated
+ assert 'Concept = "This should NOT be migrated because it\'s inside a multiline string"' in result
+
+ # Should not have the old syntax for valid concepts
+ assert 'Concept = "This should be migrated"' not in result
+ assert 'Concept = "This should also be migrated"' not in result
+
+ def test_multiline_string_detection_with_single_quotes(self, migrator: TomlMigrator) -> None:
+ """Test that multiline string detection works with both double and single quotes."""
+ test_content = """domain = "test"
+
+[concept.ValidConcept]
+Concept = "Should be migrated"
+
+[pipe.test_pipe]
+prompt_template = '''
+This is a single-quote multiline string:
+Concept = "Should NOT be migrated"
+'''
+
+[concept.AnotherValidConcept]
+Concept = "Should also be migrated"
+"""
+
+ result = migrator.migrate_content(test_content)
+
+ # Valid concepts should be migrated
+ assert 'definition = "Should be migrated"' in result
+ assert 'definition = "Should also be migrated"' in result
+
+ # Concept inside single-quote multiline string should NOT be migrated
+ assert 'Concept = "Should NOT be migrated"' in result
+
+ def test_needs_migration_detection(self, migrator: TomlMigrator, sample_old_syntax_content: str, sample_new_syntax_content: str) -> None:
+ """Test needs_migration correctly identifies files that need migration."""
+ assert migrator.needs_migration(sample_old_syntax_content) is True
+ assert migrator.needs_migration(sample_new_syntax_content) is False
+
+ def test_get_migration_preview(self, migrator: TomlMigrator) -> None:
+ """Test getting preview of migration changes."""
+ test_content = """[concept.TestConcept]
+Concept = "A test concept"
+refines = "Text"
+
+[concept.AnotherConcept]
+ Concept = "Another concept"
+ refines = "Text"
+"""
+
+ changes = migrator.get_migration_preview(test_content)
+
+ assert len(changes) == 2
+ assert changes[0]["line_number"] == 2
+ assert changes[0]["old_line"] == 'Concept = "A test concept"'
+ assert changes[0]["new_line"] == 'definition = "A test concept"'
+
+ assert changes[1]["line_number"] == 6
+ assert changes[1]["old_line"] == 'Concept = "Another concept"'
+ assert changes[1]["new_line"] == 'definition = "Another concept"'
+
+ def test_migrate_file_success(
+ self, migrator: TomlMigrator, tmp_path: Path, sample_old_syntax_content: str, sample_new_syntax_content: str
+ ) -> None:
+ """Test successful file migration."""
+ test_file = tmp_path / "test.toml"
+ test_file.write_text(sample_old_syntax_content)
+
+ changes_count = migrator.migrate_file(test_file, create_backup=True)
+
+ # Check that file was migrated correctly
+ migrated_content = test_file.read_text()
+ assert migrated_content == sample_new_syntax_content
+
+ # Check that backup was created
+ backup_file = tmp_path / "test.toml.backup"
+ assert backup_file.exists()
+ backup_content = backup_file.read_text()
+ assert backup_content == sample_old_syntax_content
+
+ # Check changes count
+ assert changes_count == 2 # Two "Concept =" lines in the sample
+
+ def test_migrate_file_no_changes_needed(self, migrator: TomlMigrator, tmp_path: Path, sample_new_syntax_content: str) -> None:
+ """Test file migration when no changes are needed."""
+ test_file = tmp_path / "test.toml"
+ test_file.write_text(sample_new_syntax_content)
+
+ changes_count = migrator.migrate_file(test_file, create_backup=True)
+
+ # File should be unchanged
+ content = test_file.read_text()
+ assert content == sample_new_syntax_content
+
+ # No backup should be created
+ backup_file = tmp_path / "test.toml.backup"
+ assert not backup_file.exists()
+
+ # No changes should be reported
+ assert changes_count == 0
+
+ def test_migrate_file_without_backup(
+ self, migrator: TomlMigrator, tmp_path: Path, sample_old_syntax_content: str, sample_new_syntax_content: str
+ ) -> None:
+ """Test file migration without creating backup."""
+ test_file = tmp_path / "test.toml"
+ test_file.write_text(sample_old_syntax_content)
+
+ changes_count = migrator.migrate_file(test_file, create_backup=False)
+
+ # Check that file was migrated correctly
+ migrated_content = test_file.read_text()
+ assert migrated_content == sample_new_syntax_content
+
+ # Check that no backup was created
+ backup_file = tmp_path / "test.toml.backup"
+ assert not backup_file.exists()
+
+ # Check changes count
+ assert changes_count == 2
+
+ def test_find_toml_files(self, migrator: TomlMigrator, tmp_path: Path) -> None:
+ """Test finding TOML files in directory and subdirectories."""
+ # Create some TOML files
+ (tmp_path / "file1.toml").write_text("content1")
+ (tmp_path / "file2.toml").write_text("content2")
+
+ # Create subdirectory with TOML file
+ subdir = tmp_path / "subdir"
+ subdir.mkdir()
+ (subdir / "file3.toml").write_text("content3")
+
+ # Create non-TOML file (should be ignored)
+ (tmp_path / "file.txt").write_text("not toml")
+
+ toml_files = migrator.find_toml_files(tmp_path)
+ toml_files.sort() # Sort for consistent testing
+
+ expected_files = [
+ tmp_path / "file1.toml",
+ tmp_path / "file2.toml",
+ tmp_path / "subdir" / "file3.toml",
+ ]
+ expected_files.sort()
+
+ assert toml_files == expected_files
+
+ def test_find_toml_files_nonexistent_directory(self, migrator: TomlMigrator, tmp_path: Path) -> None:
+ """Test finding TOML files in nonexistent directory."""
+ nonexistent_dir = tmp_path / "nonexistent"
+
+ with pytest.raises(FileNotFoundError):
+ migrator.find_toml_files(nonexistent_dir)
+
+ def test_migrate_directory_success(self, migrator: TomlMigrator, tmp_path: Path) -> None:
+ """Test successful directory migration."""
+ # Create files with different states
+ file1 = tmp_path / "file1.toml"
+ file1.write_text("""[concept.Test1]
+Concept = "Old syntax file 1"
+""")
+
+ file2 = tmp_path / "file2.toml"
+ file2.write_text("""[concept.Test2]
+definition = "Already migrated file 2"
+""")
+
+ file3 = tmp_path / "file3.toml"
+ file3.write_text("""[concept.Test3]
+Concept = "Old syntax file 3"
+[concept.Test4]
+Concept = "Another old concept"
+""")
+
+ result = migrator.migrate_directory(tmp_path, create_backups=True, dry_run=False)
+
+ # Check results
+ assert result.files_processed == 3
+ assert result.files_modified == 2 # file1 and file3 needed migration
+ assert result.total_changes == 3 # 1 change in file1, 2 changes in file3
+ assert len(result.modified_files) == 2
+ assert len(result.errors) == 0
+
+ # Check that files were migrated correctly
+ assert 'definition = "Old syntax file 1"' in file1.read_text()
+ assert 'definition = "Already migrated file 2"' in file2.read_text() # Unchanged
+ assert 'definition = "Old syntax file 3"' in file3.read_text()
+ assert 'definition = "Another old concept"' in file3.read_text()
+
+ # Check backups exist for modified files only
+ assert (tmp_path / "file1.toml.backup").exists()
+ assert not (tmp_path / "file2.toml.backup").exists() # No changes needed
+ assert (tmp_path / "file3.toml.backup").exists()
+
+ def test_migrate_directory_dry_run(self, migrator: TomlMigrator, tmp_path: Path) -> None:
+ """Test directory migration in dry-run mode."""
+ # Create file with old syntax
+ test_file = tmp_path / "test.toml"
+ original_content = """[concept.Test]
+Concept = "Old syntax"
+"""
+ test_file.write_text(original_content)
+
+ result = migrator.migrate_directory(tmp_path, create_backups=False, dry_run=True)
+
+ # Check results
+ assert result.files_processed == 1
+ assert result.files_modified == 1
+ assert result.total_changes == 1
+ assert len(result.modified_files) == 1
+ assert len(result.errors) == 0
+
+ # Check that file was NOT actually modified
+ content = test_file.read_text()
+ assert content == original_content
+
+ # Check that no backup was created
+ backup_file = tmp_path / "test.toml.backup"
+ assert not backup_file.exists()
+
+ def test_migrate_directory_no_toml_files(self, migrator: TomlMigrator, tmp_path: Path) -> None:
+ """Test directory migration with no TOML files."""
+ # Create non-TOML file
+ (tmp_path / "file.txt").write_text("not toml")
+
+ result = migrator.migrate_directory(tmp_path, create_backups=True, dry_run=False)
+
+ assert result.files_processed == 0
+ assert result.files_modified == 0
+ assert result.total_changes == 0
+ assert len(result.modified_files) == 0
+ assert len(result.errors) == 0
+
+
+class TestMigrateConceptSyntax:
+ """Test the convenience function."""
+
+ def test_migrate_concept_syntax_function(self, tmp_path: Path) -> None:
+ """Test the convenience function works correctly."""
+ # Create test file
+ test_file = tmp_path / "test.toml"
+ test_file.write_text("""[concept.Test]
+Concept = "Test concept"
+""")
+
+ result = migrate_concept_syntax(tmp_path, create_backups=True, dry_run=False)
+
+ assert result.files_processed == 1
+ assert result.files_modified == 1
+ assert result.total_changes == 1
+ assert len(result.modified_files) == 1
+ assert len(result.errors) == 0
+
+ # Check migration worked
+ assert 'definition = "Test concept"' in test_file.read_text()
+ assert (tmp_path / "test.toml.backup").exists()
+
+
+class TestMigrationResult:
+ """Test the MigrationResult NamedTuple."""
+
+ def test_migration_result_creation(self) -> None:
+ """Test creating MigrationResult."""
+ result = MigrationResult(
+ files_processed=5,
+ files_modified=3,
+ total_changes=7,
+ modified_files=[Path("file1.toml"), Path("file2.toml")],
+ errors=["Error 1", "Error 2"],
+ )
+
+ assert result.files_processed == 5
+ assert result.files_modified == 3
+ assert result.total_changes == 7
+ assert len(result.modified_files) == 2
+ assert len(result.errors) == 2