Build AI methods with your coding agent, then run them anywhere: as an MCP for chatbots, as a webapp for people, or via API for your software.
Feature/ready to run py file (#366)
Thomas Hebrard7507543acec2
diff --git a/.badges/tests.json b/.badges/tests.jsonindex f748d9646..e921a4aae 100644--- a/.badges/tests.json+++ b/.badges/tests.json@@ -1,7 +1,7 @@{"schemaVersion": 1,"label": "tests",- "message": "1253",+ "message": "1272","color": "blue","cacheSeconds": 300}\ No newline at end of filediff --git a/pipelex/builder/runner_code.py b/pipelex/builder/runner_code.pynew file mode 100644index 000000000..4234f55aa--- /dev/null+++ b/pipelex/builder/runner_code.py@@ -0,0 +1,153 @@+from typing import Any++from pipelex.core.concepts.concept import Concept+from pipelex.core.pipes.pipe_abstract import PipeAbstract+++def value_to_python_code(value: Any, indent_level: int = 0) -> str:+ """Convert a value to Python code representation recursively.++ Args:+ value: The value to convert (can be str, int, dict, list, etc.)+ indent_level: Current indentation level for nested dicts++ Returns:+ String representation of Python code+ """+ indent = " " * indent_level++ if isinstance(value, dict) and "_class" in value:+ # Special handling for Content class instantiation (e.g., PDFContent, ImageContent)+ class_name = value["_class"] # pyright: ignore[reportUnknownVariableType]+ if class_name in {"PDFContent", "ImageContent"}:+ url = value.get("url", "your_url") # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType, reportUnknownVariableType]+ return f'{class_name}(url="{url}")'+ return str(value) # pyright: ignore[reportUnknownArgumentType]+ elif isinstance(value, dict) and "concept_code" in value and "content" in value:+ # Special handling for refined concepts with explicit concept_code+ # Format: {"concept": "domain.ConceptCode", "content": ContentClass(...)}+ concept_code = value["concept_code"] # pyright: ignore[reportUnknownVariableType]+ content = value["content"] # pyright: ignore[reportUnknownVariableType]++ # Generate the content part+ content_code = value_to_python_code(content, indent_level + 1)++ # Return the full format with concept and content+ return f'{{\n{indent} "concept": "{concept_code}",\n{indent} "content": {content_code},\n{indent}}}'+ elif isinstance(value, str):+ # String value - add quotes+ return f'"{value}"'+ elif isinstance(value, bool):+ # Boolean - Python True/False+ return str(value)+ elif isinstance(value, (int, float)):+ # Numeric value+ return str(value)+ elif isinstance(value, list):+ # List - recursively convert items+ if not value:+ return "[]"+ items: list[str] = [value_to_python_code(item, indent_level + 1) for item in value] # pyright: ignore[reportUnknownVariableType]+ return "[" + ", ".join(items) + "]"+ elif isinstance(value, dict):+ # Dict - recursively convert with proper formatting+ if not value:+ return "{}"+ lines_dict: list[str] = []+ for key, val in value.items(): # pyright: ignore[reportUnknownVariableType]+ val_code = value_to_python_code(val, indent_level + 1)+ lines_dict.append(f'{indent} "{key}": {val_code}')+ return "{\n" + ",\n".join(lines_dict) + f"\n{indent}}}"+ else:+ # Fallback - use repr+ return repr(value)+++def generate_compact_memory_entry(var_name: str, concept: Concept) -> str:+ """Generate the compact_memory dictionary entry for a given input."""+ example_value = concept.get_compact_memory_example(var_name)++ # Convert the example value to a Python code string+ value_str = value_to_python_code(example_value, indent_level=3)++ return f' "{var_name}": {value_str},'+++def generate_runner_code(pipe: PipeAbstract) -> str:+ """Generate the complete Python runner code for a pipe."""+ pipe_code = pipe.code+ inputs = pipe.inputs++ # Determine which imports are needed based on input concepts+ needs_pdf = False+ needs_image = False+ for input_req in inputs.root.values():+ concept = input_req.concept+ if concept.structure_class_name == "PDFContent":+ needs_pdf = True+ elif concept.structure_class_name == "ImageContent":+ needs_image = True++ # Build import section+ import_lines = ["import asyncio", ""]++ # Add content class imports if needed+ if needs_pdf:+ import_lines.append("from pipelex.core.stuffs.pdf_content import PDFContent")+ if needs_image:+ import_lines.append("from pipelex.core.stuffs.image_content import ImageContent")++ import_lines.extend(+ [+ "from pipelex.pipelex import Pipelex",+ "from pipelex.pipeline.execute import execute_pipeline",+ ]+ )++ # Build input_memory entries+ if inputs.nb_inputs > 0:+ input_memory_entries: list[str] = []+ for var_name, input_req in inputs.root.items():+ concept = input_req.concept+ entry = generate_compact_memory_entry(var_name, concept)+ input_memory_entries.append(entry)+ input_memory_block = "\n".join(input_memory_entries)+ else:+ input_memory_block = " # No inputs required"++ # Build the main function+ function_lines = [+ "",+ "",+ f"async def run_{pipe_code}():",+ " return await execute_pipeline(",+ f' pipe_code="{pipe_code}",',+ ]++ if inputs.nb_inputs > 0:+ function_lines.extend(+ [+ " input_memory={",+ input_memory_block,+ " },",+ ]+ )++ function_lines.extend(+ [+ " )",+ "",+ "",+ 'if __name__ == "__main__":',+ " # Initialize Pipelex",+ " Pipelex.make()",+ "",+ " # Run the pipeline",+ f" result = asyncio.run(run_{pipe_code}())",+ "",+ ]+ )++ # Combine everything+ code_lines = import_lines + function_lines+ return "\n".join(code_lines)diff --git a/pipelex/cli/commands/build_cmd.py b/pipelex/cli/commands/build_cmd.pyindex 32574e198..c2cc5f5cb 100644--- a/pipelex/cli/commands/build_cmd.py+++ b/pipelex/cli/commands/build_cmd.py@@ -7,7 +7,8 @@ import typerfrom pipelex import pretty_printfrom pipelex.builder.builder import PipelexBundleSpecfrom pipelex.builder.builder_loop import BuilderLoop-from pipelex.hub import get_report_delegate+from pipelex.builder.runner_code import generate_runner_code+from pipelex.hub import get_report_delegate, get_required_pipefrom pipelex.language.plx_factory import PlxFactoryfrom pipelex.pipelex import Pipelexfrom pipelex.pipeline.execute import execute_pipeline@@ -181,3 +182,51 @@ def build_partial_cmd(typer.echo(typer.style(f"\n✅ Pipeline built in {end_time - start_time:.2f} seconds", fg=typer.colors.GREEN))get_report_delegate().generate_report()+++@build_app.command("prepare", help="Prepare a Python runner file for a pipe")+def prepare_runner_cmd(+ pipe_code: Annotated[str, typer.Argument(help="The pipe code to prepare a runner for")],+ output_path: Annotated[+ str | None,+ typer.Option("--output", "-o", help="Path to save the generated Python file"),+ ] = None,+) -> None:+ """Prepare a Python runner file for a pipe.++ The generated file will include:+ - All necessary imports+ - Example input values based on the pipe's input types++ Native concept types (Text, Image, PDF, etc.) will be automatically handled.+ Custom concept types will have their structure recursively generated.+ """+ # Initialize Pipelex+ Pipelex.make()++ # Get the pipe+ try:+ pipe = get_required_pipe(pipe_code=pipe_code)+ except Exception as e:+ typer.echo(typer.style(f"❌ Error: Could not find pipe '{pipe_code}': {e}", fg=typer.colors.RED))+ raise typer.Exit(1) from e++ # Generate the code+ try:+ runner_code = generate_runner_code(pipe)+ except Exception as e:+ typer.echo(typer.style(f"❌ Error generating runner code: {e}", fg=typer.colors.RED))+ raise typer.Exit(1) from e++ # Determine output path+ if not output_path:+ output_path = f"run_{pipe_code}.py"++ # Save the file+ try:+ ensure_directory_for_file_path(file_path=output_path)+ save_text_to_path(text=runner_code, path=output_path)+ typer.echo(typer.style(f"✅ Generated runner file: {output_path}", fg=typer.colors.GREEN))+ except Exception as e:+ typer.echo(typer.style(f"❌ Error saving file: {e}", fg=typer.colors.RED))+ raise typer.Exit(1) from ediff --git a/pipelex/core/concepts/concept.py b/pipelex/core/concepts/concept.pyindex 3b981555b..f3ce38e1d 100644--- a/pipelex/core/concepts/concept.py+++ b/pipelex/core/concepts/concept.py@@ -1,3 +1,6 @@+import inspect+from typing import Any, cast, get_args, get_origin+from kajson.kajson_manager import KajsonManagerfrom pydantic import BaseModel, ConfigDict, field_validator@@ -11,6 +14,7 @@ from pipelex.core.stuffs.stuff_content import StuffContentfrom pipelex.exceptions import PipelexUnexpectedErrorfrom pipelex.tools.misc.string_utils import pascal_case_to_sentencefrom pipelex.tools.typing.class_utils import are_classes_equivalent, has_compatible_field+from pipelex.types import StrEnumclass Concept(BaseModel):@@ -103,3 +107,160 @@ class Concept(BaseModel):msg = f"Concept class '{self.structure_class_name}' is not a subclass of StuffContent"raise PipelexUnexpectedError(msg)return search_for_nested_image_fields(content_class=structure_class)++ def get_compact_memory_example(self, var_name: str) -> dict[str, Any] | str | int:+ """Generate an example value for compact memory format based on this concept.++ Compact memory follows these conventions:+ - For native concepts that can be represented as simple values (Text, Image, PDF): returns a simple string+ - For structured concepts: returns {"concept_code": "...", "content": {...}}++ The content dict is recursively generated based on the StuffContent class structure.+ """+ # Get the structure class+ structure_class = KajsonManager.get_class_registry().get_class(name=self.structure_class_name)++ # If class not found, return placeholder+ if structure_class is None:+ return {+ "concept_code": self.concept_string,+ "content": {}, # Empty dict for unknown structures+ }++ # Verify it's a subclass of StuffContent+ if not issubclass(structure_class, StuffContent):+ return {+ "concept_code": self.concept_string,+ "content": {}, # Empty dict for invalid structures+ }++ # Generate the content based on structure+ content_example = self._generate_content_example_for_class(structure_class, var_name)++ # Check if this is actually a native concept (not just using a native structure class)+ is_native = Concept.is_native_concept(self)++ # For simple native concepts ONLY - return compact format+ if is_native and self.structure_class_name == "TextContent":+ return cast("str", content_example) # Just a string+ elif is_native and self.structure_class_name == "ImageContent":+ # Return dict with class instantiation info+ return {+ "_class": "ImageContent",+ "url": cast("str", content_example),+ }+ elif is_native and self.structure_class_name == "PDFContent":+ # Return dict with class instantiation info+ return {+ "_class": "PDFContent",+ "url": cast("str", content_example),+ }+ elif is_native and self.structure_class_name == "NumberContent":+ return cast("int", content_example) # Just a number++ # For refined or complex concepts, wrap with concept_code+ # For Image/PDF content, wrap in the _class format+ if self.structure_class_name == "ImageContent":+ content_wrapped = {+ "_class": "ImageContent",+ "url": cast("str", content_example),+ }+ elif self.structure_class_name == "PDFContent":+ content_wrapped = {+ "_class": "PDFContent",+ "url": cast("str", content_example),+ }+ else:+ content_wrapped = content_example++ return {+ "concept_code": self.concept_string,+ "content": content_wrapped,+ }++ @classmethod+ def _generate_content_example_for_class(cls, content_class: type[StuffContent], var_name: str) -> Any:+ """Recursively generate example content based on a StuffContent class structure.++ Args:+ content_class: The StuffContent class to generate an example for+ var_name: Variable name for generating contextual example values++ Returns:+ Example content dict or simple value+ """+ class_name = content_class.__name__++ # Handle simple native content types+ if class_name == "TextContent":+ return f"{var_name}_text"+ elif class_name in {"ImageContent", "PDFContent"}:+ return f"{var_name}_url"+ elif class_name == "NumberContent":+ return 0++ # For structured content, inspect fields and recursively generate+ # Note: model_fields includes inherited fields from parent classes+ content_dict: dict[str, Any] = {}+ for field_name, field_info in content_class.model_fields.items():+ field_type = field_info.annotation++ # Handle Optional types (e.g., TextContent | None)+ origin = get_origin(field_type)+ args = get_args(field_type)++ if origin is type(None) or (args and type(None) in args):+ # Optional field - get the non-None type+ actual_type = next((arg for arg in args if arg is not type(None)), field_type) if args else field_type+ else:+ actual_type = field_type++ # Re-check origin after unwrapping Optional+ origin = get_origin(actual_type)+ args = get_args(actual_type)++ # Handle list types+ if origin is list:+ list_item_type = args[0] if args else str+ if hasattr(list_item_type, "__name__"):+ if list_item_type.__name__ == "ImageContent":+ content_dict[field_name] = [f"{field_name}_url_1", f"{field_name}_url_2"]+ elif list_item_type.__name__ == "TextContent":+ content_dict[field_name] = [f"{field_name}_text_1", f"{field_name}_text_2"]+ elif inspect.isclass(list_item_type) and issubclass(list_item_type, StuffContent):+ # List of StuffContent - generate examples+ content_dict[field_name] = [cls._generate_content_example_for_class(list_item_type, f"{field_name}_item")]+ else:+ content_dict[field_name] = [f"{field_name}_item_1"]+ else:+ content_dict[field_name] = []+ # Handle dict types+ elif origin is dict:+ # Simple example with one key-value pair+ content_dict[field_name] = {f"{field_name}_key": f"{field_name}_value"}+ # Handle StrEnum types+ elif inspect.isclass(actual_type) and issubclass(actual_type, StrEnum):+ # Get first enum value+ enum_values = list(actual_type)+ content_dict[field_name] = enum_values[0].value if enum_values else f"{field_name}_enum_value"+ # Handle nested StuffContent+ elif inspect.isclass(actual_type) and issubclass(actual_type, StuffContent):+ content_dict[field_name] = cls._generate_content_example_for_class(actual_type, field_name)+ # Handle basic types+ elif actual_type is str:+ content_dict[field_name] = f"{field_name}_value"+ elif actual_type is int:+ content_dict[field_name] = 0+ elif actual_type is float:+ content_dict[field_name] = 0.0+ elif actual_type is bool:+ content_dict[field_name] = False+ else:+ # For unknown types, try to get a simple repr+ try:+ type_name = getattr(actual_type, "__name__", str(actual_type))+ content_dict[field_name] = f"{field_name}_value # TODO: Fill {type_name}"+ except Exception:+ content_dict[field_name] = f"{field_name}_value"++ return content_dictdiff --git a/pipelex/system/__init__.py b/pipelex/system/__init__.pyindex 8b1378917..e69de29bb 100644--- a/pipelex/system/__init__.py+++ b/pipelex/system/__init__.py@@ -1 +0,0 @@-diff --git a/pipelex/system/registries/__init__py b/pipelex/system/registries/__init__pynew file mode 100644index 000000000..e69de29bbdiff --git a/tests/unit/core/concepts/test_concept_compact_memory.py b/tests/unit/core/concepts/test_concept_compact_memory.pynew file mode 100644index 000000000..c947e7b1b--- /dev/null+++ b/tests/unit/core/concepts/test_concept_compact_memory.py@@ -0,0 +1,127 @@+"""Unit tests for Concept compact memory generation methods."""++from __future__ import annotations++from pipelex.core.concepts.concept_factory import ConceptFactory+from pipelex.core.concepts.concept_native import NativeConceptCode+++class TestConceptCompactMemory:+ """Test Concept methods for generating compact memory examples."""++ def test_get_compact_memory_example_text(self) -> None:+ """Test compact memory example for Text concept."""+ concept = ConceptFactory.make_native_concept(NativeConceptCode.TEXT)+ result = concept.get_compact_memory_example("message")+ assert isinstance(result, str)+ assert result == "message_text"++ def test_get_compact_memory_example_number(self) -> None:+ """Test compact memory example for Number concept."""+ concept = ConceptFactory.make_native_concept(NativeConceptCode.NUMBER)+ result = concept.get_compact_memory_example("count")+ assert isinstance(result, int)+ assert result == 0++ def test_get_compact_memory_example_image(self) -> None:+ """Test compact memory example for Image concept."""+ concept = ConceptFactory.make_native_concept(NativeConceptCode.IMAGE)+ result = concept.get_compact_memory_example("photo")+ assert isinstance(result, dict)+ assert result["_class"] == "ImageContent"+ assert result["url"] == "photo_url"++ def test_get_compact_memory_example_pdf(self) -> None:+ """Test compact memory example for PDF concept."""+ concept = ConceptFactory.make_native_concept(NativeConceptCode.PDF)+ result = concept.get_compact_memory_example("document")+ assert isinstance(result, dict)+ assert result["_class"] == "PDFContent"+ assert result["url"] == "document_url"++ def test_get_compact_memory_example_text_and_images(self) -> None:+ """Test compact memory example for TextAndImages concept."""+ concept = ConceptFactory.make_native_concept(NativeConceptCode.TEXT_AND_IMAGES)++ result = concept.get_compact_memory_example("content")++ # Should return a dict with concept_code and content+ assert isinstance(result, dict)+ assert result["concept_code"] == "native.TextAndImages"+ assert "content" in result+ assert "text" in result["content"]+ assert result["content"]["text"] == "text_text" # Generated from field name "text"+ assert "images" in result["content"]+ assert isinstance(result["content"]["images"], list)++ def test_get_compact_memory_example_page(self) -> None:+ """Test compact memory example for Page concept."""+ concept = ConceptFactory.make_native_concept(NativeConceptCode.PAGE)++ result = concept.get_compact_memory_example("page")++ # Should return a dict with concept_code and content+ assert isinstance(result, dict)+ assert result["concept_code"] == "native.Page"+ assert "content" in result+ assert "text_and_images" in result["content"]+ assert isinstance(result["content"]["text_and_images"], dict)++ def test_get_compact_memory_example_custom_structured(self) -> None:+ """Test compact memory example for a custom structured concept."""+ concept = ConceptFactory.make(+ concept_code="Invoice",+ domain="accounting",+ description="Invoice data",+ structure_class_name="Invoice",+ refines=None,+ )++ result = concept.get_compact_memory_example("invoice")++ # Should return a dict with concept_code and content+ assert isinstance(result, dict)+ assert result["concept_code"] == "accounting.Invoice"+ assert "content" in result++ def test_get_compact_memory_example_for_refined_text_concept(self) -> None:+ """Test compact memory example for a concept that refines Text."""+ # Create a concept that refines Text using direct make() to control domain+ concept = ConceptFactory.make(+ domain="test_domain",+ concept_code="Question",+ description="A question",+ structure_class_name="TextContent",+ refines="native.Text",+ )++ # Test - should return full format with concept_code since it's a refined (non-native) concept+ # Even though it refines Text, it's not the native Text concept itself+ result = concept.get_compact_memory_example("question")+ assert isinstance(result, dict)+ assert "concept_code" in result+ assert result["concept_code"] == "test_domain.Question"+ assert "content" in result+ assert result["content"] == "question_text" # Content is still the simple text string++ def test_get_compact_memory_example_for_refined_image_concept(self) -> None:+ """Test compact memory example for a concept that refines Image (non-native)."""+ # Create a concept that refines Image using direct make() to control domain+ concept = ConceptFactory.make(+ domain="tables",+ concept_code="TableScreenshot",+ description="A screenshot of a table",+ structure_class_name="ImageContent",+ refines="native.Image",+ )++ # Test - should return full format with concept_code since it's a refined (non-native) concept+ result = concept.get_compact_memory_example("table_screenshot")+ assert isinstance(result, dict)+ assert "concept_code" in result+ assert result["concept_code"] == "tables.TableScreenshot"+ assert "content" in result+ # Content should be wrapped in ImageContent format+ assert isinstance(result["content"], dict)+ assert result["content"]["_class"] == "ImageContent"+ assert result["content"]["url"] == "table_screenshot_url"diff --git a/tests/unit/tools/test_runner_generator.py b/tests/unit/tools/test_runner_generator.pynew file mode 100644index 000000000..129e8c863--- /dev/null+++ b/tests/unit/tools/test_runner_generator.py@@ -0,0 +1,111 @@+"""Unit tests for runner_generator module."""++from __future__ import annotations++from pipelex.builder.runner_code import generate_compact_memory_entry, value_to_python_code+from pipelex.core.concepts.concept_factory import ConceptFactory+from pipelex.core.concepts.concept_native import NativeConceptCode+++class TestValueToPythonCode:+ """Test value_to_python_code function."""++ def test_simple_string(self) -> None:+ """Test conversion of a simple string."""+ result = value_to_python_code("hello")+ assert result == '"hello"'++ def test_simple_number(self) -> None:+ """Test conversion of a number."""+ result = value_to_python_code(42)+ assert result == "42"++ def test_simple_boolean(self) -> None:+ """Test conversion of a boolean."""+ result = value_to_python_code(True)+ assert result == "True"++ def test_dict_with_class_image_content(self) -> None:+ """Test conversion of ImageContent dict."""+ value = {"_class": "ImageContent", "url": "test_url"}+ result = value_to_python_code(value)+ assert result == 'ImageContent(url="test_url")'++ def test_dict_with_class_pdf_content(self) -> None:+ """Test conversion of PDFContent dict."""+ value = {"_class": "PDFContent", "url": "test_url"}+ result = value_to_python_code(value)+ assert result == 'PDFContent(url="test_url")'++ def test_dict_with_concept_code_and_content_simple(self) -> None:+ """Test conversion of refined concept with simple content."""+ value = {+ "concept_code": "test_domain.Question",+ "content": "question_text",+ }+ result = value_to_python_code(value, indent_level=3)+ expected = '{\n "concept": "test_domain.Question",\n "content": "question_text",\n }'+ assert result == expected++ def test_dict_with_concept_code_and_content_image(self) -> None:+ """Test conversion of refined Image concept."""+ value = {+ "concept_code": "tables.TableScreenshot",+ "content": {"_class": "ImageContent", "url": "table_screenshot_url"},+ }+ result = value_to_python_code(value, indent_level=3)+ expected = (+ "{\n"+ ' "concept": "tables.TableScreenshot",\n'+ ' "content": ImageContent(url="table_screenshot_url"),\n'+ " }"+ )+ assert result == expected+++class TestGenerateCompactMemoryEntry:+ """Test generate_compact_memory_entry function."""++ def test_generate_entry_for_native_text(self) -> None:+ """Test generating entry for native Text concept."""+ concept = ConceptFactory.make_native_concept(NativeConceptCode.TEXT)+ result = generate_compact_memory_entry("message", concept)+ assert result == ' "message": "message_text",'++ def test_generate_entry_for_native_image(self) -> None:+ """Test generating entry for native Image concept."""+ concept = ConceptFactory.make_native_concept(NativeConceptCode.IMAGE)+ result = generate_compact_memory_entry("photo", concept)+ assert result == ' "photo": ImageContent(url="photo_url"),'++ def test_generate_entry_for_refined_image(self) -> None:+ """Test generating entry for a concept that refines Image."""+ concept = ConceptFactory.make(+ domain="tables",+ concept_code="TableScreenshot",+ description="A screenshot of a table",+ structure_class_name="ImageContent",+ refines="native.Image",+ )+ result = generate_compact_memory_entry("table_screenshot", concept)++ # Should generate the full format with concept and content+ assert '"concept": "tables.TableScreenshot"' in result+ assert 'ImageContent(url="table_screenshot_url")' in result+ assert "table_screenshot" in result++ def test_generate_entry_for_refined_text(self) -> None:+ """Test generating entry for a concept that refines Text."""+ concept = ConceptFactory.make(+ domain="test_domain",+ concept_code="Question",+ description="A question",+ structure_class_name="TextContent",+ refines="native.Text",+ )+ result = generate_compact_memory_entry("question", concept)++ # Should generate the full format with concept and content+ assert '"concept": "test_domain.Question"' in result+ assert '"question_text"' in result+ assert "question" in result