Skip to content

Build AI methods with your coding agent, then run them anywhere: as an MCP for chatbots, as a webapp for people, or via API for your software.

feature/structure-generator (#504)

Thomas Hebrardb0120f933ebe
diff --git a/.gitignore b/.gitignore
index 00a8bc1ba..3584b117f 100644
--- a/.gitignore
+++ b/.gitignore
@@ -7,7 +7,6 @@ __pycache__/
# Distribution / packaging
dist/
-build/
*.egg-info/
# mkdocs
@@ -53,3 +52,4 @@ gcp_credentials.json
/pipelex.toml
settings.local.json
+structures/
\ No newline at end of file
diff --git a/CHANGELOG.md b/CHANGELOG.md
index a693d7b10..f26fb2116 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,5 +1,22 @@
# Changelog
+## Unreleased
+
+### Added
+
+- Can now generate the pydantic structure of a concept with the cli: `pipelex build structures /library_dir/`.
+- The Pipelex CLI `pipelex build runner` that creates the python file to run a pipe now runs the `structures` build command and then imports those structures in the runner file.
+- Added validation for output concepts of `PipeCompose`: must be strictly compatible with the Text concept.
+- Added validation for the Pipelex Bundle on the concept keys: Cannot create a native concept.
+
+### Changed
+
+- The Pipelex cli for the `runner`, `structures`, and `inputs` commands now accept a `--output-dir` option to specify the output directory, by default it in the `target_dir`
+
+### Refactored
+
+- Refactored the `ConceptFactory.make_from_blueprint` method: Now it correctly handles native concepts.
+
## [v0.17.3] - 2025-12-01
### Fixed
diff --git a/pipelex/builder/runner_code.py b/pipelex/builder/runner_code.py
index 14a1bdab4..1d0b010af 100644
--- a/pipelex/builder/runner_code.py
+++ b/pipelex/builder/runner_code.py
@@ -1,351 +1,257 @@
"""Runner code generation utilities for Pipelex pipelines.
-This module provides functions to generate executable Python code and JSON input examples
-for Pipelex pipelines. It supports two output formats:
-
-1. **Python Code Format**: Generates Python code with class instantiations
- Example: PDFContent(url="document.pdf")
-
-2. **JSON Format**: Generates pure JSON with concept metadata
- Example: {"concept": "native.PDF", "content": {"url": "document.pdf"}}
+This module provides functions to generate executable Python code for Pipelex pipelines.
Main Functions:
--------------
- generate_runner_code(pipe): Generate complete executable Python script
-- generate_input_memory_block_python(inputs): Generate just the Python inputs dict
-- generate_input_memory_json(inputs): Generate JSON inputs as dict
-- generate_input_memory_json_string(inputs): Generate JSON inputs as formatted string
-
-Example Usage:
--------------
-```python
-from pipelex.hub import get_required_pipe
-from pipelex.builder.runner_code import (
- generate_runner_code,
- generate_input_memory_block_python,
- generate_input_memory_json,
- generate_input_memory_json_string,
-)
-
-# Get a pipe
-pipe = get_required_pipe("my_pipe")
-
-# Generate complete Python runner code
-python_code = generate_runner_code(pipe)
-print(python_code)
-
-# Generate just the Python input memory block
-python_inputs = generate_input_memory_block_python(pipe.inputs)
-print(f"inputs = {python_inputs}")
-
-# Generate JSON input memory (as dict)
-json_inputs = generate_input_memory_json(pipe.inputs)
-print(json_inputs)
-
-# Generate JSON input memory (as formatted string)
-json_string = generate_input_memory_json_string(pipe.inputs, indent=2)
-print(json_string)
-```
-
-Python Output Example:
----------------------
-```python
-import asyncio
+"""
-from pipelex.core.stuffs.image_content import ImageContent
-from pipelex.pipelex import Pipelex
-from pipelex.pipeline.execute import execute_pipeline
+from dataclasses import dataclass
+from typing import Any
+from pipelex.core.concepts.concept import Concept
+from pipelex.core.concepts.concept_representation_generator import (
+ ConceptRepresentationFormat,
+ ConceptRepresentationGenerator,
+)
+from pipelex.core.concepts.native.concept_native import NativeConceptCode
+from pipelex.core.domains.domain import SpecialDomain
+from pipelex.core.pipes.inputs.input_requirements import InputRequirements
+from pipelex.core.pipes.pipe_abstract import PipeAbstract
+from pipelex.core.pipes.variable_multiplicity import VariableMultiplicity
-async def run_my_pipe():
- return await execute_pipeline(
- pipe_code="my_pipe",
- inputs={
- "image": ImageContent(url="image_url"),
- },
- )
+@dataclass
+class CustomClassInfo:
+ """Information about a custom structure class for import generation."""
-if __name__ == "__main__":
- Pipelex.make()
- result = asyncio.run(run_my_pipe())
-```
-
-JSON Output Example:
--------------------
-```json
-{
- "image": {
- "concept": "native.Image",
- "content": {
- "url": "image_url"
- }
- }
-}
-```
-"""
+ class_name: str
+ domain: str
+ concept_code: str
-import json
-from typing import Any, cast
+ @property
+ def module_name(self) -> str:
+ """Get the module name (filename without .py) for this class."""
+ return f"{self.domain}_{self.concept_code}"
-from pipelex.core.concepts.concept import Concept
-from pipelex.core.pipes.inputs.input_requirements import InputRequirements
-from pipelex.core.pipes.pipe_abstract import PipeAbstract
+ @property
+ def import_statement(self) -> str:
+ """Get the import statement for this class (uses sys.path from script directory)."""
+ return f"from structures.{self.module_name} import {self.class_name}"
-def value_to_json(value: Any) -> Any:
- """Convert a value to pure JSON representation (dict/list/primitives only).
+def _is_multiple(multiplicity: VariableMultiplicity | None) -> bool:
+ """Check if the multiplicity indicates multiple items.
Args:
- value: The value to convert
+ multiplicity: The multiplicity value (None, bool, or int)
Returns:
- Pure JSON-serializable representation (no Python classes)
+ True if multiple items are expected
"""
- if isinstance(value, dict) and "_class" in value:
- # Convert Content class instantiation to pure JSON
- # E.g., {"_class": "PDFContent", "url": "..."} -> {"url": "..."}
- class_name = value["_class"] # pyright: ignore[reportUnknownVariableType]
- if class_name in {"PDFContent", "ImageContent"}:
- url = value.get("url", "your_url") # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType, reportUnknownVariableType]
- return {"url": cast("str", url)}
- # For other classes, remove the _class key and return the rest
- return {k: value_to_json(v) for k, v in value.items() if k != "_class"} # pyright: ignore[reportUnknownVariableType, reportUnknownArgumentType]
- elif isinstance(value, dict) and "concept_code" in value and "content" in value:
- # Convert concept_code format to concept format with pure JSON content
- concept_code = value["concept_code"] # pyright: ignore[reportUnknownVariableType]
- content = value["content"] # pyright: ignore[reportUnknownVariableType]
-
- # Recursively convert content to JSON
- content_json = value_to_json(content)
-
- return {"concept": cast("str", concept_code), "content": content_json}
- elif isinstance(value, str | bool | int | float):
- return value
- elif isinstance(value, list):
- return [value_to_json(item) for item in value] # pyright: ignore[reportUnknownVariableType]
- elif isinstance(value, dict):
- return {k: value_to_json(v) for k, v in value.items()} # pyright: ignore[reportUnknownVariableType, reportUnknownArgumentType]
- else:
- # For other types, try to convert to string
- return str(value)
+ if multiplicity is None:
+ return False
+ if isinstance(multiplicity, bool):
+ return multiplicity
+ # int means specific count, which is multiple
+ return multiplicity > 1
-def value_to_python_code(value: Any, indent_level: int = 0) -> str:
- """Convert a value to Python code representation recursively.
+def _format_representation_as_python(representation: dict[str, Any], is_multiple: bool = False) -> str:
+ """Format a representation dict as Python code.
Args:
- value: The value to convert (can be str, int, dict, list, etc.)
- indent_level: Current indentation level for nested dicts
+ representation: Dict with concept and content
+ is_multiple: If True, wrap content in a list
Returns:
- String representation of Python code
+ Python code string
"""
- indent = " " * indent_level
-
- if isinstance(value, dict) and "_class" in value:
- # Special handling for Content class instantiation (e.g., PDFContent, ImageContent)
- class_name = value["_class"] # pyright: ignore[reportUnknownVariableType]
- if class_name in {"PDFContent", "ImageContent"}:
- url = value.get("url", "your_url") # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType, reportUnknownVariableType]
- return f'{class_name}(url="{url}")'
- return str(value) # pyright: ignore[reportUnknownArgumentType]
- elif isinstance(value, dict) and "concept_code" in value and "content" in value:
- # Special handling for refined concepts with explicit concept_code
- # Format: {"concept": "domain.ConceptCode", "content": ContentClass(...)}
- concept_code = value["concept_code"] # pyright: ignore[reportUnknownVariableType]
- content = value["content"] # pyright: ignore[reportUnknownVariableType]
-
- # Generate the content part
- content_code = value_to_python_code(content, indent_level + 1)
-
- # Return the full format with concept and content
- return f'{{\n{indent} "concept": "{concept_code}",\n{indent} "content": {content_code},\n{indent}}}'
- elif isinstance(value, str):
- # String value - add quotes
- return f'"{value}"'
- elif isinstance(value, bool):
- # Boolean - Python True/False
- return str(value)
- elif isinstance(value, (int, float)):
- # Numeric value
- return str(value)
- elif isinstance(value, list):
- # List - recursively convert items
- if not value:
- return "[]"
- items: list[str] = [value_to_python_code(item, indent_level + 1) for item in value] # pyright: ignore[reportUnknownVariableType]
- return "[" + ", ".join(items) + "]"
- elif isinstance(value, dict):
- # Dict - recursively convert with proper formatting
- if not value:
- return "{}"
- lines_dict: list[str] = []
- for key, val in value.items(): # pyright: ignore[reportUnknownVariableType]
- val_code = value_to_python_code(val, indent_level + 1)
- lines_dict.append(f'{indent} "{key}": {val_code}')
- return "{\n" + ",\n".join(lines_dict) + f"\n{indent}}}"
- else:
- # Fallback - use repr
- return repr(value)
+ concept = representation["concept"]
+ content = representation["content"]
+ # If multiple, wrap content in a list
+ if is_multiple:
+ content = f"[{content}]"
-def generate_compact_memory_entry(var_name: str, concept: Concept) -> str:
- """Generate the pipeline_inputs dictionary entry for a given input (Python code version)."""
- example_value = concept.get_compact_memory_example(var_name)
+ # Content is already a Python instantiation string from the generator
+ return f'{{\n "concept": "{concept}",\n "content": {content},\n }}'
- # Convert the example value to a Python code string
- value_str = value_to_python_code(example_value, indent_level=3)
- return f' "{var_name}": {value_str},'
+def _get_structure_class_import(class_name: str) -> str | None:
+ """Get the import statement for a native structure class.
+ The import statement is generated dynamically from the class's actual module location,
+ so it will automatically reflect any file relocations.
-def generate_json_memory_entry(var_name: str, concept: Concept) -> dict[str, Any]:
- """Generate the pipeline_inputs dictionary entry for a given input (pure JSON version).
+ Args:
+ class_name: The name of the structure class
Returns:
- A tuple of (var_name, json_value) where json_value is pure JSON (dict/list/primitives)
+ Import statement string, or None if not a native structure class
"""
- example_value = concept.get_compact_memory_example(var_name)
-
- # Convert to JSON - always wrap with concept if not already present
- json_value = value_to_json(example_value)
-
- # For simple values (strings for Text, numbers for Number), wrap with concept
- if isinstance(json_value, str):
- # This is a native Text concept - wrap it
- return {
- "concept": concept.concept_string,
- "content": json_value,
- }
- elif isinstance(json_value, (int, float)) and not isinstance(json_value, bool):
- # This is a native Number concept - wrap it
- return {
- "concept": concept.concept_string,
- "content": json_value,
- }
- elif isinstance(json_value, dict) and "concept" not in json_value:
- # Not yet wrapped with concept - wrap it now
- return {
- "concept": concept.concept_string,
- "content": json_value,
- }
- else:
- # Already has concept key or is properly formatted
- # At this point, json_value should be a dict with "concept" key
- return cast("dict[str, Any]", json_value)
+ cls = NativeConceptCode.get_native_structure_class(class_name)
+ if cls is None:
+ return None
+ return f"from {cls.__module__} import {cls.__name__}"
-def generate_input_memory_block_python(inputs: InputRequirements) -> str:
- """Generate just the Python input memory block (without surrounding code).
+def _collect_concept_info(concept: Concept) -> CustomClassInfo | None:
+ """Collect information about a concept for import generation.
Args:
- inputs: The pipe inputs
+ concept: The concept to collect info for
Returns:
- Python code string representing the inputs dictionary
+ CustomClassInfo if it's a custom concept, None if native
"""
- if inputs.nb_inputs == 0:
- return "{}"
-
- input_memory_entries: list[str] = []
- for var_name, input_req in inputs.root.items():
- concept = input_req.concept
- entry = generate_compact_memory_entry(var_name, concept)
- input_memory_entries.append(entry)
-
- # Join entries and format as a dictionary
- entries_str = "\n".join(input_memory_entries)
- return f"{{\n{entries_str}\n }}"
+ if SpecialDomain.is_native(concept.domain):
+ return None
+
+ # For custom concepts, use the concept code as the class name
+ # The structure class name should match the concept code
+ return CustomClassInfo(
+ class_name=concept.structure_class_name,
+ domain=concept.domain,
+ concept_code=concept.code,
+ )
-def generate_input_memory_json(inputs: InputRequirements) -> dict[str, Any]:
- """Generate the input memory in pure JSON format.
+def _collect_imports_for_inputs(inputs: InputRequirements) -> tuple[set[str], dict[str, CustomClassInfo]]:
+ """Collect all imports needed for a pipe's inputs.
Args:
inputs: The pipe inputs
Returns:
- Dictionary with pure JSON values (no Python classes)
+ Tuple of (set of native class names, dict mapping class name to CustomClassInfo)
"""
- if inputs.nb_inputs == 0:
- return {}
+ native_classes: set[str] = set()
+ custom_classes: dict[str, CustomClassInfo] = {}
- json_inputs: dict[str, Any] = {}
- for var_name, input_req in inputs.root.items():
+ for input_req in inputs.root.values():
concept = input_req.concept
- json_value = generate_json_memory_entry(var_name, concept)
- json_inputs[var_name] = json_value
+ structure_class = concept.get_structure_class()
- return json_inputs
+ # Get imports from the representation generator
+ generator = ConceptRepresentationGenerator(ConceptRepresentationFormat.PYTHON)
+ generator.generate_representation(concept.concept_string, structure_class)
+ for class_name in generator.imports_needed:
+ if NativeConceptCode.is_native_structure_class(class_name):
+ native_classes.add(class_name)
+ else:
+ # For custom classes, we need the concept info
+ # The class name should be the structure class name which is the concept code
+ custom_info = _collect_concept_info(concept)
+ if custom_info and custom_info.class_name == class_name:
+ custom_classes[class_name] = custom_info
-def generate_input_memory_json_string(inputs: InputRequirements, indent: int = 2) -> str:
- """Generate the input memory in pure JSON format as a formatted string.
+ return native_classes, custom_classes
- Args:
- inputs: The pipe inputs
- indent: Number of spaces for indentation (default: 2)
- Returns:
- Formatted JSON string
- """
- json_inputs = generate_input_memory_json(inputs)
- return json.dumps(json_inputs, indent=indent, ensure_ascii=False)
+def generate_runner_code(pipe: PipeAbstract, output_multiplicity: bool = False) -> str:
+ """Generate the complete Python runner code for a pipe.
+ This generates a runnable Python script with:
+ - Import statements for all required structure classes
+ - An async function to run the pipeline with proper return type
+ - Example input values based on the pipe's input concepts
+ - Output handling with main_stuff_as
-def generate_runner_code(pipe: PipeAbstract) -> str:
- """Generate the complete Python runner code for a pipe."""
- pipe_code = pipe.code
- inputs = pipe.inputs
+ Args:
+ pipe: The pipe to generate runner code for
+ output_multiplicity: Whether the output is a list (e.g., Text[])
+ """
+ # Get output information
+ structure_class_name = pipe.output.structure_class_name
+ is_native = NativeConceptCode.is_native_structure_class(structure_class_name)
+ custom_info = None if is_native else _collect_concept_info(pipe.output)
- # Determine which imports are needed based on input concepts
- needs_pdf = False
- needs_image = False
- for input_req in inputs.root.values():
- concept = input_req.concept
- if concept.structure_class_name == "PDFContent":
- needs_pdf = True
- elif concept.structure_class_name == "ImageContent":
- needs_image = True
+ # Collect all imports needed for inputs
+ native_classes, custom_classes = _collect_imports_for_inputs(pipe.inputs)
+
+ # Add output class to appropriate set
+ if is_native:
+ native_classes.add(structure_class_name)
+ elif custom_info:
+ custom_classes[structure_class_name] = custom_info
# Build import section
- import_lines = ["import asyncio", ""]
+ import_lines: list[str] = []
+
+ # Add path setup for custom structure imports (must be before other imports)
+ if custom_classes:
+ import_lines.extend(
+ [
+ "import sys",
+ "from pathlib import Path",
+ "",
+ "# Add script directory to path for local imports",
+ "sys.path.insert(0, str(Path(__file__).parent))",
+ "",
+ ]
+ )
- # Add content class imports if needed
- if needs_pdf:
- import_lines.append("from pipelex.core.stuffs.pdf_content import PDFContent")
- if needs_image:
- import_lines.append("from pipelex.core.stuffs.image_content import ImageContent")
+ import_lines.extend(["import asyncio", ""])
+
+ # Add native content class imports
+ native_imports: list[str] = []
+ for class_name in sorted(native_classes):
+ import_stmt = _get_structure_class_import(class_name)
+ if import_stmt:
+ native_imports.append(import_stmt)
+ import_lines.extend(native_imports)
+
+ # Add custom structure class imports from structures folder
+ if custom_classes:
+ import_lines.append("")
+ for class_name in sorted(custom_classes.keys()):
+ custom_info = custom_classes[class_name]
+ import_lines.append(custom_info.import_statement)
import_lines.extend(
[
+ "",
"from pipelex.pipelex import Pipelex",
"from pipelex.pipeline.execute import execute_pipeline",
]
)
# Build inputs entries
- if inputs.nb_inputs > 0:
- input_memory_entries: list[str] = []
- for var_name, input_req in inputs.root.items():
- concept = input_req.concept
- entry = generate_compact_memory_entry(var_name, concept)
- input_memory_entries.append(entry)
- input_memory_block = "\n".join(input_memory_entries)
+ if pipe.inputs.nb_inputs > 0:
+ input_entries: list[str] = []
+ for var_name, input_req in pipe.inputs.root.items():
+ is_multiple = _is_multiple(input_req.multiplicity)
+ result, _ = input_req.concept.generate_input_representation(
+ output_format=ConceptRepresentationFormat.PYTHON,
+ is_multiple=is_multiple,
+ )
+ python_code = _format_representation_as_python(result, is_multiple=is_multiple)
+ input_entries.append(f' "{var_name}": {python_code},')
+ input_memory_block = "\n".join(input_entries)
+ else:
+ input_memory_block = " # No inputs required"
+
+ # Determine return type annotation
+ if output_multiplicity:
+ return_type = f"list[{structure_class_name}]"
+ result_call = f"pipe_output.main_stuff_as_items(item_type={structure_class_name})"
else:
- input_memory_block = " # No inputs required"
+ return_type = structure_class_name
+ result_call = f"pipe_output.main_stuff_as(content_type={structure_class_name})"
# Build the main function
function_lines = [
"",
"",
- f"async def run_{pipe_code}():",
- " return await execute_pipeline(",
- f' pipe_code="{pipe_code}",',
+ f"async def run_{pipe.code}() -> {return_type}:",
+ " pipe_output = await execute_pipeline(",
+ f' pipe_code="{pipe.code}",',
]
- if inputs.nb_inputs > 0:
+ if pipe.inputs.nb_inputs > 0:
function_lines.extend(
[
" inputs={",
@@ -357,6 +263,7 @@ def generate_runner_code(pipe: PipeAbstract) -> str:
function_lines.extend(
[
" )",
+ f" return {result_call}",
"",
"",
'if __name__ == "__main__":',
@@ -364,7 +271,7 @@ def generate_runner_code(pipe: PipeAbstract) -> str:
" Pipelex.make()",
"",
" # Run the pipeline",
- f" result = asyncio.run(run_{pipe_code}())",
+ f" result = asyncio.run(run_{pipe.code}())",
"",
]
)
diff --git a/pipelex/cli/_cli.py b/pipelex/cli/_cli.py
index e75239b2a..1fbbc36f7 100644
--- a/pipelex/cli/_cli.py
+++ b/pipelex/cli/_cli.py
@@ -5,7 +5,7 @@ from click import Command, Context
from typer.core import TyperGroup
from typing_extensions import override
-from pipelex.cli.commands.build_cmd import build_app
+from pipelex.cli.commands.build import build_app
from pipelex.cli.commands.doctor_cmd import doctor_cmd
from pipelex.cli.commands.init.command import init_cmd
from pipelex.cli.commands.init.ui.types import InitFocus
diff --git a/pipelex/cli/commands/build/__init__.py b/pipelex/cli/commands/build/__init__.py
new file mode 100644
index 000000000..9fe929468
--- /dev/null
+++ b/pipelex/cli/commands/build/__init__.py
@@ -0,0 +1,6 @@
+# Import command modules to register them with build_app
+from pipelex.cli.commands.build import inputs_cmd as inputs_cmd
+from pipelex.cli.commands.build import pipe_cmd as pipe_cmd
+from pipelex.cli.commands.build import runner_cmd as runner_cmd
+from pipelex.cli.commands.build import structures_cmd as structures_cmd
+from pipelex.cli.commands.build.app import build_app as build_app
diff --git a/pipelex/cli/commands/build/app.py b/pipelex/cli/commands/build/app.py
new file mode 100644
index 000000000..6b1274219
--- /dev/null
+++ b/pipelex/cli/commands/build/app.py
@@ -0,0 +1,3 @@
+import typer
+
+build_app = typer.Typer(help="Build working pipelines from natural language requirements", no_args_is_help=True)
diff --git a/pipelex/cli/commands/build/inputs_cmd.py b/pipelex/cli/commands/build/inputs_cmd.py
new file mode 100644
index 000000000..f4a84e52d
--- /dev/null
+++ b/pipelex/cli/commands/build/inputs_cmd.py
@@ -0,0 +1,207 @@
+import asyncio
+from pathlib import Path
+from typing import Annotated
+
+import click
+import typer
+from posthog import tag
+
+from pipelex.cli.commands.build.app import build_app
+from pipelex.cli.error_handlers import (
+ ErrorContext,
+ handle_model_availability_error,
+ handle_model_choice_error,
+ handle_model_deck_preset_error,
+)
+from pipelex.cogt.exceptions import ModelDeckPresetValidatonError
+from pipelex.core.pipes.exceptions import PipeOperatorModelChoiceError
+from pipelex.core.pipes.inputs.exceptions import PipeInputError
+from pipelex.hub import get_required_pipe, get_telemetry_manager
+from pipelex.pipe_operators.exceptions import PipeOperatorModelAvailabilityError
+from pipelex.pipelex import PACKAGE_VERSION, Pipelex
+from pipelex.pipeline.validate_bundle import ValidateBundleError, validate_bundle
+from pipelex.system.runtime import IntegrationMode
+from pipelex.system.telemetry.events import EventProperty
+from pipelex.tools.misc.file_utils import (
+ ensure_directory_for_file_path,
+ save_text_to_path,
+)
+
+COMMAND = "build"
+SUB_COMMAND_INPUTS = "inputs"
+
+
+@build_app.command(SUB_COMMAND_INPUTS, help="Generate example input JSON for a pipe")
+def generate_inputs_cmd(
+ target: Annotated[
+ str | None,
+ typer.Argument(help="Pipe code or bundle file path (auto-detected)"),
+ ] = None,
+ pipe: Annotated[
+ str | None,
+ typer.Option("--pipe", help="Pipe code to use, can be omitted if you specify a bundle (.plx) that declares a main pipe"),
+ ] = None,
+ bundle: Annotated[
+ str | None,
+ typer.Option("--bundle", help="Bundle file path (.plx) - uses its main_pipe unless you specify a pipe code"),
+ ] = None,
+ output_path: Annotated[
+ str | None,
+ typer.Option(
+ "--output", "-o", help="Path to save the generated JSON file (defaults to bundle's directory if bundle provided, otherwise 'results/')"
+ ),
+ ] = None,
+) -> None:
+ """Generate example input JSON for a pipe.
+
+ The generated JSON file will include example values for all pipe inputs
+ based on their concept types.
+
+ Examples:
+ pipelex build inputs my_pipe
+ pipelex build inputs --bundle my_bundle.plx
+ pipelex build inputs --bundle my_bundle.plx --pipe my_pipe
+ pipelex build inputs my_bundle.plx
+ pipelex build inputs my_pipe --output custom_inputs.json
+ """
+ # Validate mutual exclusivity
+ provided_options = sum([target is not None, pipe is not None, bundle is not None])
+ if provided_options == 0:
+ ctx: click.Context = click.get_current_context()
+ typer.echo(ctx.get_help())
+ raise typer.Exit(0)
+
+ # Let's analyze the options and determine what pipe code to use and if we need to load a bundle
+ pipe_code: str | None = None
+ bundle_path: str | None = None
+
+ # Determine source:
+ if target:
+ # Check if target is a directory (not allowed for inputs command)
+ target_path = Path(target)
+ if target_path.is_dir():
+ typer.secho(
+ f"Failed to run: '{target}' is a directory. The inputs command requires a .plx file or a pipe code, not a directory.",
+ fg=typer.colors.RED,
+ err=True,
+ )
+ raise typer.Exit(1)
+
+ if target.endswith(".plx"):
+ bundle_path = target
+ if bundle:
+ typer.secho(
+ "Failed to run: cannot use option --bundle if you're already passing a bundle file (.plx) as positional argument",
+ fg=typer.colors.RED,
+ err=True,
+ )
+ raise typer.Exit(1)
+ else:
+ pipe_code = target
+ if pipe:
+ typer.secho(
+ "Failed to run: cannot use option --pipe if you're already passing a pipe code as positional argument",
+ fg=typer.colors.RED,
+ err=True,
+ )
+ raise typer.Exit(1)
+
+ if bundle:
+ assert not bundle_path, "bundle_path should be None at this stage if --bundle is provided"
+ bundle_path = bundle
+
+ if pipe:
+ assert not pipe_code, "pipe_code should be None at this stage if --pipe is provided"
+ pipe_code = pipe
+
+ if not pipe_code and not bundle_path:
+ typer.secho("Failed to run: no pipe code or bundle file specified", fg=typer.colors.RED, err=True)
+ raise typer.Exit(1)
+
+ async def generate_inputs(pipe_code: str | None = None, bundle_path: str | None = None):
+ if bundle_path:
+ try:
+ validate_bundle_result = await validate_bundle(plx_file_path=bundle_path)
+ bundle_blueprint = validate_bundle_result.blueprints[0]
+ if not pipe_code:
+ main_pipe_code = bundle_blueprint.main_pipe
+ if not main_pipe_code:
+ # Fall back to first pipe if no main_pipe declared
+ if bundle_blueprint.pipe:
+ main_pipe_code = next(iter(bundle_blueprint.pipe.keys()))
+ typer.echo(f"No main_pipe declared, using first pipe '{main_pipe_code}' from bundle '{bundle_path}'")
+ else:
+ typer.secho(f"Bundle '{bundle_path}' has no pipes defined", fg=typer.colors.RED, err=True)
+ raise typer.Exit(1)
+ else:
+ typer.echo(f"Using main pipe '{main_pipe_code}' from bundle '{bundle_path}'")
+ pipe_code = main_pipe_code
+ else:
+ typer.echo(f"Using pipe '{pipe_code}' from bundle '{bundle_path}'")
+ except FileNotFoundError as exc:
+ typer.secho(f"Failed to load bundle '{bundle_path}': {exc}", fg=typer.colors.RED, err=True)
+ raise typer.Exit(1) from exc
+ except ValidateBundleError as exc:
+ typer.secho(f"Failed to load bundle '{bundle_path}': {exc}", fg=typer.colors.RED, err=True)
+ raise typer.Exit(1) from exc
+ except PipeInputError as exc:
+ typer.secho(f"Failed to load bundle '{bundle_path}': {exc}", fg=typer.colors.RED, err=True)
+ raise typer.Exit(1) from exc
+ elif not pipe_code:
+ typer.secho("Failed to run: no pipe code specified", fg=typer.colors.RED, err=True)
+ raise typer.Exit(1)
+
+ # Get the pipe
+ try:
+ the_pipe = get_required_pipe(pipe_code=pipe_code)
+ except Exception as exc:
+ typer.secho(f"❌ Error: Could not find pipe '{pipe_code}': {exc}", fg=typer.colors.RED)
+ raise typer.Exit(1) from exc
+
+ # Generate the input JSON
+ try:
+ inputs_json_str = the_pipe.inputs.generate_json_string(indent=2)
+ except Exception as exc:
+ typer.secho(f"❌ Error generating input JSON: {exc}", fg=typer.colors.RED)
+ raise typer.Exit(1) from exc
+
+ # Determine output path - use bundle's directory if bundle provided, otherwise results/
+ if output_path:
+ final_output_path = output_path
+ elif bundle_path:
+ # Place inputs.json in the same directory as the PLX file
+ bundle_dir = Path(bundle_path).parent
+ final_output_path = str(bundle_dir / "inputs.json")
+ else:
+ final_output_path = "results/inputs.json"
+
+ # Save the file
+ try:
+ ensure_directory_for_file_path(file_path=final_output_path)
+ save_text_to_path(text=inputs_json_str, path=final_output_path)
+ typer.secho(f"✅ Generated input JSON file: {final_output_path}", fg=typer.colors.GREEN)
+ except Exception as exc:
+ typer.secho(f"❌ Error saving file: {exc}", fg=typer.colors.RED)
+ raise typer.Exit(1) from exc
+
+ try:
+ pipelex_instance = Pipelex.make(integration_mode=IntegrationMode.CLI)
+ except ModelDeckPresetValidatonError as model_deck_error:
+ handle_model_deck_preset_error(model_deck_error, context=ErrorContext.VALIDATION_BEFORE_BUILD_INPUTS)
+
+ try:
+ with get_telemetry_manager().telemetry_context():
+ tag(name=EventProperty.INTEGRATION, value=IntegrationMode.CLI)
+ tag(name=EventProperty.PIPELEX_VERSION, value=PACKAGE_VERSION)
+ tag(name=EventProperty.CLI_COMMAND, value=f"{COMMAND} {SUB_COMMAND_INPUTS}")
+
+ asyncio.run(generate_inputs(pipe_code=pipe_code, bundle_path=bundle_path))
+
+ except PipeOperatorModelChoiceError as exc:
+ handle_model_choice_error(exc, context=ErrorContext.BUILD)
+
+ except PipeOperatorModelAvailabilityError as exc:
+ handle_model_availability_error(exc, context=ErrorContext.BUILD)
+
+ finally:
+ pipelex_instance.teardown()
diff --git a/pipelex/cli/commands/build_cmd.py b/pipelex/cli/commands/build/pipe_cmd.py
similarity index 57%
rename from pipelex/cli/commands/build_cmd.py
rename to pipelex/cli/commands/build/pipe_cmd.py
index 33cbd3106..a985280d9 100644
--- a/pipelex/cli/commands/build_cmd.py
+++ b/pipelex/cli/commands/build/pipe_cmd.py
@@ -1,9 +1,9 @@
import asyncio
import os
import time
+from pathlib import Path
from typing import TYPE_CHECKING, Annotated
-import click
import typer
from posthog import tag
@@ -11,7 +11,8 @@ from pipelex import pretty_print
from pipelex.builder.builder import PipelexBundleSpec
from pipelex.builder.builder_errors import PipeBuilderError
from pipelex.builder.builder_loop import BuilderLoop
-from pipelex.builder.runner_code import generate_input_memory_json_string, generate_runner_code
+from pipelex.builder.runner_code import generate_runner_code
+from pipelex.cli.commands.build.app import build_app
from pipelex.cli.error_handlers import (
ErrorContext,
handle_model_availability_error,
@@ -21,14 +22,13 @@ from pipelex.cli.error_handlers import (
from pipelex.cogt.exceptions import ModelDeckPresetValidatonError
from pipelex.config import get_config
from pipelex.core.pipes.exceptions import PipeOperatorModelChoiceError
-from pipelex.core.pipes.inputs.exceptions import PipeInputError
+from pipelex.core.pipes.variable_multiplicity import parse_concept_with_multiplicity
from pipelex.hub import get_console, get_report_delegate, get_required_pipe, get_telemetry_manager
from pipelex.language.plx_factory import PlxFactory
from pipelex.pipe_operators.exceptions import PipeOperatorModelAvailabilityError
from pipelex.pipelex import PACKAGE_VERSION, Pipelex
from pipelex.pipeline.exceptions import PipelineExecutionError
from pipelex.pipeline.execute import execute_pipeline
-from pipelex.pipeline.validate_bundle import ValidateBundleError, validate_bundle
from pipelex.system.runtime import IntegrationMode
from pipelex.system.telemetry.events import EventProperty
from pipelex.tools.misc.file_utils import (
@@ -43,7 +43,10 @@ from pipelex.tools.misc.pretty import PrettyPrinter
if TYPE_CHECKING:
from pipelex.client.protocol import PipelineInputs
-build_app = typer.Typer(help="Build working pipelines from natural language requirements", no_args_is_help=True)
+COMMAND = "build"
+SUB_COMMAND_PIPE = "pipe"
+SUB_COMMAND_ONE_SHOT_PIPE = "one-shot-pipe"
+SUB_COMMAND_PARTIAL_PIPE = "partial-pipe"
"""
Today's example:
@@ -70,14 +73,6 @@ pipelex build pipe "Given an RDFP PDF, build a compliance matrix"
pipelex build pipe "Given a theme, write a Haiku"
"""
-COMMAND = "build"
-
-SUB_COMMAND_PIPE = "pipe"
-SUB_COMMAND_RUNNER = "runner"
-SUB_COMMAND_INPUTS = "inputs"
-SUB_COMMAND_ONE_SHOT_PIPE = "one-shot-pipe"
-SUB_COMMAND_PARTIAL_PIPE = "partial-pipe"
-
@build_app.command(SUB_COMMAND_PIPE, help="Build a Pipelex bundle with one validation/fix loop correcting deterministic issues")
def build_pipe_cmd(
@@ -106,6 +101,9 @@ def build_pipe_cmd(
typer.Option("--no-extras", help="Skip generating inputs.json and runner.py, only generate the PLX file"),
] = False,
) -> None:
+ # Import here to avoid circular imports
+ from pipelex.cli.commands.build.structures_cmd import generate_structures_from_blueprints # noqa: PLC0415
+
try:
pipelex_instance = Pipelex.make(integration_mode=IntegrationMode.CLI)
except ModelDeckPresetValidatonError as model_deck_error:
@@ -196,18 +194,40 @@ def build_pipe_cmd(
pipe = get_required_pipe(pipe_code=main_pipe_code)
+ # Generate structures folder FIRST (before runner, since runner imports from structures)
+ structures_output_dir = Path(extras_output_dir) / "structures"
+ generated_structures = generate_structures_from_blueprints(
+ blueprints=[pipelex_bundle_spec.to_blueprint()],
+ output_directory=structures_output_dir,
+ skip_existing_check=True,
+ )
+ if generated_structures:
+ typer.secho(f"✅ Generated {len(generated_structures)} structure(s) in: {structures_output_dir}", fg=typer.colors.GREEN)
+
# Generate inputs.json
- inputs_json_str = generate_input_memory_json_string(pipe.inputs, indent=2)
+ inputs_json_str = pipe.inputs.generate_json_string(indent=2)
inputs_json_path = os.path.join(extras_output_dir, "inputs.json")
save_text_to_path(text=inputs_json_str, path=inputs_json_path)
typer.secho(f"✅ Inputs template saved to: {inputs_json_path}", fg=typer.colors.GREEN)
- # Generate runner.py
- runner_code = generate_runner_code(pipe)
+ # Determine if output is a list from the bundle spec
+ main_pipe_spec = pipelex_bundle_spec.pipe[main_pipe_code] if pipelex_bundle_spec.pipe else None
+ output_is_list = False
+ if main_pipe_spec:
+ output_parse = parse_concept_with_multiplicity(main_pipe_spec.output)
+ output_is_list = output_parse.multiplicity is not None
+
+ # Generate runner.py (after structures are generated)
+ runner_code = generate_runner_code(pipe, output_multiplicity=output_is_list)
runner_path = os.path.join(extras_output_dir, f"run_{main_pipe_code}.py")
save_text_to_path(text=runner_code, path=runner_path)
typer.secho(f"✅ Python runner script saved to: {runner_path}", fg=typer.colors.GREEN)
+ # Generate empty __init__.py to make it a proper Python package
+ init_path = os.path.join(extras_output_dir, "__init__.py")
+ save_text_to_path(text="", path=init_path)
+ typer.secho(f"✅ Package init file saved to: {init_path}", fg=typer.colors.GREEN)
+
end_time = time.time()
typer.secho(f"\n✅ Pipeline built in {end_time - start_time:.2f} seconds\n", fg=typer.colors.WHITE)
@@ -240,314 +260,6 @@ def build_pipe_cmd(
pipelex_instance.teardown()
-@build_app.command(SUB_COMMAND_RUNNER, help="Build the Python code to run a pipe with the necessary inputs")
-def prepare_runner_cmd(
- target: Annotated[
- str | None,
- typer.Argument(help="Pipe code or bundle file path (auto-detected)"),
- ] = None,
- pipe: Annotated[
- str | None,
- typer.Option("--pipe", help="Pipe code to use, can be omitted if you specify a bundle (.plx) that declares a main pipe"),
- ] = None,
- bundle: Annotated[
- str | None,
- typer.Option("--bundle", help="Bundle file path (.plx) - uses its main_pipe unless you specify a pipe code"),
- ] = None,
- output_path: Annotated[
- str | None,
- typer.Option("--output", "-o", help="Path to save the generated Python file, defaults to 'results/run_{pipe_code}.py'"),
- ] = None,
-) -> None:
- """Prepare a Python runner file for a pipe.
-
- The generated file will include:
- - All necessary imports
- - Example input values based on the pipe's input types
-
- Native concept types (Text, Image, PDF, etc.) will be automatically handled.
- Custom concept types will have their structure recursively generated.
-
- Examples:
- pipelex build runner my_pipe
- pipelex build runner --bundle my_bundle.plx
- pipelex build runner --bundle my_bundle.plx --pipe my_pipe
- pipelex build runner my_bundle.plx
- pipelex build runner my_pipe --output runner.py
- """
- # Validate mutual exclusivity
- provided_options = sum([target is not None, pipe is not None, bundle is not None])
- if provided_options == 0:
- ctx: click.Context = click.get_current_context()
- typer.echo(ctx.get_help())
- raise typer.Exit(0)
-
- # Let's analyze the options and determine what pipe code to use and if we need to load a bundle
- pipe_code: str | None = None
- bundle_path: str | None = None
-
- # Determine source:
- if target:
- if target.endswith(".plx"):
- bundle_path = target
- if bundle:
- typer.secho(
- "Failed to run: cannot use option --bundle if you're already passing a bundle file (.plx) as positional argument",
- fg=typer.colors.RED,
- err=True,
- )
- raise typer.Exit(1)
- else:
- pipe_code = target
- if pipe:
- typer.secho(
- "Failed to run: cannot use option --pipe if you're already passing a pipe code as positional argument",
- fg=typer.colors.RED,
- err=True,
- )
- raise typer.Exit(1)
-
- if bundle:
- assert not bundle_path, "bundle_path should be None at this stage if --bundle is provided"
- bundle_path = bundle
-
- if pipe:
- assert not pipe_code, "pipe_code should be None at this stage if --pipe is provided"
- pipe_code = pipe
-
- if not pipe_code and not bundle_path:
- typer.secho("Failed to run: no pipe code or bundle file specified", fg=typer.colors.RED, err=True)
- raise typer.Exit(1)
-
- async def prepare_runner(pipe_code: str | None = None, bundle_path: str | None = None):
- if bundle_path:
- try:
- validate_bundle_result = await validate_bundle(plx_file_path=bundle_path)
- if not pipe_code:
- main_pipe_code = validate_bundle_result.blueprints[0].main_pipe
- if not main_pipe_code:
- typer.secho(f"Bundle '{bundle_path}' does not declare a main_pipe", fg=typer.colors.RED, err=True)
- raise typer.Exit(1)
- pipe_code = main_pipe_code
- typer.echo(f"Using main pipe '{pipe_code}' from bundle '{bundle_path}'")
- else:
- typer.echo(f"Using pipe '{pipe_code}' from bundle '{bundle_path}'")
- except FileNotFoundError as exc:
- typer.secho(f"Failed to load bundle '{bundle_path}': {exc}", fg=typer.colors.RED, err=True)
- raise typer.Exit(1) from exc
- except ValidateBundleError as exc:
- typer.secho(f"Failed to load bundle '{bundle_path}': {exc}", fg=typer.colors.RED, err=True)
- raise typer.Exit(1) from exc
- except PipeInputError as exc:
- typer.secho(f"Failed to load bundle '{bundle_path}': {exc}", fg=typer.colors.RED, err=True)
- raise typer.Exit(1) from exc
- elif not pipe_code:
- typer.secho("Failed to run: no pipe code specified", fg=typer.colors.RED, err=True)
- raise typer.Exit(1)
-
- # Get the pipe
- try:
- pipe = get_required_pipe(pipe_code=pipe_code)
- except Exception as exc:
- typer.secho(f"❌ Error: Could not find pipe '{pipe_code}': {exc}", fg=typer.colors.RED)
- raise typer.Exit(1) from exc
-
- # Generate the code
- try:
- runner_code = generate_runner_code(pipe)
- except Exception as exc:
- typer.secho(f"❌ Error generating runner code: {exc}", fg=typer.colors.RED)
- raise typer.Exit(1) from exc
-
- # Determine output path
- final_output_path = output_path or get_incremental_file_path(
- base_path="results",
- base_name=f"run_{pipe_code}",
- extension="py",
- )
-
- # Save the file
- try:
- ensure_directory_for_file_path(file_path=final_output_path)
- save_text_to_path(text=runner_code, path=final_output_path)
- typer.secho(f"✅ Generated runner file: {final_output_path}", fg=typer.colors.GREEN)
- except Exception as exc:
- typer.secho(f"❌ Error saving file: {exc}", fg=typer.colors.RED)
- raise typer.Exit(1) from exc
-
- try:
- pipelex_instance = Pipelex.make(integration_mode=IntegrationMode.CLI)
- except ModelDeckPresetValidatonError as model_deck_error:
- handle_model_deck_preset_error(model_deck_error, context=ErrorContext.VALIDATION_BEFORE_BUILD_RUNNER)
-
- try:
- with get_telemetry_manager().telemetry_context():
- tag(name=EventProperty.INTEGRATION, value=IntegrationMode.CLI)
- tag(name=EventProperty.PIPELEX_VERSION, value=PACKAGE_VERSION)
- tag(name=EventProperty.CLI_COMMAND, value=f"{COMMAND} {SUB_COMMAND_RUNNER}")
-
- asyncio.run(prepare_runner(pipe_code=pipe_code, bundle_path=bundle_path))
-
- except PipeOperatorModelChoiceError as exc:
- handle_model_choice_error(exc, context=ErrorContext.BUILD)
-
- except PipeOperatorModelAvailabilityError as exc:
- handle_model_availability_error(exc, context=ErrorContext.BUILD)
-
- finally:
- pipelex_instance.teardown()
-
-
-@build_app.command(SUB_COMMAND_INPUTS, help="Generate example input JSON for a pipe")
-def generate_inputs_cmd(
- target: Annotated[
- str | None,
- typer.Argument(help="Pipe code or bundle file path (auto-detected)"),
- ] = None,
- pipe: Annotated[
- str | None,
- typer.Option("--pipe", help="Pipe code to use, can be omitted if you specify a bundle (.plx) that declares a main pipe"),
- ] = None,
- bundle: Annotated[
- str | None,
- typer.Option("--bundle", help="Bundle file path (.plx) - uses its main_pipe unless you specify a pipe code"),
- ] = None,
- output_path: Annotated[
- str | None,
- typer.Option("--output", "-o", help="Path to save the generated JSON file, defaults to 'results/inputs.json'"),
- ] = None,
-) -> None:
- """Generate example input JSON for a pipe.
-
- The generated JSON file will include example values for all pipe inputs
- based on their concept types.
-
- Examples:
- pipelex build inputs my_pipe
- pipelex build inputs --bundle my_bundle.plx
- pipelex build inputs --bundle my_bundle.plx --pipe my_pipe
- pipelex build inputs my_bundle.plx
- pipelex build inputs my_pipe --output custom_inputs.json
- """
- # Validate mutual exclusivity
- provided_options = sum([target is not None, pipe is not None, bundle is not None])
- if provided_options == 0:
- ctx: click.Context = click.get_current_context()
- typer.echo(ctx.get_help())
- raise typer.Exit(0)
-
- # Let's analyze the options and determine what pipe code to use and if we need to load a bundle
- pipe_code: str | None = None
- bundle_path: str | None = None
-
- # Determine source:
- if target:
- if target.endswith(".plx"):
- bundle_path = target
- if bundle:
- typer.secho(
- "Failed to run: cannot use option --bundle if you're already passing a bundle file (.plx) as positional argument",
- fg=typer.colors.RED,
- err=True,
- )
- raise typer.Exit(1)
- else:
- pipe_code = target
- if pipe:
- typer.secho(
- "Failed to run: cannot use option --pipe if you're already passing a pipe code as positional argument",
- fg=typer.colors.RED,
- err=True,
- )
- raise typer.Exit(1)
-
- if bundle:
- assert not bundle_path, "bundle_path should be None at this stage if --bundle is provided"
- bundle_path = bundle
-
- if pipe:
- assert not pipe_code, "pipe_code should be None at this stage if --pipe is provided"
- pipe_code = pipe
-
- if not pipe_code and not bundle_path:
- typer.secho("Failed to run: no pipe code or bundle file specified", fg=typer.colors.RED, err=True)
- raise typer.Exit(1)
-
- async def generate_inputs(pipe_code: str | None = None, bundle_path: str | None = None):
- if bundle_path:
- try:
- validate_bundle_result = await validate_bundle(plx_file_path=bundle_path)
- if not pipe_code:
- main_pipe_code = validate_bundle_result.blueprints[0].main_pipe
- if not main_pipe_code:
- typer.secho(f"Bundle '{bundle_path}' does not declare a main_pipe", fg=typer.colors.RED, err=True)
- raise typer.Exit(1)
- pipe_code = main_pipe_code
- typer.echo(f"Using main pipe '{pipe_code}' from bundle '{bundle_path}'")
- else:
- typer.echo(f"Using pipe '{pipe_code}' from bundle '{bundle_path}'")
- except FileNotFoundError as exc:
- typer.secho(f"Failed to load bundle '{bundle_path}': {exc}", fg=typer.colors.RED, err=True)
- raise typer.Exit(1) from exc
- except ValidateBundleError as exc:
- typer.secho(f"Failed to load bundle '{bundle_path}': {exc}", fg=typer.colors.RED, err=True)
- raise typer.Exit(1) from exc
- except PipeInputError as exc:
- typer.secho(f"Failed to load bundle '{bundle_path}': {exc}", fg=typer.colors.RED, err=True)
- raise typer.Exit(1) from exc
- elif not pipe_code:
- typer.secho("Failed to run: no pipe code specified", fg=typer.colors.RED, err=True)
- raise typer.Exit(1)
-
- # Get the pipe
- try:
- pipe = get_required_pipe(pipe_code=pipe_code)
- except Exception as exc:
- typer.secho(f"❌ Error: Could not find pipe '{pipe_code}': {exc}", fg=typer.colors.RED)
- raise typer.Exit(1) from exc
-
- # Generate the input JSON
- try:
- inputs_json_str = generate_input_memory_json_string(pipe.inputs, indent=2)
- except Exception as exc:
- typer.secho(f"❌ Error generating input JSON: {exc}", fg=typer.colors.RED)
- raise typer.Exit(1) from exc
-
- # Determine output path
- final_output_path = output_path or "results/inputs.json"
-
- # Save the file
- try:
- ensure_directory_for_file_path(file_path=final_output_path)
- save_text_to_path(text=inputs_json_str, path=final_output_path)
- typer.secho(f"✅ Generated input JSON file: {final_output_path}", fg=typer.colors.GREEN)
- except Exception as exc:
- typer.secho(f"❌ Error saving file: {exc}", fg=typer.colors.RED)
- raise typer.Exit(1) from exc
-
- try:
- pipelex_instance = Pipelex.make(integration_mode=IntegrationMode.CLI)
- except ModelDeckPresetValidatonError as model_deck_error:
- handle_model_deck_preset_error(model_deck_error, context=ErrorContext.VALIDATION_BEFORE_BUILD_INPUTS)
-
- try:
- with get_telemetry_manager().telemetry_context():
- tag(name=EventProperty.INTEGRATION, value=IntegrationMode.CLI)
- tag(name=EventProperty.PIPELEX_VERSION, value=PACKAGE_VERSION)
- tag(name=EventProperty.CLI_COMMAND, value=f"{COMMAND} {SUB_COMMAND_INPUTS}")
-
- asyncio.run(generate_inputs(pipe_code=pipe_code, bundle_path=bundle_path))
-
- except PipeOperatorModelChoiceError as exc:
- handle_model_choice_error(exc, context=ErrorContext.BUILD)
-
- except PipeOperatorModelAvailabilityError as exc:
- handle_model_availability_error(exc, context=ErrorContext.BUILD)
-
- finally:
- pipelex_instance.teardown()
-
-
@build_app.command(SUB_COMMAND_ONE_SHOT_PIPE, help="Developer utility for contributors: deliver pipeline in one shot, without validation loop")
def build_one_shot_cmd(
brief: Annotated[
diff --git a/pipelex/cli/commands/build/runner_cmd.py b/pipelex/cli/commands/build/runner_cmd.py
new file mode 100644
index 000000000..3d1abbdac
--- /dev/null
+++ b/pipelex/cli/commands/build/runner_cmd.py
@@ -0,0 +1,241 @@
+import asyncio
+from pathlib import Path
+from typing import Annotated
+
+import click
+import typer
+from posthog import tag
+
+from pipelex.builder.runner_code import generate_runner_code
+from pipelex.cli.commands.build.app import build_app
+from pipelex.cli.error_handlers import (
+ ErrorContext,
+ handle_model_availability_error,
+ handle_model_choice_error,
+ handle_model_deck_preset_error,
+)
+from pipelex.cogt.exceptions import ModelDeckPresetValidatonError
+from pipelex.core.pipes.exceptions import PipeOperatorModelChoiceError
+from pipelex.core.pipes.inputs.exceptions import PipeInputError
+from pipelex.core.pipes.variable_multiplicity import parse_concept_with_multiplicity
+from pipelex.hub import get_required_pipe, get_telemetry_manager
+from pipelex.pipe_operators.exceptions import PipeOperatorModelAvailabilityError
+from pipelex.pipelex import PACKAGE_VERSION, Pipelex
+from pipelex.pipeline.validate_bundle import ValidateBundleError, validate_bundle
+from pipelex.system.runtime import IntegrationMode
+from pipelex.system.telemetry.events import EventProperty
+from pipelex.tools.misc.file_utils import (
+ ensure_directory_for_file_path,
+ get_incremental_file_path,
+ save_text_to_path,
+)
+
+COMMAND = "build"
+SUB_COMMAND_RUNNER = "runner"
+
+
+@build_app.command(SUB_COMMAND_RUNNER, help="Build the Python code to run a pipe with the necessary inputs")
+def prepare_runner_cmd(
+ target: Annotated[
+ str | None,
+ typer.Argument(help="Pipe code or bundle file path (auto-detected)"),
+ ] = None,
+ pipe: Annotated[
+ str | None,
+ typer.Option("--pipe", help="Pipe code to use, can be omitted if you specify a bundle (.plx) that declares a main pipe"),
+ ] = None,
+ bundle: Annotated[
+ str | None,
+ typer.Option("--bundle", help="Bundle file path (.plx) - uses its main_pipe unless you specify a pipe code"),
+ ] = None,
+ output_path: Annotated[
+ str | None,
+ typer.Option(
+ "--output", "-o", help="Path to save the generated Python file (defaults to bundle's directory if bundle provided, otherwise 'results/')"
+ ),
+ ] = None,
+) -> None:
+ """Prepare a Python runner file for a pipe.
+
+ The generated file will include:
+ - All necessary imports
+ - Example input values based on the pipe's input types
+
+ Native concept types (Text, Image, PDF, etc.) will be automatically handled.
+ Custom concept types will have their structure recursively generated.
+
+ Examples:
+ pipelex build runner my_pipe
+ pipelex build runner --bundle my_bundle.plx
+ pipelex build runner --bundle my_bundle.plx --pipe my_pipe
+ pipelex build runner my_bundle.plx
+ pipelex build runner my_pipe --output runner.py
+ """
+ # Import here to avoid circular imports
+ from pipelex.cli.commands.build.structures_cmd import generate_structures_from_blueprints # noqa: PLC0415
+
+ # Validate mutual exclusivity
+ provided_options = sum([target is not None, pipe is not None, bundle is not None])
+ if provided_options == 0:
+ ctx: click.Context = click.get_current_context()
+ typer.echo(ctx.get_help())
+ raise typer.Exit(0)
+
+ # Let's analyze the options and determine what pipe code to use and if we need to load a bundle
+ pipe_code: str | None = None
+ bundle_path: str | None = None
+
+ # Determine source:
+ if target:
+ # Check if target is a directory (not allowed for runner command)
+ target_path = Path(target)
+ if target_path.is_dir():
+ typer.secho(
+ f"Failed to run: '{target}' is a directory. The runner command requires a .plx file or a pipe code, not a directory.",
+ fg=typer.colors.RED,
+ err=True,
+ )
+ raise typer.Exit(1)
+
+ if target.endswith(".plx"):
+ bundle_path = target
+ if bundle:
+ typer.secho(
+ "Failed to run: cannot use option --bundle if you're already passing a bundle file (.plx) as positional argument",
+ fg=typer.colors.RED,
+ err=True,
+ )
+ raise typer.Exit(1)
+ else:
+ pipe_code = target
+ if pipe:
+ typer.secho(
+ "Failed to run: cannot use option --pipe if you're already passing a pipe code as positional argument",
+ fg=typer.colors.RED,
+ err=True,
+ )
+ raise typer.Exit(1)
+
+ if bundle:
+ assert not bundle_path, "bundle_path should be None at this stage if --bundle is provided"
+ bundle_path = bundle
+
+ if pipe:
+ assert not pipe_code, "pipe_code should be None at this stage if --pipe is provided"
+ pipe_code = pipe
+
+ if not pipe_code and not bundle_path:
+ typer.secho("Failed to run: no pipe code or bundle file specified", fg=typer.colors.RED, err=True)
+ raise typer.Exit(1)
+
+ async def prepare_runner(pipe_code: str | None = None, bundle_path: str | None = None):
+ bundle_blueprint = None
+ if bundle_path:
+ try:
+ validate_bundle_result = await validate_bundle(plx_file_path=bundle_path)
+ bundle_blueprint = validate_bundle_result.blueprints[0]
+ if not pipe_code:
+ main_pipe_code = bundle_blueprint.main_pipe
+ if not main_pipe_code:
+ # Fall back to first pipe if no main_pipe declared
+ if bundle_blueprint.pipe:
+ main_pipe_code = next(iter(bundle_blueprint.pipe.keys()))
+ typer.echo(f"No main_pipe declared, using first pipe '{main_pipe_code}' from bundle '{bundle_path}'")
+ else:
+ typer.secho(f"Bundle '{bundle_path}' has no pipes defined", fg=typer.colors.RED, err=True)
+ raise typer.Exit(1)
+ else:
+ typer.echo(f"Using main pipe '{main_pipe_code}' from bundle '{bundle_path}'")
+ pipe_code = main_pipe_code
+ else:
+ typer.echo(f"Using pipe '{pipe_code}' from bundle '{bundle_path}'")
+ except FileNotFoundError as exc:
+ typer.secho(f"Failed to load bundle '{bundle_path}': {exc}", fg=typer.colors.RED, err=True)
+ raise typer.Exit(1) from exc
+ except ValidateBundleError as exc:
+ typer.secho(f"Failed to load bundle '{bundle_path}': {exc}", fg=typer.colors.RED, err=True)
+ raise typer.Exit(1) from exc
+ except PipeInputError as exc:
+ typer.secho(f"Failed to load bundle '{bundle_path}': {exc}", fg=typer.colors.RED, err=True)
+ raise typer.Exit(1) from exc
+ elif not pipe_code:
+ typer.secho("Failed to run: no pipe code specified", fg=typer.colors.RED, err=True)
+ raise typer.Exit(1)
+
+ # Get the pipe
+ try:
+ the_pipe = get_required_pipe(pipe_code=pipe_code)
+ except Exception as exc:
+ typer.secho(f"❌ Error: Could not find pipe '{pipe_code}': {exc}", fg=typer.colors.RED)
+ raise typer.Exit(1) from exc
+
+ # Determine if output is a list from the bundle blueprint
+ output_is_list = False
+ if bundle_blueprint and bundle_blueprint.pipe and pipe_code in bundle_blueprint.pipe:
+ pipe_blueprint = bundle_blueprint.pipe[pipe_code]
+ output_parse = parse_concept_with_multiplicity(pipe_blueprint.output)
+ output_is_list = output_parse.multiplicity is not None
+
+ # Determine output path - use bundle's directory if bundle provided, otherwise results/
+ if output_path:
+ final_output_path = output_path
+ elif bundle_path:
+ # Place runner in the same directory as the PLX file
+ bundle_dir = Path(bundle_path).parent
+ final_output_path = str(bundle_dir / f"run_{pipe_code}.py")
+ else:
+ final_output_path = get_incremental_file_path(
+ base_path="results",
+ base_name=f"run_{pipe_code}",
+ extension="py",
+ )
+ output_dir = Path(final_output_path).parent
+
+ # Generate structures folder FIRST (before runner, since runner imports from structures)
+ if bundle_blueprint:
+ structures_output_dir = output_dir / "structures"
+ generated_structures = generate_structures_from_blueprints(
+ blueprints=[bundle_blueprint],
+ output_directory=structures_output_dir,
+ target_path=output_dir, # Check for existing structures in the bundle's directory
+ )
+ if generated_structures:
+ typer.secho(f"✅ Generated {len(generated_structures)} structure(s) in: {structures_output_dir}", fg=typer.colors.GREEN)
+
+ # Generate the runner code
+ try:
+ runner_code = generate_runner_code(the_pipe, output_multiplicity=output_is_list)
+ except Exception as exc:
+ typer.secho(f"❌ Error generating runner code: {exc}", fg=typer.colors.RED)
+ raise typer.Exit(1) from exc
+
+ # Save the runner file
+ try:
+ ensure_directory_for_file_path(file_path=final_output_path)
+ save_text_to_path(text=runner_code, path=final_output_path)
+ typer.secho(f"✅ Generated runner file: {final_output_path}", fg=typer.colors.GREEN)
+ except Exception as exc:
+ typer.secho(f"❌ Error saving file: {exc}", fg=typer.colors.RED)
+ raise typer.Exit(1) from exc
+
+ try:
+ pipelex_instance = Pipelex.make(integration_mode=IntegrationMode.CLI)
+ except ModelDeckPresetValidatonError as model_deck_error:
+ handle_model_deck_preset_error(model_deck_error, context=ErrorContext.VALIDATION_BEFORE_BUILD_RUNNER)
+
+ try:
+ with get_telemetry_manager().telemetry_context():
+ tag(name=EventProperty.INTEGRATION, value=IntegrationMode.CLI)
+ tag(name=EventProperty.PIPELEX_VERSION, value=PACKAGE_VERSION)
+ tag(name=EventProperty.CLI_COMMAND, value=f"{COMMAND} {SUB_COMMAND_RUNNER}")
+
+ asyncio.run(prepare_runner(pipe_code=pipe_code, bundle_path=bundle_path))
+
+ except PipeOperatorModelChoiceError as exc:
+ handle_model_choice_error(exc, context=ErrorContext.BUILD)
+
+ except PipeOperatorModelAvailabilityError as exc:
+ handle_model_availability_error(exc, context=ErrorContext.BUILD)
+
+ finally:
+ pipelex_instance.teardown()
diff --git a/pipelex/cli/commands/build/structures_cmd.py b/pipelex/cli/commands/build/structures_cmd.py
new file mode 100644
index 000000000..f916611cf
--- /dev/null
+++ b/pipelex/cli/commands/build/structures_cmd.py
@@ -0,0 +1,260 @@
+import asyncio
+from pathlib import Path
+from typing import TYPE_CHECKING, Annotated
+
+import typer
+from kajson.kajson_manager import KajsonManager
+
+from pipelex import log
+from pipelex.base_exceptions import PipelexError
+from pipelex.cli.commands.build.app import build_app
+from pipelex.core.concepts.concept_factory import ConceptFactory
+from pipelex.core.concepts.helpers import normalize_structure_blueprint
+from pipelex.core.concepts.structure_generation.exceptions import ConceptStructureGeneratorError
+from pipelex.core.concepts.structure_generation.generator import StructureGenerator
+from pipelex.core.stuffs.structured_content import StructuredContent
+from pipelex.core.stuffs.text_content import TextContent
+from pipelex.pipelex import Pipelex
+from pipelex.pipeline.validate_bundle import validate_bundle, validate_bundles_from_directory
+from pipelex.system.registries.class_registry_utils import ClassRegistryUtils
+from pipelex.system.runtime import IntegrationMode
+
+if TYPE_CHECKING:
+ from pipelex.core.bundles.pipelex_bundle_blueprint import PipelexBundleBlueprint
+
+SUB_COMMAND_STRUCTURES = "structures"
+
+
+def generate_structures_from_blueprints(
+ blueprints: list["PipelexBundleBlueprint"],
+ output_directory: Path,
+ target_path: Path | None = None,
+ skip_existing_check: bool = False,
+) -> list[tuple[str, str]]:
+ """Generate Python structure files from blueprint concept definitions.
+
+ Args:
+ blueprints: List of PipelexBundleBlueprint containing concept definitions
+ output_directory: Directory where structure files will be generated
+ target_path: Optional path to scan for manually-created structure classes
+ skip_existing_check: If True, always generate structures without checking if they exist
+
+ Returns:
+ List of (domain, concept_code) tuples for generated files
+ """
+ output_directory.mkdir(parents=True, exist_ok=True)
+
+ # Only check for existing classes if we're not skipping and have a target path
+ check_existing = not skip_existing_check and target_path is not None
+ class_registry = KajsonManager.get_class_registry()
+ if check_existing:
+ class_registry.teardown()
+ class_registry.setup()
+ ClassRegistryUtils.register_classes_in_folder(
+ folder_path=str(target_path),
+ base_class=StructuredContent,
+ force_exclude_dirs=[str(output_directory.resolve())],
+ )
+
+ generated_files: list[tuple[str, str]] = []
+
+ typer.echo(f"\n📝 Generating structures in: {output_directory}")
+
+ for blueprint in blueprints:
+ if blueprint.domain == "native":
+ continue
+
+ if not blueprint.concept:
+ continue
+
+ for concept_code, concept_blueprint in blueprint.concept.items():
+ # Check if structure class was manually created (only when check_existing is enabled)
+ if check_existing and class_registry.has_class(name=concept_code):
+ existing_class = class_registry.get_class(name=concept_code)
+ if existing_class:
+ import inspect # noqa: PLC0415
+
+ try:
+ source_file = inspect.getfile(existing_class)
+ log.warning(
+ f"Skipping Generation for '{concept_code}' (domain '{blueprint.domain}'): "
+ f"manually-created class exists at '{source_file}'"
+ )
+ except (TypeError, OSError):
+ module_name = existing_class.__module__ if hasattr(existing_class, "__module__") else "unknown"
+ log.warning(
+ f"Skipping '{concept_code}' (domain '{blueprint.domain}'): manually-created class exists in module '{module_name}'"
+ )
+ continue
+
+ # Handle simple string concept definitions (description only, refines Text by default)
+ if isinstance(concept_blueprint, str):
+ try:
+ generated_code, _ = StructureGenerator().generate_from_structure_blueprint(
+ class_name=concept_code,
+ structure_blueprint={},
+ base_class_name=TextContent.__name__,
+ )
+ except ConceptStructureGeneratorError as exc:
+ msg = f"Error generating structure class for concept '{concept_code}' in domain '{blueprint.domain}': {exc}"
+ raise PipelexError(msg) from exc
+
+ output_file = output_directory / f"{blueprint.domain}_{concept_code}.py"
+ output_file.write_text(generated_code)
+ generated_files.append((blueprint.domain, concept_code))
+ typer.secho(f" ✅ Generated {output_file.name}", fg=typer.colors.GREEN)
+ continue
+
+ # Handle concepts with explicit structure definition
+ if concept_blueprint.structure:
+ if isinstance(concept_blueprint.structure, str):
+ continue
+ normalized_structure = normalize_structure_blueprint(concept_blueprint.structure)
+
+ try:
+ generated_code, _ = StructureGenerator().generate_from_structure_blueprint(
+ class_name=concept_code,
+ structure_blueprint=normalized_structure,
+ )
+ except ConceptStructureGeneratorError as exc:
+ msg = f"Error generating python code for structure class of concept '{concept_code}' in domain '{blueprint.domain}': {exc}"
+ raise PipelexError(msg) from exc
+
+ output_file = output_directory / f"{blueprint.domain}_{concept_code}.py"
+ output_file.write_text(generated_code)
+ generated_files.append((blueprint.domain, concept_code))
+ typer.secho(f" ✅ Generated {output_file.name}", fg=typer.colors.GREEN)
+
+ # Handle concepts with refines
+ elif concept_blueprint.refines:
+ try:
+ current_refine = ConceptFactory.make_refine(refine=concept_blueprint.refines)
+ except Exception as exc:
+ msg = (
+ f"Could not validate refine '{concept_blueprint.refines}' for concept '{concept_code}' in domain '{blueprint.domain}': {exc}"
+ )
+ raise PipelexError(msg) from exc
+
+ refined_structure_class_name = current_refine.split(".")[1] + "Content" if current_refine else TextContent.__name__
+
+ try:
+ generated_code, _ = StructureGenerator().generate_from_structure_blueprint(
+ class_name=concept_code,
+ structure_blueprint={},
+ base_class_name=refined_structure_class_name,
+ )
+ except ConceptStructureGeneratorError as exc:
+ msg = (
+ f"Error generating python code for structure class of concept '{concept_code}' "
+ f"refining '{refined_structure_class_name}' in domain '{blueprint.domain}': {exc}"
+ )
+ raise PipelexError(msg) from exc
+
+ output_file = output_directory / f"{blueprint.domain}_{concept_code}.py"
+ output_file.write_text(generated_code)
+ generated_files.append((blueprint.domain, concept_code))
+ typer.secho(f" ✅ Generated {output_file.name}", fg=typer.colors.GREEN)
+
+ # Handle concepts with neither structure nor refines - defaults to TextContent
+ else:
+ try:
+ generated_code, _ = StructureGenerator().generate_from_structure_blueprint(
+ class_name=concept_code,
+ structure_blueprint={},
+ base_class_name=TextContent.__name__,
+ )
+ except ConceptStructureGeneratorError as exc:
+ msg = f"Error generating structure class for concept '{concept_code}' in domain '{blueprint.domain}': {exc}"
+ raise PipelexError(msg) from exc
+
+ output_file = output_directory / f"{blueprint.domain}_{concept_code}.py"
+ output_file.write_text(generated_code)
+ generated_files.append((blueprint.domain, concept_code))
+ typer.secho(f" ✅ Generated {output_file.name}", fg=typer.colors.GREEN)
+
+ # Generate empty __init__.py to make structures importable
+ if generated_files:
+ init_file = output_directory / "__init__.py"
+ init_file.write_text("")
+ typer.secho(" ✅ Generated __init__.py", fg=typer.colors.GREEN)
+
+ return generated_files
+
+
+@build_app.command(SUB_COMMAND_STRUCTURES, help="Generate Python structure files from concept definitions in PLX files")
+def build_structures_command(
+ target: Annotated[
+ str,
+ typer.Argument(help="Target directory to scan for PLX files, or a specific .plx file"),
+ ],
+ output_dir: Annotated[
+ str | None,
+ typer.Option("--output-dir", "-o", help="Output directory for generated structures (default: structures/ in target's directory)"),
+ ] = None,
+) -> None:
+ """Generate Python structure files from concept definitions in PLX files."""
+
+ async def _build_structures_cmd():
+ target_path = Path(target).resolve()
+
+ if not target_path.exists():
+ typer.secho(f"❌ Target does not exist: {target_path}", fg=typer.colors.RED, err=True)
+ raise typer.Exit(1)
+
+ # Determine if target is a file or directory
+ is_plx_file = target_path.is_file() and target_path.suffix == ".plx"
+
+ pipelex_instance = Pipelex.make(integration_mode=IntegrationMode.CLI)
+
+ try:
+ if is_plx_file:
+ # Single PLX file: output to parent directory
+ base_dir = target_path.parent
+ output_directory = Path(output_dir) if output_dir else base_dir / "structures"
+
+ typer.echo(f"🔍 Validating bundle: {target_path}")
+
+ # Validate single bundle
+ validate_result = await validate_bundle(plx_file_path=str(target_path))
+ all_blueprints: list[PipelexBundleBlueprint] = validate_result.blueprints
+
+ typer.echo(f"✅ Validated {len(all_blueprints)} blueprint(s)")
+
+ # Generate structures using the helper function
+ generated_files = generate_structures_from_blueprints(
+ blueprints=all_blueprints,
+ output_directory=output_directory,
+ target_path=base_dir,
+ )
+ else:
+ # Directory: scan for all PLX files
+ if not target_path.is_dir():
+ typer.secho(f"❌ Target is not a directory or .plx file: {target_path}", fg=typer.colors.RED, err=True)
+ raise typer.Exit(1)
+
+ output_directory = Path(output_dir) if output_dir else target_path / "structures"
+
+ typer.echo(f"🔍 Validating bundles in: {target_path}")
+
+ # Validate bundles from directory
+ validate_result = await validate_bundles_from_directory(directory=target_path)
+ all_blueprints = validate_result.blueprints
+
+ typer.echo(f"✅ Validated {len(all_blueprints)} blueprint(s)")
+
+ # Generate structures using the helper function
+ generated_files = generate_structures_from_blueprints(
+ blueprints=all_blueprints,
+ output_directory=output_directory,
+ target_path=target_path,
+ )
+
+ if generated_files:
+ typer.secho(f"\n✨ Done! Generated {len(generated_files)} structure(s) in: {output_directory}", fg=typer.colors.GREEN)
+ else:
+ typer.secho("\n✨ Done! No structures to generate.", fg=typer.colors.GREEN)
+
+ finally:
+ pipelex_instance.teardown()
+
+ asyncio.run(_build_structures_cmd())
diff --git a/pipelex/core/bundles/pipelex_bundle_blueprint.py b/pipelex/core/bundles/pipelex_bundle_blueprint.py
index 5e046061b..b35e3b735 100644
--- a/pipelex/core/bundles/pipelex_bundle_blueprint.py
+++ b/pipelex/core/bundles/pipelex_bundle_blueprint.py
@@ -3,6 +3,8 @@ from typing import Annotated
from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
from pipelex.core.concepts.concept_blueprint import ConceptBlueprint
+from pipelex.core.concepts.native.concept_native import NativeConceptCode
+from pipelex.core.concepts.validation import is_concept_code_valid
from pipelex.core.domains.exceptions import DomainCodeError
from pipelex.core.domains.validation import validate_domain_code
from pipelex.pipe_controllers.batch.pipe_batch_blueprint import PipeBatchBlueprint
@@ -14,6 +16,7 @@ from pipelex.pipe_operators.extract.pipe_extract_blueprint import PipeExtractBlu
from pipelex.pipe_operators.func.pipe_func_blueprint import PipeFuncBlueprint
from pipelex.pipe_operators.img_gen.pipe_img_gen_blueprint import PipeImgGenBlueprint
from pipelex.pipe_operators.llm.pipe_llm_blueprint import PipeLLMBlueprint
+from pipelex.urls import URLs
PipeBlueprintUnion = Annotated[
PipeFuncBlueprint
@@ -45,7 +48,6 @@ class PipelexBundleBlueprint(BaseModel):
@field_validator("domain", mode="before")
@classmethod
def validate_domain_syntax(cls, domain: str) -> str:
- # Then validate the domain code format
try:
validate_domain_code(code=domain)
except DomainCodeError as exc:
@@ -53,6 +55,25 @@ class PipelexBundleBlueprint(BaseModel):
raise ValueError(msg) from exc
return domain
+ @field_validator("concept", mode="before")
+ @classmethod
+ def validate_concept_keys(cls, concept: dict[str, ConceptBlueprint | str] | None) -> dict[str, ConceptBlueprint | str] | None:
+ if concept is None:
+ return None
+ native_concept_codes = [native.value for native in NativeConceptCode.values_list()]
+ for concept_code in concept:
+ if not is_concept_code_valid(concept_code=concept_code):
+ msg = f"Concept code '{concept_code}' is not a valid concept code"
+ raise ValueError(msg)
+ if concept_code in native_concept_codes:
+ msg = (
+ f"Cannot declare a concept named '{concept_code}' because it is natively available in Pipelex. "
+ f"Native concepts are: {', '.join(native_concept_codes)}. "
+ f"See {URLs.native_concepts_docs}"
+ )
+ raise ValueError(msg)
+ return concept
+
@model_validator(mode="after")
def validate_main_pipe(self) -> "PipelexBundleBlueprint":
if self.main_pipe and (not self.pipe or (self.main_pipe not in self.pipe)):
diff --git a/pipelex/core/concepts/concept.py b/pipelex/core/concepts/concept.py
index 4f2f33b6b..03c144c61 100644
--- a/pipelex/core/concepts/concept.py
+++ b/pipelex/core/concepts/concept.py
@@ -1,21 +1,23 @@
-import inspect
-from typing import Any, cast, get_args, get_origin
+from typing import Any
from kajson.kajson_manager import KajsonManager
from pydantic import BaseModel, ConfigDict, field_validator
from pipelex import log
from pipelex.base_exceptions import PipelexUnexpectedError
-from pipelex.core.concepts.exceptions import ConceptCodeError, ConceptStringError, ConceptValueError
+from pipelex.core.concepts.concept_representation_generator import (
+ ConceptRepresentationFormat,
+ ConceptRepresentationGenerator,
+)
+from pipelex.core.concepts.exceptions import ConceptCodeError, ConceptValueError
from pipelex.core.concepts.native.concept_native import NativeConceptCode
-from pipelex.core.concepts.validation import validate_concept_code, validate_concept_string
+from pipelex.core.concepts.validation import validate_concept_code
from pipelex.core.domains.exceptions import DomainCodeError
from pipelex.core.domains.validation import validate_domain_code
from pipelex.core.stuffs.image_field_search import search_for_nested_image_fields
from pipelex.core.stuffs.stuff_content import StuffContent
from pipelex.tools.misc.string_utils import pascal_case_to_sentence
from pipelex.tools.typing.class_utils import are_classes_equivalent, has_compatible_field
-from pipelex.types import StrEnum
class Concept(BaseModel):
@@ -25,6 +27,7 @@ class Concept(BaseModel):
domain: str
description: str
structure_class_name: str
+ # TODO: rethink this refines field here.
refines: str | None = None
@field_validator("code")
@@ -52,11 +55,10 @@ class Concept(BaseModel):
def validate_refines(cls, refines: str | None) -> str | None:
if refines is None:
return None
- try:
- validate_concept_string(concept_string=refines)
- except ConceptStringError as exc:
- msg = f"Refines '{refines}' is not a valid concept string for concept '{cls.concept_string}'"
- raise ConceptValueError(msg) from exc
+ if not NativeConceptCode.is_valid_native_concept_string(concept_string=refines):
+ valid_native_concepts = ", ".join(native.concept_string for native in NativeConceptCode.values_list())
+ msg = f"Refines '{refines}' is not a valid native concept string. Valid options are: {valid_native_concepts}"
+ raise ConceptValueError(msg)
return refines
@property
@@ -82,30 +84,37 @@ class Concept(BaseModel):
if concept_1.structure_class_name == concept_2.structure_class_name:
return True
- # If concept_1 refines concept_2, they are strictly compatible
+ # If concept_1 refines concept_2 by string, they are strictly compatible
if concept_1.refines is not None and concept_1.refines == concept_2.concept_string:
return True
- if concept_1.refines is None and concept_2.refines is None:
- concept_1_class = KajsonManager.get_class_registry().get_class(name=concept_1.structure_class_name)
- concept_2_class = KajsonManager.get_class_registry().get_class(name=concept_2.structure_class_name)
+ # If both concepts refine the same concept, they are compatible
+ if concept_1.refines is not None and concept_2.refines is not None and concept_1.refines == concept_2.refines:
+ return True
- if concept_1_class is None or concept_2_class is None:
- return False
+ # Check class-based compatibility
+ # This now works even when one or both concepts have refines, since we generate
+ # structure classes that inherit from the refined concept's class
+ concept_1_class = KajsonManager.get_class_registry().get_class(name=concept_1.structure_class_name)
+ concept_2_class = KajsonManager.get_class_registry().get_class(name=concept_2.structure_class_name)
- if strict:
- # Check if classes are equivalent (same fields, types, descriptions)
- return are_classes_equivalent(concept_1_class, concept_2_class)
- # Check if concept_1 is a subclass of concept_2
- try:
- if issubclass(concept_1_class, concept_2_class):
- return True
- except TypeError:
- pass
+ if concept_1_class is None or concept_2_class is None:
+ return False
- # Check if concept_1 has compatible fields with concept_2
- return has_compatible_field(concept_1_class, concept_2_class)
- return False
+ if strict:
+ # Check if classes are equivalent (same fields, types, descriptions)
+ return are_classes_equivalent(concept_1_class, concept_2_class)
+
+ # Check if concept_1 is a subclass of concept_2
+ # This handles inheritance from refined concepts (e.g., RefusalEmail inherits from TextContent)
+ try:
+ if issubclass(concept_1_class, concept_2_class):
+ return True
+ except TypeError:
+ pass
+
+ # Check if concept_1 has compatible fields with concept_2
+ return has_compatible_field(concept_1_class, concept_2_class)
@classmethod
def is_valid_structure_class(cls, structure_class_name: str) -> bool:
@@ -119,6 +128,18 @@ class Concept(BaseModel):
log.warning(f"Concept class '{structure_class_name}' is registered but it's not a subclass of StuffContent")
return False
+ def get_structure_class(self) -> type[StuffContent]:
+ """Get the structure class for this concept.
+
+ Returns:
+ The StuffContent subclass, or None if not found
+ """
+ structure_class = KajsonManager.get_class_registry().get_class(name=self.structure_class_name)
+ if structure_class is None:
+ msg = f"Concept class '{self.structure_class_name}' not found"
+ raise ConceptValueError(msg)
+ return structure_class
+
def search_for_nested_image_fields_in_structure_class(self) -> list[str]:
"""Recursively search for image fields in a structure class."""
structure_class = KajsonManager.get_class_registry().get_required_subclass(name=self.structure_class_name, base_class=StuffContent)
@@ -127,159 +148,31 @@ class Concept(BaseModel):
raise PipelexUnexpectedError(msg)
return search_for_nested_image_fields(content_class=structure_class)
- def get_compact_memory_example(self, var_name: str) -> dict[str, Any] | str | int:
- """Generate an example value for compact memory format based on this concept.
-
- Compact memory follows these conventions:
- - For native concepts that can be represented as simple values (Text, Image, PDF): returns a simple string
- - For structured concepts: returns {"concept_code": "...", "content": {...}}
-
- The content dict is recursively generated based on the StuffContent class structure.
- """
- # Get the structure class
- structure_class = KajsonManager.get_class_registry().get_class(name=self.structure_class_name)
-
- # If class not found, return placeholder
- if structure_class is None:
- return {
- "concept_code": self.concept_string,
- "content": {}, # Empty dict for unknown structures
- }
-
- # Verify it's a subclass of StuffContent
- if not issubclass(structure_class, StuffContent):
- return {
- "concept_code": self.concept_string,
- "content": {}, # Empty dict for invalid structures
- }
-
- # Generate the content based on structure
- content_example = self._generate_content_example_for_class(structure_class, var_name)
-
- # Check if this is actually a native concept (not just using a native structure class)
- is_native = Concept.is_native_concept(self)
-
- # For simple native concepts ONLY - return compact format
- if is_native and self.structure_class_name == "TextContent":
- return cast("str", content_example) # Just a string
- elif is_native and self.structure_class_name == "ImageContent":
- # Return dict with class instantiation info
- return {
- "_class": "ImageContent",
- "url": cast("str", content_example),
- }
- elif is_native and self.structure_class_name == "PDFContent":
- # Return dict with class instantiation info
- return {
- "_class": "PDFContent",
- "url": cast("str", content_example),
- }
- elif is_native and self.structure_class_name == "NumberContent":
- return cast("int", content_example) # Just a number
-
- # For refined or complex concepts, wrap with concept_code
- # For Image/PDF content, wrap in the _class format
- if self.structure_class_name == "ImageContent":
- content_wrapped = {
- "_class": "ImageContent",
- "url": cast("str", content_example),
- }
- elif self.structure_class_name == "PDFContent":
- content_wrapped = {
- "_class": "PDFContent",
- "url": cast("str", content_example),
- }
- else:
- content_wrapped = content_example
-
- return {
- "concept_code": self.concept_string,
- "content": content_wrapped,
- }
-
- @classmethod
- def _generate_content_example_for_class(cls, content_class: type[StuffContent], var_name: str) -> Any:
- """Recursively generate example content based on a StuffContent class structure.
+ def generate_input_representation(
+ self,
+ output_format: ConceptRepresentationFormat,
+ is_multiple: bool = False,
+ ) -> tuple[dict[str, Any], set[str]]:
+ """Generate a representation for this concept's input.
Args:
- content_class: The StuffContent class to generate an example for
- var_name: Variable name for generating contextual example values
+ output_format: The format to generate (JSON or PYTHON)
+ is_multiple: If True, wrap content in a list (only for JSON format)
Returns:
- Example content dict or simple value
+ Tuple of (representation dict, imports_needed set)
+ - For JSON: content is a dict (or list of dicts if is_multiple)
+ - For Python: content is a class instantiation string (wrapping handled by caller)
"""
- class_name = content_class.__name__
-
- # Handle simple native content types
- if class_name == "TextContent":
- return f"{var_name}_text"
- elif class_name in {"ImageContent", "PDFContent"}:
- return f"{var_name}_url"
- elif class_name == "NumberContent":
- return 0
-
- # For structured content, inspect fields and recursively generate
- # Note: model_fields includes inherited fields from parent classes
- content_dict: dict[str, Any] = {}
- for field_name, field_info in content_class.model_fields.items():
- field_type = field_info.annotation
-
- # Handle Optional types (e.g., TextContent | None)
- origin = get_origin(field_type)
- args = get_args(field_type)
-
- if origin is type(None) or (args and type(None) in args):
- # Optional field - get the non-None type
- actual_type = next((arg for arg in args if arg is not type(None)), field_type) if args else field_type
- else:
- actual_type = field_type
+ structure_class = self.get_structure_class()
- # Re-check origin after unwrapping Optional
- origin = get_origin(actual_type)
- args = get_args(actual_type)
+ generator = ConceptRepresentationGenerator(output_format)
+ # For inputs, we only want required fields (not optional ones)
+ result = generator.generate_representation(self.concept_string, structure_class, include_optional=False)
- # Handle list types
- if origin is list:
- list_item_type = args[0] if args else str
- if hasattr(list_item_type, "__name__"):
- if list_item_type.__name__ == "ImageContent":
- content_dict[field_name] = [f"{field_name}_url_1", f"{field_name}_url_2"]
- elif list_item_type.__name__ == "TextContent":
- content_dict[field_name] = [f"{field_name}_text_1", f"{field_name}_text_2"]
- elif inspect.isclass(list_item_type) and issubclass(list_item_type, StuffContent):
- # List of StuffContent - generate examples
- content_dict[field_name] = [cls._generate_content_example_for_class(list_item_type, f"{field_name}_item")]
- else:
- content_dict[field_name] = [f"{field_name}_item_1"]
- else:
- content_dict[field_name] = []
- # Handle dict types
- elif origin is dict:
- # Simple example with one key-value pair
- content_dict[field_name] = {f"{field_name}_key": f"{field_name}_value"}
- # Handle StrEnum types
- elif inspect.isclass(actual_type) and issubclass(actual_type, StrEnum):
- # Get first enum value
- enum_values = list(actual_type)
- content_dict[field_name] = enum_values[0].value if enum_values else f"{field_name}_enum_value"
- # Handle nested StuffContent
- elif inspect.isclass(actual_type) and issubclass(actual_type, StuffContent):
- content_dict[field_name] = cls._generate_content_example_for_class(actual_type, field_name)
- # Handle basic types
- elif actual_type is str:
- content_dict[field_name] = f"{field_name}_value"
- elif actual_type is int:
- content_dict[field_name] = 0
- elif actual_type is float:
- content_dict[field_name] = 0.0
- elif actual_type is bool:
- content_dict[field_name] = False
- else:
- # For unknown types, try to get a simple repr
- try:
- type_name = getattr(actual_type, "__name__", str(actual_type))
- content_dict[field_name] = f"{field_name}_value # TODO: Fill {type_name}"
- except Exception:
- content_dict[field_name] = f"{field_name}_value"
+ # If multiple and JSON format, wrap content in a list
+ # For Python format, the caller handles wrapping since content is a string
+ if is_multiple and output_format == ConceptRepresentationFormat.JSON:
+ result["content"] = [result["content"]]
- return content_dict
+ return result, generator.imports_needed
diff --git a/pipelex/core/concepts/concept_blueprint.py b/pipelex/core/concepts/concept_blueprint.py
index 38cd4140a..f2753b6f9 100644
--- a/pipelex/core/concepts/concept_blueprint.py
+++ b/pipelex/core/concepts/concept_blueprint.py
@@ -64,11 +64,13 @@ class ConceptBlueprint(BaseModel):
@model_validator(mode="before")
@classmethod
- def validate_blueprint(cls, values: dict[str, Any] | str) -> dict[str, Any] | str:
+ def validate_mutually_exclusive_fields(cls, values: dict[str, Any] | str) -> dict[str, Any] | str:
+ """Validate that refines and structure are not both set."""
if isinstance(values, dict) and values.get("refines") and values.get("structure"):
msg = (
- f"Forbidden to have refines and structure at the same time: `{values.get('refines')}` "
- f"and `{values.get('structure')}` for concept that has the definition `{values.get('description')}`"
+ f"A concept cannot have both 'refines' and 'structure'. "
+ f"Got refines='{values.get('refines')}' and structure='{values.get('structure')}'. "
+ f"Concept description: '{values.get('description')}'"
)
raise ValueError(msg)
return values
diff --git a/pipelex/core/concepts/concept_factory.py b/pipelex/core/concepts/concept_factory.py
index e966c5e8e..742d83420 100644
--- a/pipelex/core/concepts/concept_factory.py
+++ b/pipelex/core/concepts/concept_factory.py
@@ -1,56 +1,75 @@
+from typing import cast
+
from kajson.kajson_manager import KajsonManager
from pydantic import BaseModel
from pipelex.core.concepts.concept import Concept
from pipelex.core.concepts.concept_blueprint import ConceptBlueprint
-from pipelex.core.concepts.concept_structure_blueprint import ConceptStructureBlueprint, ConceptStructureBlueprintFieldType
from pipelex.core.concepts.exceptions import (
ConceptFactoryError,
ConceptRefineError,
ConceptStringError,
)
+from pipelex.core.concepts.helpers import normalize_structure_blueprint
from pipelex.core.concepts.native.concept_native import NativeConceptCode
from pipelex.core.concepts.structure_generation.exceptions import ConceptStructureGeneratorError
from pipelex.core.concepts.structure_generation.generator import StructureGenerator
-from pipelex.core.concepts.validation import is_concept_code_valid, validate_concept_string_or_code
+from pipelex.core.concepts.validation import validate_concept_string_or_code
from pipelex.core.domains.domain import SpecialDomain
from pipelex.core.stuffs.text_content import TextContent
+from pipelex.types import StrEnum
-class DomainAndConceptCode(BaseModel):
- """Small model to represent domain and concept code pair."""
+class ConceptDeclarationType(StrEnum):
+ """Enum representing the 5 ways a concept can be declared in PLX files.
- domain: str
- concept_code: str
+ Option 1: STRING - Concept is defined as a string
+ Example:
+ [concept]
+ Concept1 = "Definition of Concept1"
+ Option 2: BASIC_BLUEPRINT - Concept is defined with a basic blueprint
+ Example:
+ [concept.Concept2]
+ description = "Definition of Concept2"
-class ConceptFactory:
- @classmethod
- def normalize_structure_blueprint(cls, structure_dict: dict[str, str | ConceptStructureBlueprint]) -> dict[str, ConceptStructureBlueprint]:
- """Convert a mixed structure dictionary to a proper ConceptStructureBlueprint dictionary.
+ Option 3: REFINES - Concept refines another concept
+ Example:
+ [concept.Concept3]
+ description = "A concept3"
+ refines = "native.Text"
- Args:
- structure_dict: Dictionary that may contain strings or ConceptStructureBlueprint objects
+ Option 4: STRUCTURE_WITH_CLASSNAME
+ Example:
+ [concept.Concept4]
+ description = "A concept4"
+ structure = "ExistingClassName"
- Returns:
- Dictionary with all values as ConceptStructureBlueprint objects
+ Option 5: BLUEPRINT_WITH_STRUCTURE - Concept is defined with a blueprint and a structure
+ Example:
+ [concept.Concept5]
+ description = "A concept5"
+
+ [concept.Concept5.structure]
+ field1 = "A field1"
+ field2 = {type = "text", description = "A field2"}
+ """
+
+ STRING = "string"
+ BASIC_BLUEPRINT = "basic_blueprint"
+ REFINES = "refines"
+ STRUCTURE_WITH_CLASSNAME = "structure_with_classname"
+ BLUEPRINT_WITH_STRUCTURE = "blueprint_with_structure"
- """
- normalized: dict[str, ConceptStructureBlueprint] = {}
-
- for field_name, field_value in structure_dict.items():
- if isinstance(field_value, str):
- # Convert string definition to ConceptStructureBlueprint for text field
- normalized[field_name] = ConceptStructureBlueprint(
- description=field_value,
- type=ConceptStructureBlueprintFieldType.TEXT, # Explicitly set as text field
- required=True, # Default for simple string definitions
- )
- else:
- normalized[field_name] = field_value
- return normalized
+class DomainAndConceptCode(BaseModel):
+ """Small model to represent domain and concept code pair."""
+ domain: str
+ concept_code: str
+
+
+class ConceptFactory:
@classmethod
def make(cls, concept_code: str, domain: str, description: str, structure_class_name: str, refines: str | None = None) -> Concept:
return Concept(
@@ -136,10 +155,6 @@ class ConceptFactory:
structure_class_name=structure_class_name,
)
- @classmethod
- def make_all_native_concepts(cls) -> list[Concept]:
- return [cls.make_native_concept(native_concept_code=native_concept) for native_concept in NativeConceptCode.values_list()]
-
@classmethod
def make_domain_and_concept_code_from_concept_string_or_code(
cls,
@@ -204,102 +219,240 @@ class ConceptFactory:
return NativeConceptCode.get_validated_native_concept_string(concept_string_or_code=refine)
@classmethod
- def make_from_blueprint_or_description(
+ def _handle_structure_with_classname(
cls,
+ blueprint: ConceptBlueprint,
+ concept_code: str,
domain: str,
+ ) -> str:
+ """Handle STRUCTURE_WITH_CLASSNAME declaration type.
+
+ Structure is defined as a string class name - check if the class is in the registry and is valid.
+
+ Returns:
+ The structure class name
+ """
+ structure_class_name = blueprint.structure
+ if not isinstance(structure_class_name, str):
+ msg = f"Expected structure to be a string, got {type(structure_class_name)}"
+ raise ConceptFactoryError(msg)
+ if not Concept.is_valid_structure_class(structure_class_name=structure_class_name):
+ msg = (
+ f"Structure class '{structure_class_name}' set for concept '{concept_code}' in domain '{domain}' "
+ "is not a registered subclass of StuffContent, or was not found in the library."
+ )
+ raise ConceptFactoryError(msg)
+ return structure_class_name
+
+ @classmethod
+ def _handle_blueprint_with_structure(
+ cls,
+ blueprint: ConceptBlueprint,
concept_code: str,
- concept_blueprint_or_description: ConceptBlueprint | str,
- ) -> Concept:
- blueprint: ConceptBlueprint
- if isinstance(concept_blueprint_or_description, str):
- blueprint = ConceptBlueprint(description=concept_blueprint_or_description)
- else:
- blueprint = concept_blueprint_or_description
- return cls.make_from_blueprint(
- domain=domain,
- concept_code=concept_code,
- blueprint=blueprint,
- )
+ domain: str,
+ ) -> str:
+ """Handle BLUEPRINT_WITH_STRUCTURE declaration type.
+
+ Structure is defined as a ConceptStructureBlueprint dict - run the structure generator
+ and register it in the class registry.
+
+ Returns:
+ The structure class name (which is the concept_code)
+ """
+ if not isinstance(blueprint.structure, dict):
+ msg = f"Expected structure to be a dict, got {type(blueprint.structure)}"
+ raise ConceptFactoryError(msg)
+
+ # Normalize the structure blueprint to ensure all values are ConceptStructureBlueprint objects
+ normalized_structure = normalize_structure_blueprint(blueprint.structure)
+
+ try:
+ _, the_generated_class = StructureGenerator().generate_from_structure_blueprint(
+ class_name=concept_code,
+ structure_blueprint=normalized_structure,
+ )
+ except ConceptStructureGeneratorError as exc:
+ msg = f"Error generating python code for structure class of concept '{concept_code}' in domain '{domain}': {exc}"
+ raise ConceptFactoryError(msg) from exc
+
+ # Register the generated class
+ KajsonManager.get_class_registry().register_class(the_generated_class)
+
+ return concept_code
+
+ @classmethod
+ def _handle_basic_blueprint(
+ cls,
+ concept_code: str,
+ domain: str,
+ ) -> tuple[str, str | None]:
+ """Handle BASIC_BLUEPRINT declaration type.
+
+ Returns:
+ Tuple of (structure_class_name, refine_string)
+ """
+ # Generate a new class that inherits from TextContent and register it
+ # (unless a valid structure class already exists for this concept_code)
+ if Concept.is_valid_structure_class(structure_class_name=concept_code):
+ return concept_code, None
+
+ # Because native concepts have structure class names diffrent than other (with "Content")
+ if concept_code in NativeConceptCode.values_list():
+ return NativeConceptCode.TEXT.structure_class_name, NativeConceptCode.TEXT.concept_string
+
+ try:
+ _, the_generated_class = StructureGenerator().generate_from_structure_blueprint(
+ class_name=concept_code,
+ structure_blueprint={},
+ base_class_name=TextContent.__name__,
+ )
+ except ConceptStructureGeneratorError as exc:
+ msg = f"Error generating structure class for concept '{concept_code}' in domain '{domain}': {exc}"
+ raise ConceptFactoryError(msg) from exc
+ # Register the generated class
+ KajsonManager.get_class_registry().register_class(the_generated_class)
+
+ return concept_code, NativeConceptCode.TEXT.concept_string
+
+ @classmethod
+ def _handle_refines(
+ cls,
+ blueprint: ConceptBlueprint,
+ concept_code: str,
+ domain: str,
+ ) -> tuple[str, str]:
+ """Handle REFINES declaration type.
+
+ Concept refines another concept - generate a new class that inherits from the refined
+ structure class.
+
+ Returns:
+ Tuple of (structure_class_name, refine_string)
+ """
+ if blueprint.refines is None:
+ msg = "Expected refines to be set"
+ raise ConceptFactoryError(msg)
+
+ try:
+ current_refine = cls.make_refine(refine=blueprint.refines)
+ except ConceptRefineError as exc:
+ msg = f"Could not validate refine '{blueprint.refines}' for concept '{concept_code}' in domain '{domain}': {exc}"
+ raise ConceptFactoryError(msg) from exc
+
+ # Get the refined concept's structure class name
+ refined_structure_class_name = current_refine.split(".")[1] + "Content"
+
+ # Generate a new class that inherits from the refined structure class
+ # This creates an empty class that can be extended with additional fields in the future
+ try:
+ _, the_generated_class = StructureGenerator().generate_from_structure_blueprint(
+ class_name=concept_code,
+ structure_blueprint={}, # Empty structure - just inherits from refined class
+ base_class_name=refined_structure_class_name,
+ )
+ except ConceptStructureGeneratorError as exc:
+ msg = (
+ f"Error generating structure class for concept '{concept_code}' refining '{refined_structure_class_name}' in domain '{domain}': {exc}"
+ )
+ raise ConceptFactoryError(msg) from exc
+
+ # Register the generated class
+ KajsonManager.get_class_registry().register_class(the_generated_class)
+
+ return concept_code, current_refine
@classmethod
def make_from_blueprint(
cls,
domain: str,
concept_code: str,
- blueprint: ConceptBlueprint,
+ blueprint_or_string_description: ConceptBlueprint | str,
) -> Concept:
- if not is_concept_code_valid(concept_code=concept_code):
- msg = f"Concept code '{concept_code}' is not a valid concept code"
- raise ConceptFactoryError(msg)
- structure_class_name: str
- current_refine: str | None = None
-
- # Handle structure definition
- if blueprint.structure:
- if isinstance(blueprint.structure, str):
- # Structure is defined as a string - check if the class is in the registry and is valid
- if not Concept.is_valid_structure_class(structure_class_name=blueprint.structure):
- msg = (
- f"Structure class '{blueprint.structure}' set for concept '{concept_code}' in domain '{domain}' "
- "is not a registered subclass of StuffContent"
- )
- raise ConceptFactoryError(msg)
- structure_class_name = blueprint.structure
+ # Determine declaration type
+ declaration_type: ConceptDeclarationType
+ if isinstance(blueprint_or_string_description, str):
+ declaration_type = ConceptDeclarationType.STRING
+ elif blueprint_or_string_description.structure is not None:
+ if isinstance(blueprint_or_string_description.structure, str):
+ declaration_type = ConceptDeclarationType.STRUCTURE_WITH_CLASSNAME
else:
- # Structure is defined as a ConceptStructureBlueprint - run the structure generator and put it in the class registry
- # Normalize the structure blueprint to ensure all values are ConceptStructureBlueprint objects
- normalized_structure = cls.normalize_structure_blueprint(blueprint.structure)
-
- try:
- _, the_generated_class = StructureGenerator().generate_from_structure_blueprint(
- class_name=concept_code,
- structure_blueprint=normalized_structure,
- )
- except ConceptStructureGeneratorError as exc:
- msg = f"Error generating python code for structure class of concept '{concept_code}' in domain '{domain}': {exc}"
- raise ConceptFactoryError(
- msg,
- ) from exc
-
- # Register the generated class
- KajsonManager.get_class_registry().register_class(the_generated_class)
-
- # The structure_class_name of the concept is the concept_code
- structure_class_name = concept_code
-
- # Handle refines definition
- elif blueprint.refines:
- # If we have refines, validate that there is not already a structure related to this concept code in the class registry
- # TODO: This test should NOT BE HERE, but in the `validate_with_libraries`.
- if Concept.is_valid_structure_class(structure_class_name=concept_code):
- msg = (
- f"Concept '{concept_code}' in domain '{domain}' has refines but also has a structure class registered. "
- "A concept cannot have both structure and refines."
- )
- raise ConceptFactoryError(msg)
- try:
- current_refine = cls.make_refine(refine=blueprint.refines)
- except ConceptRefineError as exc:
- msg = f"Could not validate refine '{blueprint.refines}' for concept '{concept_code}' in domain '{domain}': {exc}"
- raise ConceptFactoryError(msg) from exc
- structure_class_name = current_refine.split(".")[1] + "Content" if current_refine else TextContent.__name__
- # Handle neither structure nor refines - check the class registry
- # If there is a class, use it. structure_class_name is then the concept_code
- elif Concept.is_valid_structure_class(structure_class_name=concept_code):
- structure_class_name = concept_code
+ declaration_type = ConceptDeclarationType.BLUEPRINT_WITH_STRUCTURE
+ elif blueprint_or_string_description.refines is not None:
+ declaration_type = ConceptDeclarationType.REFINES
else:
- # If there is NO class, the fallback class is TextContent.__name__
- structure_class_name = TextContent.__name__
+ declaration_type = ConceptDeclarationType.BASIC_BLUEPRINT
domain_and_concept_code = cls.make_domain_and_concept_code_from_concept_string_or_code(
concept_string_or_code=concept_code,
domain=domain,
)
- return Concept(
- domain=domain_and_concept_code.domain,
- code=domain_and_concept_code.concept_code,
- description=blueprint.description,
- structure_class_name=structure_class_name,
- refines=current_refine,
- )
+ match declaration_type:
+ case ConceptDeclarationType.STRING:
+ structure_class_name, refines = cls._handle_basic_blueprint(
+ concept_code=concept_code,
+ domain=domain,
+ )
+ return Concept(
+ domain=domain_and_concept_code.domain,
+ code=domain_and_concept_code.concept_code,
+ description=cast("str", blueprint_or_string_description),
+ structure_class_name=structure_class_name,
+ refines=refines,
+ )
+
+ case ConceptDeclarationType.BASIC_BLUEPRINT:
+ structure_class_name, refines = cls._handle_basic_blueprint(
+ concept_code=concept_code,
+ domain=domain,
+ )
+ return Concept(
+ domain=domain_and_concept_code.domain,
+ code=domain_and_concept_code.concept_code,
+ description=cast("ConceptBlueprint", blueprint_or_string_description).description,
+ structure_class_name=structure_class_name,
+ refines=refines,
+ )
+
+ case ConceptDeclarationType.STRUCTURE_WITH_CLASSNAME:
+ blueprint = cast("ConceptBlueprint", blueprint_or_string_description)
+ return Concept(
+ domain=domain_and_concept_code.domain,
+ code=domain_and_concept_code.concept_code,
+ description=blueprint.description,
+ structure_class_name=cls._handle_structure_with_classname(
+ blueprint=blueprint,
+ concept_code=concept_code,
+ domain=domain,
+ ),
+ refines=None,
+ )
+
+ case ConceptDeclarationType.BLUEPRINT_WITH_STRUCTURE:
+ blueprint = cast("ConceptBlueprint", blueprint_or_string_description)
+ return Concept(
+ domain=domain_and_concept_code.domain,
+ code=domain_and_concept_code.concept_code,
+ description=blueprint.description,
+ structure_class_name=cls._handle_blueprint_with_structure(
+ blueprint=blueprint,
+ concept_code=concept_code,
+ domain=domain,
+ ),
+ refines=None,
+ )
+
+ case ConceptDeclarationType.REFINES:
+ blueprint = cast("ConceptBlueprint", blueprint_or_string_description)
+ structure_class_name, current_refine = cls._handle_refines(
+ blueprint=blueprint,
+ concept_code=concept_code,
+ domain=domain,
+ )
+ return Concept(
+ domain=domain_and_concept_code.domain,
+ code=domain_and_concept_code.concept_code,
+ description=blueprint.description,
+ structure_class_name=structure_class_name,
+ refines=current_refine,
+ )
diff --git a/pipelex/core/concepts/concept_representation_generator.py b/pipelex/core/concepts/concept_representation_generator.py
new file mode 100644
index 000000000..857ec6a6e
--- /dev/null
+++ b/pipelex/core/concepts/concept_representation_generator.py
@@ -0,0 +1,379 @@
+"""Generator for concept representation values in JSON and Python formats.
+
+This module provides recursive generation of example representations for concept structures.
+It supports two output formats: JSON (dict) and Python (class instantiation strings).
+"""
+
+import inspect
+from typing import Any, cast, get_args, get_origin
+
+from pydantic import BaseModel
+
+from pipelex.core.stuffs.stuff_content import StuffContent
+from pipelex.types import StrEnum
+
+
+class ConceptRepresentationFormat(StrEnum):
+ """Output format for concept representations."""
+
+ JSON = "json"
+ PYTHON = "python"
+
+
+class ConceptRepresentationGenerator:
+ """Generates representation values for concepts in different formats.
+
+ Supports two output formats (JSON and Python).
+ The generation is recursive: nested StuffContent classes are fully expanded.
+
+ JSON format:
+ {"concept": "domain.ConceptCode", "content": {"field1": "value1", ...}}
+
+ Python format:
+ {"concept": "domain.ConceptCode", "content": "MyClass(field1='value1', ...)"}
+ """
+
+ def __init__(self, output_format: ConceptRepresentationFormat):
+ self.output_format = output_format
+ self._imports_needed: set[str] = set()
+
+ @property
+ def imports_needed(self) -> set[str]:
+ """Returns the set of class names that need to be imported."""
+ return self._imports_needed
+
+ def generate_representation(
+ self,
+ concept_string: str,
+ structure_class: type[StuffContent],
+ include_optional: bool = True,
+ ) -> dict[str, Any]:
+ """Generate a representation value for a concept.
+
+ This is the main entry point. It wraps the generated content with concept.
+
+ Args:
+ concept_string: The concept string (e.g., "domain.ConceptCode")
+ structure_class: The StuffContent class to generate representation for
+ include_optional: If False, exclude fields with default values (optional fields)
+
+ Returns:
+ Dict with concept and content keys
+ """
+ self._imports_needed.clear()
+
+ content = self.generate_class_representation(structure_class, include_optional=include_optional)
+
+ return {"concept": concept_string, "content": content}
+
+ def generate_class_representation(
+ self,
+ content_class: type[StuffContent],
+ include_optional: bool = True,
+ ) -> dict[str, Any] | str:
+ """Generate representation for a StuffContent class (recursive).
+
+ For JSON format: returns a dict with all fields
+ For Python format: returns a string like "ClassName(field1=..., field2=...)"
+
+ Args:
+ content_class: The StuffContent class to generate representation for
+ include_optional: If False, exclude fields with default values (optional fields)
+
+ Returns:
+ Dict (JSON) or string (Python) representing the class
+ """
+ class_name = content_class.__name__
+ self._imports_needed.add(class_name)
+
+ fields_dict = self._generate_fields_dict(content_class, include_optional=include_optional)
+
+ match self.output_format:
+ case ConceptRepresentationFormat.JSON:
+ return fields_dict
+ case ConceptRepresentationFormat.PYTHON:
+ return self._format_as_python(class_name, fields_dict)
+
+ def _generate_fields_dict(
+ self,
+ content_class: type[StuffContent],
+ include_optional: bool = True,
+ ) -> dict[str, Any]:
+ """Generate a dict with field values for a class.
+
+ Args:
+ content_class: The class to generate fields for
+ include_optional: If False, exclude fields with default values (optional fields)
+
+ Returns:
+ Dict mapping field names to their generated values
+ """
+ fields_dict: dict[str, Any] = {}
+
+ for field_name, field_info in content_class.model_fields.items():
+ # Skip optional fields if include_optional is False
+ if not include_optional and not field_info.is_required():
+ continue
+
+ field_type = field_info.annotation
+ field_value = self.generate_field_value(field_type, field_name)
+ fields_dict[field_name] = field_value
+
+ return fields_dict
+
+ def generate_field_value(self, field_type: Any, field_name: str) -> Any:
+ """Generate a representation value for a field based on its type (recursive).
+
+ Handles:
+ - Basic types (str, int, float, bool)
+ - StrEnum types (returns first enum value)
+ - list[T] types (recursively generates item)
+ - dict types (generates placeholder)
+ - Nested StuffContent/BaseModel classes (recursive call)
+
+ Args:
+ field_type: The type annotation of the field
+ field_name: Name of the field (used for generating placeholder values)
+
+ Returns:
+ Generated value appropriate for the field type
+ """
+ actual_type = self._unwrap_optional(field_type)
+ origin = get_origin(actual_type)
+ args = get_args(actual_type)
+
+ # Handle list types
+ if origin is list:
+ return self._generate_list_value(args, field_name)
+
+ # Handle dict types
+ if origin is dict:
+ return self._generate_dict_value(field_name)
+
+ # Handle StrEnum types
+ if inspect.isclass(actual_type) and issubclass(actual_type, StrEnum):
+ return self._generate_enum_value(actual_type, field_name)
+
+ # Handle nested StuffContent (recursive)
+ if inspect.isclass(actual_type) and issubclass(actual_type, StuffContent):
+ return self.generate_class_representation(actual_type)
+
+ # Handle nested BaseModel that is NOT StuffContent
+ if inspect.isclass(actual_type) and issubclass(actual_type, BaseModel):
+ return self._generate_basemodel_representation(actual_type)
+
+ # Handle basic types
+ return self._generate_basic_value(actual_type, field_name)
+
+ def _unwrap_optional(self, field_type: Any) -> Any:
+ """Unwrap Optional[T] to get T.
+
+ Args:
+ field_type: The type to unwrap
+
+ Returns:
+ The inner type if Optional, otherwise the original type
+ """
+ origin = get_origin(field_type)
+ args = get_args(field_type)
+
+ if origin is type(None) or (args and type(None) in args):
+ return next((arg for arg in args if arg is not type(None)), field_type) if args else field_type
+ return field_type
+
+ def _generate_list_value(self, args: tuple[Any, ...], field_name: str) -> list[Any]:
+ """Generate a list value with one example item.
+
+ Args:
+ args: Type arguments of the list (e.g., (str,) for list[str])
+ field_name: Name of the field
+
+ Returns:
+ List with one generated item
+ """
+ if not args:
+ return [f"{field_name}_item"]
+
+ item_type = args[0]
+
+ # Handle list of StuffContent (recursive)
+ if inspect.isclass(item_type) and issubclass(item_type, StuffContent):
+ item_repr = self.generate_class_representation(item_type)
+ return [item_repr]
+
+ # Handle list of BaseModel (not StuffContent)
+ if inspect.isclass(item_type) and issubclass(item_type, BaseModel):
+ item_repr = self._generate_basemodel_representation(item_type)
+ return [item_repr]
+
+ # Handle list of basic types
+ return [self._generate_basic_value(item_type, f"{field_name}_item")]
+
+ def _generate_dict_value(self, field_name: str) -> dict[str, str]:
+ """Generate a placeholder dict value.
+
+ Args:
+ field_name: Name of the field
+
+ Returns:
+ Dict with placeholder key/value
+ """
+ return {f"{field_name}_key": f"{field_name}_value"}
+
+ def _generate_enum_value(self, enum_type: type[StrEnum], field_name: str) -> str:
+ """Generate a value from a StrEnum type.
+
+ Args:
+ enum_type: The StrEnum class
+ field_name: Name of the field (used as fallback)
+
+ Returns:
+ First enum value or placeholder if empty
+ """
+ enum_values = list(enum_type)
+ return enum_values[0].value if enum_values else f"{field_name}_enum"
+
+ def _generate_basemodel_representation(self, model_class: type[BaseModel]) -> dict[str, Any] | str:
+ """Generate representation for a BaseModel that is not StuffContent.
+
+ Args:
+ model_class: The BaseModel class
+
+ Returns:
+ Dict (JSON) or string (Python)
+ """
+ class_name = model_class.__name__
+ self._imports_needed.add(class_name)
+
+ fields_dict: dict[str, Any] = {}
+ for field_name, field_info in model_class.model_fields.items():
+ field_type = field_info.annotation
+ field_value = self.generate_field_value(field_type, field_name)
+ fields_dict[field_name] = field_value
+
+ match self.output_format:
+ case ConceptRepresentationFormat.JSON:
+ return fields_dict
+ case ConceptRepresentationFormat.PYTHON:
+ return self._format_as_python(class_name, fields_dict)
+
+ def _generate_basic_value(self, actual_type: Any, field_name: str) -> Any:
+ """Generate a value for basic Python types.
+
+ Args:
+ actual_type: The type (str, int, float, bool, or unknown)
+ field_name: Name of the field (used for generating placeholder)
+
+ Returns:
+ Appropriate placeholder value for the type
+ """
+ if actual_type is str:
+ return f"{field_name}_value"
+ elif actual_type is int:
+ return 0
+ elif actual_type is float:
+ return 0.0
+ elif actual_type is bool:
+ return False
+ else:
+ type_name = getattr(actual_type, "__name__", str(actual_type))
+ return f"{field_name}_{type_name}"
+
+ def _format_as_python(self, class_name: str, fields: dict[str, Any]) -> str:
+ """Format a class instantiation as Python code string.
+
+ Args:
+ class_name: Name of the class
+ fields: Dict of field names to values
+
+ Returns:
+ String like "ClassName(field1=..., field2=...)"
+ """
+ args_parts: list[str] = []
+ for field_name, value in fields.items():
+ formatted_value = self._format_python_value(value)
+ args_parts.append(f"{field_name}={formatted_value}")
+
+ args = ", ".join(args_parts)
+ return f"{class_name}({args})"
+
+ def _format_python_value(self, value: Any) -> str:
+ """Format a value for Python code representation.
+
+ Args:
+ value: The value to format
+
+ Returns:
+ Python code string representation
+
+ Examples:
+ >>> self._format_python_value("hello")
+ '"hello"'
+ >>> self._format_python_value("MyClass(arg=value)") # Already a Python expression
+ 'MyClass(arg=value)'
+ >>> self._format_python_value(True)
+ 'True'
+ >>> self._format_python_value(42)
+ '42'
+ >>> self._format_python_value(3.14)
+ '3.14'
+ >>> self._format_python_value({"key": "val"})
+ '{"key": "val"}'
+ >>> self._format_python_value([1, 2, 3])
+ '[1, 2, 3]'
+ """
+ if isinstance(value, str):
+ # Check if it's already a Python expression (class instantiation)
+ if "(" in value and ")" in value and "=" in value:
+ return value
+ return f'"{value}"'
+ elif isinstance(value, (bool, int, float)):
+ return str(value)
+ elif isinstance(value, dict):
+ dict_value = cast("dict[str, Any]", value)
+ items = ", ".join(f"{self._format_python_value(key)}: {self._format_python_value(val)}" for key, val in dict_value.items())
+ return "{" + items + "}"
+ elif isinstance(value, list):
+ list_value = cast("list[Any]", value)
+ items = ", ".join(self._format_python_value(item) for item in list_value)
+ return "[" + items + "]"
+ else:
+ return str(value)
+
+
+# Convenience functions
+
+
+def generate_json_representation(
+ concept_string: str,
+ structure_class: type[StuffContent],
+) -> dict[str, Any]:
+ """Convenience function to generate a JSON format representation.
+
+ Args:
+ concept_string: The concept string (e.g., "domain.ConceptCode")
+ structure_class: The StuffContent class
+
+ Returns:
+ Dict with concept and content
+ """
+ generator = ConceptRepresentationGenerator(ConceptRepresentationFormat.JSON)
+ return generator.generate_representation(concept_string, structure_class)
+
+
+def generate_python_representation(
+ concept_string: str,
+ structure_class: type[StuffContent],
+) -> tuple[dict[str, Any], set[str]]:
+ """Convenience function to generate a Python format representation.
+
+ Args:
+ concept_string: The concept string (e.g., "domain.ConceptCode")
+ structure_class: The StuffContent class
+
+ Returns:
+ Tuple of (representation dict, imports_needed set)
+ """
+ generator = ConceptRepresentationGenerator(ConceptRepresentationFormat.PYTHON)
+ representation = generator.generate_representation(concept_string, structure_class)
+ return representation, generator.imports_needed
diff --git a/pipelex/core/concepts/helpers.py b/pipelex/core/concepts/helpers.py
index 3945fb5c8..a7f48a0ef 100644
--- a/pipelex/core/concepts/helpers.py
+++ b/pipelex/core/concepts/helpers.py
@@ -1,3 +1,6 @@
+from pipelex.core.concepts.concept_structure_blueprint import ConceptStructureBlueprint, ConceptStructureBlueprintFieldType
+
+
def strip_multiplicity_from_concept_string_or_code(concept_string_or_code: str) -> str:
"""Strip multiplicity from a concept string or code.
@@ -10,3 +13,26 @@ def strip_multiplicity_from_concept_string_or_code(concept_string_or_code: str)
if "[" in concept_string_or_code:
return concept_string_or_code.split("[", maxsplit=1)[0]
return concept_string_or_code
+
+
+def normalize_structure_blueprint(structure_dict: dict[str, str | ConceptStructureBlueprint]) -> dict[str, ConceptStructureBlueprint]:
+ """Convert a mixed structure dictionary to a proper ConceptStructureBlueprint dictionary.
+
+ Args:
+ structure_dict: Dictionary that may contain strings or ConceptStructureBlueprint objects
+
+ Returns:
+ Dictionary with all values as ConceptStructureBlueprint objects
+ """
+ normalized: dict[str, ConceptStructureBlueprint] = {}
+
+ for field_name, field_value in structure_dict.items():
+ if isinstance(field_value, str):
+ normalized[field_name] = ConceptStructureBlueprint(
+ description=field_value,
+ type=ConceptStructureBlueprintFieldType.TEXT,
+ )
+ else:
+ normalized[field_name] = field_value
+
+ return normalized
diff --git a/pipelex/core/concepts/native/concept_native.py b/pipelex/core/concepts/native/concept_native.py
index 1de58771f..960af642e 100644
--- a/pipelex/core/concepts/native/concept_native.py
+++ b/pipelex/core/concepts/native/concept_native.py
@@ -1,6 +1,14 @@
from pipelex.core.concepts.native.exceptions import NativeConceptDefinitionError
from pipelex.core.concepts.validation import is_concept_string_or_code_valid
from pipelex.core.domains.domain import SpecialDomain
+from pipelex.core.stuffs.dynamic_content import DynamicContent
+from pipelex.core.stuffs.image_content import ImageContent
+from pipelex.core.stuffs.json_content import JSONContent
+from pipelex.core.stuffs.number_content import NumberContent
+from pipelex.core.stuffs.page_content import PageContent
+from pipelex.core.stuffs.pdf_content import PDFContent
+from pipelex.core.stuffs.text_and_images_content import TextAndImagesContent
+from pipelex.core.stuffs.text_content import TextContent
from pipelex.types import StrEnum
@@ -28,6 +36,64 @@ class NativeConceptCode(StrEnum):
def structure_class_name(self) -> str:
return f"{self.value}Content"
+ @property
+ def structure_class(self) -> type | None:
+ """Get the structure class for this native concept.
+
+ Returns:
+ The structure class, or None if this concept doesn't have a dedicated content class
+ """
+ match self:
+ case NativeConceptCode.DYNAMIC:
+ return DynamicContent
+ case NativeConceptCode.TEXT:
+ return TextContent
+ case NativeConceptCode.IMAGE:
+ return ImageContent
+ case NativeConceptCode.PDF:
+ return PDFContent
+ case NativeConceptCode.TEXT_AND_IMAGES:
+ return TextAndImagesContent
+ case NativeConceptCode.NUMBER:
+ return NumberContent
+ case NativeConceptCode.PAGE:
+ return PageContent
+ case NativeConceptCode.JSON:
+ return JSONContent
+ case NativeConceptCode.IMG_GEN_PROMPT | NativeConceptCode.ANYTHING:
+ # These don't have dedicated content classes
+ return None
+
+ @classmethod
+ def is_native_structure_class(cls, class_name: str) -> bool:
+ """Check if a class name is a native structure class.
+
+ Args:
+ class_name: The class name to check
+
+ Returns:
+ True if it's a native structure class
+ """
+ return class_name in cls.native_concept_class_names()
+
+ @classmethod
+ def get_native_structure_class(cls, class_name: str) -> type | None:
+ """Get the native structure class by name.
+
+ Args:
+ class_name: The class name to look up
+
+ Returns:
+ The structure class, or None if not found
+ """
+ if not cls.is_native_structure_class(class_name):
+ msg = f"Class name '{class_name}' is not a native structure class"
+ raise NativeConceptDefinitionError(msg)
+ for native_code in cls:
+ if native_code.structure_class_name == class_name:
+ return native_code.structure_class
+ return None
+
@classmethod
def is_text_concept(cls, concept_code: str) -> bool:
try:
@@ -92,6 +158,24 @@ class NativeConceptCode(StrEnum):
return SpecialDomain.is_native(domain=domain_code) and concept_code in cls.values_list()
return concept_string_or_code in cls.values_list()
+ @classmethod
+ def is_valid_native_concept_string(cls, concept_string: str) -> bool:
+ """Check if the string is a valid native concept string (e.g., native.Text, native.Image).
+
+ Unlike is_native_concept_string_or_code, this method requires the full concept string
+ with the native domain prefix (e.g., "native.Text" is valid, but "Text" alone is not).
+
+ Args:
+ concept_string: The concept string to validate
+
+ Returns:
+ True if the string is a valid native concept string with domain prefix
+ """
+ if "." not in concept_string:
+ return False
+ domain_code, concept_code = concept_string.split(".", 1)
+ return SpecialDomain.is_native(domain=domain_code) and concept_code in cls.values_list()
+
@classmethod
def validate_native_concept_string_or_code(cls, concept_string_or_code: str) -> None:
if not cls.is_native_concept_string_or_code(concept_string_or_code=concept_string_or_code):
diff --git a/pipelex/core/concepts/structure_generation/generator.py b/pipelex/core/concepts/structure_generation/generator.py
index 3abcf24b3..5429cb63c 100644
--- a/pipelex/core/concepts/structure_generation/generator.py
+++ b/pipelex/core/concepts/structure_generation/generator.py
@@ -6,8 +6,9 @@ from typing import Any, Literal, Optional
from pydantic import Field
from pipelex.core.concepts.concept_structure_blueprint import ConceptStructureBlueprint, ConceptStructureBlueprintFieldType
+from pipelex.core.concepts.native.concept_native import NativeConceptCode
from pipelex.core.concepts.structure_generation.exceptions import ConceptStructureGeneratorError, ConceptStructureValidationError, SyntaxErrorData
-from pipelex.core.stuffs.structured_content import StructuredContent
+from pipelex.core.stuffs.stuff_content import StuffContent
class StructureGenerator:
@@ -23,12 +24,18 @@ class StructureGenerator:
}
self.enum_definitions: dict[str, dict[str, Any]] = {} # Store enum definitions
- def generate_from_structure_blueprint(self, class_name: str, structure_blueprint: dict[str, ConceptStructureBlueprint]) -> tuple[str, type]:
+ def generate_from_structure_blueprint(
+ self,
+ class_name: str,
+ structure_blueprint: dict[str, ConceptStructureBlueprint],
+ base_class_name: str | None = None,
+ ) -> tuple[str, type]:
"""Generate Python module content from structure blueprint.
Args:
class_name: Name of the class to generate
structure_blueprint: Dictionary mapping field names to their ConceptStructureBlueprint definitions
+ base_class_name: Optional base class name to inherit from (defaults to StructuredContent)
Returns:
Generated Python module content, the generated class
@@ -38,17 +45,33 @@ class StructureGenerator:
"""
# Generate the class
- class_code = self._generate_class_source_code_from_blueprint(class_name, structure_blueprint)
+ class_code = self._generate_class_source_code_from_blueprint(class_name, structure_blueprint, base_class_name)
- # Generate the complete module
+ # Generate the complete module with a header comment
imports_section = "\n".join(sorted(self.imports))
- generated_code = f"{imports_section}\n\n\n{class_code}\n"
+ autogen_header = (
+ '"""\n'
+ "AUTOGENERATED CODE - DO NOT EDIT\n"
+ "\n"
+ "If you want to customize this structure:\n"
+ " 1. Copy this file to your own module\n"
+ " 2. Remove the 'structure' or 'refines' declaration from the concept in the PLX file\n"
+ " and declare it in inline mode (see https://docs.pipelex.com/home/6-build-reliable-ai-workflows/concepts/define_your_concepts/#basic-concept-definition)\n"
+ " 3. Make sure your custom class is importable and registered\n"
+ "\n"
+ "To regenerate: pipelex build structures <target_directory>\n"
+ '"""\n\n'
+ )
+
+ generated_code = f"{autogen_header}{imports_section}\n\n\n{class_code}\n"
# Validate the generated code
try:
the_class = self.validate_generated_code(
- python_code=generated_code, expected_class_name=class_name, required_base_class=StructuredContent
+ python_code=generated_code,
+ expected_class_name=class_name,
+ base_class_name=base_class_name,
)
except SyntaxError as syntax_error:
msg = f"Error validating generated code: {syntax_error}"
@@ -62,13 +85,18 @@ class StructureGenerator:
return generated_code, the_class
- def validate_generated_code(self, python_code: str, expected_class_name: str, required_base_class: type) -> type:
+ def validate_generated_code(
+ self,
+ python_code: str,
+ expected_class_name: str,
+ base_class_name: str | None = None,
+ ) -> type:
"""Validate that the generated Python code is syntactically correct and executable.
Args:
python_code: The generated Python code to validate
expected_class_name: The name of the class that should be created
- required_base_class: The base class that the generated class should inherit from
+ base_class_name: Optional name of a custom base class to use
"""
ast.parse(python_code)
@@ -78,7 +106,7 @@ class StructureGenerator:
return self._validate_execution(
python_code=python_code,
expected_class_name=expected_class_name,
- required_base_class=required_base_class,
+ base_class_name=base_class_name,
)
############################################################
@@ -123,19 +151,36 @@ class StructureGenerator:
return self._escape_string_for_python(value)
return repr(value)
- def _generate_class_source_code_from_blueprint(self, class_name: str, structure_blueprint: dict[str, ConceptStructureBlueprint]) -> str:
+ def _generate_class_source_code_from_blueprint(
+ self,
+ class_name: str,
+ structure_blueprint: dict[str, ConceptStructureBlueprint],
+ base_class_name: str | None = None,
+ ) -> str:
"""Generate a class definition from ConceptStructureBlueprint.
Args:
class_name: Name of the class
structure_blueprint: Dictionary mapping field names to their ConceptStructureBlueprint definitions
+ base_class_name: Optional base class name to inherit from (defaults to StructuredContent)
Returns:
- Generated class code
+ Generated class code with markers for regeneration
"""
- # Generate class header
- class_header = f'class {class_name}(StructuredContent):\n """Generated {class_name} class"""\n'
+ # Determine base class
+ base_class = base_class_name or "StructuredContent"
+
+ # Add import for the base class if it's a native content class (other than StructuredContent which is already imported)
+ if NativeConceptCode.is_native_structure_class(base_class):
+ cls = NativeConceptCode.get_native_structure_class(base_class)
+ if not cls:
+ msg = f"Native structure class '{base_class}' not found"
+ raise ConceptStructureGeneratorError(msg)
+ self.imports.add(f"from {cls.__module__} import {cls.__name__}")
+
+ # Generate class header with docstring
+ class_header = f'class {class_name}({base_class}):\n """Generated {class_name} class"""\n'
# Generate fields
field_definitions: list[str] = []
@@ -144,8 +189,8 @@ class StructureGenerator:
field_definitions.append(field_code)
if not field_definitions:
- # Empty class with just pass
- return class_header + "\n pass"
+ # Empty class - docstring is sufficient, no need for pass
+ return class_header
fields_code = "\n".join(field_definitions)
return class_header + "\n" + fields_code
@@ -366,14 +411,23 @@ class StructureGenerator:
# Unknown FieldType, assume it's a custom type
return str(field_type)
- def _validate_execution(self, python_code: str, expected_class_name: str, required_base_class: type) -> type:
+ def _validate_execution(
+ self,
+ python_code: str,
+ expected_class_name: str,
+ base_class_name: str | None = None,
+ ) -> type:
"""Validate that the code executes and creates the expected class."""
# Import necessary modules for the execution context
from typing import Any # noqa: PLC0415
- # Provide necessary imports in the execution context
- exec_globals = {
+ from kajson.kajson_manager import KajsonManager # noqa: PLC0415
+
+ # exec_globals with basic types but let native class imports execute naturally
+ # This ensures we use the same class objects for inheritance checks
+ exec_globals: dict[str, Any] = {
"__builtins__": __builtins__,
+ # Provide these directly as they're used in generated code but not imported from pipelex
"datetime": datetime,
"Enum": Enum,
"Optional": Optional,
@@ -382,8 +436,18 @@ class StructureGenerator:
"Any": Any,
"Literal": Literal,
"Field": Field,
- "StructuredContent": StructuredContent,
}
+
+ # If a custom base class is specified that's not a native class, get it from the registry
+ if base_class_name:
+ if not NativeConceptCode.is_native_structure_class(base_class_name):
+ # Not a native class, provide it from registry
+ custom_base_class = KajsonManager.get_class_registry().get_class(name=base_class_name)
+ if custom_base_class is None:
+ msg = f"Base class '{base_class_name}' not found in native classes or class registry"
+ raise ConceptStructureValidationError(msg)
+ exec_globals[base_class_name] = custom_base_class # type: ignore[assignment]
+
exec_locals: dict[str, Any] = {}
exec(python_code, exec_globals, exec_locals)
@@ -400,8 +464,12 @@ class StructureGenerator:
raise ConceptStructureValidationError(msg)
# Verify it inherits from the required base class
- if not issubclass(the_class, required_base_class):
- msg = f"'{expected_class_name}' does not inherit from {required_base_class.__name__}"
+ # Check the MRO (Method Resolution Order) by class name to handle cases
+ # where the class object references might differ due to exec() context
+ mro_class_names = [cls.__name__ for cls in the_class.__mro__]
+
+ if StuffContent.__name__ not in mro_class_names:
+ msg = f"'{expected_class_name}' does not inherit from StuffContent. MRO: {mro_class_names}"
raise ConceptStructureValidationError(msg)
return the_class
diff --git a/pipelex/core/pipes/inputs/input_requirements.py b/pipelex/core/pipes/inputs/input_requirements.py
index 5f4eb61f5..f80950be2 100644
--- a/pipelex/core/pipes/inputs/input_requirements.py
+++ b/pipelex/core/pipes/inputs/input_requirements.py
@@ -1,9 +1,12 @@
+import json
from collections.abc import Callable
+from typing import Any
from pydantic import BaseModel, Field, RootModel, field_validator
from pipelex import log
from pipelex.core.concepts.concept import Concept
+from pipelex.core.concepts.concept_representation_generator import ConceptRepresentationFormat
from pipelex.core.pipes.inputs.exceptions import PipeInputNotFoundError
from pipelex.core.pipes.variable_multiplicity import VariableMultiplicity
from pipelex.core.stuffs.stuff_content import StuffContent
@@ -125,3 +128,40 @@ class InputRequirements(RootModel[InputRequirementsRoot]):
@property
def nb_inputs(self) -> int:
return len(self.root)
+
+ def _is_multiple(self, multiplicity: VariableMultiplicity | None) -> bool:
+ """Check if multiplicity indicates multiple items."""
+ if multiplicity is None:
+ return False
+ if isinstance(multiplicity, bool):
+ return multiplicity
+ return multiplicity > 1
+
+ def generate_json_representation(self) -> dict[str, Any]:
+ """Generate a JSON representation for all inputs.
+
+ Returns:
+ Dictionary with JSON representations for each input
+ """
+ json_inputs: dict[str, Any] = {}
+ for var_name, input_req in self.root.items():
+ is_multiple = self._is_multiple(input_req.multiplicity)
+ json_value, _ = input_req.concept.generate_input_representation(
+ output_format=ConceptRepresentationFormat.JSON,
+ is_multiple=is_multiple,
+ )
+ json_inputs[var_name] = json_value
+
+ return json_inputs
+
+ def generate_json_string(self, indent: int = 2) -> str:
+ """Generate a JSON representation for all inputs as a formatted string.
+
+ Args:
+ indent: Number of spaces for indentation (default: 2)
+
+ Returns:
+ Formatted JSON string
+ """
+ json_inputs = self.generate_json_representation()
+ return json.dumps(json_inputs, indent=indent, ensure_ascii=False)
diff --git a/pipelex/core/stuffs/stuff.py b/pipelex/core/stuffs/stuff.py
index 4d9ab5681..132dd163d 100644
--- a/pipelex/core/stuffs/stuff.py
+++ b/pipelex/core/stuffs/stuff.py
@@ -148,11 +148,12 @@ Forbidden fields are: 'stuff_name', 'content_class', 'concept_code', 'stuff_code
"""
list_content = cast("ListContent[StuffContentType]", self.content_as(content_type=ListContent))
- # Validate all items are of the expected type
+ converted_items: list[StuffContentType] = []
for item in list_content.items:
- self.verify_content_type(item, item_type)
+ converted_item = self.verify_content_type(item, item_type)
+ converted_items.append(converted_item)
- return list_content
+ return ListContent[StuffContentType](items=converted_items)
@property
def as_text(self) -> TextContent:
diff --git a/pipelex/core/stuffs/stuff_factory.py b/pipelex/core/stuffs/stuff_factory.py
index 084cf1572..491922fa0 100644
--- a/pipelex/core/stuffs/stuff_factory.py
+++ b/pipelex/core/stuffs/stuff_factory.py
@@ -10,7 +10,6 @@ from pipelex.core.concepts.concept_factory import ConceptFactory
from pipelex.core.concepts.native.concept_native import NativeConceptCode
from pipelex.core.concepts.validation import validate_concept_string
from pipelex.core.stuffs.list_content import ListContent
-from pipelex.core.stuffs.structured_content import StructuredContent
from pipelex.core.stuffs.stuff import DictStuff, Stuff
from pipelex.core.stuffs.stuff_content import StuffContent
from pipelex.core.stuffs.text_content import TextContent
@@ -376,8 +375,8 @@ class StuffFactory:
)
raise StuffFactoryError(msg)
- # Case 2.3: content is a StructuredContent object
- if isinstance(content, StructuredContent):
+ # Case 2.3: content is a StuffContent object (includes both native and StructuredContent)
+ if isinstance(content, StuffContent):
if concept.structure_class_name != content.__class__.__name__:
msg = (
f"Trying to create a Stuff '{name}' in the inputs of your pipe, from a dict that should represent a StuffContentOrData "
@@ -441,12 +440,12 @@ class StuffFactory:
msg = f"Concept '{concept_string}' is not compatible with list of text content"
raise StuffFactoryError(msg)
- # Case 2.4: list[StructuredContent]
- if isinstance(first_item, StructuredContent):
+ # Case 2.4: list[StuffContent] (includes both native and StructuredContent)
+ if isinstance(first_item, StuffContent):
for item in list_content_2:
if not isinstance(item, type(first_item)):
msg = (
- f"Trying to create a Stuff '{name}' in the inputs of your pipe, from a list of StructuredContent "
+ f"Trying to create a Stuff '{name}' in the inputs of your pipe, from a list of StuffContent "
"but the items are not of the same type. Every items of the list should be a identical type. "
f"If its a '{type(first_item).__name__}', everything should be a '{type(first_item).__name__}'."
)
@@ -454,7 +453,7 @@ class StuffFactory:
return cls.make_stuff(
concept=concept,
- content=ListContent(items=cast("list[StructuredContent]", list_content_2)),
+ content=ListContent(items=cast("list[StuffContent]", list_content_2)),
name=name,
code=code,
)
diff --git a/pipelex/libraries/concept/concept_library.py b/pipelex/libraries/concept/concept_library.py
index ffeba20fd..0e40ef885 100644
--- a/pipelex/libraries/concept/concept_library.py
+++ b/pipelex/libraries/concept/concept_library.py
@@ -49,8 +49,9 @@ class ConceptLibrary(RootModel[ConceptLibraryRoot], ConceptLibraryAbstract):
def make_empty_with_native_concepts(cls) -> Self:
library = cls(root={})
library.setup()
- all_native_concepts = ConceptFactory.make_all_native_concepts()
- library.add_concepts(concepts=all_native_concepts)
+ library.add_concepts(
+ concepts=[ConceptFactory.make_native_concept(native_concept_code=native_concept) for native_concept in NativeConceptCode.values_list()]
+ )
return library
@override
diff --git a/pipelex/libraries/library_manager.py b/pipelex/libraries/library_manager.py
index 4474ad12a..afc91753d 100644
--- a/pipelex/libraries/library_manager.py
+++ b/pipelex/libraries/library_manager.py
@@ -113,7 +113,7 @@ class LibraryManager(LibraryManagerAbstract):
library_id: str,
library_dirs: list[Path] | None = None,
library_file_paths: list[Path] | None = None,
- ) -> None:
+ ) -> list[PipeAbstract]:
# Ensure libraries exist for this library_id
if library_id not in self._libraries:
msg = f"Trying to load a library that does not exist: '{library_id}'"
@@ -167,7 +167,7 @@ class LibraryManager(LibraryManagerAbstract):
# Load PLX files into the specific library
- self._load_plx_files_into_library(library_id=library_id, valid_plx_paths=valid_plx_paths)
+ return self._load_plx_files_into_library(library_id=library_id, valid_plx_paths=valid_plx_paths)
@override
def load_from_blueprints(self, library_id: str, blueprints: list[PipelexBundleBlueprint]) -> list[PipeAbstract]:
@@ -202,11 +202,11 @@ class LibraryManager(LibraryManagerAbstract):
for blueprint in blueprints:
if blueprint.concept is not None:
concepts: list[Concept] = []
- for concept_code, concept_blueprint_or_description in blueprint.concept.items():
- concept = ConceptFactory.make_from_blueprint_or_description(
+ for concept_code, concept_blueprint in blueprint.concept.items():
+ concept = ConceptFactory.make_from_blueprint(
domain=blueprint.domain,
concept_code=concept_code,
- concept_blueprint_or_description=concept_blueprint_or_description,
+ blueprint_or_string_description=concept_blueprint,
)
concepts.append(concept)
all_concepts.extend(concepts)
@@ -235,7 +235,7 @@ class LibraryManager(LibraryManagerAbstract):
# Private helper methods
############################################################
- def _load_plx_files_into_library(self, library_id: str, valid_plx_paths: list[Path]) -> None:
+ def _load_plx_files_into_library(self, library_id: str, valid_plx_paths: list[Path]) -> list[PipeAbstract]:
"""Load PLX files into a specific library.
This method:
@@ -264,7 +264,7 @@ class LibraryManager(LibraryManagerAbstract):
blueprints.append(blueprint)
self.loaded_plx_paths.extend([str(plx_file_path) for plx_file_path in valid_plx_paths])
- self.load_from_blueprints(library_id=library_id, blueprints=blueprints)
+ return self.load_from_blueprints(library_id=library_id, blueprints=blueprints)
def _remove_pipes_from_blueprint(self, blueprint: PipelexBundleBlueprint) -> None:
library = self.get_current_library()
diff --git a/pipelex/libraries/library_manager_abstract.py b/pipelex/libraries/library_manager_abstract.py
index a6e2061ae..9cf2c4d2e 100644
--- a/pipelex/libraries/library_manager_abstract.py
+++ b/pipelex/libraries/library_manager_abstract.py
@@ -52,5 +52,5 @@ class LibraryManagerAbstract(ABC):
library_id: str,
library_dirs: list[Path] | None = None,
library_file_paths: list[Path] | None = None,
- ) -> None:
+ ) -> list[PipeAbstract]:
pass
diff --git a/pipelex/pipe_operators/compose/pipe_compose.py b/pipelex/pipe_operators/compose/pipe_compose.py
index 2171fc4bd..e5c4c9c30 100644
--- a/pipelex/pipe_operators/compose/pipe_compose.py
+++ b/pipelex/pipe_operators/compose/pipe_compose.py
@@ -13,12 +13,13 @@ from pipelex.core.concepts.concept import Concept
from pipelex.core.concepts.concept_factory import ConceptFactory
from pipelex.core.concepts.native.concept_native import NativeConceptCode
from pipelex.core.memory.working_memory import WorkingMemory
+from pipelex.core.pipes.exceptions import PipeValidationError, PipeValidationErrorType
from pipelex.core.pipes.inputs.input_requirements import InputRequirements
from pipelex.core.pipes.inputs.input_requirements_factory import InputRequirementsFactory
from pipelex.core.pipes.pipe_output import PipeOutput
+from pipelex.core.stuffs.stuff_content import StuffContent
from pipelex.core.stuffs.stuff_factory import StuffFactory
-from pipelex.core.stuffs.text_content import TextContent
-from pipelex.hub import get_content_generator
+from pipelex.hub import get_class_registry, get_concept_library, get_content_generator, get_native_concept
from pipelex.pipe_operators.pipe_operator import PipeOperator
from pipelex.pipe_run.exceptions import PipeRunParamsError
from pipelex.pipe_run.pipe_run_mode import PipeRunMode
@@ -88,7 +89,24 @@ class PipeCompose(PipeOperator[PipeComposeOutput]):
@override
def validate_output_with_library(self):
- pass
+ if not get_concept_library().is_compatible(
+ tested_concept=self.output,
+ wanted_concept=get_native_concept(native_concept=NativeConceptCode.TEXT),
+ strict=True,
+ ):
+ msg = (
+ f"The output of a PipeCompose must be strictly compatible with the Text concept. "
+ f"In the pipe '{self.code}' the output is '{self.output.concept_string}'. "
+ "Make sure this concept refines the native Text concept."
+ )
+ raise PipeValidationError(
+ message=msg,
+ error_type=PipeValidationErrorType.INADEQUATE_OUTPUT_CONCEPT,
+ domain=self.domain,
+ pipe_code=self.code,
+ provided_concept_code=self.output.concept_string,
+ required_concept_codes=[NativeConceptCode.TEXT.concept_string],
+ )
@override
async def _run_operator_pipe(
@@ -118,7 +136,13 @@ class PipeCompose(PipeOperator[PipeComposeOutput]):
)
log.verbose(f"Jinja2 rendered text:\n{jinja2_text}")
assert isinstance(jinja2_text, str)
- the_content = TextContent(text=jinja2_text)
+
+ # Get the structure class from the registry (might be a subclass of TextContent)
+ structure_class = get_class_registry().get_required_subclass(
+ name=self.output.structure_class_name,
+ base_class=StuffContent,
+ )
+ the_content = structure_class(text=jinja2_text)
output_stuff = StuffFactory.make_stuff(concept=self.output, content=the_content, name=output_name)
diff --git a/pipelex/pipe_operators/img_gen/pipe_img_gen.py b/pipelex/pipe_operators/img_gen/pipe_img_gen.py
index 129e3eb0a..a50c19345 100644
--- a/pipelex/pipe_operators/img_gen/pipe_img_gen.py
+++ b/pipelex/pipe_operators/img_gen/pipe_img_gen.py
@@ -1,4 +1,4 @@
-from typing import TYPE_CHECKING, Literal
+from typing import Literal
from pydantic import Field, model_validator
from typing_extensions import Self, override
@@ -23,8 +23,9 @@ from pipelex.core.pipes.variable_multiplicity import VariableMultiplicity
from pipelex.core.stuffs.exceptions import StuffContentTypeError
from pipelex.core.stuffs.image_content import ImageContent
from pipelex.core.stuffs.list_content import ListContent
+from pipelex.core.stuffs.stuff_content import StuffContent
from pipelex.core.stuffs.stuff_factory import StuffFactory
-from pipelex.hub import get_concept_library, get_content_generator, get_model_deck, get_native_concept
+from pipelex.hub import get_class_registry, get_concept_library, get_content_generator, get_model_deck, get_native_concept
from pipelex.pipe_operators.img_gen.exceptions import PipeImgGenRunError
from pipelex.pipe_operators.pipe_operator import PipeOperator
from pipelex.pipe_run.exceptions import PipeRunParamsError
@@ -33,9 +34,6 @@ from pipelex.pipe_run.pipe_run_params import PipeRunParams, output_multiplicity_
from pipelex.pipe_run.pipe_run_params_factory import PipeRunParamsFactory
from pipelex.pipeline.job_metadata import JobMetadata
-if TYPE_CHECKING:
- from pipelex.core.stuffs.stuff_content import StuffContent
-
class PipeImgGenOutput(PipeOutput):
@property
@@ -149,7 +147,7 @@ class PipeImgGen(PipeOperator[PipeImgGenOutput]):
strict=True,
):
msg = (
- f"The output of a ImgGen pipe must be compatible with the Image concept. "
+ f"The output of a PipeImgGen must be compatible with the Image concept. "
f"In the pipe '{self.code}' the output is '{self.output.concept_string}'"
)
raise PipeValidationError(
@@ -258,6 +256,12 @@ class PipeImgGen(PipeOperator[PipeImgGenOutput]):
else:
nb_images = 1
+ # Get the structure class from the registry (might be a subclass of ImageContent)
+ structure_class = get_class_registry().get_required_subclass(
+ name=self.output.structure_class_name,
+ base_class=StuffContent,
+ )
+
if nb_images > 1:
generated_image_list = await content_generator.make_image_list(
job_metadata=job_metadata,
@@ -272,7 +276,7 @@ class PipeImgGen(PipeOperator[PipeImgGenOutput]):
image_content_items: list[StuffContent] = []
for generated_image in generated_image_list:
image_content_items.append(
- ImageContent(
+ structure_class(
url=generated_image.url,
source_prompt=img_gen_prompt_text,
),
@@ -296,7 +300,7 @@ class PipeImgGen(PipeOperator[PipeImgGenOutput]):
generated_image_url = generated_image.url
image_urls = [generated_image_url]
- the_content = ImageContent(
+ the_content = structure_class(
url=generated_image_url,
source_prompt=img_gen_prompt_text,
)
diff --git a/pipelex/pipe_operators/llm/pipe_llm.py b/pipelex/pipe_operators/llm/pipe_llm.py
index 3591505a9..316c46169 100644
--- a/pipelex/pipe_operators/llm/pipe_llm.py
+++ b/pipelex/pipe_operators/llm/pipe_llm.py
@@ -14,6 +14,7 @@ from pipelex.cogt.llm.llm_setting import LLMModelChoice, LLMSetting, LLMSettingC
from pipelex.cogt.models.model_deck_check import check_llm_choice_with_deck
from pipelex.cogt.templating.template_category import TemplateCategory
from pipelex.config import get_config
+from pipelex.core.concepts.concept import Concept
from pipelex.core.concepts.concept_factory import ConceptFactory
from pipelex.core.concepts.native.concept_native import NativeConceptCode
from pipelex.core.domains.domain import SpecialDomain
@@ -26,7 +27,6 @@ from pipelex.core.pipes.variable_multiplicity import VariableMultiplicity
from pipelex.core.stuffs.list_content import ListContent
from pipelex.core.stuffs.stuff_content import StuffContent
from pipelex.core.stuffs.stuff_factory import StuffFactory
-from pipelex.core.stuffs.text_content import TextContent
from pipelex.hub import (
get_class_registry,
get_concept_library,
@@ -208,7 +208,11 @@ class PipeLLM(PipeOperator[PipeLLMOutput]):
# we acknowledge the code here with llm_prompt_1 and llm_prompt_2 is overly complex and should be refactored.
the_content: StuffContent
- if output_concept.structure_class_name == "TextContent" and not is_multiple_output:
+
+ if (
+ Concept.are_concept_compatible(concept_1=output_concept, concept_2=get_native_concept(NativeConceptCode.TEXT), strict=True)
+ and not is_multiple_output
+ ):
llm_prompt_1_for_text = await self.llm_prompt_spec.make_llm_prompt(
output_concept_string=output_concept.concept_string,
context_provider=working_memory,
@@ -227,8 +231,13 @@ class PipeLLM(PipeOperator[PipeLLMOutput]):
msg = f"Error generating text with LLM {location}: {error_details}"
raise PipeRunError(message=msg, run_mode=pipe_run_params.run_mode) from exc
+ structure_class = get_class_registry().get_required_subclass(
+ name=output_concept.structure_class_name,
+ base_class=StuffContent,
+ )
+
try:
- the_content = TextContent(
+ the_content = structure_class(
text=generated_text,
)
except ValidationError as exc:
diff --git a/pipelex/pipeline/validate_bundle.py b/pipelex/pipeline/validate_bundle.py
index aa0ddbd1f..7d7bc17cc 100644
--- a/pipelex/pipeline/validate_bundle.py
+++ b/pipelex/pipeline/validate_bundle.py
@@ -1,3 +1,5 @@
+from pathlib import Path
+
from pydantic import BaseModel, ValidationError
from pipelex.base_exceptions import PipelexError
@@ -14,6 +16,7 @@ from pipelex.core.pipes.handle_pipe_errors import (
from pipelex.core.pipes.pipe_abstract import PipeAbstract
from pipelex.core.validation import report_validation_error
from pipelex.hub import get_library_manager, set_current_library
+from pipelex.libraries.library_utils import get_pipelex_plx_files_from_dirs
from pipelex.pipe_run.dry_run import DryRunError, DryRunOutput, dry_run_pipes
@@ -129,3 +132,45 @@ async def validate_bundle(
) from dry_run_error
return ValidateBundleResult(blueprints=loaded_blueprints, pipes=loaded_pipes, dry_run_result=dry_run_results)
+
+
+async def validate_bundles_from_directory(directory: Path) -> ValidateBundleResult:
+ plx_files = get_pipelex_plx_files_from_dirs(dirs={directory})
+ all_blueprints: list[PipelexBundleBlueprint] = []
+
+ library_manager = get_library_manager()
+ library_id, _ = library_manager.open_library()
+ set_current_library(library_id=library_id)
+ try:
+ for plx_file in plx_files:
+ blueprint = PipelexInterpreter.make_pipelex_bundle_blueprint(bundle_path=str(plx_file))
+ all_blueprints.append(blueprint)
+
+ loaded_pipes = library_manager.load_libraries(library_id=library_id, library_dirs=[Path(directory)])
+ dry_run_results = await dry_run_pipes(pipes=loaded_pipes, raise_on_failure=True)
+ except PipelexInterpreterError as interpreter_error:
+ raise ValidateBundleError(
+ message=interpreter_error.message,
+ pipelex_bundle_blueprint_validation_errors=interpreter_error.validation_errors,
+ ) from interpreter_error
+ except PipeValidationError as pipe_error:
+ pipe_error_data = categorize_pipe_validation_with_libraries_error(pipe_error=pipe_error)
+ raise ValidateBundleError(
+ message=f"Pipe validation failed: {pipe_error}",
+ pipe_validation_errors=[pipe_error_data],
+ ) from pipe_error
+ except ValidationError as validation_error:
+ pipe_validation_errors = categorize_pipe_validation_error(validation_error=validation_error)
+ validation_error_msg = report_validation_error(category="plx", validation_error=validation_error)
+ msg = f"Could not load blueprints because of: {validation_error_msg}"
+ raise ValidateBundleError(
+ message=msg,
+ pipe_validation_errors=pipe_validation_errors,
+ ) from validation_error
+ except DryRunError as dry_run_error:
+ raise ValidateBundleError(
+ message=dry_run_error.message,
+ dry_run_error_message=dry_run_error.message,
+ ) from dry_run_error
+
+ return ValidateBundleResult(blueprints=all_blueprints, pipes=loaded_pipes, dry_run_result=dry_run_results)
diff --git a/pipelex/system/registries/class_registry_utils.py b/pipelex/system/registries/class_registry_utils.py
index 26c61135f..cc62f280e 100644
--- a/pipelex/system/registries/class_registry_utils.py
+++ b/pipelex/system/registries/class_registry_utils.py
@@ -43,6 +43,7 @@ class ClassRegistryUtils:
base_class: type[Any] | None = None,
is_recursive: bool = True,
is_include_imported: bool = False,
+ force_exclude_dirs: list[str] | None = None,
) -> None:
"""Registers all classes in Python files within folders that are subclasses of base_class.
If base_class is None, registers all classes.
@@ -52,13 +53,14 @@ class ClassRegistryUtils:
base_class: Optional base class to filter registerable classes
is_recursive: Whether to search recursively in subdirectories
is_include_imported: Whether to include classes imported from other modules
+ force_exclude_dirs: List of absolute directory paths to exclude from scanning
"""
python_files = find_files_in_dir(
dir_path=folder_path,
pattern="*.py",
is_recursive=is_recursive,
- excluded_dirs=list(get_config().pipelex.scan_config.excluded_dirs),
+ excluded_dirs=list(get_config().pipelex.scan_config.excluded_dirs) + (force_exclude_dirs or []),
)
for python_file in python_files:
diff --git a/pipelex/tools/misc/file_utils.py b/pipelex/tools/misc/file_utils.py
index 7e67ffb11..0301d74ef 100644
--- a/pipelex/tools/misc/file_utils.py
+++ b/pipelex/tools/misc/file_utils.py
@@ -375,7 +375,25 @@ def find_files_in_dir(
files = list(path.glob(pattern))
for file in files:
- is_excluded = excluded_dirs is not None and any(excluded_dir in file.parts for excluded_dir in excluded_dirs)
+ # Check if file is under any excluded directory
+ is_excluded = False
+ if excluded_dirs is not None:
+ for excluded_dir in excluded_dirs:
+ excluded_path = Path(excluded_dir)
+ # If excluded_dir is an absolute path, check if file is under it
+ if excluded_path.is_absolute():
+ try:
+ # Resolve both paths to handle symlinks (e.g., /var -> /private/var on macOS)
+ file.resolve().relative_to(excluded_path.resolve())
+ is_excluded = True
+ break
+ except ValueError:
+ # file is not relative to excluded_path
+ continue
+ # If excluded_dir is just a directory name, check if it's in the file path parts
+ elif excluded_dir in file.parts:
+ is_excluded = True
+ break
# Check if file is in a force include directory (forced inclusion despite exclusions)
# Force include dirs can be full paths or directory names
diff --git a/pipelex/urls.py b/pipelex/urls.py
index 09674bdc9..b917cddcb 100644
--- a/pipelex/urls.py
+++ b/pipelex/urls.py
@@ -6,3 +6,4 @@ class URLs:
discord = "https://go.pipelex.com/discord"
pipe_func_docs = "https://docs.pipelex.com/home/6-build-reliable-ai-workflows/pipes/pipe-operators/PipeFunc/"
backend_provider_docs = "https://docs.pipelex.com/home/5-setup/configure-ai-providers/"
+ native_concepts_docs = "https://docs.pipelex.com/home/6-build-reliable-ai-workflows/concepts/native-concepts/"
diff --git a/tests/integration/pipelex/core/concepts/test_concept_representation_generator.py b/tests/integration/pipelex/core/concepts/test_concept_representation_generator.py
new file mode 100644
index 000000000..734f6e73c
--- /dev/null
+++ b/tests/integration/pipelex/core/concepts/test_concept_representation_generator.py
@@ -0,0 +1,260 @@
+"""Integration tests for ConceptRepresentationGenerator with complex nested structures."""
+
+from pydantic import Field
+
+from pipelex.core.concepts.concept_representation_generator import (
+ ConceptRepresentationFormat,
+ ConceptRepresentationGenerator,
+ generate_json_representation,
+ generate_python_representation,
+)
+from pipelex.core.stuffs.structured_content import StructuredContent
+from pipelex.tools.typing.pydantic_utils import empty_list_factory_of
+
+# =============================================================================
+# Complex nested structures for integration testing
+# =============================================================================
+
+
+class Attachment(StructuredContent):
+ """Represents a message attachment."""
+
+ name: str = Field(..., description="Name of the attachment file")
+ url: str = Field(..., description="URL of the attachment")
+
+
+class Embed(StructuredContent):
+ """Represents a message embed."""
+
+ title: str = Field(..., description="Title of the embed")
+ description: str = Field(..., description="Description of the embed content")
+ embed_type: str = Field(..., description="Type of the embed")
+
+
+class Message(StructuredContent):
+ """Represents a message with nested attachments and embeds."""
+
+ author: str = Field(..., description="Author of the message")
+ content: str = Field(..., description="Content of the message")
+ attachments: list[Attachment] = Field(default_factory=empty_list_factory_of(Attachment), description="List of attachments")
+ embeds: list[Embed] = Field(default_factory=empty_list_factory_of(Embed), description="List of embeds")
+ link: str = Field(..., description="Link to the message")
+
+
+class Channel(StructuredContent):
+ """Represents a channel with messages."""
+
+ name: str = Field(..., description="Name of the channel")
+ position: int = Field(..., description="Position of the channel")
+ messages: list[Message] = Field(default_factory=empty_list_factory_of(Message), description="List of messages")
+
+
+class ChannelSummary(StructuredContent):
+ """Represents a summarized channel."""
+
+ channel_name: str = Field(..., description="Name of the channel")
+ summary_items: list[str] = Field(..., description="Summaries of the channel's activity")
+
+
+class Newsletter(StructuredContent):
+ """Represents a newsletter with multiple sections."""
+
+ weekly_summary: str = Field(..., description="Summary of weekly content")
+ new_members: list[str] = Field(default_factory=list, description="New member introductions")
+ channel_sections: list[ChannelSummary] = Field(default_factory=empty_list_factory_of(ChannelSummary), description="Channel summaries")
+
+
+# =============================================================================
+# Integration tests with exact expected results
+# =============================================================================
+
+
+class TestComplexNestedStructuresJson:
+ """Integration tests for complex nested structures with JSON output."""
+
+ def test_attachment_json(self) -> None:
+ """Test simple Attachment generates correct JSON."""
+ generator = ConceptRepresentationGenerator(ConceptRepresentationFormat.JSON)
+ result = generator.generate_representation("test.Attachment", Attachment)
+ expected = {
+ "concept": "test.Attachment",
+ "content": {
+ "name": "name_value",
+ "url": "url_value",
+ },
+ }
+ assert result == expected
+
+ def test_embed_json(self) -> None:
+ """Test Embed with multiple string fields generates correct JSON."""
+ generator = ConceptRepresentationGenerator(ConceptRepresentationFormat.JSON)
+ result = generator.generate_representation("test.Embed", Embed)
+ expected = {
+ "concept": "test.Embed",
+ "content": {
+ "title": "title_value",
+ "description": "description_value",
+ "embed_type": "embed_type_value",
+ },
+ }
+ assert result == expected
+
+ def test_message_json_recursive(self) -> None:
+ """Test Message with nested Attachment and Embed lists generates correct recursive JSON."""
+ generator = ConceptRepresentationGenerator(ConceptRepresentationFormat.JSON)
+ result = generator.generate_representation("test.Message", Message)
+ expected = {
+ "concept": "test.Message",
+ "content": {
+ "author": "author_value",
+ "content": "content_value",
+ "attachments": [{"name": "name_value", "url": "url_value"}],
+ "embeds": [
+ {
+ "title": "title_value",
+ "description": "description_value",
+ "embed_type": "embed_type_value",
+ }
+ ],
+ "link": "link_value",
+ },
+ }
+ assert result == expected
+
+ def test_channel_json_deeply_nested(self) -> None:
+ """Test Channel with deeply nested messages generates correct JSON."""
+ generator = ConceptRepresentationGenerator(ConceptRepresentationFormat.JSON)
+ result = generator.generate_representation("test.Channel", Channel)
+ expected = {
+ "concept": "test.Channel",
+ "content": {
+ "name": "name_value",
+ "position": 0,
+ "messages": [
+ {
+ "author": "author_value",
+ "content": "content_value",
+ "attachments": [{"name": "name_value", "url": "url_value"}],
+ "embeds": [
+ {
+ "title": "title_value",
+ "description": "description_value",
+ "embed_type": "embed_type_value",
+ }
+ ],
+ "link": "link_value",
+ }
+ ],
+ },
+ }
+ assert result == expected
+
+ def test_newsletter_json_multiple_lists(self) -> None:
+ """Test Newsletter with multiple list fields generates correct JSON."""
+ generator = ConceptRepresentationGenerator(ConceptRepresentationFormat.JSON)
+ result = generator.generate_representation("test.Newsletter", Newsletter)
+ expected = {
+ "concept": "test.Newsletter",
+ "content": {
+ "weekly_summary": "weekly_summary_value",
+ "new_members": ["new_members_item_value"],
+ "channel_sections": [
+ {
+ "channel_name": "channel_name_value",
+ "summary_items": ["summary_items_item_value"],
+ }
+ ],
+ },
+ }
+ assert result == expected
+
+
+class TestComplexNestedStructuresPython:
+ """Integration tests for complex nested structures with Python output."""
+
+ def test_attachment_python(self) -> None:
+ """Test simple Attachment generates correct Python instantiation."""
+ generator = ConceptRepresentationGenerator(ConceptRepresentationFormat.PYTHON)
+ result = generator.generate_representation("test.Attachment", Attachment)
+ expected = {
+ "concept": "test.Attachment",
+ "content": 'Attachment(name="name_value", url="url_value")',
+ }
+ assert result == expected
+
+ def test_message_python_recursive(self) -> None:
+ """Test Message with nested structures generates correct Python."""
+ generator = ConceptRepresentationGenerator(ConceptRepresentationFormat.PYTHON)
+ result = generator.generate_representation("test.Message", Message)
+ expected_content = (
+ 'Message(author="author_value", content="content_value", '
+ 'attachments=[Attachment(name="name_value", url="url_value")], '
+ 'embeds=[Embed(title="title_value", description="description_value", embed_type="embed_type_value")], '
+ 'link="link_value")'
+ )
+ expected = {
+ "concept": "test.Message",
+ "content": expected_content,
+ }
+ assert result == expected
+
+ def test_channel_python_deeply_nested(self) -> None:
+ """Test Channel with deeply nested structures generates correct Python."""
+ generator = ConceptRepresentationGenerator(ConceptRepresentationFormat.PYTHON)
+ result = generator.generate_representation("test.Channel", Channel)
+
+ # Build expected content string
+ message_content = (
+ 'Message(author="author_value", content="content_value", '
+ 'attachments=[Attachment(name="name_value", url="url_value")], '
+ 'embeds=[Embed(title="title_value", description="description_value", embed_type="embed_type_value")], '
+ 'link="link_value")'
+ )
+ expected_content = f'Channel(name="name_value", position=0, messages=[{message_content}])'
+
+ expected = {
+ "concept": "test.Channel",
+ "content": expected_content,
+ }
+ assert result == expected
+
+
+class TestImportsTrackingIntegration:
+ """Integration tests for import tracking with complex structures."""
+
+ def test_channel_tracks_all_nested_imports(self) -> None:
+ """Test that Channel tracks imports for all nested classes."""
+ generator = ConceptRepresentationGenerator(ConceptRepresentationFormat.JSON)
+ generator.generate_representation("test.Channel", Channel)
+
+ expected_imports = {"Channel", "Message", "Attachment", "Embed"}
+ assert generator.imports_needed == expected_imports
+
+ def test_newsletter_tracks_all_imports(self) -> None:
+ """Test that Newsletter tracks imports for all nested classes."""
+ generator = ConceptRepresentationGenerator(ConceptRepresentationFormat.JSON)
+ generator.generate_representation("test.Newsletter", Newsletter)
+
+ expected_imports = {"Newsletter", "ChannelSummary"}
+ assert generator.imports_needed == expected_imports
+
+
+class TestConvenienceFunctionsIntegration:
+ """Integration tests for convenience functions with complex structures."""
+
+ def test_generate_json_representation_complex(self) -> None:
+ """Test generate_json_representation with complex structure."""
+ result = generate_json_representation("test.Message", Message)
+ assert result["concept"] == "test.Message"
+ assert "attachments" in result["content"]
+ assert result["content"]["attachments"] == [{"name": "name_value", "url": "url_value"}]
+
+ def test_generate_python_representation_complex(self) -> None:
+ """Test generate_python_representation with complex structure."""
+ result, imports = generate_python_representation("test.Message", Message)
+ assert result["concept"] == "test.Message"
+ assert "Message(" in result["content"]
+ assert "Attachment(" in result["content"]
+ assert "Message" in imports
+ assert "Attachment" in imports
+ assert "Embed" in imports
diff --git a/tests/integration/pipelex/pipes/controller/pipe_batch/test_pipe_batch_simple.py b/tests/integration/pipelex/pipes/controller/pipe_batch/test_pipe_batch_simple.py
index 68179b903..8c1d8f942 100644
--- a/tests/integration/pipelex/pipes/controller/pipe_batch/test_pipe_batch_simple.py
+++ b/tests/integration/pipelex/pipes/controller/pipe_batch/test_pipe_batch_simple.py
@@ -39,12 +39,12 @@ class TestPipeBatchSimple:
concept_1 = ConceptFactory.make_from_blueprint(
concept_code="TestConcept1",
domain=domain,
- blueprint=ConceptBlueprint(description="Lorem Ipsum"),
+ blueprint_or_string_description=ConceptBlueprint(description="Lorem Ipsum"),
)
concept_2 = ConceptFactory.make_from_blueprint(
concept_code="TestConcept2",
domain=domain,
- blueprint=ConceptBlueprint(description="Lorem Ipsum"),
+ blueprint_or_string_description=ConceptBlueprint(description="Lorem Ipsum"),
)
concept_library = get_concept_library()
concept_library.add_concepts([concept_1, concept_2])
diff --git a/tests/integration/pipelex/pipes/controller/pipe_condition/test_pipe_condition_validation.py b/tests/integration/pipelex/pipes/controller/pipe_condition/test_pipe_condition_validation.py
index 57463e50d..1b0db25a2 100644
--- a/tests/integration/pipelex/pipes/controller/pipe_condition/test_pipe_condition_validation.py
+++ b/tests/integration/pipelex/pipes/controller/pipe_condition/test_pipe_condition_validation.py
@@ -19,12 +19,12 @@ class TestPipeConditionValidation:
concept_1 = ConceptFactory.make_from_blueprint(
concept_code="TestConcept",
domain=domain,
- blueprint=ConceptBlueprint(description="Lorem Ipsum"),
+ blueprint_or_string_description=ConceptBlueprint(description="Lorem Ipsum"),
)
concept_2 = ConceptFactory.make_from_blueprint(
concept_code="Result",
domain=domain,
- blueprint=ConceptBlueprint(description="Lorem Ipsum"),
+ blueprint_or_string_description=ConceptBlueprint(description="Lorem Ipsum"),
)
concept_library = get_concept_library()
concept_library.add_concepts([concept_1, concept_2])
@@ -61,12 +61,12 @@ class TestPipeConditionValidation:
concept_1 = ConceptFactory.make_from_blueprint(
concept_code="TestConcept",
domain=domain,
- blueprint=ConceptBlueprint(description="Lorem Ipsum"),
+ blueprint_or_string_description=ConceptBlueprint(description="Lorem Ipsum"),
)
concept_2 = ConceptFactory.make_from_blueprint(
concept_code="Result",
domain=domain,
- blueprint=ConceptBlueprint(description="Lorem Ipsum"),
+ blueprint_or_string_description=ConceptBlueprint(description="Lorem Ipsum"),
)
concept_library.add_concepts([concept_1, concept_2])
diff --git a/tests/integration/pipelex/pipes/controller/pipe_parallel/test_pipe_parallel_validation.py b/tests/integration/pipelex/pipes/controller/pipe_parallel/test_pipe_parallel_validation.py
index fad2c1f6e..97ba2a7a1 100644
--- a/tests/integration/pipelex/pipes/controller/pipe_parallel/test_pipe_parallel_validation.py
+++ b/tests/integration/pipelex/pipes/controller/pipe_parallel/test_pipe_parallel_validation.py
@@ -24,17 +24,17 @@ class TestPipeParallelValidation:
concept_1 = ConceptFactory.make_from_blueprint(
domain=domain,
concept_code="Document",
- blueprint=concept_blueprint,
+ blueprint_or_string_description=concept_blueprint,
)
concept_2 = ConceptFactory.make_from_blueprint(
domain=domain,
concept_code="Context",
- blueprint=concept_blueprint,
+ blueprint_or_string_description=concept_blueprint,
)
concept_3 = ConceptFactory.make_from_blueprint(
domain=domain,
concept_code="Analysis",
- blueprint=concept_blueprint,
+ blueprint_or_string_description=concept_blueprint,
)
concept_library.add_concepts(concepts=[concept_1, concept_2, concept_3])
@@ -105,17 +105,17 @@ class TestPipeParallelValidation:
concept_1 = ConceptFactory.make_from_blueprint(
domain=domain,
concept_code="Document",
- blueprint=concept_blueprint,
+ blueprint_or_string_description=concept_blueprint,
)
concept_2 = ConceptFactory.make_from_blueprint(
domain=domain,
concept_code="Context",
- blueprint=concept_blueprint,
+ blueprint_or_string_description=concept_blueprint,
)
concept_3 = ConceptFactory.make_from_blueprint(
domain=domain,
concept_code="ProcessedAnalysis",
- blueprint=concept_blueprint,
+ blueprint_or_string_description=concept_blueprint,
)
concept_library.add_concepts(concepts=[concept_1, concept_2, concept_3])
@@ -156,17 +156,17 @@ class TestPipeParallelValidation:
concept_1 = ConceptFactory.make_from_blueprint(
domain=domain,
concept_code="Document",
- blueprint=concept_blueprint,
+ blueprint_or_string_description=concept_blueprint,
)
concept_2 = ConceptFactory.make_from_blueprint(
domain=domain,
concept_code="Context",
- blueprint=concept_blueprint,
+ blueprint_or_string_description=concept_blueprint,
)
concept_3 = ConceptFactory.make_from_blueprint(
domain=domain,
concept_code="ProcessedAnalysis",
- blueprint=concept_blueprint,
+ blueprint_or_string_description=concept_blueprint,
)
concept_library.add_concepts(concepts=[concept_1, concept_2, concept_3])
diff --git a/tests/integration/pipelex/pipes/controller/pipe_sequence/test_pipe_sequence_validation.py b/tests/integration/pipelex/pipes/controller/pipe_sequence/test_pipe_sequence_creation.py
similarity index 52%
rename from tests/integration/pipelex/pipes/controller/pipe_sequence/test_pipe_sequence_validation.py
rename to tests/integration/pipelex/pipes/controller/pipe_sequence/test_pipe_sequence_creation.py
index a98605fe3..59b0917fe 100644
--- a/tests/integration/pipelex/pipes/controller/pipe_sequence/test_pipe_sequence_validation.py
+++ b/tests/integration/pipelex/pipes/controller/pipe_sequence/test_pipe_sequence_creation.py
@@ -9,8 +9,8 @@ from pipelex.pipe_controllers.sequence.pipe_sequence_blueprint import PipeSequen
from pipelex.pipe_controllers.sub_pipe_factory import SubPipeBlueprint
-class TestPipeSequenceValidation:
- """Tests for PipeSequence validate_inputs method"""
+class TestPipeSequenceCreation:
+ """Tests for basic PipeSequence creation"""
def test_pipe_sequence_creation(self, load_test_library: Callable[[list[Path]], None]):
"""Test basic PipeSequence creation"""
@@ -19,12 +19,12 @@ class TestPipeSequenceValidation:
concept_1 = ConceptFactory.make_from_blueprint(
concept_code="TestConcept",
domain=domain,
- blueprint=ConceptBlueprint(description="Lorem Ipsum"),
+ blueprint_or_string_description=ConceptBlueprint(description="Lorem Ipsum"),
)
concept_2 = ConceptFactory.make_from_blueprint(
concept_code="ProcessedText",
domain=domain,
- blueprint=ConceptBlueprint(description="Lorem Ipsum"),
+ blueprint_or_string_description=ConceptBlueprint(description="Lorem Ipsum"),
)
concept_library = get_concept_library()
concept_library.add_concepts([concept_1, concept_2])
@@ -47,41 +47,4 @@ class TestPipeSequenceValidation:
assert len(pipe_sequence.sequential_sub_pipes) == 1
assert pipe_sequence.sequential_sub_pipes[0].pipe_code == "test_pipe_1"
assert pipe_sequence.sequential_sub_pipes[0].output_name == "intermediate_result"
-
- concept_library.teardown()
-
- def test_pipe_sequence_multiple_sub_pipes(self, load_test_library: Callable[[list[Path]], None]):
- """Test PipeSequence with multiple sequential sub-pipes"""
- load_test_library([Path("tests/integration/pipelex/pipes/controller/pipe_sequence")])
- domain = "test_domain"
- concept_1 = ConceptFactory.make_from_blueprint(
- concept_code="TestConcept",
- domain=domain,
- blueprint=ConceptBlueprint(description="Lorem Ipsum"),
- )
- concept_2 = ConceptFactory.make_from_blueprint(
- concept_code="ProcessedText",
- domain=domain,
- blueprint=ConceptBlueprint(description="Lorem Ipsum"),
- )
- concept_library = get_concept_library()
- concept_library.add_concepts([concept_1, concept_2])
-
- pipe_sequence_blueprint = PipeSequenceBlueprint(
- description="Test sequence with multiple steps",
- inputs={"initial_input": concept_1.concept_string},
- output=concept_2.concept_string,
- steps=[SubPipeBlueprint(pipe="step_1", result="intermediate"), SubPipeBlueprint(pipe="step_2", result="final_output")],
- )
-
- pipe_sequence = PipeFactory[PipeSequence].make_from_blueprint(
- domain_code=domain,
- pipe_code="test_sequence",
- blueprint=pipe_sequence_blueprint,
- )
-
- assert pipe_sequence.code == "test_sequence"
- assert len(pipe_sequence.sequential_sub_pipes) == 2
- assert pipe_sequence.inputs.root["initial_input"].concept.code == concept_1.code
-
concept_library.teardown()
diff --git a/tests/integration/pipelex/pipes/controller/pipe_sequence/test_pipe_sequence_dry_run.py b/tests/integration/pipelex/pipes/controller/pipe_sequence/test_pipe_sequence_dry_run.py
index 2cadd4715..06299143e 100644
--- a/tests/integration/pipelex/pipes/controller/pipe_sequence/test_pipe_sequence_dry_run.py
+++ b/tests/integration/pipelex/pipes/controller/pipe_sequence/test_pipe_sequence_dry_run.py
@@ -8,6 +8,7 @@ from pipelex import log, pretty_print
from pipelex.core.concepts.concept_factory import ConceptFactory
from pipelex.core.memory.working_memory_factory import WorkingMemoryFactory
from pipelex.core.stuffs.list_content import ListContent
+from pipelex.core.stuffs.stuff import Stuff
from pipelex.core.stuffs.stuff_factory import StuffFactory
from pipelex.hub import get_pipe_router, get_required_pipe
from pipelex.pipe_run.pipe_job_factory import PipeJobFactory
@@ -19,7 +20,6 @@ from tests.integration.pipelex.pipes.controller.pipe_sequence.discord_newsletter
if TYPE_CHECKING:
from pipelex.core.memory.working_memory import WorkingMemory
- from pipelex.core.stuffs.stuff import Stuff
@pytest.mark.dry_runnable
@@ -104,14 +104,20 @@ class TestPipeSequenceDryRun:
assert channel_summaries_stuff is not None, "channel_summaries should be in working memory"
assert channel_summaries_stuff.concept.code == "ChannelSummary"
assert channel_summaries_stuff.concept.domain == "discord_newsletter"
+ assert isinstance(channel_summaries_stuff, Stuff), "ChannelSurrmary Stuff is not a Stuff"
# Verify channel_summaries is also a ListContent with multiple ChannelSummary items
channel_summaries_list: ListContent[ChannelSummary] = channel_summaries_stuff.as_list_of_fixed_content_type(item_type=ChannelSummary)
+
+ pretty_print(channel_summaries_stuff, "jiodisjdqosj")
+ pretty_print(channel_summaries_list, title="Channel Summaries List")
assert isinstance(channel_summaries_list, ListContent)
assert len(channel_summaries_list.items) > 1, "Should have multiple ChannelSummary items from batch processing"
# Verify each summary item (these should be proper ChannelSummary objects from the LLM mock)
for i, summary in enumerate(channel_summaries_list.items):
+ print(summary)
+ print(type(summary))
assert isinstance(summary, ChannelSummary), f"Summary {i} should be ChannelSummary"
assert len(summary.channel_name) > 0, f"Summary {i} should have a non-empty channel name"
assert isinstance(summary.summary_items, list), f"Summary {i} should have a list of summary items"
diff --git a/tests/integration/pipelex/pipes/controller/pipe_sequence/test_pipe_sequence_multiple_steps.py b/tests/integration/pipelex/pipes/controller/pipe_sequence/test_pipe_sequence_multiple_steps.py
new file mode 100644
index 000000000..31bf09bba
--- /dev/null
+++ b/tests/integration/pipelex/pipes/controller/pipe_sequence/test_pipe_sequence_multiple_steps.py
@@ -0,0 +1,49 @@
+from pathlib import Path
+from typing import Callable
+
+from pipelex.core.concepts.concept_factory import ConceptBlueprint, ConceptFactory
+from pipelex.core.pipes.pipe_factory import PipeFactory
+from pipelex.hub import get_concept_library
+from pipelex.pipe_controllers.sequence.pipe_sequence import PipeSequence
+from pipelex.pipe_controllers.sequence.pipe_sequence_blueprint import PipeSequenceBlueprint
+from pipelex.pipe_controllers.sub_pipe_factory import SubPipeBlueprint
+
+
+class TestPipeSequenceMultipleSteps:
+ """Tests for PipeSequence with multiple steps"""
+
+ def test_pipe_sequence_multiple_sub_pipes(self, load_test_library: Callable[[list[Path]], None]):
+ """Test PipeSequence with multiple sequential sub-pipes"""
+ load_test_library([Path("tests/integration/pipelex/pipes/controller/pipe_sequence")])
+ domain = "test_domain"
+ concept_1 = ConceptFactory.make_from_blueprint(
+ concept_code="TestConcept",
+ domain=domain,
+ blueprint_or_string_description=ConceptBlueprint(description="Lorem Ipsum"),
+ )
+ concept_2 = ConceptFactory.make_from_blueprint(
+ concept_code="ProcessedText",
+ domain=domain,
+ blueprint_or_string_description=ConceptBlueprint(description="Lorem Ipsum"),
+ )
+ concept_library = get_concept_library()
+ concept_library.add_concepts([concept_1, concept_2])
+
+ pipe_sequence_blueprint = PipeSequenceBlueprint(
+ description="Test sequence with multiple steps",
+ inputs={"initial_input": concept_1.concept_string},
+ output=concept_2.concept_string,
+ steps=[SubPipeBlueprint(pipe="step_1", result="intermediate"), SubPipeBlueprint(pipe="step_2", result="final_output")],
+ )
+
+ pipe_sequence = PipeFactory[PipeSequence].make_from_blueprint(
+ domain_code=domain,
+ pipe_code="test_sequence",
+ blueprint=pipe_sequence_blueprint,
+ )
+
+ assert pipe_sequence.code == "test_sequence"
+ assert len(pipe_sequence.sequential_sub_pipes) == 2
+ assert pipe_sequence.inputs.root["initial_input"].concept.code == concept_1.code
+
+ concept_library.teardown()
diff --git a/tests/integration/pipelex/pipes/controller/pipe_sequence/test_pipe_sequence_simple.py b/tests/integration/pipelex/pipes/controller/pipe_sequence/test_pipe_sequence_simple.py
index 96b847208..59f2640d1 100644
--- a/tests/integration/pipelex/pipes/controller/pipe_sequence/test_pipe_sequence_simple.py
+++ b/tests/integration/pipelex/pipes/controller/pipe_sequence/test_pipe_sequence_simple.py
@@ -36,7 +36,7 @@ class TestPipeSequenceSimple:
concept_1 = ConceptFactory.make_from_blueprint(
concept_code="TestConcept1",
domain=domain,
- blueprint=ConceptBlueprint(description="Lorem Ipsum"),
+ blueprint_or_string_description=ConceptBlueprint(description="Lorem Ipsum"),
)
concept_library = get_concept_library()
concept_library.add_concepts([concept_1])
diff --git a/tests/integration/pipelex/pipes/operator/pipe_extract/test_pipe_extract.py b/tests/integration/pipelex/pipes/operator/pipe_extract/test_pipe_extract.py
index d5c5cd4b6..bbea9f6cf 100644
--- a/tests/integration/pipelex/pipes/operator/pipe_extract/test_pipe_extract.py
+++ b/tests/integration/pipelex/pipes/operator/pipe_extract/test_pipe_extract.py
@@ -36,7 +36,7 @@ class TestPipeExtract:
concept_1 = ConceptFactory.make_from_blueprint(
concept_code="PageScan",
domain="extract",
- blueprint=ConceptBlueprint(description="Lorem Ipsum"),
+ blueprint_or_string_description=ConceptBlueprint(description="Lorem Ipsum"),
)
concept_library.add_new_concept(concept=concept_1)
diff --git a/tests/integration/pipelex/pipes/operator/pipe_llm/test_pipe_llm.py b/tests/integration/pipelex/pipes/operator/pipe_llm/test_pipe_llm.py
index 68e4e1253..f23bf78a1 100644
--- a/tests/integration/pipelex/pipes/operator/pipe_llm/test_pipe_llm.py
+++ b/tests/integration/pipelex/pipes/operator/pipe_llm/test_pipe_llm.py
@@ -89,7 +89,6 @@ class TestPipeLLMBasic:
):
load_test_library([Path("tests/integration/pipelex/pipes/operator/pipe_llm")])
# TODO: Add assertion on generated objects vs expected results
- pretty_print(data, title="data")
working_memory = WorkingMemoryFactory.make_from_single_stuff(
stuff=StuffFactory.make_stuff(
concept=get_native_concept(NativeConceptCode.TEXT),
@@ -109,6 +108,7 @@ class TestPipeLLMBasic:
structuring_method=structuring_method,
)
+ pretty_print(pipe_llm_blueprint, title="pipe_llm_blueprint")
pipe_code = f"extract_{topic}_{concept}_{structuring_method}"
pipe_code = pipe_code.lower().replace(" ", "_")
pipe = PipeFactory[PipeLLM].make_from_blueprint(
@@ -124,8 +124,11 @@ class TestPipeLLMBasic:
working_memory=working_memory,
pipe_run_params=PipeRunParamsFactory.make_run_params(pipe_run_mode=pipe_run_mode),
)
+
pipe_llm_output = await get_pipe_router().run(pipe_job=pipe_job)
+ pretty_print(pipe_llm_output, title="pipe_llm_dzqdzqoutput")
+
# Log test information
log.verbose(f"Testing {topic} with {structuring_method} method", title="Test Case")
diff --git a/tests/integration/pipelex/pipes/test_bracket_notation_controllers.py b/tests/integration/pipelex/pipes/test_bracket_notation_controllers.py
index 32b2536c2..46fdf5f55 100644
--- a/tests/integration/pipelex/pipes/test_bracket_notation_controllers.py
+++ b/tests/integration/pipelex/pipes/test_bracket_notation_controllers.py
@@ -25,13 +25,13 @@ class TestBracketNotationInControllers:
concept_data_item = ConceptFactory.make_from_blueprint(
concept_code="DataItem",
domain=domain,
- blueprint=ConceptBlueprint(description="Data item"),
+ blueprint_or_string_description=ConceptBlueprint(description="Data item"),
)
concept_library.add_concepts([concept_data_item])
concept_processed_data = ConceptFactory.make_from_blueprint(
concept_code="ProcessedData",
domain=domain,
- blueprint=ConceptBlueprint(description="Processed data"),
+ blueprint_or_string_description=ConceptBlueprint(description="Processed data"),
)
concept_library.add_concepts([concept_processed_data])
@@ -64,12 +64,12 @@ class TestBracketNotationInControllers:
concept_1 = ConceptFactory.make_from_blueprint(
concept_code="Category",
domain=domain,
- blueprint=ConceptBlueprint(description="Category"),
+ blueprint_or_string_description=ConceptBlueprint(description="Category"),
)
concept_2 = ConceptFactory.make_from_blueprint(
concept_code="Result",
domain=domain,
- blueprint=ConceptBlueprint(description="Result"),
+ blueprint_or_string_description=ConceptBlueprint(description="Result"),
)
concept_library.add_concepts([concept_1, concept_2])
@@ -103,12 +103,12 @@ class TestBracketNotationInControllers:
concept_item = ConceptFactory.make_from_blueprint(
concept_code="Item",
domain=domain,
- blueprint=ConceptBlueprint(description="Item"),
+ blueprint_or_string_description=ConceptBlueprint(description="Item"),
)
concept_processed_item = ConceptFactory.make_from_blueprint(
concept_code="ProcessedItem",
domain=domain,
- blueprint=ConceptBlueprint(description="Processed item"),
+ blueprint_or_string_description=ConceptBlueprint(description="Processed item"),
)
concept_library.add_concepts([concept_item, concept_processed_item])
diff --git a/tests/integration/pipelex/pipes/test_flow_factory/discord_newsletter.py b/tests/integration/pipelex/pipes/test_flow_factory/discord_newsletter.py
index c11ac4010..5730daff5 100644
--- a/tests/integration/pipelex/pipes/test_flow_factory/discord_newsletter.py
+++ b/tests/integration/pipelex/pipes/test_flow_factory/discord_newsletter.py
@@ -42,16 +42,3 @@ class ChannelSummaryFlow(StructuredContent):
channel_name: str = Field(..., description="Name of the Discord channel")
summary_items: list[str] = Field(..., description="Well-written summaries of the channel's activity")
-
-
-class HtmlNewsletterFlow(StructuredContent):
- """Represents the final newsletter content"""
-
- weekly_summary: str = Field(..., description="200 character summary of weekly Share channel content")
- new_members: list[str] = Field(default_factory=empty_list_factory_of(str), description="New member introductions in bullet points")
- channel_sections: list[ChannelSummaryFlow] = Field(
- default_factory=empty_list_factory_of(ChannelSummaryFlow), description="Ordered channel summaries"
- )
- geographic_hubs: list[ChannelSummaryFlow] = Field(
- default_factory=empty_list_factory_of(ChannelSummaryFlow), description="Geographic hub channels grouped at end"
- )
diff --git a/tests/integration/pipelex/test_inline_structure_concepts.py b/tests/integration/pipelex/test_inline_structure_concepts.py
index f15d2806d..fa2ecc59e 100644
--- a/tests/integration/pipelex/test_inline_structure_concepts.py
+++ b/tests/integration/pipelex/test_inline_structure_concepts.py
@@ -50,7 +50,7 @@ class TestInlineStructureConcepts:
concept = ConceptFactory.make_from_blueprint(
domain="test_domain",
concept_code="TestFeatureAnalysis",
- blueprint=blueprint,
+ blueprint_or_string_description=blueprint,
)
# Verify concept properties
@@ -90,7 +90,7 @@ class TestInlineStructureConcepts:
concept = ConceptFactory.make_from_blueprint(
domain="test_domain",
concept_code="TestStringRef",
- blueprint=blueprint,
+ blueprint_or_string_description=blueprint,
)
# Verify concept properties
@@ -108,11 +108,12 @@ class TestInlineStructureConcepts:
concept = ConceptFactory.make_from_blueprint(
domain="test_domain",
concept_code="TestAutoDetect",
- blueprint=blueprint,
+ blueprint_or_string_description=blueprint,
)
# Should default to TextContent since TestAutoDetect is not a registered class
- assert concept.structure_class_name == "TextContent"
+ assert concept.structure_class_name == "TestAutoDetect"
+ assert concept.refines == "native.Text"
def test_inline_structure_with_complex_types(self):
"""Test inline structure with complex field types."""
@@ -145,7 +146,7 @@ class TestInlineStructureConcepts:
concept = ConceptFactory.make_from_blueprint(
domain="test_domain",
concept_code="ComplexDocument",
- blueprint=blueprint,
+ blueprint_or_string_description=blueprint,
)
# Verify concept creation
@@ -183,7 +184,7 @@ not a registered subclass of StuffContent",
_ = ConceptFactory.make_from_blueprint(
domain="test_domain",
concept_code="TestInvalidRef",
- blueprint=ConceptBlueprint(description="Test invalid reference", structure="NonExistentClass"),
+ blueprint_or_string_description=ConceptBlueprint(description="Test invalid reference", structure="NonExistentClass"),
)
def test_multiple_inline_structures_do_not_conflict(self):
@@ -198,7 +199,7 @@ not a registered subclass of StuffContent",
concept1 = ConceptFactory.make_from_blueprint(
domain="test_domain",
concept_code="Person",
- blueprint=blueprint1,
+ blueprint_or_string_description=blueprint1,
)
# Second structure with same field names but different context
@@ -211,7 +212,7 @@ not a registered subclass of StuffContent",
concept2 = ConceptFactory.make_from_blueprint(
domain="test_domain",
concept_code="Product",
- blueprint=blueprint2,
+ blueprint_or_string_description=blueprint2,
)
# Both should be created successfully
@@ -261,7 +262,7 @@ not a registered subclass of StuffContent",
concept = ConceptFactory.make_from_blueprint(
domain="test_domain",
concept_code="AnimalInfo",
- blueprint=blueprint,
+ blueprint_or_string_description=blueprint,
)
# Verify concept properties
diff --git a/tests/unit/pipelex/builder/test_runner_generator.py b/tests/unit/pipelex/builder/test_runner_generator.py
index fb97efb0f..dfc01f031 100644
--- a/tests/unit/pipelex/builder/test_runner_generator.py
+++ b/tests/unit/pipelex/builder/test_runner_generator.py
@@ -1,101 +1,80 @@
-"""Unit tests for runner_generator module."""
+"""Unit tests for runner_code module."""
from __future__ import annotations
-from pipelex.builder.runner_code import generate_compact_memory_entry, value_to_python_code
-from pipelex.core.concepts.concept_factory import ConceptFactory
-from pipelex.core.concepts.native.concept_native import NativeConceptCode
+from typing import Any, cast
+from unittest.mock import MagicMock
+import pytest
-class TestValueToPythonCode:
- """Test value_to_python_code function."""
-
- def test_simple_string(self) -> None:
- """Test conversion of a simple string."""
- result = value_to_python_code("hello")
- assert result == '"hello"'
-
- def test_simple_number(self) -> None:
- """Test conversion of a number."""
- result = value_to_python_code(42)
- assert result == "42"
-
- def test_simple_boolean(self) -> None:
- """Test conversion of a boolean."""
- result = value_to_python_code(True)
- assert result == "True"
-
- def test_dict_with_class_image_content(self) -> None:
- """Test conversion of ImageContent dict."""
- value = {"_class": "ImageContent", "url": "test_url"}
- result = value_to_python_code(value)
- assert result == 'ImageContent(url="test_url")'
-
- def test_dict_with_class_pdf_content(self) -> None:
- """Test conversion of PDFContent dict."""
- value = {"_class": "PDFContent", "url": "test_url"}
- result = value_to_python_code(value)
- assert result == 'PDFContent(url="test_url")'
-
- def test_dict_with_concept_code_and_content_simple(self) -> None:
- """Test conversion of refined concept with simple content."""
- value = {
- "concept_code": "test_domain.Question",
- "content": "question_text",
- }
- result = value_to_python_code(value, indent_level=3)
- expected = '{\n "concept": "test_domain.Question",\n "content": "question_text",\n }'
- assert result == expected
-
- def test_dict_with_concept_code_and_content_image(self) -> None:
- """Test conversion of refined Image concept."""
- value = {
- "concept_code": "tables.TableScreenshot",
- "content": {"_class": "ImageContent", "url": "table_screenshot_url"},
- }
- result = value_to_python_code(value, indent_level=3)
- expected = (
- "{\n"
- ' "concept": "tables.TableScreenshot",\n'
- ' "content": ImageContent(url="table_screenshot_url"),\n'
- " }"
- )
- assert result == expected
+from pipelex.builder.runner_code import generate_runner_code
+from pipelex.core.concepts.concept_factory import ConceptFactory
+from pipelex.core.concepts.concept_representation_generator import ConceptRepresentationFormat
+from pipelex.core.concepts.native.concept_native import NativeConceptCode
+from pipelex.core.pipes.inputs.input_requirements import InputRequirement, InputRequirements
-class TestGenerateCompactMemoryEntry:
- """Test generate_compact_memory_entry function."""
+class TestConceptGenerateInputRepresentationJson:
+ """Test Concept.generate_input_representation for JSON format."""
- def test_generate_entry_for_native_text(self) -> None:
- """Test generating entry for native Text concept."""
+ def test_native_text_json(self) -> None:
+ """Test JSON representation for native Text concept."""
concept = ConceptFactory.make_native_concept(NativeConceptCode.TEXT)
- result = generate_compact_memory_entry("message", concept)
- assert result == ' "message": "message_text",'
+ result, _ = concept.generate_input_representation(ConceptRepresentationFormat.JSON)
+ assert result["concept"] == "native.Text"
+ assert "content" in result
+ assert "text" in result["content"]
- def test_generate_entry_for_native_image(self) -> None:
- """Test generating entry for native Image concept."""
+ def test_native_image_json(self) -> None:
+ """Test JSON representation for native Image concept."""
concept = ConceptFactory.make_native_concept(NativeConceptCode.IMAGE)
- result = generate_compact_memory_entry("photo", concept)
- assert result == ' "photo": ImageContent(url="photo_url"),'
+ result, _ = concept.generate_input_representation(ConceptRepresentationFormat.JSON)
+ assert result["concept"] == "native.Image"
+ assert "content" in result
+ assert "url" in result["content"]
+
+ def test_native_number_json(self) -> None:
+ """Test JSON representation for native Number concept."""
+ concept = ConceptFactory.make_native_concept(NativeConceptCode.NUMBER)
+ result, _ = concept.generate_input_representation(ConceptRepresentationFormat.JSON)
+ assert result["concept"] == "native.Number"
+ assert "content" in result
+ assert "number" in result["content"]
+
+ def test_native_pdf_json_with_multiplicity(self) -> None:
+ """Test JSON representation for PDF with multiplicity - content should be a list."""
+ concept = ConceptFactory.make_native_concept(NativeConceptCode.PDF)
+ result, _ = concept.generate_input_representation(ConceptRepresentationFormat.JSON, is_multiple=True)
+ assert result["concept"] == "native.PDF"
+ assert "content" in result
+ # Content should be a list
+ content = cast("list[dict[str, Any]]", result["content"])
+ assert isinstance(content, list)
+ assert len(content) == 1
+ assert "url" in content[0]
- def test_generate_entry_for_refined_image(self) -> None:
- """Test generating entry for a concept that refines Image."""
- concept = ConceptFactory.make(
- domain="tables",
- concept_code="TableScreenshot",
- description="A screenshot of a table",
- structure_class_name="ImageContent",
- refines="native.Image",
- )
- result = generate_compact_memory_entry("table_screenshot", concept)
- # Should generate the full format with concept and content
- assert '"concept": "tables.TableScreenshot"' in result
- assert 'ImageContent(url="table_screenshot_url")' in result
- assert "table_screenshot" in result
+class TestConceptGenerateInputRepresentationPython:
+ """Test Concept.generate_input_representation for Python format."""
- def test_generate_entry_for_refined_text(self) -> None:
- """Test generating entry for a concept that refines Text."""
+ def test_native_text_python(self) -> None:
+ """Test Python representation for native Text concept."""
+ concept = ConceptFactory.make_native_concept(NativeConceptCode.TEXT)
+ result, imports = concept.generate_input_representation(ConceptRepresentationFormat.PYTHON)
+ assert result["concept"] == "native.Text"
+ assert "TextContent" in result["content"]
+ assert "TextContent" in imports
+
+ def test_native_image_python(self) -> None:
+ """Test Python representation for native Image concept."""
+ concept = ConceptFactory.make_native_concept(NativeConceptCode.IMAGE)
+ result, imports = concept.generate_input_representation(ConceptRepresentationFormat.PYTHON)
+ assert result["concept"] == "native.Image"
+ assert "ImageContent" in result["content"]
+ assert "ImageContent" in imports
+
+ def test_refined_text_python(self) -> None:
+ """Test Python representation for a concept that refines Text."""
concept = ConceptFactory.make(
domain="test_domain",
concept_code="Question",
@@ -103,9 +82,204 @@ class TestGenerateCompactMemoryEntry:
structure_class_name="TextContent",
refines="native.Text",
)
- result = generate_compact_memory_entry("question", concept)
+ result, imports = concept.generate_input_representation(ConceptRepresentationFormat.PYTHON)
+ assert result["concept"] == "test_domain.Question"
+ assert "TextContent" in result["content"]
+ assert "TextContent" in imports
+
+ def test_native_pdf_python_with_multiplicity(self) -> None:
+ """Test Python representation for PDF with multiplicity - for Python format, wrapping is handled by caller."""
+ concept = ConceptFactory.make_native_concept(NativeConceptCode.PDF)
+ result, imports = concept.generate_input_representation(ConceptRepresentationFormat.PYTHON, is_multiple=True)
+ # For Python format, is_multiple doesn't wrap content (caller handles it)
+ assert result["concept"] == "native.PDF"
+ assert "PDFContent" in result["content"]
+ assert "PDFContent" in imports
+
+
+class TestInputRequirementsGenerateJsonRepresentation:
+ """Test InputRequirements.generate_json_representation method."""
+
+ def test_empty_inputs(self) -> None:
+ """Test JSON generation with no inputs."""
+ inputs = InputRequirements(root={})
+ result = inputs.generate_json_representation()
+ assert result == {}
+
+ def test_single_input(self) -> None:
+ """Test JSON generation with a single input."""
+ concept = ConceptFactory.make_native_concept(NativeConceptCode.TEXT)
+ inputs = InputRequirements(root={"message": InputRequirement(concept=concept)})
+ result = inputs.generate_json_representation()
+ assert "message" in result
+ assert result["message"]["concept"] == "native.Text"
+
+ def test_multiple_inputs(self) -> None:
+ """Test JSON generation with multiple inputs."""
+ text_concept = ConceptFactory.make_native_concept(NativeConceptCode.TEXT)
+ image_concept = ConceptFactory.make_native_concept(NativeConceptCode.IMAGE)
+ inputs = InputRequirements(
+ root={
+ "text_input": InputRequirement(concept=text_concept),
+ "image_input": InputRequirement(concept=image_concept),
+ }
+ )
+ result = inputs.generate_json_representation()
+ assert "text_input" in result
+ assert "image_input" in result
+ assert result["text_input"]["concept"] == "native.Text"
+ assert result["image_input"]["concept"] == "native.Image"
+
+
+class TestInputRequirementsJsonWithMultiplicity:
+ """Test InputRequirements.generate_json_representation with multiplicity - content is wrapped in list."""
+
+ def test_single_item_no_multiplicity(self) -> None:
+ """Test that single item (no multiplicity) has content as dict, not list."""
+ concept = ConceptFactory.make_native_concept(NativeConceptCode.PDF)
+ inputs = InputRequirements(root={"document": InputRequirement(concept=concept, multiplicity=None)})
+ result = inputs.generate_json_representation()
+ assert "document" in result
+ # concept should be present
+ assert result["document"]["concept"] == "native.PDF"
+ # content should NOT be a list
+ content = result["document"]["content"]
+ assert isinstance(content, dict)
+ assert "url" in content
+
+ def test_multiple_items_true_multiplicity(self) -> None:
+ """Test that multiplicity=True wraps content in list."""
+ concept = ConceptFactory.make_native_concept(NativeConceptCode.PDF)
+ inputs = InputRequirements(root={"documents": InputRequirement(concept=concept, multiplicity=True)})
+ result = inputs.generate_json_representation()
+ assert "documents" in result
+ # concept should be at the top level
+ assert result["documents"]["concept"] == "native.PDF"
+ # content should be a list
+ content = cast("list[dict[str, Any]]", result["documents"]["content"])
+ assert isinstance(content, list)
+ assert len(content) == 1
+ assert "url" in content[0]
+
+ def test_multiple_items_int_multiplicity(self) -> None:
+ """Test that multiplicity=N (int > 1) wraps content in list."""
+ concept = ConceptFactory.make_native_concept(NativeConceptCode.IMAGE)
+ inputs = InputRequirements(root={"images": InputRequirement(concept=concept, multiplicity=3)})
+ result = inputs.generate_json_representation()
+ assert "images" in result
+ # concept should be at the top level
+ assert result["images"]["concept"] == "native.Image"
+ # content should be a list
+ content = cast("list[dict[str, Any]]", result["images"]["content"])
+ assert isinstance(content, list)
+ assert len(content) == 1
+ assert "url" in content[0]
+
+ def test_mixed_multiplicity(self) -> None:
+ """Test mixed single and multiple inputs."""
+ pdf_concept = ConceptFactory.make_native_concept(NativeConceptCode.PDF)
+ inputs = InputRequirements(
+ root={
+ "single_pdf": InputRequirement(concept=pdf_concept, multiplicity=None),
+ "multiple_pdfs": InputRequirement(concept=pdf_concept, multiplicity=True),
+ }
+ )
+ result = inputs.generate_json_representation()
+ # single_pdf content should be a dict
+ assert isinstance(result["single_pdf"]["content"], dict)
+ # multiple_pdfs content should be a list
+ assert isinstance(result["multiple_pdfs"]["content"], list)
+
+
+class TestGenerateRunnerCode:
+ """Test generate_runner_code function."""
+
+ @pytest.fixture
+ def mock_pipe_single_output(self) -> MagicMock:
+ """Create a mock pipe with a single output."""
+ text_concept = ConceptFactory.make_native_concept(NativeConceptCode.TEXT)
+ pdf_concept = ConceptFactory.make_native_concept(NativeConceptCode.PDF)
+
+ mock_pipe = MagicMock()
+ mock_pipe.code = "test_pipe"
+ mock_pipe.output = text_concept
+ mock_pipe.inputs = InputRequirements(root={"document": InputRequirement(concept=pdf_concept)})
+ return mock_pipe
+
+ @pytest.fixture
+ def mock_pipe_list_output(self) -> MagicMock:
+ """Create a mock pipe with a list output."""
+ text_concept = ConceptFactory.make_native_concept(NativeConceptCode.TEXT)
+ pdf_concept = ConceptFactory.make_native_concept(NativeConceptCode.PDF)
+
+ mock_pipe = MagicMock()
+ mock_pipe.code = "test_pipe_list"
+ mock_pipe.output = text_concept
+ mock_pipe.inputs = InputRequirements(root={"documents": InputRequirement(concept=pdf_concept, multiplicity=True)})
+ return mock_pipe
+
+ def test_runner_code_includes_imports(self, mock_pipe_single_output: MagicMock) -> None:
+ """Test that generated runner code includes necessary imports."""
+ runner_code = generate_runner_code(mock_pipe_single_output)
+ assert "import asyncio" in runner_code
+ assert "from pipelex.pipelex import Pipelex" in runner_code
+ assert "from pipelex.pipeline.execute import execute_pipeline" in runner_code
+
+ def test_runner_code_includes_structure_imports(self, mock_pipe_single_output: MagicMock) -> None:
+ """Test that generated runner code includes structure class imports."""
+ runner_code = generate_runner_code(mock_pipe_single_output)
+ assert "from pipelex.core.stuffs.pdf_content import PDFContent" in runner_code
+ assert "from pipelex.core.stuffs.text_content import TextContent" in runner_code
+
+ def test_runner_code_single_output_return_type(self, mock_pipe_single_output: MagicMock) -> None:
+ """Test that generated runner code has correct return type for single output."""
+ runner_code = generate_runner_code(mock_pipe_single_output, output_multiplicity=False)
+ assert "async def run_test_pipe() -> TextContent:" in runner_code
+ assert "pipe_output.main_stuff_as(content_type=TextContent)" in runner_code
+
+ def test_runner_code_list_output_return_type(self, mock_pipe_list_output: MagicMock) -> None:
+ """Test that generated runner code has correct return type for list output."""
+ runner_code = generate_runner_code(mock_pipe_list_output, output_multiplicity=True)
+ assert "async def run_test_pipe_list() -> list[TextContent]:" in runner_code
+ assert "pipe_output.main_stuff_as_items(item_type=TextContent)" in runner_code
+
+ def test_runner_code_includes_inputs(self, mock_pipe_single_output: MagicMock) -> None:
+ """Test that generated runner code includes input values."""
+ runner_code = generate_runner_code(mock_pipe_single_output)
+ assert '"document":' in runner_code
+ assert '"concept": "native.PDF"' in runner_code
+ assert "PDFContent(" in runner_code
+
+ def test_runner_code_includes_main_block(self, mock_pipe_single_output: MagicMock) -> None:
+ """Test that generated runner code includes main block."""
+ runner_code = generate_runner_code(mock_pipe_single_output)
+ assert 'if __name__ == "__main__":' in runner_code
+ assert "Pipelex.make()" in runner_code
+ assert "asyncio.run(run_test_pipe())" in runner_code
+
+ def test_runner_code_multiplicity_input(self, mock_pipe_list_output: MagicMock) -> None:
+ """Test that generated runner code handles input multiplicity correctly."""
+ runner_code = generate_runner_code(mock_pipe_list_output)
+ # Should have list-wrapped content for multiplicity input
+ assert "[PDFContent(" in runner_code
+
+ def test_runner_code_custom_class_import_format(self) -> None:
+ """Test that custom class imports use domain_conceptCode format for standalone scripts."""
+ # Create a custom concept (non-native)
+ custom_concept = ConceptFactory.make(
+ domain="test_domain",
+ concept_code="CustomOutput",
+ description="A custom output",
+ structure_class_name="CustomOutput",
+ )
+ pdf_concept = ConceptFactory.make_native_concept(NativeConceptCode.PDF)
+
+ mock_pipe = MagicMock()
+ mock_pipe.code = "custom_pipe"
+ mock_pipe.output = custom_concept
+ mock_pipe.inputs = InputRequirements(root={"document": InputRequirement(concept=pdf_concept)})
- # Should generate the full format with concept and content
- assert '"concept": "test_domain.Question"' in result
- assert '"question_text"' in result
- assert "question" in result
+ runner_code = generate_runner_code(mock_pipe)
+ # Custom imports should NOT use relative import (no leading dot) for standalone scripts
+ assert "from structures.test_domain_CustomOutput import CustomOutput" in runner_code
+ assert "from .structures" not in runner_code
diff --git a/tests/unit/pipelex/core/concepts/concept_factory/data.py b/tests/unit/pipelex/core/concepts/concept_factory/data.py
index c51ac573e..084bb59ff 100644
--- a/tests/unit/pipelex/core/concepts/concept_factory/data.py
+++ b/tests/unit/pipelex/core/concepts/concept_factory/data.py
@@ -73,7 +73,7 @@ class TestCases:
code="Text",
description="Native text concept",
structure_class_name=NativeConceptCode.TEXT.structure_class_name,
- refines=None,
+ refines=NativeConceptCode.TEXT.concept_string,
),
),
# Test case 6: Concept code from same domain
@@ -86,8 +86,8 @@ class TestCases:
domain="my_domain",
code="DomainConcept",
description="A concept from same domain",
- structure_class_name=NativeConceptCode.TEXT.structure_class_name,
- refines=None,
+ structure_class_name="DomainConcept",
+ refines="native.Text",
),
),
# Test case 9: Blueprint with dict structure (same domain)
@@ -126,7 +126,7 @@ class TestCases:
domain="my_domain",
code="ConceptWithRefines",
description="A concept with refines",
- structure_class_name=NativeConceptCode.IMAGE.structure_class_name,
+ structure_class_name="ConceptWithRefines",
refines="native.Image",
),
),
diff --git a/tests/unit/pipelex/core/concepts/concept_factory/test_concept_factory.py b/tests/unit/pipelex/core/concepts/concept_factory/test_concept_factory.py
index d42ba6d8c..785e37ad9 100644
--- a/tests/unit/pipelex/core/concepts/concept_factory/test_concept_factory.py
+++ b/tests/unit/pipelex/core/concepts/concept_factory/test_concept_factory.py
@@ -1,5 +1,6 @@
import pytest
+from pipelex import pretty_print
from pipelex.core.concepts.concept import Concept
from pipelex.core.concepts.concept_blueprint import (
ConceptBlueprint,
@@ -8,6 +9,7 @@ from pipelex.core.concepts.concept_blueprint import (
from pipelex.core.concepts.concept_factory import ConceptFactory
from pipelex.core.concepts.concept_structure_blueprint import ConceptStructureBlueprintFieldType
from pipelex.core.concepts.exceptions import ConceptFactoryError
+from pipelex.core.concepts.helpers import normalize_structure_blueprint
from tests.unit.pipelex.core.concepts.concept_factory.data import TestCases
@@ -54,7 +56,7 @@ class TestConceptFactory:
),
}
- assert ConceptFactory.normalize_structure_blueprint(mixed_structure_blueprint) == expected_structure
+ assert normalize_structure_blueprint(mixed_structure_blueprint) == expected_structure
mixed_structure_blueprint2: dict[str, str | ConceptStructureBlueprint] = {
"name": ConceptStructureBlueprint(description="The name of the person", type=ConceptStructureBlueprintFieldType.TEXT, required=True),
@@ -78,7 +80,7 @@ class TestConceptFactory:
),
}
- assert ConceptFactory.normalize_structure_blueprint(mixed_structure_blueprint2) == expected_structure2
+ assert normalize_structure_blueprint(mixed_structure_blueprint2) == expected_structure2
@pytest.mark.parametrize(
("domain", "concept_string_or_code", "expected_result"),
@@ -112,8 +114,10 @@ class TestConceptFactory:
result = ConceptFactory.make_from_blueprint(
domain=domain,
concept_code=concept_code,
- blueprint=blueprint,
+ blueprint_or_string_description=blueprint,
)
+ pretty_print(result)
+ pretty_print(expected_concept)
assert result == expected_concept, f"Concept mismatch for {test_name}"
@@ -125,5 +129,5 @@ class TestConceptFactory:
ConceptFactory.make_from_blueprint(
domain="my_domain",
concept_code="TestConcept",
- blueprint=blueprint,
+ blueprint_or_string_description=blueprint,
)
diff --git a/tests/unit/pipelex/core/concepts/concept_library/test_concept_library.py b/tests/unit/pipelex/core/concepts/concept_library/test_concept_library.py
index a9d3ef239..818c15fe7 100644
--- a/tests/unit/pipelex/core/concepts/concept_library/test_concept_library.py
+++ b/tests/unit/pipelex/core/concepts/concept_library/test_concept_library.py
@@ -15,7 +15,7 @@ class TestConceptLibrary:
concept_1 = ConceptFactory.make_from_blueprint(
domain="test",
concept_code="TestConcept",
- blueprint=ConceptBlueprint(
+ blueprint_or_string_description=ConceptBlueprint(
description="Lorem Ipsum",
structure="ImageContent",
),
@@ -24,7 +24,7 @@ class TestConceptLibrary:
concept_2 = ConceptFactory.make_from_blueprint(
domain="test",
concept_code="TestConcept2",
- blueprint=ConceptBlueprint(
+ blueprint_or_string_description=ConceptBlueprint(
description="Lorem Ipsum",
refines="native.Image",
),
@@ -32,7 +32,7 @@ class TestConceptLibrary:
concept_3 = ConceptFactory.make_from_blueprint(
domain="test",
concept_code="TestConcept2",
- blueprint=ConceptBlueprint(
+ blueprint_or_string_description=ConceptBlueprint(
description="Lorem Ipsum",
refines="Image",
),
@@ -41,7 +41,7 @@ class TestConceptLibrary:
concept_4 = ConceptFactory.make_from_blueprint(
domain="test",
concept_code="TestConcept4",
- blueprint=ConceptBlueprint(
+ blueprint_or_string_description=ConceptBlueprint(
description="Lorem Ipsum",
structure="ImageContent",
),
@@ -50,7 +50,7 @@ class TestConceptLibrary:
concept_5 = ConceptFactory.make_from_blueprint(
domain="test",
concept_code="TestConcept5",
- blueprint=ConceptBlueprint(
+ blueprint_or_string_description=ConceptBlueprint(
description="Lorem Ipsum",
structure="TextContent",
),
@@ -59,7 +59,7 @@ class TestConceptLibrary:
concept_6 = ConceptFactory.make_from_blueprint(
domain="test",
concept_code="TestConcept6",
- blueprint=ConceptBlueprint(
+ blueprint_or_string_description=ConceptBlueprint(
description="Lorem Ipsum",
structure="PDFContent",
),
diff --git a/tests/unit/pipelex/builder/test_pipelex_bundle_blueprint.py b/tests/unit/pipelex/core/concepts/structure_generation/__init__.py
similarity index 100%
rename from tests/unit/pipelex/builder/test_pipelex_bundle_blueprint.py
rename to tests/unit/pipelex/core/concepts/structure_generation/__init__.py
diff --git a/tests/unit/pipelex/core/concepts/test_concept.py b/tests/unit/pipelex/core/concepts/test_concept.py
index 98d37dbb0..6b9d64303 100644
--- a/tests/unit/pipelex/core/concepts/test_concept.py
+++ b/tests/unit/pipelex/core/concepts/test_concept.py
@@ -82,7 +82,7 @@ class TestConcept:
ConceptFactory.make_from_blueprint(
concept_code=NativeConceptCode.TEXT,
domain=valid_domain,
- blueprint=ConceptBlueprint(description=valid_definition),
+ blueprint_or_string_description=ConceptBlueprint(description=valid_definition),
),
)
is True
@@ -92,7 +92,7 @@ class TestConcept:
ConceptFactory.make_from_blueprint(
concept_code=NativeConceptCode.TEXT,
domain=SpecialDomain.NATIVE,
- blueprint=ConceptBlueprint(description=valid_definition),
+ blueprint_or_string_description=ConceptBlueprint(description=valid_definition),
),
)
is True
@@ -102,7 +102,7 @@ class TestConcept:
ConceptFactory.make_from_blueprint(
concept_code=NativeConceptCode.IMAGE,
domain=valid_domain,
- blueprint=ConceptBlueprint(description=valid_definition),
+ blueprint_or_string_description=ConceptBlueprint(description=valid_definition),
),
)
is True
@@ -112,7 +112,7 @@ class TestConcept:
ConceptFactory.make_from_blueprint(
concept_code=NativeConceptCode.PDF,
domain=valid_domain,
- blueprint=ConceptBlueprint(description=valid_definition),
+ blueprint_or_string_description=ConceptBlueprint(description=valid_definition),
),
)
is True
@@ -122,7 +122,7 @@ class TestConcept:
ConceptFactory.make_from_blueprint(
concept_code=NativeConceptCode.TEXT_AND_IMAGES,
domain=valid_domain,
- blueprint=ConceptBlueprint(description=valid_definition),
+ blueprint_or_string_description=ConceptBlueprint(description=valid_definition),
),
)
is True
@@ -132,7 +132,7 @@ class TestConcept:
ConceptFactory.make_from_blueprint(
concept_code=NativeConceptCode.NUMBER,
domain=valid_domain,
- blueprint=ConceptBlueprint(description=valid_definition),
+ blueprint_or_string_description=ConceptBlueprint(description=valid_definition),
),
)
is True
@@ -142,7 +142,7 @@ class TestConcept:
ConceptFactory.make_from_blueprint(
concept_code=NativeConceptCode.ANYTHING,
domain=valid_domain,
- blueprint=ConceptBlueprint(description=valid_definition),
+ blueprint_or_string_description=ConceptBlueprint(description=valid_definition),
),
)
is True
@@ -152,7 +152,7 @@ class TestConcept:
ConceptFactory.make_from_blueprint(
concept_code="RandomConcept",
domain=valid_domain,
- blueprint=ConceptBlueprint(description=valid_definition),
+ blueprint_or_string_description=ConceptBlueprint(description=valid_definition),
),
)
is False
@@ -218,22 +218,22 @@ class TestConcept:
concept1 = ConceptFactory.make_from_blueprint(
concept_code="Code1",
domain="domain1",
- blueprint=ConceptBlueprint(description="Lorem Ipsum", refines=NativeConceptCode.TEXT),
+ blueprint_or_string_description=ConceptBlueprint(description="Lorem Ipsum", refines=NativeConceptCode.TEXT),
)
concept2 = ConceptFactory.make_from_blueprint(
concept_code="Code2",
domain="domain1",
- blueprint=ConceptBlueprint(description="Lorem Ipsum", refines=NativeConceptCode.TEXT),
+ blueprint_or_string_description=ConceptBlueprint(description="Lorem Ipsum", refines=NativeConceptCode.TEXT),
)
concept3 = ConceptFactory.make_from_blueprint(
concept_code="Code3",
domain="domain2",
- blueprint=ConceptBlueprint(description="Lorem Ipsum", structure="TextContent"),
+ blueprint_or_string_description=ConceptBlueprint(description="Lorem Ipsum", structure="TextContent"),
)
concept4 = ConceptFactory.make_from_blueprint(
concept_code="Code4",
domain="domain1",
- blueprint=ConceptBlueprint(description="Lorem Ipsum", structure="ImageContent"),
+ blueprint_or_string_description=ConceptBlueprint(description="Lorem Ipsum", structure="ImageContent"),
)
concept_5 = ConceptFactory.make_native_concept(
@@ -247,7 +247,7 @@ class TestConcept:
concept_7 = ConceptFactory.make_from_blueprint(
concept_code="VisualDescription",
domain="images",
- blueprint=ConceptBlueprint(description="Lorem Ipsum"),
+ blueprint_or_string_description=ConceptBlueprint(description="Lorem Ipsum"),
)
assert Concept.are_concept_compatible(concept_7, concept_6, strict=True) is False
diff --git a/tests/unit/pipelex/core/concepts/test_concept_compact_memory.py b/tests/unit/pipelex/core/concepts/test_concept_compact_memory.py
deleted file mode 100644
index ec1f903a2..000000000
--- a/tests/unit/pipelex/core/concepts/test_concept_compact_memory.py
+++ /dev/null
@@ -1,127 +0,0 @@
-"""Unit tests for Concept compact memory generation methods."""
-
-from __future__ import annotations
-
-from pipelex.core.concepts.concept_factory import ConceptFactory
-from pipelex.core.concepts.native.concept_native import NativeConceptCode
-
-
-class TestConceptCompactMemory:
- """Test Concept methods for generating compact memory examples."""
-
- def test_get_compact_memory_example_text(self) -> None:
- """Test compact memory example for Text concept."""
- concept = ConceptFactory.make_native_concept(NativeConceptCode.TEXT)
- result = concept.get_compact_memory_example("message")
- assert isinstance(result, str)
- assert result == "message_text"
-
- def test_get_compact_memory_example_number(self) -> None:
- """Test compact memory example for Number concept."""
- concept = ConceptFactory.make_native_concept(NativeConceptCode.NUMBER)
- result = concept.get_compact_memory_example("count")
- assert isinstance(result, int)
- assert result == 0
-
- def test_get_compact_memory_example_image(self) -> None:
- """Test compact memory example for Image concept."""
- concept = ConceptFactory.make_native_concept(NativeConceptCode.IMAGE)
- result = concept.get_compact_memory_example("photo")
- assert isinstance(result, dict)
- assert result["_class"] == "ImageContent"
- assert result["url"] == "photo_url"
-
- def test_get_compact_memory_example_pdf(self) -> None:
- """Test compact memory example for PDF concept."""
- concept = ConceptFactory.make_native_concept(NativeConceptCode.PDF)
- result = concept.get_compact_memory_example("document")
- assert isinstance(result, dict)
- assert result["_class"] == "PDFContent"
- assert result["url"] == "document_url"
-
- def test_get_compact_memory_example_text_and_images(self) -> None:
- """Test compact memory example for TextAndImages concept."""
- concept = ConceptFactory.make_native_concept(NativeConceptCode.TEXT_AND_IMAGES)
-
- result = concept.get_compact_memory_example("content")
-
- # Should return a dict with concept_code and content
- assert isinstance(result, dict)
- assert result["concept_code"] == "native.TextAndImages"
- assert "content" in result
- assert "text" in result["content"]
- assert result["content"]["text"] == "text_text" # Generated from field name "text"
- assert "images" in result["content"]
- assert isinstance(result["content"]["images"], list)
-
- def test_get_compact_memory_example_page(self) -> None:
- """Test compact memory example for Page concept."""
- concept = ConceptFactory.make_native_concept(NativeConceptCode.PAGE)
-
- result = concept.get_compact_memory_example("page")
-
- # Should return a dict with concept_code and content
- assert isinstance(result, dict)
- assert result["concept_code"] == "native.Page"
- assert "content" in result
- assert "text_and_images" in result["content"]
- assert isinstance(result["content"]["text_and_images"], dict)
-
- def test_get_compact_memory_example_custom_structured(self) -> None:
- """Test compact memory example for a custom structured concept."""
- concept = ConceptFactory.make(
- concept_code="Invoice",
- domain="accounting",
- description="Invoice data",
- structure_class_name="Invoice",
- refines=None,
- )
-
- result = concept.get_compact_memory_example("invoice")
-
- # Should return a dict with concept_code and content
- assert isinstance(result, dict)
- assert result["concept_code"] == "accounting.Invoice"
- assert "content" in result
-
- def test_get_compact_memory_example_for_refined_text_concept(self) -> None:
- """Test compact memory example for a concept that refines Text."""
- # Create a concept that refines Text using direct make() to control domain
- concept = ConceptFactory.make(
- domain="test_domain",
- concept_code="Question",
- description="A question",
- structure_class_name="TextContent",
- refines="native.Text",
- )
-
- # Test - should return full format with concept_code since it's a refined (non-native) concept
- # Even though it refines Text, it's not the native Text concept itself
- result = concept.get_compact_memory_example("question")
- assert isinstance(result, dict)
- assert "concept_code" in result
- assert result["concept_code"] == "test_domain.Question"
- assert "content" in result
- assert result["content"] == "question_text" # Content is still the simple text string
-
- def test_get_compact_memory_example_for_refined_image_concept(self) -> None:
- """Test compact memory example for a concept that refines Image (non-native)."""
- # Create a concept that refines Image using direct make() to control domain
- concept = ConceptFactory.make(
- domain="tables",
- concept_code="TableScreenshot",
- description="A screenshot of a table",
- structure_class_name="ImageContent",
- refines="native.Image",
- )
-
- # Test - should return full format with concept_code since it's a refined (non-native) concept
- result = concept.get_compact_memory_example("table_screenshot")
- assert isinstance(result, dict)
- assert "concept_code" in result
- assert result["concept_code"] == "tables.TableScreenshot"
- assert "content" in result
- # Content should be wrapped in ImageContent format
- assert isinstance(result["content"], dict)
- assert result["content"]["_class"] == "ImageContent"
- assert result["content"]["url"] == "table_screenshot_url"
diff --git a/tests/unit/pipelex/core/concepts/test_concept_representation_generator.py b/tests/unit/pipelex/core/concepts/test_concept_representation_generator.py
new file mode 100644
index 000000000..f8ece2df7
--- /dev/null
+++ b/tests/unit/pipelex/core/concepts/test_concept_representation_generator.py
@@ -0,0 +1,461 @@
+"""Unit tests for ConceptRepresentationGenerator."""
+
+from pydantic import Field
+
+from pipelex.core.concepts.concept_representation_generator import (
+ ConceptRepresentationFormat,
+ ConceptRepresentationGenerator,
+ generate_json_representation,
+ generate_python_representation,
+)
+from pipelex.core.stuffs.structured_content import StructuredContent
+from pipelex.types import StrEnum
+
+# =============================================================================
+# Test Fixtures - Simple classes for unit testing
+# =============================================================================
+
+
+class SimpleContent(StructuredContent):
+ """Simple content with basic types."""
+
+ name: str = Field(..., description="A name")
+ count: int = Field(..., description="A count")
+ rate: float = Field(..., description="A rate")
+ active: bool = Field(..., description="Is active")
+
+
+class StatusEnum(StrEnum):
+ """Status enum for testing."""
+
+ PENDING = "pending"
+ DONE = "done"
+
+
+class ContentWithEnum(StructuredContent):
+ """Content with an enum field."""
+
+ status: StatusEnum = Field(..., description="Status")
+
+
+class NestedChild(StructuredContent):
+ """A child content class."""
+
+ value: str = Field(..., description="A value")
+
+
+class ContentWithNestedClass(StructuredContent):
+ """Content with a nested StuffContent field."""
+
+ child: NestedChild = Field(..., description="Nested child")
+
+
+class ContentWithList(StructuredContent):
+ """Content with list fields."""
+
+ tags: list[str] = Field(..., description="List of tags")
+
+
+class ContentWithNestedList(StructuredContent):
+ """Content with a list of nested content."""
+
+ children: list[NestedChild] = Field(..., description="List of children")
+
+
+class ContentWithDict(StructuredContent):
+ """Content with a dict field."""
+
+ metadata: dict[str, str] = Field(..., description="Metadata dict")
+
+
+class ContentWithOptional(StructuredContent):
+ """Content with an optional field."""
+
+ maybe_name: str | None = Field(None, description="Optional name")
+
+
+class ContentWithRequiredAndOptional(StructuredContent):
+ """Content with both required and optional fields (like ImageContent)."""
+
+ url: str = Field(..., description="Required URL")
+ source_prompt: str | None = Field(None, description="Optional prompt")
+ caption: str | None = Field(None, description="Optional caption")
+ base_64: str | None = Field(None, description="Optional base64")
+
+
+# =============================================================================
+# Tests for ConceptRepresentationFormat enum
+# =============================================================================
+
+
+class TestConceptRepresentationFormat:
+ """Test the ConceptRepresentationFormat enum."""
+
+ def test_enum_values(self) -> None:
+ """Test that the enum has the expected values."""
+ assert ConceptRepresentationFormat.JSON.value == "json"
+ assert ConceptRepresentationFormat.PYTHON.value == "python"
+
+ def test_enum_iteration(self) -> None:
+ """Test that we can iterate over enum values."""
+ values = list(ConceptRepresentationFormat)
+ assert len(values) == 2
+
+
+# =============================================================================
+# Tests for generate_field_value - basic types
+# =============================================================================
+
+
+class TestGenerateFieldValueBasicTypes:
+ """Test generate_field_value for basic Python types."""
+
+ def test_string_field(self) -> None:
+ """String field generates 'fieldname_value'."""
+ generator = ConceptRepresentationGenerator(ConceptRepresentationFormat.JSON)
+ result = generator.generate_field_value(str, "my_field")
+ assert result == "my_field_value"
+
+ def test_int_field(self) -> None:
+ """Int field generates 0."""
+ generator = ConceptRepresentationGenerator(ConceptRepresentationFormat.JSON)
+ result = generator.generate_field_value(int, "count")
+ assert result == 0
+
+ def test_float_field(self) -> None:
+ """Float field generates 0.0."""
+ generator = ConceptRepresentationGenerator(ConceptRepresentationFormat.JSON)
+ result = generator.generate_field_value(float, "rate")
+ assert result == 0.0
+
+ def test_bool_field(self) -> None:
+ """Bool field generates False."""
+ generator = ConceptRepresentationGenerator(ConceptRepresentationFormat.JSON)
+ result = generator.generate_field_value(bool, "active")
+ assert result is False
+
+
+# =============================================================================
+# Tests for generate_field_value - complex types
+# =============================================================================
+
+
+class TestGenerateFieldValueComplexTypes:
+ """Test generate_field_value for complex types (enums, lists, dicts, nested)."""
+
+ def test_enum_field(self) -> None:
+ """Enum field generates first enum value."""
+ generator = ConceptRepresentationGenerator(ConceptRepresentationFormat.JSON)
+ result = generator.generate_field_value(StatusEnum, "status")
+ assert result == "pending"
+
+ def test_list_of_strings(self) -> None:
+ """List[str] generates list with one placeholder."""
+ generator = ConceptRepresentationGenerator(ConceptRepresentationFormat.JSON)
+ result = generator.generate_field_value(list[str], "tags")
+ assert result == ["tags_item_value"]
+
+ def test_list_of_nested_content_json(self) -> None:
+ """List[StuffContent] generates list with nested dict (JSON)."""
+ generator = ConceptRepresentationGenerator(ConceptRepresentationFormat.JSON)
+ result = generator.generate_field_value(list[NestedChild], "children")
+ expected = [{"value": "value_value"}]
+ assert result == expected
+
+ def test_list_of_nested_content_python(self) -> None:
+ """List[StuffContent] generates list with instantiation (Python)."""
+ generator = ConceptRepresentationGenerator(ConceptRepresentationFormat.PYTHON)
+ result = generator.generate_field_value(list[NestedChild], "children")
+ expected = ['NestedChild(value="value_value")']
+ assert result == expected
+
+ def test_dict_field(self) -> None:
+ """Dict field generates placeholder dict."""
+ generator = ConceptRepresentationGenerator(ConceptRepresentationFormat.JSON)
+ result = generator.generate_field_value(dict[str, str], "metadata")
+ expected = {"metadata_key": "metadata_value"}
+ assert result == expected
+
+ def test_nested_content_json(self) -> None:
+ """Nested StuffContent generates dict (JSON)."""
+ generator = ConceptRepresentationGenerator(ConceptRepresentationFormat.JSON)
+ result = generator.generate_field_value(NestedChild, "child")
+ expected = {"value": "value_value"}
+ assert result == expected
+
+ def test_nested_content_python(self) -> None:
+ """Nested StuffContent generates instantiation (Python)."""
+ generator = ConceptRepresentationGenerator(ConceptRepresentationFormat.PYTHON)
+ result = generator.generate_field_value(NestedChild, "child")
+ expected = 'NestedChild(value="value_value")'
+ assert result == expected
+
+ def test_optional_field_unwrapped(self) -> None:
+ """Optional[str] is unwrapped and handled as str."""
+ generator = ConceptRepresentationGenerator(ConceptRepresentationFormat.JSON)
+ result = generator.generate_field_value(str | None, "name")
+ assert result == "name_value"
+
+
+# =============================================================================
+# Tests for generate_class_representation - JSON format
+# =============================================================================
+
+
+class TestGenerateClassRepresentationJson:
+ """Test generate_class_representation with JSON output."""
+
+ def test_simple_content(self) -> None:
+ """SimpleContent generates correct JSON dict."""
+ generator = ConceptRepresentationGenerator(ConceptRepresentationFormat.JSON)
+ result = generator.generate_class_representation(SimpleContent)
+ expected = {
+ "name": "name_value",
+ "count": 0,
+ "rate": 0.0,
+ "active": False,
+ }
+ assert result == expected
+
+ def test_content_with_enum(self) -> None:
+ """ContentWithEnum generates correct JSON."""
+ generator = ConceptRepresentationGenerator(ConceptRepresentationFormat.JSON)
+ result = generator.generate_class_representation(ContentWithEnum)
+ expected = {"status": "pending"}
+ assert result == expected
+
+ def test_content_with_nested_class(self) -> None:
+ """ContentWithNestedClass generates nested dict."""
+ generator = ConceptRepresentationGenerator(ConceptRepresentationFormat.JSON)
+ result = generator.generate_class_representation(ContentWithNestedClass)
+ expected = {"child": {"value": "value_value"}}
+ assert result == expected
+
+ def test_content_with_list(self) -> None:
+ """ContentWithList generates list field."""
+ generator = ConceptRepresentationGenerator(ConceptRepresentationFormat.JSON)
+ result = generator.generate_class_representation(ContentWithList)
+ expected = {"tags": ["tags_item_value"]}
+ assert result == expected
+
+ def test_content_with_nested_list(self) -> None:
+ """ContentWithNestedList generates list of nested dicts."""
+ generator = ConceptRepresentationGenerator(ConceptRepresentationFormat.JSON)
+ result = generator.generate_class_representation(ContentWithNestedList)
+ expected = {"children": [{"value": "value_value"}]}
+ assert result == expected
+
+ def test_content_with_dict(self) -> None:
+ """ContentWithDict generates dict field."""
+ generator = ConceptRepresentationGenerator(ConceptRepresentationFormat.JSON)
+ result = generator.generate_class_representation(ContentWithDict)
+ expected = {"metadata": {"metadata_key": "metadata_value"}}
+ assert result == expected
+
+ def test_content_with_optional(self) -> None:
+ """ContentWithOptional handles optional field."""
+ generator = ConceptRepresentationGenerator(ConceptRepresentationFormat.JSON)
+ result = generator.generate_class_representation(ContentWithOptional)
+ expected = {"maybe_name": "maybe_name_value"}
+ assert result == expected
+
+
+# =============================================================================
+# Tests for generate_class_representation - Python format
+# =============================================================================
+
+
+class TestGenerateClassRepresentationPython:
+ """Test generate_class_representation with Python output."""
+
+ def test_simple_content(self) -> None:
+ """SimpleContent generates correct Python instantiation."""
+ generator = ConceptRepresentationGenerator(ConceptRepresentationFormat.PYTHON)
+ result = generator.generate_class_representation(SimpleContent)
+ expected = 'SimpleContent(name="name_value", count=0, rate=0.0, active=False)'
+ assert result == expected
+
+ def test_content_with_nested_class(self) -> None:
+ """ContentWithNestedClass generates nested instantiation."""
+ generator = ConceptRepresentationGenerator(ConceptRepresentationFormat.PYTHON)
+ result = generator.generate_class_representation(ContentWithNestedClass)
+ expected = 'ContentWithNestedClass(child=NestedChild(value="value_value"))'
+ assert result == expected
+
+ def test_content_with_nested_list(self) -> None:
+ """ContentWithNestedList generates list of instantiations."""
+ generator = ConceptRepresentationGenerator(ConceptRepresentationFormat.PYTHON)
+ result = generator.generate_class_representation(ContentWithNestedList)
+ expected = 'ContentWithNestedList(children=[NestedChild(value="value_value")])'
+ assert result == expected
+
+
+# =============================================================================
+# Tests for generate_representation (main entry point)
+# =============================================================================
+
+
+class TestGenerateRepresentation:
+ """Test the main generate_representation method."""
+
+ def test_wraps_with_concept_json(self) -> None:
+ """Result is wrapped with concept (JSON)."""
+ generator = ConceptRepresentationGenerator(ConceptRepresentationFormat.JSON)
+ result = generator.generate_representation("test.SimpleContent", SimpleContent)
+ expected = {
+ "concept": "test.SimpleContent",
+ "content": {
+ "name": "name_value",
+ "count": 0,
+ "rate": 0.0,
+ "active": False,
+ },
+ }
+ assert result == expected
+
+ def test_wraps_with_concept_python(self) -> None:
+ """Result is wrapped with concept (Python)."""
+ generator = ConceptRepresentationGenerator(ConceptRepresentationFormat.PYTHON)
+ result = generator.generate_representation("test.SimpleContent", SimpleContent)
+ expected = {
+ "concept": "test.SimpleContent",
+ "content": 'SimpleContent(name="name_value", count=0, rate=0.0, active=False)',
+ }
+ assert result == expected
+
+ def test_nested_structure_wrapped(self) -> None:
+ """Nested structure is wrapped correctly."""
+ generator = ConceptRepresentationGenerator(ConceptRepresentationFormat.JSON)
+ result = generator.generate_representation("test.Nested", ContentWithNestedClass)
+ expected = {
+ "concept": "test.Nested",
+ "content": {"child": {"value": "value_value"}},
+ }
+ assert result == expected
+
+
+# =============================================================================
+# Tests for imports tracking
+# =============================================================================
+
+
+class TestImportsTracking:
+ """Test that imports_needed tracks used classes."""
+
+ def test_tracks_main_class(self) -> None:
+ """Tracks the main class."""
+ generator = ConceptRepresentationGenerator(ConceptRepresentationFormat.JSON)
+ generator.generate_representation("test.Simple", SimpleContent)
+ assert "SimpleContent" in generator.imports_needed
+
+ def test_tracks_nested_classes(self) -> None:
+ """Tracks nested classes."""
+ generator = ConceptRepresentationGenerator(ConceptRepresentationFormat.JSON)
+ generator.generate_representation("test.Nested", ContentWithNestedClass)
+ assert "ContentWithNestedClass" in generator.imports_needed
+ assert "NestedChild" in generator.imports_needed
+
+ def test_clears_on_new_generation(self) -> None:
+ """Clears imports on new generation call."""
+ generator = ConceptRepresentationGenerator(ConceptRepresentationFormat.JSON)
+ generator.generate_representation("test.Nested", ContentWithNestedClass)
+ generator.generate_representation("test.Simple", SimpleContent)
+ assert "NestedChild" not in generator.imports_needed
+ assert "SimpleContent" in generator.imports_needed
+
+
+# =============================================================================
+# Tests for convenience functions
+# =============================================================================
+
+
+class TestConvenienceFunctions:
+ """Test the convenience functions."""
+
+ def test_generate_json_representation(self) -> None:
+ """generate_json_representation returns correct result."""
+ result = generate_json_representation("test.Simple", SimpleContent)
+ expected = {
+ "concept": "test.Simple",
+ "content": {
+ "name": "name_value",
+ "count": 0,
+ "rate": 0.0,
+ "active": False,
+ },
+ }
+ assert result == expected
+
+ def test_generate_python_representation(self) -> None:
+ """generate_python_representation returns correct result and imports."""
+ result, imports = generate_python_representation("test.Simple", SimpleContent)
+ expected = {
+ "concept": "test.Simple",
+ "content": 'SimpleContent(name="name_value", count=0, rate=0.0, active=False)',
+ }
+ assert result == expected
+ assert "SimpleContent" in imports
+
+
+# =============================================================================
+# Tests for include_optional parameter
+# =============================================================================
+
+
+class TestIncludeOptionalParameter:
+ """Test that optional fields can be excluded from generation."""
+
+ def test_excludes_optional_fields_json(self) -> None:
+ """With include_optional=False, optional fields are excluded (JSON)."""
+ generator = ConceptRepresentationGenerator(ConceptRepresentationFormat.JSON)
+ result = generator.generate_class_representation(ContentWithRequiredAndOptional, include_optional=False)
+ expected = {"url": "url_value"}
+ assert result == expected
+ # Verify optional fields are NOT present
+ assert "source_prompt" not in result
+ assert "caption" not in result
+ assert "base_64" not in result
+
+ def test_excludes_optional_fields_python(self) -> None:
+ """With include_optional=False, optional fields are excluded (Python)."""
+ generator = ConceptRepresentationGenerator(ConceptRepresentationFormat.PYTHON)
+ result = generator.generate_class_representation(ContentWithRequiredAndOptional, include_optional=False)
+ expected = 'ContentWithRequiredAndOptional(url="url_value")'
+ assert result == expected
+
+ def test_includes_optional_fields_by_default(self) -> None:
+ """By default, optional fields ARE included (backward compatibility)."""
+ generator = ConceptRepresentationGenerator(ConceptRepresentationFormat.JSON)
+ result = generator.generate_class_representation(ContentWithRequiredAndOptional)
+ # All fields should be present by default
+ assert "url" in result
+ assert "source_prompt" in result
+ assert "caption" in result
+ assert "base_64" in result
+
+ def test_all_optional_returns_empty_dict_json(self) -> None:
+ """Content with only optional fields returns empty dict when include_optional=False."""
+ generator = ConceptRepresentationGenerator(ConceptRepresentationFormat.JSON)
+ result = generator.generate_class_representation(ContentWithOptional, include_optional=False)
+ expected: dict[str, str] = {}
+ assert result == expected
+
+ def test_all_optional_returns_empty_instantiation_python(self) -> None:
+ """Content with only optional fields returns empty instantiation when include_optional=False."""
+ generator = ConceptRepresentationGenerator(ConceptRepresentationFormat.PYTHON)
+ result = generator.generate_class_representation(ContentWithOptional, include_optional=False)
+ expected = "ContentWithOptional()"
+ assert result == expected
+
+ def test_image_content_only_url_required(self) -> None:
+ """ImageContent should only have url when include_optional=False."""
+ from pipelex.core.stuffs.image_content import ImageContent # noqa: PLC0415
+
+ generator = ConceptRepresentationGenerator(ConceptRepresentationFormat.JSON)
+ result = generator.generate_class_representation(ImageContent, include_optional=False)
+ expected = {"url": "url_value"}
+ assert result == expected
+ # Verify optional fields are NOT present
+ assert "source_prompt" not in result
+ assert "caption" not in result
+ assert "base_64" not in result
diff --git a/tests/unit/pipelex/core/concepts/test_special_args_concept.py b/tests/unit/pipelex/core/concepts/test_special_args_concept.py
index 1236579e5..134d29147 100644
--- a/tests/unit/pipelex/core/concepts/test_special_args_concept.py
+++ b/tests/unit/pipelex/core/concepts/test_special_args_concept.py
@@ -48,7 +48,7 @@ class TestSpecialArgsConcept:
concept = ConceptFactory.make_from_blueprint(
domain=domain,
concept_code="ContentConcept",
- blueprint=concept_blueprint,
+ blueprint_or_string_description=concept_blueprint,
)
concept_library.add_new_concept(concept)
diff --git a/tests/unit/pipelex/core/concepts/test_structured_output_generator.py b/tests/unit/pipelex/core/concepts/test_structured_output_generator.py
index 8cb417264..57e7ec154 100644
--- a/tests/unit/pipelex/core/concepts/test_structured_output_generator.py
+++ b/tests/unit/pipelex/core/concepts/test_structured_output_generator.py
@@ -1,8 +1,8 @@
import pytest
from pipelex import pretty_print
-from pipelex.core.concepts.concept_factory import ConceptFactory
from pipelex.core.concepts.concept_structure_blueprint import ConceptStructureBlueprint, ConceptStructureBlueprintFieldType
+from pipelex.core.concepts.helpers import normalize_structure_blueprint
from pipelex.core.concepts.structure_generation.exceptions import ConceptStructureValidationError
from pipelex.core.concepts.structure_generation.generator import StructureGenerator
from pipelex.core.stuffs.structured_content import StructuredContent
@@ -27,21 +27,13 @@ class TestStructureGenerator:
pretty_print(structure_blueprint, title="Source Blueprint")
pretty_print(result, title="Generated Result")
- # Check the complete generated structure
- expected_result = (
- "from enum import Enum\n"
- "from pipelex.core.stuffs.structured_content import StructuredContent\n"
- "from pydantic import Field\n"
- "from typing import Optional, List, Dict, Any, Literal\n"
- "\n\n"
- "class TestModel(StructuredContent):\n"
- ' """Generated TestModel class"""\n'
- "\n"
- ' name: str = Field(..., description="Name field")\n'
- ' age: Optional[int] = Field(default=None, description="Age field")\n'
- ' active: Optional[bool] = Field(default=True, description="Active status")\n'
- )
- assert result == expected_result
+ # Check that generated code contains required elements
+ assert "class TestModel(StructuredContent):" in result
+ assert ' name: str = Field(..., description="Name field")' in result
+ assert ' age: Optional[int] = Field(default=None, description="Age field")' in result
+ assert ' active: Optional[bool] = Field(default=True, description="Active status")' in result
+ # Check for autogen header
+ assert "AUTOGENERATED CODE - DO NOT EDIT" in result
def test_complex_types_generation(self):
"""Test generation with complex types like lists and dicts."""
@@ -72,21 +64,13 @@ class TestStructureGenerator:
pretty_print(structure_blueprint, title="Source Blueprint")
pretty_print(result, title="Generated Result")
- # Check the complete generated structure
- expected_result = (
- "from enum import Enum\n"
- "from pipelex.core.stuffs.structured_content import StructuredContent\n"
- "from pydantic import Field\n"
- "from typing import Optional, List, Dict, Any, Literal\n"
- "\n\n"
- "class ComplexModel(StructuredContent):\n"
- ' """Generated ComplexModel class"""\n'
- "\n"
- ' tags: Optional[List[str]] = Field(default=None, description="List of tags")\n'
- ' metadata: Optional[Dict[str, str]] = Field(default=None, description="Metadata dictionary")\n'
- ' scores: List[float] = Field(..., description="List of scores")\n'
- )
- assert result == expected_result
+ # Check that generated code contains required elements
+ assert "class ComplexModel(StructuredContent):" in result
+ assert ' tags: Optional[List[str]] = Field(default=None, description="List of tags")' in result
+ assert ' metadata: Optional[Dict[str, str]] = Field(default=None, description="Metadata dictionary")' in result
+ assert ' scores: List[float] = Field(..., description="List of scores")' in result
+ # Check for autogen header
+ assert "AUTOGENERATED CODE - DO NOT EDIT" in result
def test_choices_generation(self):
"""Test generation with inline choices (Literal type)."""
@@ -103,21 +87,13 @@ class TestStructureGenerator:
pretty_print(structure_blueprint, title="Source Blueprint")
pretty_print(result, title="Generated Result")
- # Check the complete generated structure with Literal types
- expected_result = (
- "from enum import Enum\n"
- "from pipelex.core.stuffs.structured_content import StructuredContent\n"
- "from pydantic import Field\n"
- "from typing import Optional, List, Dict, Any, Literal\n"
- "\n\n"
- "class Product(StructuredContent):\n"
- ' """Generated Product class"""\n'
- "\n"
- ' name: Optional[str] = Field(default=None, description="Product name")\n'
- " category: Literal['electronics', 'clothing', 'food', 'books'] = Field(..., description=\"Product category\")\n"
- " size: Optional[Literal['XS', 'S', 'M', 'L', 'XL']] = Field(default=None, description=\"Size of the product\")\n"
- )
- assert result == expected_result
+ # Check that generated code contains required elements
+ assert "class Product(StructuredContent):" in result
+ assert ' name: Optional[str] = Field(default=None, description="Product name")' in result
+ assert " category: Literal['electronics', 'clothing', 'food', 'books']" in result
+ assert " size: Optional[Literal['XS', 'S', 'M', 'L', 'XL']]" in result
+ # Check for autogen header
+ assert "AUTOGENERATED CODE - DO NOT EDIT" in result
def test_empty_structure(self):
"""Test generation of structure with no fields."""
@@ -128,19 +104,12 @@ class TestStructureGenerator:
pretty_print(structure_blueprint, title="Source Blueprint")
pretty_print(result, title="Generated Result")
- # Check the complete generated structure for empty model
- expected_result = (
- "from enum import Enum\n"
- "from pipelex.core.stuffs.structured_content import StructuredContent\n"
- "from pydantic import Field\n"
- "from typing import Optional, List, Dict, Any, Literal\n"
- "\n\n"
- "class EmptyModel(StructuredContent):\n"
- ' """Generated EmptyModel class"""\n'
- "\n"
- " pass\n"
- )
- assert result == expected_result
+ # Check that generated code contains required elements
+ assert "class EmptyModel(StructuredContent):" in result
+ assert '"""Generated EmptyModel class"""' in result
+ # No pass needed when there's a docstring
+ # Check for autogen header
+ assert "AUTOGENERATED CODE - DO NOT EDIT" in result
def test_concept_get_structure_method(self):
"""Test the get_structure method on Concept class."""
@@ -154,20 +123,12 @@ class TestStructureGenerator:
pretty_print(structure_blueprint, title="Source Blueprint")
pretty_print(result, title="Generated Result")
- # Check the complete generated structure
- expected_result = (
- "from enum import Enum\n"
- "from pipelex.core.stuffs.structured_content import StructuredContent\n"
- "from pydantic import Field\n"
- "from typing import Optional, List, Dict, Any, Literal\n"
- "\n\n"
- "class DocumentInfo(StructuredContent):\n"
- ' """Generated DocumentInfo class"""\n'
- "\n"
- ' title: str = Field(..., description="Document title")\n'
- ' page_count: Optional[int] = Field(default=None, description="Number of pages")\n'
- )
- assert result == expected_result
+ # Check that generated code contains required elements
+ assert "class DocumentInfo(StructuredContent):" in result
+ assert ' title: str = Field(..., description="Document title")' in result
+ assert ' page_count: Optional[int] = Field(default=None, description="Number of pages")' in result
+ # Check for autogen header
+ assert "AUTOGENERATED CODE - DO NOT EDIT" in result
def test_generate_from_blueprint_dict_function(self):
"""Test the convenience function for generating from blueprint dict."""
@@ -180,19 +141,11 @@ class TestStructureGenerator:
pretty_print(structure_blueprint, title="Source Blueprint")
pretty_print(result, title="Generated Result")
- # Check the complete generated structure
- expected_result = (
- "from enum import Enum\n"
- "from pipelex.core.stuffs.structured_content import StructuredContent\n"
- "from pydantic import Field\n"
- "from typing import Optional, List, Dict, Any, Literal\n"
- "\n\n"
- "class ConvenienceTest(StructuredContent):\n"
- ' """Generated ConvenienceTest class"""\n'
- "\n"
- ' value: str = Field(..., description="Test value")\n'
- )
- assert result == expected_result
+ # Check that generated code contains required elements
+ assert "class ConvenienceTest(StructuredContent):" in result
+ assert ' value: str = Field(..., description="Test value")' in result
+ # Check for autogen header
+ assert "AUTOGENERATED CODE - DO NOT EDIT" in result
def test_all_field_types(self):
"""Test that all field types are properly handled."""
@@ -221,24 +174,16 @@ class TestStructureGenerator:
pretty_print(structure_blueprint, title="Source Blueprint")
pretty_print(result, title="Generated Result")
- # Check the complete generated structure with all field types
- expected_result = (
- "from enum import Enum\n"
- "from pipelex.core.stuffs.structured_content import StructuredContent\n"
- "from pydantic import Field\n"
- "from typing import Optional, List, Dict, Any, Literal\n"
- "\n\n"
- "class TypeMappingTest(StructuredContent):\n"
- ' """Generated TypeMappingTest class"""\n'
- "\n"
- ' text_field: Optional[str] = Field(default=None, description="Text field")\n'
- ' number_field: Optional[float] = Field(default=None, description="Number field")\n'
- ' integer_field: Optional[int] = Field(default=None, description="Integer field")\n'
- ' boolean_field: Optional[bool] = Field(default=None, description="Boolean field")\n'
- ' list_field: Optional[List[str]] = Field(default=None, description="List field")\n'
- ' dict_field: Optional[Dict[str, int]] = Field(default=None, description="Dict field")\n'
- )
- assert result == expected_result
+ # Check that generated code contains all required field types
+ assert "class TypeMappingTest(StructuredContent):" in result
+ assert ' text_field: Optional[str] = Field(default=None, description="Text field")' in result
+ assert ' number_field: Optional[float] = Field(default=None, description="Number field")' in result
+ assert ' integer_field: Optional[int] = Field(default=None, description="Integer field")' in result
+ assert ' boolean_field: Optional[bool] = Field(default=None, description="Boolean field")' in result
+ assert ' list_field: Optional[List[str]] = Field(default=None, description="List field")' in result
+ assert ' dict_field: Optional[Dict[str, int]] = Field(default=None, description="Dict field")' in result
+ # Check for autogen header
+ assert "AUTOGENERATED CODE - DO NOT EDIT" in result
def test_required_vs_optional_fields(self):
"""Test that fields can be marked as required vs optional."""
@@ -252,20 +197,12 @@ class TestStructureGenerator:
pretty_print(structure_blueprint, title="Source Blueprint")
pretty_print(result, title="Generated Result")
- # Check the complete generated structure with required vs optional fields
- expected_result = (
- "from enum import Enum\n"
- "from pipelex.core.stuffs.structured_content import StructuredContent\n"
- "from pydantic import Field\n"
- "from typing import Optional, List, Dict, Any, Literal\n"
- "\n\n"
- "class RequiredFieldsModel(StructuredContent):\n"
- ' """Generated RequiredFieldsModel class"""\n'
- "\n"
- ' title: str = Field(..., description="Required title")\n'
- ' optional_field: Optional[str] = Field(default=None, description="Optional field")\n'
- )
- assert result == expected_result
+ # Check that generated code contains required elements
+ assert "class RequiredFieldsModel(StructuredContent):" in result
+ assert ' title: str = Field(..., description="Required title")' in result
+ assert ' optional_field: Optional[str] = Field(default=None, description="Optional field")' in result
+ # Check for autogen header
+ assert "AUTOGENERATED CODE - DO NOT EDIT" in result
def test_default_values(self):
"""Test fields with default values."""
@@ -295,21 +232,13 @@ class TestStructureGenerator:
pretty_print(structure_blueprint, title="Source Blueprint")
pretty_print(result, title="Generated Result")
- # Check the complete generated structure with default values
- expected_result = (
- "from enum import Enum\n"
- "from pipelex.core.stuffs.structured_content import StructuredContent\n"
- "from pydantic import Field\n"
- "from typing import Optional, List, Dict, Any, Literal\n"
- "\n\n"
- "class PersonWithDefaults(StructuredContent):\n"
- ' """Generated PersonWithDefaults class"""\n'
- "\n"
- ' name: Optional[str] = Field(default="Anonymous", description="Person name")\n'
- ' age: Optional[int] = Field(default=0, description="Person age")\n'
- ' active: Optional[bool] = Field(default=True, description="Is active")\n'
- )
- assert result == expected_result
+ # Check that generated code contains required elements with default values
+ assert "class PersonWithDefaults(StructuredContent):" in result
+ assert ' name: Optional[str] = Field(default="Anonymous", description="Person name")' in result
+ assert ' age: Optional[int] = Field(default=0, description="Person age")' in result
+ assert ' active: Optional[bool] = Field(default=True, description="Is active")' in result
+ # Check for autogen header
+ assert "AUTOGENERATED CODE - DO NOT EDIT" in result
def test_nested_list_types(self):
"""Test nested list types with different item types."""
@@ -339,21 +268,13 @@ class TestStructureGenerator:
pretty_print(structure_blueprint, title="Source Blueprint")
pretty_print(result, title="Generated Result")
- # Check the complete generated structure with nested list types
- expected_result = (
- "from enum import Enum\n"
- "from pipelex.core.stuffs.structured_content import StructuredContent\n"
- "from pydantic import Field\n"
- "from typing import Optional, List, Dict, Any, Literal\n"
- "\n\n"
- "class ListTypesModel(StructuredContent):\n"
- ' """Generated ListTypesModel class"""\n'
- "\n"
- ' text_list: Optional[List[str]] = Field(default=None, description="List of text items")\n'
- ' number_list: List[float] = Field(..., description="List of numbers")\n'
- ' integer_list: Optional[List[int]] = Field(default=None, description="List of integers")\n'
- )
- assert result == expected_result
+ # Check that generated code contains required list types
+ assert "class ListTypesModel(StructuredContent):" in result
+ assert ' text_list: Optional[List[str]] = Field(default=None, description="List of text items")' in result
+ assert ' number_list: List[float] = Field(..., description="List of numbers")' in result
+ assert ' integer_list: Optional[List[int]] = Field(default=None, description="List of integers")' in result
+ # Check for autogen header
+ assert "AUTOGENERATED CODE - DO NOT EDIT" in result
def test_nested_dict_types(self):
"""Test nested dict types with different key/value combinations."""
@@ -386,21 +307,13 @@ class TestStructureGenerator:
pretty_print(structure_blueprint, title="Source Blueprint")
pretty_print(result, title="Generated Result")
- # Check the complete generated structure with nested dict types
- expected_result = (
- "from enum import Enum\n"
- "from pipelex.core.stuffs.structured_content import StructuredContent\n"
- "from pydantic import Field\n"
- "from typing import Optional, List, Dict, Any, Literal\n"
- "\n\n"
- "class DictTypesModel(StructuredContent):\n"
- ' """Generated DictTypesModel class"""\n'
- "\n"
- ' string_to_string: Optional[Dict[str, str]] = Field(default=None, description="String to string mapping")\n'
- ' string_to_number: Dict[str, float] = Field(..., description="String to number mapping")\n'
- ' string_to_integer: Optional[Dict[str, int]] = Field(default=None, description="String to integer mapping")\n'
- )
- assert result == expected_result
+ # Check that generated code contains required dict types
+ assert "class DictTypesModel(StructuredContent):" in result
+ assert ' string_to_string: Optional[Dict[str, str]] = Field(default=None, description="String to string mapping")' in result
+ assert ' string_to_number: Dict[str, float] = Field(..., description="String to number mapping")' in result
+ assert ' string_to_integer: Optional[Dict[str, int]] = Field(default=None, description="String to integer mapping")' in result
+ # Check for autogen header
+ assert "AUTOGENERATED CODE - DO NOT EDIT" in result
def test_mixed_complexity_structure(self):
"""Test a structure with mixed complexity - simple and complex types together."""
@@ -439,24 +352,16 @@ class TestStructureGenerator:
pretty_print(structure_blueprint, title="Source Blueprint")
pretty_print(result, title="Generated Result")
- # Check the complete generated structure with mixed complexity
- expected_result = (
- "from enum import Enum\n"
- "from pipelex.core.stuffs.structured_content import StructuredContent\n"
- "from pydantic import Field\n"
- "from typing import Optional, List, Dict, Any, Literal\n"
- "\n\n"
- "class ComplexItem(StructuredContent):\n"
- ' """Generated ComplexItem class"""\n'
- "\n"
- ' id: int = Field(..., description="Unique identifier")\n'
- ' name: str = Field(..., description="Display name")\n'
- ' tags: Optional[List[str]] = Field(default=None, description="Associated tags")\n'
- ' metadata: Optional[Dict[str, str]] = Field(default=None, description="Additional metadata")\n'
- ' active: Optional[bool] = Field(default=True, description="Whether item is active")\n'
- " priority: Optional[Literal['low', 'medium', 'high', 'urgent']] = Field(default=\"medium\", description=\"Priority level\")\n"
- )
- assert result == expected_result
+ # Check that generated code contains all required complex elements
+ assert "class ComplexItem(StructuredContent):" in result
+ assert ' id: int = Field(..., description="Unique identifier")' in result
+ assert ' name: str = Field(..., description="Display name")' in result
+ assert ' tags: Optional[List[str]] = Field(default=None, description="Associated tags")' in result
+ assert ' metadata: Optional[Dict[str, str]] = Field(default=None, description="Additional metadata")' in result
+ assert ' active: Optional[bool] = Field(default=True, description="Whether item is active")' in result
+ assert "priority: Optional[Literal['low', 'medium', 'high', 'urgent']]" in result
+ # Check for autogen header
+ assert "AUTOGENERATED CODE - DO NOT EDIT" in result
def test_mixed_structure_blueprint_normalization(self):
"""Test that mixed structure blueprints (strings and ConceptStructureBlueprint objects) are properly normalized."""
@@ -471,7 +376,7 @@ class TestStructureGenerator:
),
}
- normalized_structure = ConceptFactory.normalize_structure_blueprint(mixed_structure_blueprint)
+ normalized_structure = normalize_structure_blueprint(mixed_structure_blueprint)
result, _ = StructureGenerator().generate_from_structure_blueprint("PersonInfo", normalized_structure)
@@ -479,22 +384,14 @@ class TestStructureGenerator:
pretty_print(normalized_structure, title="Normalized Blueprint")
pretty_print(result, title="Generated Result")
- # Check the complete generated structure with mixed field types
- expected_result = (
- "from datetime import datetime\n"
- "from enum import Enum\n"
- "from pipelex.core.stuffs.structured_content import StructuredContent\n"
- "from pydantic import Field\n"
- "from typing import Optional, List, Dict, Any, Literal\n"
- "\n\n"
- "class PersonInfo(StructuredContent):\n"
- ' """Generated PersonInfo class"""\n'
- "\n"
- ' name: str = Field(..., description="The name of the person")\n'
- ' age: float = Field(..., description="The age of the person")\n'
- ' birthdate: datetime = Field(..., description="The birthdate of the person")\n'
- )
- assert result == expected_result
+ # Check that generated code contains required elements including datetime
+ assert "from datetime import datetime" in result
+ assert "class PersonInfo(StructuredContent):" in result
+ assert ' name: str = Field(..., description="The name of the person")' in result
+ assert ' age: float = Field(..., description="The age of the person")' in result
+ assert ' birthdate: datetime = Field(..., description="The birthdate of the person")' in result
+ # Check for autogen header
+ assert "AUTOGENERATED CODE - DO NOT EDIT" in result
# Verify that the string was properly converted to ConceptStructureBlueprint
assert isinstance(normalized_structure["name"], ConceptStructureBlueprint)
@@ -517,7 +414,7 @@ class TestStructureGenerator:
assert issubclass(the_class, StructuredContent)
# Test validation directly
- generator.validate_generated_code(python_code=python_code, expected_class_name="ValidTestModel", required_base_class=StructuredContent)
+ generator.validate_generated_code(python_code=python_code, expected_class_name="ValidTestModel")
def test_code_validation_syntax_error(self):
"""Test that invalid syntax fails validation."""
@@ -531,7 +428,7 @@ class InvalidModel(StructuredContent):
"""
with pytest.raises(SyntaxError):
- generator.validate_generated_code(python_code=invalid_code, expected_class_name="InvalidModel", required_base_class=StructuredContent)
+ generator.validate_generated_code(python_code=invalid_code, expected_class_name="InvalidModel")
def test_code_validation_missing_class(self):
"""Test that code without the expected class fails validation."""
@@ -545,6 +442,250 @@ class WrongClassName(StructuredContent):
"""
with pytest.raises(ConceptStructureValidationError):
- generator.validate_generated_code(
- python_code=code_without_expected_class, expected_class_name="ExpectedClassName", required_base_class=StructuredContent
+ generator.validate_generated_code(python_code=code_without_expected_class, expected_class_name="ExpectedClassName")
+
+ def test_inheritance_from_text_content(self):
+ """Test generating a class that inherits from TextContent."""
+ structure_blueprint = {
+ "author": ConceptStructureBlueprint(
+ description="The author of the question",
+ type=ConceptStructureBlueprintFieldType.TEXT,
+ required=False,
+ ),
+ }
+
+ generator = StructureGenerator()
+ generated_code, generated_class = generator.generate_from_structure_blueprint(
+ "Question",
+ structure_blueprint,
+ base_class_name="TextContent",
+ )
+
+ pretty_print(structure_blueprint, title="Source Blueprint")
+ pretty_print(generated_code, title="Generated Result")
+
+ # Verify the code compiles
+ import ast # noqa: PLC0415
+
+ ast.parse(generated_code)
+ compile(generated_code, "<generated>", "exec")
+
+ # Verify correct import is present
+ assert "from pipelex.core.stuffs.text_content import TextContent" in generated_code
+
+ # Verify class inherits from TextContent
+ assert "class Question(TextContent):" in generated_code
+
+ # Verify the generated class is valid
+ from pipelex.core.stuffs.text_content import TextContent # noqa: PLC0415
+
+ assert issubclass(generated_class, TextContent)
+ from pipelex.core.stuffs.stuff_content import StuffContent # noqa: PLC0415
+
+ assert issubclass(generated_class, StuffContent)
+
+ # Test instantiation (TextContent requires 'text' field)
+ instance = generated_class(text="Sample text", author="John Doe") # pyright: ignore[reportCallIssue]
+ assert instance.author == "John Doe" # type: ignore[attr-defined] # pyright: ignore[reportAttributeAccessIssue]
+
+ def test_inheritance_from_image_content(self):
+ """Test generating a class that inherits from ImageContent."""
+ structure_blueprint = {
+ "table_name": ConceptStructureBlueprint(
+ description="Name of the table in the screenshot",
+ type=ConceptStructureBlueprintFieldType.TEXT,
+ required=False,
+ ),
+ "row_count": ConceptStructureBlueprint(
+ description="Number of rows in the table",
+ type=ConceptStructureBlueprintFieldType.INTEGER,
+ required=False,
+ ),
+ }
+
+ generator = StructureGenerator()
+ generated_code, generated_class = generator.generate_from_structure_blueprint(
+ "TableScreenshot",
+ structure_blueprint,
+ base_class_name="ImageContent",
+ )
+
+ pretty_print(structure_blueprint, title="Source Blueprint")
+ pretty_print(generated_code, title="Generated Result")
+
+ # Verify correct import is present
+ assert "from pipelex.core.stuffs.image_content import ImageContent" in generated_code
+
+ # Verify class inherits from ImageContent
+ assert "class TableScreenshot(ImageContent):" in generated_code
+
+ # Verify the generated class is valid
+ from pipelex.core.stuffs.image_content import ImageContent # noqa: PLC0415
+
+ assert issubclass(generated_class, ImageContent)
+ from pipelex.core.stuffs.stuff_content import StuffContent # noqa: PLC0415
+
+ assert issubclass(generated_class, StuffContent)
+
+ def test_inheritance_from_number_content(self):
+ """Test generating a class that inherits from NumberContent."""
+ structure_blueprint = {
+ "unit": ConceptStructureBlueprint(
+ description="Unit of measurement",
+ type=ConceptStructureBlueprintFieldType.TEXT,
+ required=False,
+ ),
+ }
+
+ generator = StructureGenerator()
+ generated_code, generated_class = generator.generate_from_structure_blueprint(
+ "Temperature",
+ structure_blueprint,
+ base_class_name="NumberContent",
+ )
+
+ # Verify correct import is present
+ assert "from pipelex.core.stuffs.number_content import NumberContent" in generated_code
+
+ # Verify class inherits from NumberContent
+ assert "class Temperature(NumberContent):" in generated_code
+
+ # Verify the generated class is valid
+ from pipelex.core.stuffs.number_content import NumberContent # noqa: PLC0415
+
+ assert issubclass(generated_class, NumberContent)
+ from pipelex.core.stuffs.stuff_content import StuffContent # noqa: PLC0415
+
+ assert issubclass(generated_class, StuffContent)
+
+ def test_inheritance_from_json_content(self):
+ """Test generating a class that inherits from JSONContent."""
+ structure_blueprint = {
+ "schema_version": ConceptStructureBlueprint(
+ description="Version of the schema",
+ type=ConceptStructureBlueprintFieldType.TEXT,
+ required=False,
+ ),
+ }
+
+ generator = StructureGenerator()
+ generated_code, generated_class = generator.generate_from_structure_blueprint(
+ "ConfigData",
+ structure_blueprint,
+ base_class_name="JSONContent",
+ )
+
+ # Verify correct import is present
+ assert "from pipelex.core.stuffs.json_content import JSONContent" in generated_code
+
+ # Verify class inherits from JSONContent
+ assert "class ConfigData(JSONContent):" in generated_code
+
+ # Verify the generated class is valid
+ from pipelex.core.stuffs.json_content import JSONContent # noqa: PLC0415
+
+ assert issubclass(generated_class, JSONContent)
+ from pipelex.core.stuffs.stuff_content import StuffContent # noqa: PLC0415
+
+ assert issubclass(generated_class, StuffContent)
+
+ def test_inheritance_with_empty_structure(self):
+ """Test generating a class with no additional fields that just inherits from a base class."""
+ structure_blueprint: dict[str, ConceptStructureBlueprint] = {}
+
+ generator = StructureGenerator()
+ generated_code, generated_class = generator.generate_from_structure_blueprint(
+ "EnhancedText",
+ structure_blueprint,
+ base_class_name="TextContent",
+ )
+
+ pretty_print(generated_code, title="Generated Result with Empty Structure")
+
+ # Verify correct import is present
+ assert "from pipelex.core.stuffs.text_content import TextContent" in generated_code
+
+ # Verify class inherits from TextContent
+ assert "class EnhancedText(TextContent):" in generated_code
+
+ # Verify it has docstring (no pass needed)
+ assert '"""Generated EnhancedText class"""' in generated_code
+
+ # Verify the generated class is valid
+ from pipelex.core.stuffs.text_content import TextContent # noqa: PLC0415
+
+ assert issubclass(generated_class, TextContent)
+ from pipelex.core.stuffs.stuff_content import StuffContent # noqa: PLC0415
+
+ assert issubclass(generated_class, StuffContent)
+
+ # Test instantiation works (TextContent requires 'text' field)
+ instance = generated_class(text="Sample text") # pyright: ignore[reportCallIssue]
+ assert instance is not None
+
+ def test_inheritance_from_pdf_content(self):
+ """Test generating a class that inherits from PDFContent."""
+ structure_blueprint = {
+ "document_type": ConceptStructureBlueprint(
+ description="Type of PDF document",
+ type=ConceptStructureBlueprintFieldType.TEXT,
+ required=False,
+ ),
+ }
+
+ generator = StructureGenerator()
+ generated_code, generated_class = generator.generate_from_structure_blueprint(
+ "Invoice",
+ structure_blueprint,
+ base_class_name="PDFContent",
+ )
+
+ # Verify correct import is present
+ assert "from pipelex.core.stuffs.pdf_content import PDFContent" in generated_code
+
+ # Verify class inherits from PDFContent
+ assert "class Invoice(PDFContent):" in generated_code
+
+ # Verify the generated class is valid
+ from pipelex.core.stuffs.pdf_content import PDFContent # noqa: PLC0415
+
+ assert issubclass(generated_class, PDFContent)
+ from pipelex.core.stuffs.stuff_content import StuffContent # noqa: PLC0415
+
+ assert issubclass(generated_class, StuffContent)
+
+ def test_multiple_inheritance_scenarios(self):
+ """Test multiple classes with different inheritance scenarios."""
+ from pipelex.core.stuffs.stuff_content import StuffContent # noqa: PLC0415
+
+ test_cases = [
+ ("TextContent", "text_content", "Question", {"source": "The source of the question"}),
+ ("ImageContent", "image_content", "Diagram", {"diagram_type": "Type of diagram"}),
+ ("NumberContent", "number_content", "Score", {"max_value": "Maximum possible score"}),
+ ]
+
+ for base_class_name, module_name, class_name, field_dict in test_cases:
+ structure_blueprint = {
+ field_name: ConceptStructureBlueprint(
+ description=description,
+ type=ConceptStructureBlueprintFieldType.TEXT,
+ required=False,
+ )
+ for field_name, description in field_dict.items()
+ }
+
+ generator = StructureGenerator()
+ generated_code, generated_class = generator.generate_from_structure_blueprint(
+ class_name,
+ structure_blueprint,
+ base_class_name=base_class_name,
)
+
+ # Verify correct import is present
+ assert f"from pipelex.core.stuffs.{module_name} import {base_class_name}" in generated_code
+
+ # Verify class inherits from correct base
+ assert f"class {class_name}({base_class_name}):" in generated_code
+
+ # Verify it's a valid subclass of StuffContent
+ assert issubclass(generated_class, StuffContent)
diff --git a/tests/unit/pipelex/core/stuffs/test_special_args.py b/tests/unit/pipelex/core/stuffs/test_special_args.py
index bd3b6339d..bd3fcc572 100644
--- a/tests/unit/pipelex/core/stuffs/test_special_args.py
+++ b/tests/unit/pipelex/core/stuffs/test_special_args.py
@@ -71,7 +71,7 @@ class TestSpecialArgsStuff:
concept = ConceptFactory.make_from_blueprint(
domain=domain,
concept_code=f"Concept{field_name.replace('_', '').title()}",
- blueprint=concept_blueprint,
+ blueprint_or_string_description=concept_blueprint,
)
concept_library.add_new_concept(concept)
diff --git a/tests/unit/pipelex/tools/misc/test_find_files_in_dir.py b/tests/unit/pipelex/tools/misc/test_find_files_in_dir.py
index 67725a2c2..65f541114 100644
--- a/tests/unit/pipelex/tools/misc/test_find_files_in_dir.py
+++ b/tests/unit/pipelex/tools/misc/test_find_files_in_dir.py
@@ -447,3 +447,104 @@ class TestFindFilesInDir:
# Empty include list should exclude all venv files
assert len(files) == 1
assert files[0].name == "file1.py"
+
+ def test_exclude_using_absolute_path_structures_directory(self):
+ """Test excluding structures directory using absolute path (build_structures_cmd use case)."""
+ with tempfile.TemporaryDirectory() as temp_dir:
+ temp_path = Path(temp_dir)
+
+ # Create Python files in library directory
+ (temp_path / "concept_a.py").write_text("# concept_a")
+ (temp_path / "concept_b.py").write_text("# concept_b")
+
+ # Create structures subdirectory with generated files (should be excluded)
+ structures_dir = temp_path / "structures"
+ structures_dir.mkdir()
+ (structures_dir / "__init__.py").write_text("")
+ (structures_dir / "domain_Generated1.py").write_text("# generated")
+ (structures_dir / "domain_Generated2.py").write_text("# generated")
+
+ # Exclude structures using its absolute path
+ result = find_files_in_dir(
+ str(temp_path),
+ "*.py",
+ is_recursive=True,
+ excluded_dirs=[str(structures_dir.resolve())],
+ )
+
+ # Should find only the manually-created files, not generated ones
+ assert len(result) == 2
+ result_names = [f.name for f in result]
+ assert "concept_a.py" in result_names
+ assert "concept_b.py" in result_names
+ # Verify no files from structures/ are included
+ assert not any("structures" in str(f) for f in result)
+
+ def test_exclude_absolute_path_with_trailing_slash(self):
+ """Test that absolute path exclusion works with or without trailing slash."""
+ with tempfile.TemporaryDirectory() as temp_dir:
+ temp_path = Path(temp_dir)
+
+ (temp_path / "main.py").write_text("# main")
+
+ structures_dir = temp_path / "structures"
+ structures_dir.mkdir()
+ (structures_dir / "gen.py").write_text("# gen")
+
+ # Test with trailing slash
+ result_with_slash = find_files_in_dir(
+ str(temp_path),
+ "*.py",
+ is_recursive=True,
+ excluded_dirs=[str(structures_dir.resolve()) + "/"],
+ )
+
+ # Test without trailing slash
+ result_without_slash = find_files_in_dir(
+ str(temp_path),
+ "*.py",
+ is_recursive=True,
+ excluded_dirs=[str(structures_dir.resolve())],
+ )
+
+ # Both should produce same result
+ assert len(result_with_slash) == 1
+ assert len(result_without_slash) == 1
+ assert result_with_slash[0].name == "main.py"
+ assert result_without_slash[0].name == "main.py"
+
+ def test_mixed_name_and_absolute_path_exclusions(self):
+ """Test mixing directory name and absolute path exclusions."""
+ with tempfile.TemporaryDirectory() as temp_dir:
+ temp_path = Path(temp_dir)
+
+ (temp_path / "main.py").write_text("# main")
+
+ # Exclude by name
+ venv_dir = temp_path / ".venv"
+ venv_dir.mkdir()
+ (venv_dir / "venv.py").write_text("# venv")
+
+ # Exclude by absolute path
+ structures_dir = temp_path / "structures"
+ structures_dir.mkdir()
+ (structures_dir / "gen.py").write_text("# gen")
+
+ # Regular directory
+ src_dir = temp_path / "src"
+ src_dir.mkdir()
+ (src_dir / "code.py").write_text("# code")
+
+ result = find_files_in_dir(
+ str(temp_path),
+ "*.py",
+ is_recursive=True,
+ excluded_dirs=[".venv", str(structures_dir.resolve())],
+ )
+
+ assert len(result) == 2
+ result_names = [f.name for f in result]
+ assert "main.py" in result_names
+ assert "code.py" in result_names
+ assert "venv.py" not in result_names
+ assert "gen.py" not in result_names