Build AI methods with your coding agent, then run them anywhere: as an MCP for chatbots, as a webapp for people, or via API for your software.
feature/structure-generator (#504)
Thomas Hebrardb0120f933ebe
diff --git a/.gitignore b/.gitignoreindex 00a8bc1ba..3584b117f 100644--- a/.gitignore+++ b/.gitignore@@ -7,7 +7,6 @@ __pycache__/# Distribution / packagingdist/-build/*.egg-info/# mkdocs@@ -53,3 +52,4 @@ gcp_credentials.json/pipelex.tomlsettings.local.json+structures/\ No newline at end of filediff --git a/CHANGELOG.md b/CHANGELOG.mdindex a693d7b10..f26fb2116 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -1,5 +1,22 @@# Changelog+## Unreleased++### Added++- Can now generate the pydantic structure of a concept with the cli: `pipelex build structures /library_dir/`.+- The Pipelex CLI `pipelex build runner` that creates the python file to run a pipe now runs the `structures` build command and then imports those structures in the runner file.+- Added validation for output concepts of `PipeCompose`: must be strictly compatible with the Text concept.+- Added validation for the Pipelex Bundle on the concept keys: Cannot create a native concept.++### Changed++- The Pipelex cli for the `runner`, `structures`, and `inputs` commands now accept a `--output-dir` option to specify the output directory, by default it in the `target_dir`++### Refactored++- Refactored the `ConceptFactory.make_from_blueprint` method: Now it correctly handles native concepts.+## [v0.17.3] - 2025-12-01### Fixeddiff --git a/pipelex/builder/runner_code.py b/pipelex/builder/runner_code.pyindex 14a1bdab4..1d0b010af 100644--- a/pipelex/builder/runner_code.py+++ b/pipelex/builder/runner_code.py@@ -1,351 +1,257 @@"""Runner code generation utilities for Pipelex pipelines.-This module provides functions to generate executable Python code and JSON input examples-for Pipelex pipelines. It supports two output formats:--1. **Python Code Format**: Generates Python code with class instantiations- Example: PDFContent(url="document.pdf")--2. **JSON Format**: Generates pure JSON with concept metadata- Example: {"concept": "native.PDF", "content": {"url": "document.pdf"}}+This module provides functions to generate executable Python code for Pipelex pipelines.Main Functions:--------------- generate_runner_code(pipe): Generate complete executable Python script-- generate_input_memory_block_python(inputs): Generate just the Python inputs dict-- generate_input_memory_json(inputs): Generate JSON inputs as dict-- generate_input_memory_json_string(inputs): Generate JSON inputs as formatted string--Example Usage:---------------```python-from pipelex.hub import get_required_pipe-from pipelex.builder.runner_code import (- generate_runner_code,- generate_input_memory_block_python,- generate_input_memory_json,- generate_input_memory_json_string,-)--# Get a pipe-pipe = get_required_pipe("my_pipe")--# Generate complete Python runner code-python_code = generate_runner_code(pipe)-print(python_code)--# Generate just the Python input memory block-python_inputs = generate_input_memory_block_python(pipe.inputs)-print(f"inputs = {python_inputs}")--# Generate JSON input memory (as dict)-json_inputs = generate_input_memory_json(pipe.inputs)-print(json_inputs)--# Generate JSON input memory (as formatted string)-json_string = generate_input_memory_json_string(pipe.inputs, indent=2)-print(json_string)-```--Python Output Example:-----------------------```python-import asyncio+"""-from pipelex.core.stuffs.image_content import ImageContent-from pipelex.pipelex import Pipelex-from pipelex.pipeline.execute import execute_pipeline+from dataclasses import dataclass+from typing import Any+from pipelex.core.concepts.concept import Concept+from pipelex.core.concepts.concept_representation_generator import (+ ConceptRepresentationFormat,+ ConceptRepresentationGenerator,+)+from pipelex.core.concepts.native.concept_native import NativeConceptCode+from pipelex.core.domains.domain import SpecialDomain+from pipelex.core.pipes.inputs.input_requirements import InputRequirements+from pipelex.core.pipes.pipe_abstract import PipeAbstract+from pipelex.core.pipes.variable_multiplicity import VariableMultiplicity-async def run_my_pipe():- return await execute_pipeline(- pipe_code="my_pipe",- inputs={- "image": ImageContent(url="image_url"),- },- )+@dataclass+class CustomClassInfo:+ """Information about a custom structure class for import generation."""-if __name__ == "__main__":- Pipelex.make()- result = asyncio.run(run_my_pipe())-```--JSON Output Example:---------------------```json-{- "image": {- "concept": "native.Image",- "content": {- "url": "image_url"- }- }-}-```-"""+ class_name: str+ domain: str+ concept_code: str-import json-from typing import Any, cast+ @property+ def module_name(self) -> str:+ """Get the module name (filename without .py) for this class."""+ return f"{self.domain}_{self.concept_code}"-from pipelex.core.concepts.concept import Concept-from pipelex.core.pipes.inputs.input_requirements import InputRequirements-from pipelex.core.pipes.pipe_abstract import PipeAbstract+ @property+ def import_statement(self) -> str:+ """Get the import statement for this class (uses sys.path from script directory)."""+ return f"from structures.{self.module_name} import {self.class_name}"-def value_to_json(value: Any) -> Any:- """Convert a value to pure JSON representation (dict/list/primitives only).+def _is_multiple(multiplicity: VariableMultiplicity | None) -> bool:+ """Check if the multiplicity indicates multiple items.Args:- value: The value to convert+ multiplicity: The multiplicity value (None, bool, or int)Returns:- Pure JSON-serializable representation (no Python classes)+ True if multiple items are expected"""- if isinstance(value, dict) and "_class" in value:- # Convert Content class instantiation to pure JSON- # E.g., {"_class": "PDFContent", "url": "..."} -> {"url": "..."}- class_name = value["_class"] # pyright: ignore[reportUnknownVariableType]- if class_name in {"PDFContent", "ImageContent"}:- url = value.get("url", "your_url") # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType, reportUnknownVariableType]- return {"url": cast("str", url)}- # For other classes, remove the _class key and return the rest- return {k: value_to_json(v) for k, v in value.items() if k != "_class"} # pyright: ignore[reportUnknownVariableType, reportUnknownArgumentType]- elif isinstance(value, dict) and "concept_code" in value and "content" in value:- # Convert concept_code format to concept format with pure JSON content- concept_code = value["concept_code"] # pyright: ignore[reportUnknownVariableType]- content = value["content"] # pyright: ignore[reportUnknownVariableType]-- # Recursively convert content to JSON- content_json = value_to_json(content)-- return {"concept": cast("str", concept_code), "content": content_json}- elif isinstance(value, str | bool | int | float):- return value- elif isinstance(value, list):- return [value_to_json(item) for item in value] # pyright: ignore[reportUnknownVariableType]- elif isinstance(value, dict):- return {k: value_to_json(v) for k, v in value.items()} # pyright: ignore[reportUnknownVariableType, reportUnknownArgumentType]- else:- # For other types, try to convert to string- return str(value)+ if multiplicity is None:+ return False+ if isinstance(multiplicity, bool):+ return multiplicity+ # int means specific count, which is multiple+ return multiplicity > 1-def value_to_python_code(value: Any, indent_level: int = 0) -> str:- """Convert a value to Python code representation recursively.+def _format_representation_as_python(representation: dict[str, Any], is_multiple: bool = False) -> str:+ """Format a representation dict as Python code.Args:- value: The value to convert (can be str, int, dict, list, etc.)- indent_level: Current indentation level for nested dicts+ representation: Dict with concept and content+ is_multiple: If True, wrap content in a listReturns:- String representation of Python code+ Python code string"""- indent = " " * indent_level-- if isinstance(value, dict) and "_class" in value:- # Special handling for Content class instantiation (e.g., PDFContent, ImageContent)- class_name = value["_class"] # pyright: ignore[reportUnknownVariableType]- if class_name in {"PDFContent", "ImageContent"}:- url = value.get("url", "your_url") # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType, reportUnknownVariableType]- return f'{class_name}(url="{url}")'- return str(value) # pyright: ignore[reportUnknownArgumentType]- elif isinstance(value, dict) and "concept_code" in value and "content" in value:- # Special handling for refined concepts with explicit concept_code- # Format: {"concept": "domain.ConceptCode", "content": ContentClass(...)}- concept_code = value["concept_code"] # pyright: ignore[reportUnknownVariableType]- content = value["content"] # pyright: ignore[reportUnknownVariableType]-- # Generate the content part- content_code = value_to_python_code(content, indent_level + 1)-- # Return the full format with concept and content- return f'{{\n{indent} "concept": "{concept_code}",\n{indent} "content": {content_code},\n{indent}}}'- elif isinstance(value, str):- # String value - add quotes- return f'"{value}"'- elif isinstance(value, bool):- # Boolean - Python True/False- return str(value)- elif isinstance(value, (int, float)):- # Numeric value- return str(value)- elif isinstance(value, list):- # List - recursively convert items- if not value:- return "[]"- items: list[str] = [value_to_python_code(item, indent_level + 1) for item in value] # pyright: ignore[reportUnknownVariableType]- return "[" + ", ".join(items) + "]"- elif isinstance(value, dict):- # Dict - recursively convert with proper formatting- if not value:- return "{}"- lines_dict: list[str] = []- for key, val in value.items(): # pyright: ignore[reportUnknownVariableType]- val_code = value_to_python_code(val, indent_level + 1)- lines_dict.append(f'{indent} "{key}": {val_code}')- return "{\n" + ",\n".join(lines_dict) + f"\n{indent}}}"- else:- # Fallback - use repr- return repr(value)+ concept = representation["concept"]+ content = representation["content"]+ # If multiple, wrap content in a list+ if is_multiple:+ content = f"[{content}]"-def generate_compact_memory_entry(var_name: str, concept: Concept) -> str:- """Generate the pipeline_inputs dictionary entry for a given input (Python code version)."""- example_value = concept.get_compact_memory_example(var_name)+ # Content is already a Python instantiation string from the generator+ return f'{{\n "concept": "{concept}",\n "content": {content},\n }}'- # Convert the example value to a Python code string- value_str = value_to_python_code(example_value, indent_level=3)- return f' "{var_name}": {value_str},'+def _get_structure_class_import(class_name: str) -> str | None:+ """Get the import statement for a native structure class.+ The import statement is generated dynamically from the class's actual module location,+ so it will automatically reflect any file relocations.-def generate_json_memory_entry(var_name: str, concept: Concept) -> dict[str, Any]:- """Generate the pipeline_inputs dictionary entry for a given input (pure JSON version).+ Args:+ class_name: The name of the structure classReturns:- A tuple of (var_name, json_value) where json_value is pure JSON (dict/list/primitives)+ Import statement string, or None if not a native structure class"""- example_value = concept.get_compact_memory_example(var_name)-- # Convert to JSON - always wrap with concept if not already present- json_value = value_to_json(example_value)-- # For simple values (strings for Text, numbers for Number), wrap with concept- if isinstance(json_value, str):- # This is a native Text concept - wrap it- return {- "concept": concept.concept_string,- "content": json_value,- }- elif isinstance(json_value, (int, float)) and not isinstance(json_value, bool):- # This is a native Number concept - wrap it- return {- "concept": concept.concept_string,- "content": json_value,- }- elif isinstance(json_value, dict) and "concept" not in json_value:- # Not yet wrapped with concept - wrap it now- return {- "concept": concept.concept_string,- "content": json_value,- }- else:- # Already has concept key or is properly formatted- # At this point, json_value should be a dict with "concept" key- return cast("dict[str, Any]", json_value)+ cls = NativeConceptCode.get_native_structure_class(class_name)+ if cls is None:+ return None+ return f"from {cls.__module__} import {cls.__name__}"-def generate_input_memory_block_python(inputs: InputRequirements) -> str:- """Generate just the Python input memory block (without surrounding code).+def _collect_concept_info(concept: Concept) -> CustomClassInfo | None:+ """Collect information about a concept for import generation.Args:- inputs: The pipe inputs+ concept: The concept to collect info forReturns:- Python code string representing the inputs dictionary+ CustomClassInfo if it's a custom concept, None if native"""- if inputs.nb_inputs == 0:- return "{}"-- input_memory_entries: list[str] = []- for var_name, input_req in inputs.root.items():- concept = input_req.concept- entry = generate_compact_memory_entry(var_name, concept)- input_memory_entries.append(entry)-- # Join entries and format as a dictionary- entries_str = "\n".join(input_memory_entries)- return f"{{\n{entries_str}\n }}"+ if SpecialDomain.is_native(concept.domain):+ return None++ # For custom concepts, use the concept code as the class name+ # The structure class name should match the concept code+ return CustomClassInfo(+ class_name=concept.structure_class_name,+ domain=concept.domain,+ concept_code=concept.code,+ )-def generate_input_memory_json(inputs: InputRequirements) -> dict[str, Any]:- """Generate the input memory in pure JSON format.+def _collect_imports_for_inputs(inputs: InputRequirements) -> tuple[set[str], dict[str, CustomClassInfo]]:+ """Collect all imports needed for a pipe's inputs.Args:inputs: The pipe inputsReturns:- Dictionary with pure JSON values (no Python classes)+ Tuple of (set of native class names, dict mapping class name to CustomClassInfo)"""- if inputs.nb_inputs == 0:- return {}+ native_classes: set[str] = set()+ custom_classes: dict[str, CustomClassInfo] = {}- json_inputs: dict[str, Any] = {}- for var_name, input_req in inputs.root.items():+ for input_req in inputs.root.values():concept = input_req.concept- json_value = generate_json_memory_entry(var_name, concept)- json_inputs[var_name] = json_value+ structure_class = concept.get_structure_class()- return json_inputs+ # Get imports from the representation generator+ generator = ConceptRepresentationGenerator(ConceptRepresentationFormat.PYTHON)+ generator.generate_representation(concept.concept_string, structure_class)+ for class_name in generator.imports_needed:+ if NativeConceptCode.is_native_structure_class(class_name):+ native_classes.add(class_name)+ else:+ # For custom classes, we need the concept info+ # The class name should be the structure class name which is the concept code+ custom_info = _collect_concept_info(concept)+ if custom_info and custom_info.class_name == class_name:+ custom_classes[class_name] = custom_info-def generate_input_memory_json_string(inputs: InputRequirements, indent: int = 2) -> str:- """Generate the input memory in pure JSON format as a formatted string.+ return native_classes, custom_classes- Args:- inputs: The pipe inputs- indent: Number of spaces for indentation (default: 2)- Returns:- Formatted JSON string- """- json_inputs = generate_input_memory_json(inputs)- return json.dumps(json_inputs, indent=indent, ensure_ascii=False)+def generate_runner_code(pipe: PipeAbstract, output_multiplicity: bool = False) -> str:+ """Generate the complete Python runner code for a pipe.+ This generates a runnable Python script with:+ - Import statements for all required structure classes+ - An async function to run the pipeline with proper return type+ - Example input values based on the pipe's input concepts+ - Output handling with main_stuff_as-def generate_runner_code(pipe: PipeAbstract) -> str:- """Generate the complete Python runner code for a pipe."""- pipe_code = pipe.code- inputs = pipe.inputs+ Args:+ pipe: The pipe to generate runner code for+ output_multiplicity: Whether the output is a list (e.g., Text[])+ """+ # Get output information+ structure_class_name = pipe.output.structure_class_name+ is_native = NativeConceptCode.is_native_structure_class(structure_class_name)+ custom_info = None if is_native else _collect_concept_info(pipe.output)- # Determine which imports are needed based on input concepts- needs_pdf = False- needs_image = False- for input_req in inputs.root.values():- concept = input_req.concept- if concept.structure_class_name == "PDFContent":- needs_pdf = True- elif concept.structure_class_name == "ImageContent":- needs_image = True+ # Collect all imports needed for inputs+ native_classes, custom_classes = _collect_imports_for_inputs(pipe.inputs)++ # Add output class to appropriate set+ if is_native:+ native_classes.add(structure_class_name)+ elif custom_info:+ custom_classes[structure_class_name] = custom_info# Build import section- import_lines = ["import asyncio", ""]+ import_lines: list[str] = []++ # Add path setup for custom structure imports (must be before other imports)+ if custom_classes:+ import_lines.extend(+ [+ "import sys",+ "from pathlib import Path",+ "",+ "# Add script directory to path for local imports",+ "sys.path.insert(0, str(Path(__file__).parent))",+ "",+ ]+ )- # Add content class imports if needed- if needs_pdf:- import_lines.append("from pipelex.core.stuffs.pdf_content import PDFContent")- if needs_image:- import_lines.append("from pipelex.core.stuffs.image_content import ImageContent")+ import_lines.extend(["import asyncio", ""])++ # Add native content class imports+ native_imports: list[str] = []+ for class_name in sorted(native_classes):+ import_stmt = _get_structure_class_import(class_name)+ if import_stmt:+ native_imports.append(import_stmt)+ import_lines.extend(native_imports)++ # Add custom structure class imports from structures folder+ if custom_classes:+ import_lines.append("")+ for class_name in sorted(custom_classes.keys()):+ custom_info = custom_classes[class_name]+ import_lines.append(custom_info.import_statement)import_lines.extend([+ "","from pipelex.pipelex import Pipelex","from pipelex.pipeline.execute import execute_pipeline",])# Build inputs entries- if inputs.nb_inputs > 0:- input_memory_entries: list[str] = []- for var_name, input_req in inputs.root.items():- concept = input_req.concept- entry = generate_compact_memory_entry(var_name, concept)- input_memory_entries.append(entry)- input_memory_block = "\n".join(input_memory_entries)+ if pipe.inputs.nb_inputs > 0:+ input_entries: list[str] = []+ for var_name, input_req in pipe.inputs.root.items():+ is_multiple = _is_multiple(input_req.multiplicity)+ result, _ = input_req.concept.generate_input_representation(+ output_format=ConceptRepresentationFormat.PYTHON,+ is_multiple=is_multiple,+ )+ python_code = _format_representation_as_python(result, is_multiple=is_multiple)+ input_entries.append(f' "{var_name}": {python_code},')+ input_memory_block = "\n".join(input_entries)+ else:+ input_memory_block = " # No inputs required"++ # Determine return type annotation+ if output_multiplicity:+ return_type = f"list[{structure_class_name}]"+ result_call = f"pipe_output.main_stuff_as_items(item_type={structure_class_name})"else:- input_memory_block = " # No inputs required"+ return_type = structure_class_name+ result_call = f"pipe_output.main_stuff_as(content_type={structure_class_name})"# Build the main functionfunction_lines = ["","",- f"async def run_{pipe_code}():",- " return await execute_pipeline(",- f' pipe_code="{pipe_code}",',+ f"async def run_{pipe.code}() -> {return_type}:",+ " pipe_output = await execute_pipeline(",+ f' pipe_code="{pipe.code}",',]- if inputs.nb_inputs > 0:+ if pipe.inputs.nb_inputs > 0:function_lines.extend([" inputs={",@@ -357,6 +263,7 @@ def generate_runner_code(pipe: PipeAbstract) -> str:function_lines.extend([" )",+ f" return {result_call}","","",'if __name__ == "__main__":',@@ -364,7 +271,7 @@ def generate_runner_code(pipe: PipeAbstract) -> str:" Pipelex.make()",""," # Run the pipeline",- f" result = asyncio.run(run_{pipe_code}())",+ f" result = asyncio.run(run_{pipe.code}())","",])diff --git a/pipelex/cli/_cli.py b/pipelex/cli/_cli.pyindex e75239b2a..1fbbc36f7 100644--- a/pipelex/cli/_cli.py+++ b/pipelex/cli/_cli.py@@ -5,7 +5,7 @@ from click import Command, Contextfrom typer.core import TyperGroupfrom typing_extensions import override-from pipelex.cli.commands.build_cmd import build_app+from pipelex.cli.commands.build import build_appfrom pipelex.cli.commands.doctor_cmd import doctor_cmdfrom pipelex.cli.commands.init.command import init_cmdfrom pipelex.cli.commands.init.ui.types import InitFocusdiff --git a/pipelex/cli/commands/build/__init__.py b/pipelex/cli/commands/build/__init__.pynew file mode 100644index 000000000..9fe929468--- /dev/null+++ b/pipelex/cli/commands/build/__init__.py@@ -0,0 +1,6 @@+# Import command modules to register them with build_app+from pipelex.cli.commands.build import inputs_cmd as inputs_cmd+from pipelex.cli.commands.build import pipe_cmd as pipe_cmd+from pipelex.cli.commands.build import runner_cmd as runner_cmd+from pipelex.cli.commands.build import structures_cmd as structures_cmd+from pipelex.cli.commands.build.app import build_app as build_appdiff --git a/pipelex/cli/commands/build/app.py b/pipelex/cli/commands/build/app.pynew file mode 100644index 000000000..6b1274219--- /dev/null+++ b/pipelex/cli/commands/build/app.py@@ -0,0 +1,3 @@+import typer++build_app = typer.Typer(help="Build working pipelines from natural language requirements", no_args_is_help=True)diff --git a/pipelex/cli/commands/build/inputs_cmd.py b/pipelex/cli/commands/build/inputs_cmd.pynew file mode 100644index 000000000..f4a84e52d--- /dev/null+++ b/pipelex/cli/commands/build/inputs_cmd.py@@ -0,0 +1,207 @@+import asyncio+from pathlib import Path+from typing import Annotated++import click+import typer+from posthog import tag++from pipelex.cli.commands.build.app import build_app+from pipelex.cli.error_handlers import (+ ErrorContext,+ handle_model_availability_error,+ handle_model_choice_error,+ handle_model_deck_preset_error,+)+from pipelex.cogt.exceptions import ModelDeckPresetValidatonError+from pipelex.core.pipes.exceptions import PipeOperatorModelChoiceError+from pipelex.core.pipes.inputs.exceptions import PipeInputError+from pipelex.hub import get_required_pipe, get_telemetry_manager+from pipelex.pipe_operators.exceptions import PipeOperatorModelAvailabilityError+from pipelex.pipelex import PACKAGE_VERSION, Pipelex+from pipelex.pipeline.validate_bundle import ValidateBundleError, validate_bundle+from pipelex.system.runtime import IntegrationMode+from pipelex.system.telemetry.events import EventProperty+from pipelex.tools.misc.file_utils import (+ ensure_directory_for_file_path,+ save_text_to_path,+)++COMMAND = "build"+SUB_COMMAND_INPUTS = "inputs"+++@build_app.command(SUB_COMMAND_INPUTS, help="Generate example input JSON for a pipe")+def generate_inputs_cmd(+ target: Annotated[+ str | None,+ typer.Argument(help="Pipe code or bundle file path (auto-detected)"),+ ] = None,+ pipe: Annotated[+ str | None,+ typer.Option("--pipe", help="Pipe code to use, can be omitted if you specify a bundle (.plx) that declares a main pipe"),+ ] = None,+ bundle: Annotated[+ str | None,+ typer.Option("--bundle", help="Bundle file path (.plx) - uses its main_pipe unless you specify a pipe code"),+ ] = None,+ output_path: Annotated[+ str | None,+ typer.Option(+ "--output", "-o", help="Path to save the generated JSON file (defaults to bundle's directory if bundle provided, otherwise 'results/')"+ ),+ ] = None,+) -> None:+ """Generate example input JSON for a pipe.++ The generated JSON file will include example values for all pipe inputs+ based on their concept types.++ Examples:+ pipelex build inputs my_pipe+ pipelex build inputs --bundle my_bundle.plx+ pipelex build inputs --bundle my_bundle.plx --pipe my_pipe+ pipelex build inputs my_bundle.plx+ pipelex build inputs my_pipe --output custom_inputs.json+ """+ # Validate mutual exclusivity+ provided_options = sum([target is not None, pipe is not None, bundle is not None])+ if provided_options == 0:+ ctx: click.Context = click.get_current_context()+ typer.echo(ctx.get_help())+ raise typer.Exit(0)++ # Let's analyze the options and determine what pipe code to use and if we need to load a bundle+ pipe_code: str | None = None+ bundle_path: str | None = None++ # Determine source:+ if target:+ # Check if target is a directory (not allowed for inputs command)+ target_path = Path(target)+ if target_path.is_dir():+ typer.secho(+ f"Failed to run: '{target}' is a directory. The inputs command requires a .plx file or a pipe code, not a directory.",+ fg=typer.colors.RED,+ err=True,+ )+ raise typer.Exit(1)++ if target.endswith(".plx"):+ bundle_path = target+ if bundle:+ typer.secho(+ "Failed to run: cannot use option --bundle if you're already passing a bundle file (.plx) as positional argument",+ fg=typer.colors.RED,+ err=True,+ )+ raise typer.Exit(1)+ else:+ pipe_code = target+ if pipe:+ typer.secho(+ "Failed to run: cannot use option --pipe if you're already passing a pipe code as positional argument",+ fg=typer.colors.RED,+ err=True,+ )+ raise typer.Exit(1)++ if bundle:+ assert not bundle_path, "bundle_path should be None at this stage if --bundle is provided"+ bundle_path = bundle++ if pipe:+ assert not pipe_code, "pipe_code should be None at this stage if --pipe is provided"+ pipe_code = pipe++ if not pipe_code and not bundle_path:+ typer.secho("Failed to run: no pipe code or bundle file specified", fg=typer.colors.RED, err=True)+ raise typer.Exit(1)++ async def generate_inputs(pipe_code: str | None = None, bundle_path: str | None = None):+ if bundle_path:+ try:+ validate_bundle_result = await validate_bundle(plx_file_path=bundle_path)+ bundle_blueprint = validate_bundle_result.blueprints[0]+ if not pipe_code:+ main_pipe_code = bundle_blueprint.main_pipe+ if not main_pipe_code:+ # Fall back to first pipe if no main_pipe declared+ if bundle_blueprint.pipe:+ main_pipe_code = next(iter(bundle_blueprint.pipe.keys()))+ typer.echo(f"No main_pipe declared, using first pipe '{main_pipe_code}' from bundle '{bundle_path}'")+ else:+ typer.secho(f"Bundle '{bundle_path}' has no pipes defined", fg=typer.colors.RED, err=True)+ raise typer.Exit(1)+ else:+ typer.echo(f"Using main pipe '{main_pipe_code}' from bundle '{bundle_path}'")+ pipe_code = main_pipe_code+ else:+ typer.echo(f"Using pipe '{pipe_code}' from bundle '{bundle_path}'")+ except FileNotFoundError as exc:+ typer.secho(f"Failed to load bundle '{bundle_path}': {exc}", fg=typer.colors.RED, err=True)+ raise typer.Exit(1) from exc+ except ValidateBundleError as exc:+ typer.secho(f"Failed to load bundle '{bundle_path}': {exc}", fg=typer.colors.RED, err=True)+ raise typer.Exit(1) from exc+ except PipeInputError as exc:+ typer.secho(f"Failed to load bundle '{bundle_path}': {exc}", fg=typer.colors.RED, err=True)+ raise typer.Exit(1) from exc+ elif not pipe_code:+ typer.secho("Failed to run: no pipe code specified", fg=typer.colors.RED, err=True)+ raise typer.Exit(1)++ # Get the pipe+ try:+ the_pipe = get_required_pipe(pipe_code=pipe_code)+ except Exception as exc:+ typer.secho(f"❌ Error: Could not find pipe '{pipe_code}': {exc}", fg=typer.colors.RED)+ raise typer.Exit(1) from exc++ # Generate the input JSON+ try:+ inputs_json_str = the_pipe.inputs.generate_json_string(indent=2)+ except Exception as exc:+ typer.secho(f"❌ Error generating input JSON: {exc}", fg=typer.colors.RED)+ raise typer.Exit(1) from exc++ # Determine output path - use bundle's directory if bundle provided, otherwise results/+ if output_path:+ final_output_path = output_path+ elif bundle_path:+ # Place inputs.json in the same directory as the PLX file+ bundle_dir = Path(bundle_path).parent+ final_output_path = str(bundle_dir / "inputs.json")+ else:+ final_output_path = "results/inputs.json"++ # Save the file+ try:+ ensure_directory_for_file_path(file_path=final_output_path)+ save_text_to_path(text=inputs_json_str, path=final_output_path)+ typer.secho(f"✅ Generated input JSON file: {final_output_path}", fg=typer.colors.GREEN)+ except Exception as exc:+ typer.secho(f"❌ Error saving file: {exc}", fg=typer.colors.RED)+ raise typer.Exit(1) from exc++ try:+ pipelex_instance = Pipelex.make(integration_mode=IntegrationMode.CLI)+ except ModelDeckPresetValidatonError as model_deck_error:+ handle_model_deck_preset_error(model_deck_error, context=ErrorContext.VALIDATION_BEFORE_BUILD_INPUTS)++ try:+ with get_telemetry_manager().telemetry_context():+ tag(name=EventProperty.INTEGRATION, value=IntegrationMode.CLI)+ tag(name=EventProperty.PIPELEX_VERSION, value=PACKAGE_VERSION)+ tag(name=EventProperty.CLI_COMMAND, value=f"{COMMAND} {SUB_COMMAND_INPUTS}")++ asyncio.run(generate_inputs(pipe_code=pipe_code, bundle_path=bundle_path))++ except PipeOperatorModelChoiceError as exc:+ handle_model_choice_error(exc, context=ErrorContext.BUILD)++ except PipeOperatorModelAvailabilityError as exc:+ handle_model_availability_error(exc, context=ErrorContext.BUILD)++ finally:+ pipelex_instance.teardown()diff --git a/pipelex/cli/commands/build_cmd.py b/pipelex/cli/commands/build/pipe_cmd.pysimilarity index 57%rename from pipelex/cli/commands/build_cmd.pyrename to pipelex/cli/commands/build/pipe_cmd.pyindex 33cbd3106..a985280d9 100644--- a/pipelex/cli/commands/build_cmd.py+++ b/pipelex/cli/commands/build/pipe_cmd.py@@ -1,9 +1,9 @@import asyncioimport osimport time+from pathlib import Pathfrom typing import TYPE_CHECKING, Annotated-import clickimport typerfrom posthog import tag@@ -11,7 +11,8 @@ from pipelex import pretty_printfrom pipelex.builder.builder import PipelexBundleSpecfrom pipelex.builder.builder_errors import PipeBuilderErrorfrom pipelex.builder.builder_loop import BuilderLoop-from pipelex.builder.runner_code import generate_input_memory_json_string, generate_runner_code+from pipelex.builder.runner_code import generate_runner_code+from pipelex.cli.commands.build.app import build_appfrom pipelex.cli.error_handlers import (ErrorContext,handle_model_availability_error,@@ -21,14 +22,13 @@ from pipelex.cli.error_handlers import (from pipelex.cogt.exceptions import ModelDeckPresetValidatonErrorfrom pipelex.config import get_configfrom pipelex.core.pipes.exceptions import PipeOperatorModelChoiceError-from pipelex.core.pipes.inputs.exceptions import PipeInputError+from pipelex.core.pipes.variable_multiplicity import parse_concept_with_multiplicityfrom pipelex.hub import get_console, get_report_delegate, get_required_pipe, get_telemetry_managerfrom pipelex.language.plx_factory import PlxFactoryfrom pipelex.pipe_operators.exceptions import PipeOperatorModelAvailabilityErrorfrom pipelex.pipelex import PACKAGE_VERSION, Pipelexfrom pipelex.pipeline.exceptions import PipelineExecutionErrorfrom pipelex.pipeline.execute import execute_pipeline-from pipelex.pipeline.validate_bundle import ValidateBundleError, validate_bundlefrom pipelex.system.runtime import IntegrationModefrom pipelex.system.telemetry.events import EventPropertyfrom pipelex.tools.misc.file_utils import (@@ -43,7 +43,10 @@ from pipelex.tools.misc.pretty import PrettyPrinterif TYPE_CHECKING:from pipelex.client.protocol import PipelineInputs-build_app = typer.Typer(help="Build working pipelines from natural language requirements", no_args_is_help=True)+COMMAND = "build"+SUB_COMMAND_PIPE = "pipe"+SUB_COMMAND_ONE_SHOT_PIPE = "one-shot-pipe"+SUB_COMMAND_PARTIAL_PIPE = "partial-pipe""""Today's example:@@ -70,14 +73,6 @@ pipelex build pipe "Given an RDFP PDF, build a compliance matrix"pipelex build pipe "Given a theme, write a Haiku""""-COMMAND = "build"--SUB_COMMAND_PIPE = "pipe"-SUB_COMMAND_RUNNER = "runner"-SUB_COMMAND_INPUTS = "inputs"-SUB_COMMAND_ONE_SHOT_PIPE = "one-shot-pipe"-SUB_COMMAND_PARTIAL_PIPE = "partial-pipe"-@build_app.command(SUB_COMMAND_PIPE, help="Build a Pipelex bundle with one validation/fix loop correcting deterministic issues")def build_pipe_cmd(@@ -106,6 +101,9 @@ def build_pipe_cmd(typer.Option("--no-extras", help="Skip generating inputs.json and runner.py, only generate the PLX file"),] = False,) -> None:+ # Import here to avoid circular imports+ from pipelex.cli.commands.build.structures_cmd import generate_structures_from_blueprints # noqa: PLC0415+try:pipelex_instance = Pipelex.make(integration_mode=IntegrationMode.CLI)except ModelDeckPresetValidatonError as model_deck_error:@@ -196,18 +194,40 @@ def build_pipe_cmd(pipe = get_required_pipe(pipe_code=main_pipe_code)+ # Generate structures folder FIRST (before runner, since runner imports from structures)+ structures_output_dir = Path(extras_output_dir) / "structures"+ generated_structures = generate_structures_from_blueprints(+ blueprints=[pipelex_bundle_spec.to_blueprint()],+ output_directory=structures_output_dir,+ skip_existing_check=True,+ )+ if generated_structures:+ typer.secho(f"✅ Generated {len(generated_structures)} structure(s) in: {structures_output_dir}", fg=typer.colors.GREEN)+# Generate inputs.json- inputs_json_str = generate_input_memory_json_string(pipe.inputs, indent=2)+ inputs_json_str = pipe.inputs.generate_json_string(indent=2)inputs_json_path = os.path.join(extras_output_dir, "inputs.json")save_text_to_path(text=inputs_json_str, path=inputs_json_path)typer.secho(f"✅ Inputs template saved to: {inputs_json_path}", fg=typer.colors.GREEN)- # Generate runner.py- runner_code = generate_runner_code(pipe)+ # Determine if output is a list from the bundle spec+ main_pipe_spec = pipelex_bundle_spec.pipe[main_pipe_code] if pipelex_bundle_spec.pipe else None+ output_is_list = False+ if main_pipe_spec:+ output_parse = parse_concept_with_multiplicity(main_pipe_spec.output)+ output_is_list = output_parse.multiplicity is not None++ # Generate runner.py (after structures are generated)+ runner_code = generate_runner_code(pipe, output_multiplicity=output_is_list)runner_path = os.path.join(extras_output_dir, f"run_{main_pipe_code}.py")save_text_to_path(text=runner_code, path=runner_path)typer.secho(f"✅ Python runner script saved to: {runner_path}", fg=typer.colors.GREEN)+ # Generate empty __init__.py to make it a proper Python package+ init_path = os.path.join(extras_output_dir, "__init__.py")+ save_text_to_path(text="", path=init_path)+ typer.secho(f"✅ Package init file saved to: {init_path}", fg=typer.colors.GREEN)+end_time = time.time()typer.secho(f"\n✅ Pipeline built in {end_time - start_time:.2f} seconds\n", fg=typer.colors.WHITE)@@ -240,314 +260,6 @@ def build_pipe_cmd(pipelex_instance.teardown()-@build_app.command(SUB_COMMAND_RUNNER, help="Build the Python code to run a pipe with the necessary inputs")-def prepare_runner_cmd(- target: Annotated[- str | None,- typer.Argument(help="Pipe code or bundle file path (auto-detected)"),- ] = None,- pipe: Annotated[- str | None,- typer.Option("--pipe", help="Pipe code to use, can be omitted if you specify a bundle (.plx) that declares a main pipe"),- ] = None,- bundle: Annotated[- str | None,- typer.Option("--bundle", help="Bundle file path (.plx) - uses its main_pipe unless you specify a pipe code"),- ] = None,- output_path: Annotated[- str | None,- typer.Option("--output", "-o", help="Path to save the generated Python file, defaults to 'results/run_{pipe_code}.py'"),- ] = None,-) -> None:- """Prepare a Python runner file for a pipe.-- The generated file will include:- - All necessary imports- - Example input values based on the pipe's input types-- Native concept types (Text, Image, PDF, etc.) will be automatically handled.- Custom concept types will have their structure recursively generated.-- Examples:- pipelex build runner my_pipe- pipelex build runner --bundle my_bundle.plx- pipelex build runner --bundle my_bundle.plx --pipe my_pipe- pipelex build runner my_bundle.plx- pipelex build runner my_pipe --output runner.py- """- # Validate mutual exclusivity- provided_options = sum([target is not None, pipe is not None, bundle is not None])- if provided_options == 0:- ctx: click.Context = click.get_current_context()- typer.echo(ctx.get_help())- raise typer.Exit(0)-- # Let's analyze the options and determine what pipe code to use and if we need to load a bundle- pipe_code: str | None = None- bundle_path: str | None = None-- # Determine source:- if target:- if target.endswith(".plx"):- bundle_path = target- if bundle:- typer.secho(- "Failed to run: cannot use option --bundle if you're already passing a bundle file (.plx) as positional argument",- fg=typer.colors.RED,- err=True,- )- raise typer.Exit(1)- else:- pipe_code = target- if pipe:- typer.secho(- "Failed to run: cannot use option --pipe if you're already passing a pipe code as positional argument",- fg=typer.colors.RED,- err=True,- )- raise typer.Exit(1)-- if bundle:- assert not bundle_path, "bundle_path should be None at this stage if --bundle is provided"- bundle_path = bundle-- if pipe:- assert not pipe_code, "pipe_code should be None at this stage if --pipe is provided"- pipe_code = pipe-- if not pipe_code and not bundle_path:- typer.secho("Failed to run: no pipe code or bundle file specified", fg=typer.colors.RED, err=True)- raise typer.Exit(1)-- async def prepare_runner(pipe_code: str | None = None, bundle_path: str | None = None):- if bundle_path:- try:- validate_bundle_result = await validate_bundle(plx_file_path=bundle_path)- if not pipe_code:- main_pipe_code = validate_bundle_result.blueprints[0].main_pipe- if not main_pipe_code:- typer.secho(f"Bundle '{bundle_path}' does not declare a main_pipe", fg=typer.colors.RED, err=True)- raise typer.Exit(1)- pipe_code = main_pipe_code- typer.echo(f"Using main pipe '{pipe_code}' from bundle '{bundle_path}'")- else:- typer.echo(f"Using pipe '{pipe_code}' from bundle '{bundle_path}'")- except FileNotFoundError as exc:- typer.secho(f"Failed to load bundle '{bundle_path}': {exc}", fg=typer.colors.RED, err=True)- raise typer.Exit(1) from exc- except ValidateBundleError as exc:- typer.secho(f"Failed to load bundle '{bundle_path}': {exc}", fg=typer.colors.RED, err=True)- raise typer.Exit(1) from exc- except PipeInputError as exc:- typer.secho(f"Failed to load bundle '{bundle_path}': {exc}", fg=typer.colors.RED, err=True)- raise typer.Exit(1) from exc- elif not pipe_code:- typer.secho("Failed to run: no pipe code specified", fg=typer.colors.RED, err=True)- raise typer.Exit(1)-- # Get the pipe- try:- pipe = get_required_pipe(pipe_code=pipe_code)- except Exception as exc:- typer.secho(f"❌ Error: Could not find pipe '{pipe_code}': {exc}", fg=typer.colors.RED)- raise typer.Exit(1) from exc-- # Generate the code- try:- runner_code = generate_runner_code(pipe)- except Exception as exc:- typer.secho(f"❌ Error generating runner code: {exc}", fg=typer.colors.RED)- raise typer.Exit(1) from exc-- # Determine output path- final_output_path = output_path or get_incremental_file_path(- base_path="results",- base_name=f"run_{pipe_code}",- extension="py",- )-- # Save the file- try:- ensure_directory_for_file_path(file_path=final_output_path)- save_text_to_path(text=runner_code, path=final_output_path)- typer.secho(f"✅ Generated runner file: {final_output_path}", fg=typer.colors.GREEN)- except Exception as exc:- typer.secho(f"❌ Error saving file: {exc}", fg=typer.colors.RED)- raise typer.Exit(1) from exc-- try:- pipelex_instance = Pipelex.make(integration_mode=IntegrationMode.CLI)- except ModelDeckPresetValidatonError as model_deck_error:- handle_model_deck_preset_error(model_deck_error, context=ErrorContext.VALIDATION_BEFORE_BUILD_RUNNER)-- try:- with get_telemetry_manager().telemetry_context():- tag(name=EventProperty.INTEGRATION, value=IntegrationMode.CLI)- tag(name=EventProperty.PIPELEX_VERSION, value=PACKAGE_VERSION)- tag(name=EventProperty.CLI_COMMAND, value=f"{COMMAND} {SUB_COMMAND_RUNNER}")-- asyncio.run(prepare_runner(pipe_code=pipe_code, bundle_path=bundle_path))-- except PipeOperatorModelChoiceError as exc:- handle_model_choice_error(exc, context=ErrorContext.BUILD)-- except PipeOperatorModelAvailabilityError as exc:- handle_model_availability_error(exc, context=ErrorContext.BUILD)-- finally:- pipelex_instance.teardown()---@build_app.command(SUB_COMMAND_INPUTS, help="Generate example input JSON for a pipe")-def generate_inputs_cmd(- target: Annotated[- str | None,- typer.Argument(help="Pipe code or bundle file path (auto-detected)"),- ] = None,- pipe: Annotated[- str | None,- typer.Option("--pipe", help="Pipe code to use, can be omitted if you specify a bundle (.plx) that declares a main pipe"),- ] = None,- bundle: Annotated[- str | None,- typer.Option("--bundle", help="Bundle file path (.plx) - uses its main_pipe unless you specify a pipe code"),- ] = None,- output_path: Annotated[- str | None,- typer.Option("--output", "-o", help="Path to save the generated JSON file, defaults to 'results/inputs.json'"),- ] = None,-) -> None:- """Generate example input JSON for a pipe.-- The generated JSON file will include example values for all pipe inputs- based on their concept types.-- Examples:- pipelex build inputs my_pipe- pipelex build inputs --bundle my_bundle.plx- pipelex build inputs --bundle my_bundle.plx --pipe my_pipe- pipelex build inputs my_bundle.plx- pipelex build inputs my_pipe --output custom_inputs.json- """- # Validate mutual exclusivity- provided_options = sum([target is not None, pipe is not None, bundle is not None])- if provided_options == 0:- ctx: click.Context = click.get_current_context()- typer.echo(ctx.get_help())- raise typer.Exit(0)-- # Let's analyze the options and determine what pipe code to use and if we need to load a bundle- pipe_code: str | None = None- bundle_path: str | None = None-- # Determine source:- if target:- if target.endswith(".plx"):- bundle_path = target- if bundle:- typer.secho(- "Failed to run: cannot use option --bundle if you're already passing a bundle file (.plx) as positional argument",- fg=typer.colors.RED,- err=True,- )- raise typer.Exit(1)- else:- pipe_code = target- if pipe:- typer.secho(- "Failed to run: cannot use option --pipe if you're already passing a pipe code as positional argument",- fg=typer.colors.RED,- err=True,- )- raise typer.Exit(1)-- if bundle:- assert not bundle_path, "bundle_path should be None at this stage if --bundle is provided"- bundle_path = bundle-- if pipe:- assert not pipe_code, "pipe_code should be None at this stage if --pipe is provided"- pipe_code = pipe-- if not pipe_code and not bundle_path:- typer.secho("Failed to run: no pipe code or bundle file specified", fg=typer.colors.RED, err=True)- raise typer.Exit(1)-- async def generate_inputs(pipe_code: str | None = None, bundle_path: str | None = None):- if bundle_path:- try:- validate_bundle_result = await validate_bundle(plx_file_path=bundle_path)- if not pipe_code:- main_pipe_code = validate_bundle_result.blueprints[0].main_pipe- if not main_pipe_code:- typer.secho(f"Bundle '{bundle_path}' does not declare a main_pipe", fg=typer.colors.RED, err=True)- raise typer.Exit(1)- pipe_code = main_pipe_code- typer.echo(f"Using main pipe '{pipe_code}' from bundle '{bundle_path}'")- else:- typer.echo(f"Using pipe '{pipe_code}' from bundle '{bundle_path}'")- except FileNotFoundError as exc:- typer.secho(f"Failed to load bundle '{bundle_path}': {exc}", fg=typer.colors.RED, err=True)- raise typer.Exit(1) from exc- except ValidateBundleError as exc:- typer.secho(f"Failed to load bundle '{bundle_path}': {exc}", fg=typer.colors.RED, err=True)- raise typer.Exit(1) from exc- except PipeInputError as exc:- typer.secho(f"Failed to load bundle '{bundle_path}': {exc}", fg=typer.colors.RED, err=True)- raise typer.Exit(1) from exc- elif not pipe_code:- typer.secho("Failed to run: no pipe code specified", fg=typer.colors.RED, err=True)- raise typer.Exit(1)-- # Get the pipe- try:- pipe = get_required_pipe(pipe_code=pipe_code)- except Exception as exc:- typer.secho(f"❌ Error: Could not find pipe '{pipe_code}': {exc}", fg=typer.colors.RED)- raise typer.Exit(1) from exc-- # Generate the input JSON- try:- inputs_json_str = generate_input_memory_json_string(pipe.inputs, indent=2)- except Exception as exc:- typer.secho(f"❌ Error generating input JSON: {exc}", fg=typer.colors.RED)- raise typer.Exit(1) from exc-- # Determine output path- final_output_path = output_path or "results/inputs.json"-- # Save the file- try:- ensure_directory_for_file_path(file_path=final_output_path)- save_text_to_path(text=inputs_json_str, path=final_output_path)- typer.secho(f"✅ Generated input JSON file: {final_output_path}", fg=typer.colors.GREEN)- except Exception as exc:- typer.secho(f"❌ Error saving file: {exc}", fg=typer.colors.RED)- raise typer.Exit(1) from exc-- try:- pipelex_instance = Pipelex.make(integration_mode=IntegrationMode.CLI)- except ModelDeckPresetValidatonError as model_deck_error:- handle_model_deck_preset_error(model_deck_error, context=ErrorContext.VALIDATION_BEFORE_BUILD_INPUTS)-- try:- with get_telemetry_manager().telemetry_context():- tag(name=EventProperty.INTEGRATION, value=IntegrationMode.CLI)- tag(name=EventProperty.PIPELEX_VERSION, value=PACKAGE_VERSION)- tag(name=EventProperty.CLI_COMMAND, value=f"{COMMAND} {SUB_COMMAND_INPUTS}")-- asyncio.run(generate_inputs(pipe_code=pipe_code, bundle_path=bundle_path))-- except PipeOperatorModelChoiceError as exc:- handle_model_choice_error(exc, context=ErrorContext.BUILD)-- except PipeOperatorModelAvailabilityError as exc:- handle_model_availability_error(exc, context=ErrorContext.BUILD)-- finally:- pipelex_instance.teardown()--@build_app.command(SUB_COMMAND_ONE_SHOT_PIPE, help="Developer utility for contributors: deliver pipeline in one shot, without validation loop")def build_one_shot_cmd(brief: Annotated[diff --git a/pipelex/cli/commands/build/runner_cmd.py b/pipelex/cli/commands/build/runner_cmd.pynew file mode 100644index 000000000..3d1abbdac--- /dev/null+++ b/pipelex/cli/commands/build/runner_cmd.py@@ -0,0 +1,241 @@+import asyncio+from pathlib import Path+from typing import Annotated++import click+import typer+from posthog import tag++from pipelex.builder.runner_code import generate_runner_code+from pipelex.cli.commands.build.app import build_app+from pipelex.cli.error_handlers import (+ ErrorContext,+ handle_model_availability_error,+ handle_model_choice_error,+ handle_model_deck_preset_error,+)+from pipelex.cogt.exceptions import ModelDeckPresetValidatonError+from pipelex.core.pipes.exceptions import PipeOperatorModelChoiceError+from pipelex.core.pipes.inputs.exceptions import PipeInputError+from pipelex.core.pipes.variable_multiplicity import parse_concept_with_multiplicity+from pipelex.hub import get_required_pipe, get_telemetry_manager+from pipelex.pipe_operators.exceptions import PipeOperatorModelAvailabilityError+from pipelex.pipelex import PACKAGE_VERSION, Pipelex+from pipelex.pipeline.validate_bundle import ValidateBundleError, validate_bundle+from pipelex.system.runtime import IntegrationMode+from pipelex.system.telemetry.events import EventProperty+from pipelex.tools.misc.file_utils import (+ ensure_directory_for_file_path,+ get_incremental_file_path,+ save_text_to_path,+)++COMMAND = "build"+SUB_COMMAND_RUNNER = "runner"+++@build_app.command(SUB_COMMAND_RUNNER, help="Build the Python code to run a pipe with the necessary inputs")+def prepare_runner_cmd(+ target: Annotated[+ str | None,+ typer.Argument(help="Pipe code or bundle file path (auto-detected)"),+ ] = None,+ pipe: Annotated[+ str | None,+ typer.Option("--pipe", help="Pipe code to use, can be omitted if you specify a bundle (.plx) that declares a main pipe"),+ ] = None,+ bundle: Annotated[+ str | None,+ typer.Option("--bundle", help="Bundle file path (.plx) - uses its main_pipe unless you specify a pipe code"),+ ] = None,+ output_path: Annotated[+ str | None,+ typer.Option(+ "--output", "-o", help="Path to save the generated Python file (defaults to bundle's directory if bundle provided, otherwise 'results/')"+ ),+ ] = None,+) -> None:+ """Prepare a Python runner file for a pipe.++ The generated file will include:+ - All necessary imports+ - Example input values based on the pipe's input types++ Native concept types (Text, Image, PDF, etc.) will be automatically handled.+ Custom concept types will have their structure recursively generated.++ Examples:+ pipelex build runner my_pipe+ pipelex build runner --bundle my_bundle.plx+ pipelex build runner --bundle my_bundle.plx --pipe my_pipe+ pipelex build runner my_bundle.plx+ pipelex build runner my_pipe --output runner.py+ """+ # Import here to avoid circular imports+ from pipelex.cli.commands.build.structures_cmd import generate_structures_from_blueprints # noqa: PLC0415++ # Validate mutual exclusivity+ provided_options = sum([target is not None, pipe is not None, bundle is not None])+ if provided_options == 0:+ ctx: click.Context = click.get_current_context()+ typer.echo(ctx.get_help())+ raise typer.Exit(0)++ # Let's analyze the options and determine what pipe code to use and if we need to load a bundle+ pipe_code: str | None = None+ bundle_path: str | None = None++ # Determine source:+ if target:+ # Check if target is a directory (not allowed for runner command)+ target_path = Path(target)+ if target_path.is_dir():+ typer.secho(+ f"Failed to run: '{target}' is a directory. The runner command requires a .plx file or a pipe code, not a directory.",+ fg=typer.colors.RED,+ err=True,+ )+ raise typer.Exit(1)++ if target.endswith(".plx"):+ bundle_path = target+ if bundle:+ typer.secho(+ "Failed to run: cannot use option --bundle if you're already passing a bundle file (.plx) as positional argument",+ fg=typer.colors.RED,+ err=True,+ )+ raise typer.Exit(1)+ else:+ pipe_code = target+ if pipe:+ typer.secho(+ "Failed to run: cannot use option --pipe if you're already passing a pipe code as positional argument",+ fg=typer.colors.RED,+ err=True,+ )+ raise typer.Exit(1)++ if bundle:+ assert not bundle_path, "bundle_path should be None at this stage if --bundle is provided"+ bundle_path = bundle++ if pipe:+ assert not pipe_code, "pipe_code should be None at this stage if --pipe is provided"+ pipe_code = pipe++ if not pipe_code and not bundle_path:+ typer.secho("Failed to run: no pipe code or bundle file specified", fg=typer.colors.RED, err=True)+ raise typer.Exit(1)++ async def prepare_runner(pipe_code: str | None = None, bundle_path: str | None = None):+ bundle_blueprint = None+ if bundle_path:+ try:+ validate_bundle_result = await validate_bundle(plx_file_path=bundle_path)+ bundle_blueprint = validate_bundle_result.blueprints[0]+ if not pipe_code:+ main_pipe_code = bundle_blueprint.main_pipe+ if not main_pipe_code:+ # Fall back to first pipe if no main_pipe declared+ if bundle_blueprint.pipe:+ main_pipe_code = next(iter(bundle_blueprint.pipe.keys()))+ typer.echo(f"No main_pipe declared, using first pipe '{main_pipe_code}' from bundle '{bundle_path}'")+ else:+ typer.secho(f"Bundle '{bundle_path}' has no pipes defined", fg=typer.colors.RED, err=True)+ raise typer.Exit(1)+ else:+ typer.echo(f"Using main pipe '{main_pipe_code}' from bundle '{bundle_path}'")+ pipe_code = main_pipe_code+ else:+ typer.echo(f"Using pipe '{pipe_code}' from bundle '{bundle_path}'")+ except FileNotFoundError as exc:+ typer.secho(f"Failed to load bundle '{bundle_path}': {exc}", fg=typer.colors.RED, err=True)+ raise typer.Exit(1) from exc+ except ValidateBundleError as exc:+ typer.secho(f"Failed to load bundle '{bundle_path}': {exc}", fg=typer.colors.RED, err=True)+ raise typer.Exit(1) from exc+ except PipeInputError as exc:+ typer.secho(f"Failed to load bundle '{bundle_path}': {exc}", fg=typer.colors.RED, err=True)+ raise typer.Exit(1) from exc+ elif not pipe_code:+ typer.secho("Failed to run: no pipe code specified", fg=typer.colors.RED, err=True)+ raise typer.Exit(1)++ # Get the pipe+ try:+ the_pipe = get_required_pipe(pipe_code=pipe_code)+ except Exception as exc:+ typer.secho(f"❌ Error: Could not find pipe '{pipe_code}': {exc}", fg=typer.colors.RED)+ raise typer.Exit(1) from exc++ # Determine if output is a list from the bundle blueprint+ output_is_list = False+ if bundle_blueprint and bundle_blueprint.pipe and pipe_code in bundle_blueprint.pipe:+ pipe_blueprint = bundle_blueprint.pipe[pipe_code]+ output_parse = parse_concept_with_multiplicity(pipe_blueprint.output)+ output_is_list = output_parse.multiplicity is not None++ # Determine output path - use bundle's directory if bundle provided, otherwise results/+ if output_path:+ final_output_path = output_path+ elif bundle_path:+ # Place runner in the same directory as the PLX file+ bundle_dir = Path(bundle_path).parent+ final_output_path = str(bundle_dir / f"run_{pipe_code}.py")+ else:+ final_output_path = get_incremental_file_path(+ base_path="results",+ base_name=f"run_{pipe_code}",+ extension="py",+ )+ output_dir = Path(final_output_path).parent++ # Generate structures folder FIRST (before runner, since runner imports from structures)+ if bundle_blueprint:+ structures_output_dir = output_dir / "structures"+ generated_structures = generate_structures_from_blueprints(+ blueprints=[bundle_blueprint],+ output_directory=structures_output_dir,+ target_path=output_dir, # Check for existing structures in the bundle's directory+ )+ if generated_structures:+ typer.secho(f"✅ Generated {len(generated_structures)} structure(s) in: {structures_output_dir}", fg=typer.colors.GREEN)++ # Generate the runner code+ try:+ runner_code = generate_runner_code(the_pipe, output_multiplicity=output_is_list)+ except Exception as exc:+ typer.secho(f"❌ Error generating runner code: {exc}", fg=typer.colors.RED)+ raise typer.Exit(1) from exc++ # Save the runner file+ try:+ ensure_directory_for_file_path(file_path=final_output_path)+ save_text_to_path(text=runner_code, path=final_output_path)+ typer.secho(f"✅ Generated runner file: {final_output_path}", fg=typer.colors.GREEN)+ except Exception as exc:+ typer.secho(f"❌ Error saving file: {exc}", fg=typer.colors.RED)+ raise typer.Exit(1) from exc++ try:+ pipelex_instance = Pipelex.make(integration_mode=IntegrationMode.CLI)+ except ModelDeckPresetValidatonError as model_deck_error:+ handle_model_deck_preset_error(model_deck_error, context=ErrorContext.VALIDATION_BEFORE_BUILD_RUNNER)++ try:+ with get_telemetry_manager().telemetry_context():+ tag(name=EventProperty.INTEGRATION, value=IntegrationMode.CLI)+ tag(name=EventProperty.PIPELEX_VERSION, value=PACKAGE_VERSION)+ tag(name=EventProperty.CLI_COMMAND, value=f"{COMMAND} {SUB_COMMAND_RUNNER}")++ asyncio.run(prepare_runner(pipe_code=pipe_code, bundle_path=bundle_path))++ except PipeOperatorModelChoiceError as exc:+ handle_model_choice_error(exc, context=ErrorContext.BUILD)++ except PipeOperatorModelAvailabilityError as exc:+ handle_model_availability_error(exc, context=ErrorContext.BUILD)++ finally:+ pipelex_instance.teardown()diff --git a/pipelex/cli/commands/build/structures_cmd.py b/pipelex/cli/commands/build/structures_cmd.pynew file mode 100644index 000000000..f916611cf--- /dev/null+++ b/pipelex/cli/commands/build/structures_cmd.py@@ -0,0 +1,260 @@+import asyncio+from pathlib import Path+from typing import TYPE_CHECKING, Annotated++import typer+from kajson.kajson_manager import KajsonManager++from pipelex import log+from pipelex.base_exceptions import PipelexError+from pipelex.cli.commands.build.app import build_app+from pipelex.core.concepts.concept_factory import ConceptFactory+from pipelex.core.concepts.helpers import normalize_structure_blueprint+from pipelex.core.concepts.structure_generation.exceptions import ConceptStructureGeneratorError+from pipelex.core.concepts.structure_generation.generator import StructureGenerator+from pipelex.core.stuffs.structured_content import StructuredContent+from pipelex.core.stuffs.text_content import TextContent+from pipelex.pipelex import Pipelex+from pipelex.pipeline.validate_bundle import validate_bundle, validate_bundles_from_directory+from pipelex.system.registries.class_registry_utils import ClassRegistryUtils+from pipelex.system.runtime import IntegrationMode++if TYPE_CHECKING:+ from pipelex.core.bundles.pipelex_bundle_blueprint import PipelexBundleBlueprint++SUB_COMMAND_STRUCTURES = "structures"+++def generate_structures_from_blueprints(+ blueprints: list["PipelexBundleBlueprint"],+ output_directory: Path,+ target_path: Path | None = None,+ skip_existing_check: bool = False,+) -> list[tuple[str, str]]:+ """Generate Python structure files from blueprint concept definitions.++ Args:+ blueprints: List of PipelexBundleBlueprint containing concept definitions+ output_directory: Directory where structure files will be generated+ target_path: Optional path to scan for manually-created structure classes+ skip_existing_check: If True, always generate structures without checking if they exist++ Returns:+ List of (domain, concept_code) tuples for generated files+ """+ output_directory.mkdir(parents=True, exist_ok=True)++ # Only check for existing classes if we're not skipping and have a target path+ check_existing = not skip_existing_check and target_path is not None+ class_registry = KajsonManager.get_class_registry()+ if check_existing:+ class_registry.teardown()+ class_registry.setup()+ ClassRegistryUtils.register_classes_in_folder(+ folder_path=str(target_path),+ base_class=StructuredContent,+ force_exclude_dirs=[str(output_directory.resolve())],+ )++ generated_files: list[tuple[str, str]] = []++ typer.echo(f"\n📝 Generating structures in: {output_directory}")++ for blueprint in blueprints:+ if blueprint.domain == "native":+ continue++ if not blueprint.concept:+ continue++ for concept_code, concept_blueprint in blueprint.concept.items():+ # Check if structure class was manually created (only when check_existing is enabled)+ if check_existing and class_registry.has_class(name=concept_code):+ existing_class = class_registry.get_class(name=concept_code)+ if existing_class:+ import inspect # noqa: PLC0415++ try:+ source_file = inspect.getfile(existing_class)+ log.warning(+ f"Skipping Generation for '{concept_code}' (domain '{blueprint.domain}'): "+ f"manually-created class exists at '{source_file}'"+ )+ except (TypeError, OSError):+ module_name = existing_class.__module__ if hasattr(existing_class, "__module__") else "unknown"+ log.warning(+ f"Skipping '{concept_code}' (domain '{blueprint.domain}'): manually-created class exists in module '{module_name}'"+ )+ continue++ # Handle simple string concept definitions (description only, refines Text by default)+ if isinstance(concept_blueprint, str):+ try:+ generated_code, _ = StructureGenerator().generate_from_structure_blueprint(+ class_name=concept_code,+ structure_blueprint={},+ base_class_name=TextContent.__name__,+ )+ except ConceptStructureGeneratorError as exc:+ msg = f"Error generating structure class for concept '{concept_code}' in domain '{blueprint.domain}': {exc}"+ raise PipelexError(msg) from exc++ output_file = output_directory / f"{blueprint.domain}_{concept_code}.py"+ output_file.write_text(generated_code)+ generated_files.append((blueprint.domain, concept_code))+ typer.secho(f" ✅ Generated {output_file.name}", fg=typer.colors.GREEN)+ continue++ # Handle concepts with explicit structure definition+ if concept_blueprint.structure:+ if isinstance(concept_blueprint.structure, str):+ continue+ normalized_structure = normalize_structure_blueprint(concept_blueprint.structure)++ try:+ generated_code, _ = StructureGenerator().generate_from_structure_blueprint(+ class_name=concept_code,+ structure_blueprint=normalized_structure,+ )+ except ConceptStructureGeneratorError as exc:+ msg = f"Error generating python code for structure class of concept '{concept_code}' in domain '{blueprint.domain}': {exc}"+ raise PipelexError(msg) from exc++ output_file = output_directory / f"{blueprint.domain}_{concept_code}.py"+ output_file.write_text(generated_code)+ generated_files.append((blueprint.domain, concept_code))+ typer.secho(f" ✅ Generated {output_file.name}", fg=typer.colors.GREEN)++ # Handle concepts with refines+ elif concept_blueprint.refines:+ try:+ current_refine = ConceptFactory.make_refine(refine=concept_blueprint.refines)+ except Exception as exc:+ msg = (+ f"Could not validate refine '{concept_blueprint.refines}' for concept '{concept_code}' in domain '{blueprint.domain}': {exc}"+ )+ raise PipelexError(msg) from exc++ refined_structure_class_name = current_refine.split(".")[1] + "Content" if current_refine else TextContent.__name__++ try:+ generated_code, _ = StructureGenerator().generate_from_structure_blueprint(+ class_name=concept_code,+ structure_blueprint={},+ base_class_name=refined_structure_class_name,+ )+ except ConceptStructureGeneratorError as exc:+ msg = (+ f"Error generating python code for structure class of concept '{concept_code}' "+ f"refining '{refined_structure_class_name}' in domain '{blueprint.domain}': {exc}"+ )+ raise PipelexError(msg) from exc++ output_file = output_directory / f"{blueprint.domain}_{concept_code}.py"+ output_file.write_text(generated_code)+ generated_files.append((blueprint.domain, concept_code))+ typer.secho(f" ✅ Generated {output_file.name}", fg=typer.colors.GREEN)++ # Handle concepts with neither structure nor refines - defaults to TextContent+ else:+ try:+ generated_code, _ = StructureGenerator().generate_from_structure_blueprint(+ class_name=concept_code,+ structure_blueprint={},+ base_class_name=TextContent.__name__,+ )+ except ConceptStructureGeneratorError as exc:+ msg = f"Error generating structure class for concept '{concept_code}' in domain '{blueprint.domain}': {exc}"+ raise PipelexError(msg) from exc++ output_file = output_directory / f"{blueprint.domain}_{concept_code}.py"+ output_file.write_text(generated_code)+ generated_files.append((blueprint.domain, concept_code))+ typer.secho(f" ✅ Generated {output_file.name}", fg=typer.colors.GREEN)++ # Generate empty __init__.py to make structures importable+ if generated_files:+ init_file = output_directory / "__init__.py"+ init_file.write_text("")+ typer.secho(" ✅ Generated __init__.py", fg=typer.colors.GREEN)++ return generated_files+++@build_app.command(SUB_COMMAND_STRUCTURES, help="Generate Python structure files from concept definitions in PLX files")+def build_structures_command(+ target: Annotated[+ str,+ typer.Argument(help="Target directory to scan for PLX files, or a specific .plx file"),+ ],+ output_dir: Annotated[+ str | None,+ typer.Option("--output-dir", "-o", help="Output directory for generated structures (default: structures/ in target's directory)"),+ ] = None,+) -> None:+ """Generate Python structure files from concept definitions in PLX files."""++ async def _build_structures_cmd():+ target_path = Path(target).resolve()++ if not target_path.exists():+ typer.secho(f"❌ Target does not exist: {target_path}", fg=typer.colors.RED, err=True)+ raise typer.Exit(1)++ # Determine if target is a file or directory+ is_plx_file = target_path.is_file() and target_path.suffix == ".plx"++ pipelex_instance = Pipelex.make(integration_mode=IntegrationMode.CLI)++ try:+ if is_plx_file:+ # Single PLX file: output to parent directory+ base_dir = target_path.parent+ output_directory = Path(output_dir) if output_dir else base_dir / "structures"++ typer.echo(f"🔍 Validating bundle: {target_path}")++ # Validate single bundle+ validate_result = await validate_bundle(plx_file_path=str(target_path))+ all_blueprints: list[PipelexBundleBlueprint] = validate_result.blueprints++ typer.echo(f"✅ Validated {len(all_blueprints)} blueprint(s)")++ # Generate structures using the helper function+ generated_files = generate_structures_from_blueprints(+ blueprints=all_blueprints,+ output_directory=output_directory,+ target_path=base_dir,+ )+ else:+ # Directory: scan for all PLX files+ if not target_path.is_dir():+ typer.secho(f"❌ Target is not a directory or .plx file: {target_path}", fg=typer.colors.RED, err=True)+ raise typer.Exit(1)++ output_directory = Path(output_dir) if output_dir else target_path / "structures"++ typer.echo(f"🔍 Validating bundles in: {target_path}")++ # Validate bundles from directory+ validate_result = await validate_bundles_from_directory(directory=target_path)+ all_blueprints = validate_result.blueprints++ typer.echo(f"✅ Validated {len(all_blueprints)} blueprint(s)")++ # Generate structures using the helper function+ generated_files = generate_structures_from_blueprints(+ blueprints=all_blueprints,+ output_directory=output_directory,+ target_path=target_path,+ )++ if generated_files:+ typer.secho(f"\n✨ Done! Generated {len(generated_files)} structure(s) in: {output_directory}", fg=typer.colors.GREEN)+ else:+ typer.secho("\n✨ Done! No structures to generate.", fg=typer.colors.GREEN)++ finally:+ pipelex_instance.teardown()++ asyncio.run(_build_structures_cmd())diff --git a/pipelex/core/bundles/pipelex_bundle_blueprint.py b/pipelex/core/bundles/pipelex_bundle_blueprint.pyindex 5e046061b..b35e3b735 100644--- a/pipelex/core/bundles/pipelex_bundle_blueprint.py+++ b/pipelex/core/bundles/pipelex_bundle_blueprint.py@@ -3,6 +3,8 @@ from typing import Annotatedfrom pydantic import BaseModel, ConfigDict, Field, field_validator, model_validatorfrom pipelex.core.concepts.concept_blueprint import ConceptBlueprint+from pipelex.core.concepts.native.concept_native import NativeConceptCode+from pipelex.core.concepts.validation import is_concept_code_validfrom pipelex.core.domains.exceptions import DomainCodeErrorfrom pipelex.core.domains.validation import validate_domain_codefrom pipelex.pipe_controllers.batch.pipe_batch_blueprint import PipeBatchBlueprint@@ -14,6 +16,7 @@ from pipelex.pipe_operators.extract.pipe_extract_blueprint import PipeExtractBlufrom pipelex.pipe_operators.func.pipe_func_blueprint import PipeFuncBlueprintfrom pipelex.pipe_operators.img_gen.pipe_img_gen_blueprint import PipeImgGenBlueprintfrom pipelex.pipe_operators.llm.pipe_llm_blueprint import PipeLLMBlueprint+from pipelex.urls import URLsPipeBlueprintUnion = Annotated[PipeFuncBlueprint@@ -45,7 +48,6 @@ class PipelexBundleBlueprint(BaseModel):@field_validator("domain", mode="before")@classmethoddef validate_domain_syntax(cls, domain: str) -> str:- # Then validate the domain code formattry:validate_domain_code(code=domain)except DomainCodeError as exc:@@ -53,6 +55,25 @@ class PipelexBundleBlueprint(BaseModel):raise ValueError(msg) from excreturn domain+ @field_validator("concept", mode="before")+ @classmethod+ def validate_concept_keys(cls, concept: dict[str, ConceptBlueprint | str] | None) -> dict[str, ConceptBlueprint | str] | None:+ if concept is None:+ return None+ native_concept_codes = [native.value for native in NativeConceptCode.values_list()]+ for concept_code in concept:+ if not is_concept_code_valid(concept_code=concept_code):+ msg = f"Concept code '{concept_code}' is not a valid concept code"+ raise ValueError(msg)+ if concept_code in native_concept_codes:+ msg = (+ f"Cannot declare a concept named '{concept_code}' because it is natively available in Pipelex. "+ f"Native concepts are: {', '.join(native_concept_codes)}. "+ f"See {URLs.native_concepts_docs}"+ )+ raise ValueError(msg)+ return concept+@model_validator(mode="after")def validate_main_pipe(self) -> "PipelexBundleBlueprint":if self.main_pipe and (not self.pipe or (self.main_pipe not in self.pipe)):diff --git a/pipelex/core/concepts/concept.py b/pipelex/core/concepts/concept.pyindex 4f2f33b6b..03c144c61 100644--- a/pipelex/core/concepts/concept.py+++ b/pipelex/core/concepts/concept.py@@ -1,21 +1,23 @@-import inspect-from typing import Any, cast, get_args, get_origin+from typing import Anyfrom kajson.kajson_manager import KajsonManagerfrom pydantic import BaseModel, ConfigDict, field_validatorfrom pipelex import logfrom pipelex.base_exceptions import PipelexUnexpectedError-from pipelex.core.concepts.exceptions import ConceptCodeError, ConceptStringError, ConceptValueError+from pipelex.core.concepts.concept_representation_generator import (+ ConceptRepresentationFormat,+ ConceptRepresentationGenerator,+)+from pipelex.core.concepts.exceptions import ConceptCodeError, ConceptValueErrorfrom pipelex.core.concepts.native.concept_native import NativeConceptCode-from pipelex.core.concepts.validation import validate_concept_code, validate_concept_string+from pipelex.core.concepts.validation import validate_concept_codefrom pipelex.core.domains.exceptions import DomainCodeErrorfrom pipelex.core.domains.validation import validate_domain_codefrom pipelex.core.stuffs.image_field_search import search_for_nested_image_fieldsfrom pipelex.core.stuffs.stuff_content import StuffContentfrom pipelex.tools.misc.string_utils import pascal_case_to_sentencefrom pipelex.tools.typing.class_utils import are_classes_equivalent, has_compatible_field-from pipelex.types import StrEnumclass Concept(BaseModel):@@ -25,6 +27,7 @@ class Concept(BaseModel):domain: strdescription: strstructure_class_name: str+ # TODO: rethink this refines field here.refines: str | None = None@field_validator("code")@@ -52,11 +55,10 @@ class Concept(BaseModel):def validate_refines(cls, refines: str | None) -> str | None:if refines is None:return None- try:- validate_concept_string(concept_string=refines)- except ConceptStringError as exc:- msg = f"Refines '{refines}' is not a valid concept string for concept '{cls.concept_string}'"- raise ConceptValueError(msg) from exc+ if not NativeConceptCode.is_valid_native_concept_string(concept_string=refines):+ valid_native_concepts = ", ".join(native.concept_string for native in NativeConceptCode.values_list())+ msg = f"Refines '{refines}' is not a valid native concept string. Valid options are: {valid_native_concepts}"+ raise ConceptValueError(msg)return refines@property@@ -82,30 +84,37 @@ class Concept(BaseModel):if concept_1.structure_class_name == concept_2.structure_class_name:return True- # If concept_1 refines concept_2, they are strictly compatible+ # If concept_1 refines concept_2 by string, they are strictly compatibleif concept_1.refines is not None and concept_1.refines == concept_2.concept_string:return True- if concept_1.refines is None and concept_2.refines is None:- concept_1_class = KajsonManager.get_class_registry().get_class(name=concept_1.structure_class_name)- concept_2_class = KajsonManager.get_class_registry().get_class(name=concept_2.structure_class_name)+ # If both concepts refine the same concept, they are compatible+ if concept_1.refines is not None and concept_2.refines is not None and concept_1.refines == concept_2.refines:+ return True- if concept_1_class is None or concept_2_class is None:- return False+ # Check class-based compatibility+ # This now works even when one or both concepts have refines, since we generate+ # structure classes that inherit from the refined concept's class+ concept_1_class = KajsonManager.get_class_registry().get_class(name=concept_1.structure_class_name)+ concept_2_class = KajsonManager.get_class_registry().get_class(name=concept_2.structure_class_name)- if strict:- # Check if classes are equivalent (same fields, types, descriptions)- return are_classes_equivalent(concept_1_class, concept_2_class)- # Check if concept_1 is a subclass of concept_2- try:- if issubclass(concept_1_class, concept_2_class):- return True- except TypeError:- pass+ if concept_1_class is None or concept_2_class is None:+ return False- # Check if concept_1 has compatible fields with concept_2- return has_compatible_field(concept_1_class, concept_2_class)- return False+ if strict:+ # Check if classes are equivalent (same fields, types, descriptions)+ return are_classes_equivalent(concept_1_class, concept_2_class)++ # Check if concept_1 is a subclass of concept_2+ # This handles inheritance from refined concepts (e.g., RefusalEmail inherits from TextContent)+ try:+ if issubclass(concept_1_class, concept_2_class):+ return True+ except TypeError:+ pass++ # Check if concept_1 has compatible fields with concept_2+ return has_compatible_field(concept_1_class, concept_2_class)@classmethoddef is_valid_structure_class(cls, structure_class_name: str) -> bool:@@ -119,6 +128,18 @@ class Concept(BaseModel):log.warning(f"Concept class '{structure_class_name}' is registered but it's not a subclass of StuffContent")return False+ def get_structure_class(self) -> type[StuffContent]:+ """Get the structure class for this concept.++ Returns:+ The StuffContent subclass, or None if not found+ """+ structure_class = KajsonManager.get_class_registry().get_class(name=self.structure_class_name)+ if structure_class is None:+ msg = f"Concept class '{self.structure_class_name}' not found"+ raise ConceptValueError(msg)+ return structure_class+def search_for_nested_image_fields_in_structure_class(self) -> list[str]:"""Recursively search for image fields in a structure class."""structure_class = KajsonManager.get_class_registry().get_required_subclass(name=self.structure_class_name, base_class=StuffContent)@@ -127,159 +148,31 @@ class Concept(BaseModel):raise PipelexUnexpectedError(msg)return search_for_nested_image_fields(content_class=structure_class)- def get_compact_memory_example(self, var_name: str) -> dict[str, Any] | str | int:- """Generate an example value for compact memory format based on this concept.-- Compact memory follows these conventions:- - For native concepts that can be represented as simple values (Text, Image, PDF): returns a simple string- - For structured concepts: returns {"concept_code": "...", "content": {...}}-- The content dict is recursively generated based on the StuffContent class structure.- """- # Get the structure class- structure_class = KajsonManager.get_class_registry().get_class(name=self.structure_class_name)-- # If class not found, return placeholder- if structure_class is None:- return {- "concept_code": self.concept_string,- "content": {}, # Empty dict for unknown structures- }-- # Verify it's a subclass of StuffContent- if not issubclass(structure_class, StuffContent):- return {- "concept_code": self.concept_string,- "content": {}, # Empty dict for invalid structures- }-- # Generate the content based on structure- content_example = self._generate_content_example_for_class(structure_class, var_name)-- # Check if this is actually a native concept (not just using a native structure class)- is_native = Concept.is_native_concept(self)-- # For simple native concepts ONLY - return compact format- if is_native and self.structure_class_name == "TextContent":- return cast("str", content_example) # Just a string- elif is_native and self.structure_class_name == "ImageContent":- # Return dict with class instantiation info- return {- "_class": "ImageContent",- "url": cast("str", content_example),- }- elif is_native and self.structure_class_name == "PDFContent":- # Return dict with class instantiation info- return {- "_class": "PDFContent",- "url": cast("str", content_example),- }- elif is_native and self.structure_class_name == "NumberContent":- return cast("int", content_example) # Just a number-- # For refined or complex concepts, wrap with concept_code- # For Image/PDF content, wrap in the _class format- if self.structure_class_name == "ImageContent":- content_wrapped = {- "_class": "ImageContent",- "url": cast("str", content_example),- }- elif self.structure_class_name == "PDFContent":- content_wrapped = {- "_class": "PDFContent",- "url": cast("str", content_example),- }- else:- content_wrapped = content_example-- return {- "concept_code": self.concept_string,- "content": content_wrapped,- }-- @classmethod- def _generate_content_example_for_class(cls, content_class: type[StuffContent], var_name: str) -> Any:- """Recursively generate example content based on a StuffContent class structure.+ def generate_input_representation(+ self,+ output_format: ConceptRepresentationFormat,+ is_multiple: bool = False,+ ) -> tuple[dict[str, Any], set[str]]:+ """Generate a representation for this concept's input.Args:- content_class: The StuffContent class to generate an example for- var_name: Variable name for generating contextual example values+ output_format: The format to generate (JSON or PYTHON)+ is_multiple: If True, wrap content in a list (only for JSON format)Returns:- Example content dict or simple value+ Tuple of (representation dict, imports_needed set)+ - For JSON: content is a dict (or list of dicts if is_multiple)+ - For Python: content is a class instantiation string (wrapping handled by caller)"""- class_name = content_class.__name__-- # Handle simple native content types- if class_name == "TextContent":- return f"{var_name}_text"- elif class_name in {"ImageContent", "PDFContent"}:- return f"{var_name}_url"- elif class_name == "NumberContent":- return 0-- # For structured content, inspect fields and recursively generate- # Note: model_fields includes inherited fields from parent classes- content_dict: dict[str, Any] = {}- for field_name, field_info in content_class.model_fields.items():- field_type = field_info.annotation-- # Handle Optional types (e.g., TextContent | None)- origin = get_origin(field_type)- args = get_args(field_type)-- if origin is type(None) or (args and type(None) in args):- # Optional field - get the non-None type- actual_type = next((arg for arg in args if arg is not type(None)), field_type) if args else field_type- else:- actual_type = field_type+ structure_class = self.get_structure_class()- # Re-check origin after unwrapping Optional- origin = get_origin(actual_type)- args = get_args(actual_type)+ generator = ConceptRepresentationGenerator(output_format)+ # For inputs, we only want required fields (not optional ones)+ result = generator.generate_representation(self.concept_string, structure_class, include_optional=False)- # Handle list types- if origin is list:- list_item_type = args[0] if args else str- if hasattr(list_item_type, "__name__"):- if list_item_type.__name__ == "ImageContent":- content_dict[field_name] = [f"{field_name}_url_1", f"{field_name}_url_2"]- elif list_item_type.__name__ == "TextContent":- content_dict[field_name] = [f"{field_name}_text_1", f"{field_name}_text_2"]- elif inspect.isclass(list_item_type) and issubclass(list_item_type, StuffContent):- # List of StuffContent - generate examples- content_dict[field_name] = [cls._generate_content_example_for_class(list_item_type, f"{field_name}_item")]- else:- content_dict[field_name] = [f"{field_name}_item_1"]- else:- content_dict[field_name] = []- # Handle dict types- elif origin is dict:- # Simple example with one key-value pair- content_dict[field_name] = {f"{field_name}_key": f"{field_name}_value"}- # Handle StrEnum types- elif inspect.isclass(actual_type) and issubclass(actual_type, StrEnum):- # Get first enum value- enum_values = list(actual_type)- content_dict[field_name] = enum_values[0].value if enum_values else f"{field_name}_enum_value"- # Handle nested StuffContent- elif inspect.isclass(actual_type) and issubclass(actual_type, StuffContent):- content_dict[field_name] = cls._generate_content_example_for_class(actual_type, field_name)- # Handle basic types- elif actual_type is str:- content_dict[field_name] = f"{field_name}_value"- elif actual_type is int:- content_dict[field_name] = 0- elif actual_type is float:- content_dict[field_name] = 0.0- elif actual_type is bool:- content_dict[field_name] = False- else:- # For unknown types, try to get a simple repr- try:- type_name = getattr(actual_type, "__name__", str(actual_type))- content_dict[field_name] = f"{field_name}_value # TODO: Fill {type_name}"- except Exception:- content_dict[field_name] = f"{field_name}_value"+ # If multiple and JSON format, wrap content in a list+ # For Python format, the caller handles wrapping since content is a string+ if is_multiple and output_format == ConceptRepresentationFormat.JSON:+ result["content"] = [result["content"]]- return content_dict+ return result, generator.imports_neededdiff --git a/pipelex/core/concepts/concept_blueprint.py b/pipelex/core/concepts/concept_blueprint.pyindex 38cd4140a..f2753b6f9 100644--- a/pipelex/core/concepts/concept_blueprint.py+++ b/pipelex/core/concepts/concept_blueprint.py@@ -64,11 +64,13 @@ class ConceptBlueprint(BaseModel):@model_validator(mode="before")@classmethod- def validate_blueprint(cls, values: dict[str, Any] | str) -> dict[str, Any] | str:+ def validate_mutually_exclusive_fields(cls, values: dict[str, Any] | str) -> dict[str, Any] | str:+ """Validate that refines and structure are not both set."""if isinstance(values, dict) and values.get("refines") and values.get("structure"):msg = (- f"Forbidden to have refines and structure at the same time: `{values.get('refines')}` "- f"and `{values.get('structure')}` for concept that has the definition `{values.get('description')}`"+ f"A concept cannot have both 'refines' and 'structure'. "+ f"Got refines='{values.get('refines')}' and structure='{values.get('structure')}'. "+ f"Concept description: '{values.get('description')}'")raise ValueError(msg)return valuesdiff --git a/pipelex/core/concepts/concept_factory.py b/pipelex/core/concepts/concept_factory.pyindex e966c5e8e..742d83420 100644--- a/pipelex/core/concepts/concept_factory.py+++ b/pipelex/core/concepts/concept_factory.py@@ -1,56 +1,75 @@+from typing import cast+from kajson.kajson_manager import KajsonManagerfrom pydantic import BaseModelfrom pipelex.core.concepts.concept import Conceptfrom pipelex.core.concepts.concept_blueprint import ConceptBlueprint-from pipelex.core.concepts.concept_structure_blueprint import ConceptStructureBlueprint, ConceptStructureBlueprintFieldTypefrom pipelex.core.concepts.exceptions import (ConceptFactoryError,ConceptRefineError,ConceptStringError,)+from pipelex.core.concepts.helpers import normalize_structure_blueprintfrom pipelex.core.concepts.native.concept_native import NativeConceptCodefrom pipelex.core.concepts.structure_generation.exceptions import ConceptStructureGeneratorErrorfrom pipelex.core.concepts.structure_generation.generator import StructureGenerator-from pipelex.core.concepts.validation import is_concept_code_valid, validate_concept_string_or_code+from pipelex.core.concepts.validation import validate_concept_string_or_codefrom pipelex.core.domains.domain import SpecialDomainfrom pipelex.core.stuffs.text_content import TextContent+from pipelex.types import StrEnum-class DomainAndConceptCode(BaseModel):- """Small model to represent domain and concept code pair."""+class ConceptDeclarationType(StrEnum):+ """Enum representing the 5 ways a concept can be declared in PLX files.- domain: str- concept_code: str+ Option 1: STRING - Concept is defined as a string+ Example:+ [concept]+ Concept1 = "Definition of Concept1"+ Option 2: BASIC_BLUEPRINT - Concept is defined with a basic blueprint+ Example:+ [concept.Concept2]+ description = "Definition of Concept2"-class ConceptFactory:- @classmethod- def normalize_structure_blueprint(cls, structure_dict: dict[str, str | ConceptStructureBlueprint]) -> dict[str, ConceptStructureBlueprint]:- """Convert a mixed structure dictionary to a proper ConceptStructureBlueprint dictionary.+ Option 3: REFINES - Concept refines another concept+ Example:+ [concept.Concept3]+ description = "A concept3"+ refines = "native.Text"- Args:- structure_dict: Dictionary that may contain strings or ConceptStructureBlueprint objects+ Option 4: STRUCTURE_WITH_CLASSNAME+ Example:+ [concept.Concept4]+ description = "A concept4"+ structure = "ExistingClassName"- Returns:- Dictionary with all values as ConceptStructureBlueprint objects+ Option 5: BLUEPRINT_WITH_STRUCTURE - Concept is defined with a blueprint and a structure+ Example:+ [concept.Concept5]+ description = "A concept5"++ [concept.Concept5.structure]+ field1 = "A field1"+ field2 = {type = "text", description = "A field2"}+ """++ STRING = "string"+ BASIC_BLUEPRINT = "basic_blueprint"+ REFINES = "refines"+ STRUCTURE_WITH_CLASSNAME = "structure_with_classname"+ BLUEPRINT_WITH_STRUCTURE = "blueprint_with_structure"- """- normalized: dict[str, ConceptStructureBlueprint] = {}-- for field_name, field_value in structure_dict.items():- if isinstance(field_value, str):- # Convert string definition to ConceptStructureBlueprint for text field- normalized[field_name] = ConceptStructureBlueprint(- description=field_value,- type=ConceptStructureBlueprintFieldType.TEXT, # Explicitly set as text field- required=True, # Default for simple string definitions- )- else:- normalized[field_name] = field_value- return normalized+class DomainAndConceptCode(BaseModel):+ """Small model to represent domain and concept code pair."""+ domain: str+ concept_code: str+++class ConceptFactory:@classmethoddef make(cls, concept_code: str, domain: str, description: str, structure_class_name: str, refines: str | None = None) -> Concept:return Concept(@@ -136,10 +155,6 @@ class ConceptFactory:structure_class_name=structure_class_name,)- @classmethod- def make_all_native_concepts(cls) -> list[Concept]:- return [cls.make_native_concept(native_concept_code=native_concept) for native_concept in NativeConceptCode.values_list()]-@classmethoddef make_domain_and_concept_code_from_concept_string_or_code(cls,@@ -204,102 +219,240 @@ class ConceptFactory:return NativeConceptCode.get_validated_native_concept_string(concept_string_or_code=refine)@classmethod- def make_from_blueprint_or_description(+ def _handle_structure_with_classname(cls,+ blueprint: ConceptBlueprint,+ concept_code: str,domain: str,+ ) -> str:+ """Handle STRUCTURE_WITH_CLASSNAME declaration type.++ Structure is defined as a string class name - check if the class is in the registry and is valid.++ Returns:+ The structure class name+ """+ structure_class_name = blueprint.structure+ if not isinstance(structure_class_name, str):+ msg = f"Expected structure to be a string, got {type(structure_class_name)}"+ raise ConceptFactoryError(msg)+ if not Concept.is_valid_structure_class(structure_class_name=structure_class_name):+ msg = (+ f"Structure class '{structure_class_name}' set for concept '{concept_code}' in domain '{domain}' "+ "is not a registered subclass of StuffContent, or was not found in the library."+ )+ raise ConceptFactoryError(msg)+ return structure_class_name++ @classmethod+ def _handle_blueprint_with_structure(+ cls,+ blueprint: ConceptBlueprint,concept_code: str,- concept_blueprint_or_description: ConceptBlueprint | str,- ) -> Concept:- blueprint: ConceptBlueprint- if isinstance(concept_blueprint_or_description, str):- blueprint = ConceptBlueprint(description=concept_blueprint_or_description)- else:- blueprint = concept_blueprint_or_description- return cls.make_from_blueprint(- domain=domain,- concept_code=concept_code,- blueprint=blueprint,- )+ domain: str,+ ) -> str:+ """Handle BLUEPRINT_WITH_STRUCTURE declaration type.++ Structure is defined as a ConceptStructureBlueprint dict - run the structure generator+ and register it in the class registry.++ Returns:+ The structure class name (which is the concept_code)+ """+ if not isinstance(blueprint.structure, dict):+ msg = f"Expected structure to be a dict, got {type(blueprint.structure)}"+ raise ConceptFactoryError(msg)++ # Normalize the structure blueprint to ensure all values are ConceptStructureBlueprint objects+ normalized_structure = normalize_structure_blueprint(blueprint.structure)++ try:+ _, the_generated_class = StructureGenerator().generate_from_structure_blueprint(+ class_name=concept_code,+ structure_blueprint=normalized_structure,+ )+ except ConceptStructureGeneratorError as exc:+ msg = f"Error generating python code for structure class of concept '{concept_code}' in domain '{domain}': {exc}"+ raise ConceptFactoryError(msg) from exc++ # Register the generated class+ KajsonManager.get_class_registry().register_class(the_generated_class)++ return concept_code++ @classmethod+ def _handle_basic_blueprint(+ cls,+ concept_code: str,+ domain: str,+ ) -> tuple[str, str | None]:+ """Handle BASIC_BLUEPRINT declaration type.++ Returns:+ Tuple of (structure_class_name, refine_string)+ """+ # Generate a new class that inherits from TextContent and register it+ # (unless a valid structure class already exists for this concept_code)+ if Concept.is_valid_structure_class(structure_class_name=concept_code):+ return concept_code, None++ # Because native concepts have structure class names diffrent than other (with "Content")+ if concept_code in NativeConceptCode.values_list():+ return NativeConceptCode.TEXT.structure_class_name, NativeConceptCode.TEXT.concept_string++ try:+ _, the_generated_class = StructureGenerator().generate_from_structure_blueprint(+ class_name=concept_code,+ structure_blueprint={},+ base_class_name=TextContent.__name__,+ )+ except ConceptStructureGeneratorError as exc:+ msg = f"Error generating structure class for concept '{concept_code}' in domain '{domain}': {exc}"+ raise ConceptFactoryError(msg) from exc+ # Register the generated class+ KajsonManager.get_class_registry().register_class(the_generated_class)++ return concept_code, NativeConceptCode.TEXT.concept_string++ @classmethod+ def _handle_refines(+ cls,+ blueprint: ConceptBlueprint,+ concept_code: str,+ domain: str,+ ) -> tuple[str, str]:+ """Handle REFINES declaration type.++ Concept refines another concept - generate a new class that inherits from the refined+ structure class.++ Returns:+ Tuple of (structure_class_name, refine_string)+ """+ if blueprint.refines is None:+ msg = "Expected refines to be set"+ raise ConceptFactoryError(msg)++ try:+ current_refine = cls.make_refine(refine=blueprint.refines)+ except ConceptRefineError as exc:+ msg = f"Could not validate refine '{blueprint.refines}' for concept '{concept_code}' in domain '{domain}': {exc}"+ raise ConceptFactoryError(msg) from exc++ # Get the refined concept's structure class name+ refined_structure_class_name = current_refine.split(".")[1] + "Content"++ # Generate a new class that inherits from the refined structure class+ # This creates an empty class that can be extended with additional fields in the future+ try:+ _, the_generated_class = StructureGenerator().generate_from_structure_blueprint(+ class_name=concept_code,+ structure_blueprint={}, # Empty structure - just inherits from refined class+ base_class_name=refined_structure_class_name,+ )+ except ConceptStructureGeneratorError as exc:+ msg = (+ f"Error generating structure class for concept '{concept_code}' refining '{refined_structure_class_name}' in domain '{domain}': {exc}"+ )+ raise ConceptFactoryError(msg) from exc++ # Register the generated class+ KajsonManager.get_class_registry().register_class(the_generated_class)++ return concept_code, current_refine@classmethoddef make_from_blueprint(cls,domain: str,concept_code: str,- blueprint: ConceptBlueprint,+ blueprint_or_string_description: ConceptBlueprint | str,) -> Concept:- if not is_concept_code_valid(concept_code=concept_code):- msg = f"Concept code '{concept_code}' is not a valid concept code"- raise ConceptFactoryError(msg)- structure_class_name: str- current_refine: str | None = None-- # Handle structure definition- if blueprint.structure:- if isinstance(blueprint.structure, str):- # Structure is defined as a string - check if the class is in the registry and is valid- if not Concept.is_valid_structure_class(structure_class_name=blueprint.structure):- msg = (- f"Structure class '{blueprint.structure}' set for concept '{concept_code}' in domain '{domain}' "- "is not a registered subclass of StuffContent"- )- raise ConceptFactoryError(msg)- structure_class_name = blueprint.structure+ # Determine declaration type+ declaration_type: ConceptDeclarationType+ if isinstance(blueprint_or_string_description, str):+ declaration_type = ConceptDeclarationType.STRING+ elif blueprint_or_string_description.structure is not None:+ if isinstance(blueprint_or_string_description.structure, str):+ declaration_type = ConceptDeclarationType.STRUCTURE_WITH_CLASSNAMEelse:- # Structure is defined as a ConceptStructureBlueprint - run the structure generator and put it in the class registry- # Normalize the structure blueprint to ensure all values are ConceptStructureBlueprint objects- normalized_structure = cls.normalize_structure_blueprint(blueprint.structure)-- try:- _, the_generated_class = StructureGenerator().generate_from_structure_blueprint(- class_name=concept_code,- structure_blueprint=normalized_structure,- )- except ConceptStructureGeneratorError as exc:- msg = f"Error generating python code for structure class of concept '{concept_code}' in domain '{domain}': {exc}"- raise ConceptFactoryError(- msg,- ) from exc-- # Register the generated class- KajsonManager.get_class_registry().register_class(the_generated_class)-- # The structure_class_name of the concept is the concept_code- structure_class_name = concept_code-- # Handle refines definition- elif blueprint.refines:- # If we have refines, validate that there is not already a structure related to this concept code in the class registry- # TODO: This test should NOT BE HERE, but in the `validate_with_libraries`.- if Concept.is_valid_structure_class(structure_class_name=concept_code):- msg = (- f"Concept '{concept_code}' in domain '{domain}' has refines but also has a structure class registered. "- "A concept cannot have both structure and refines."- )- raise ConceptFactoryError(msg)- try:- current_refine = cls.make_refine(refine=blueprint.refines)- except ConceptRefineError as exc:- msg = f"Could not validate refine '{blueprint.refines}' for concept '{concept_code}' in domain '{domain}': {exc}"- raise ConceptFactoryError(msg) from exc- structure_class_name = current_refine.split(".")[1] + "Content" if current_refine else TextContent.__name__- # Handle neither structure nor refines - check the class registry- # If there is a class, use it. structure_class_name is then the concept_code- elif Concept.is_valid_structure_class(structure_class_name=concept_code):- structure_class_name = concept_code+ declaration_type = ConceptDeclarationType.BLUEPRINT_WITH_STRUCTURE+ elif blueprint_or_string_description.refines is not None:+ declaration_type = ConceptDeclarationType.REFINESelse:- # If there is NO class, the fallback class is TextContent.__name__- structure_class_name = TextContent.__name__+ declaration_type = ConceptDeclarationType.BASIC_BLUEPRINTdomain_and_concept_code = cls.make_domain_and_concept_code_from_concept_string_or_code(concept_string_or_code=concept_code,domain=domain,)- return Concept(- domain=domain_and_concept_code.domain,- code=domain_and_concept_code.concept_code,- description=blueprint.description,- structure_class_name=structure_class_name,- refines=current_refine,- )+ match declaration_type:+ case ConceptDeclarationType.STRING:+ structure_class_name, refines = cls._handle_basic_blueprint(+ concept_code=concept_code,+ domain=domain,+ )+ return Concept(+ domain=domain_and_concept_code.domain,+ code=domain_and_concept_code.concept_code,+ description=cast("str", blueprint_or_string_description),+ structure_class_name=structure_class_name,+ refines=refines,+ )++ case ConceptDeclarationType.BASIC_BLUEPRINT:+ structure_class_name, refines = cls._handle_basic_blueprint(+ concept_code=concept_code,+ domain=domain,+ )+ return Concept(+ domain=domain_and_concept_code.domain,+ code=domain_and_concept_code.concept_code,+ description=cast("ConceptBlueprint", blueprint_or_string_description).description,+ structure_class_name=structure_class_name,+ refines=refines,+ )++ case ConceptDeclarationType.STRUCTURE_WITH_CLASSNAME:+ blueprint = cast("ConceptBlueprint", blueprint_or_string_description)+ return Concept(+ domain=domain_and_concept_code.domain,+ code=domain_and_concept_code.concept_code,+ description=blueprint.description,+ structure_class_name=cls._handle_structure_with_classname(+ blueprint=blueprint,+ concept_code=concept_code,+ domain=domain,+ ),+ refines=None,+ )++ case ConceptDeclarationType.BLUEPRINT_WITH_STRUCTURE:+ blueprint = cast("ConceptBlueprint", blueprint_or_string_description)+ return Concept(+ domain=domain_and_concept_code.domain,+ code=domain_and_concept_code.concept_code,+ description=blueprint.description,+ structure_class_name=cls._handle_blueprint_with_structure(+ blueprint=blueprint,+ concept_code=concept_code,+ domain=domain,+ ),+ refines=None,+ )++ case ConceptDeclarationType.REFINES:+ blueprint = cast("ConceptBlueprint", blueprint_or_string_description)+ structure_class_name, current_refine = cls._handle_refines(+ blueprint=blueprint,+ concept_code=concept_code,+ domain=domain,+ )+ return Concept(+ domain=domain_and_concept_code.domain,+ code=domain_and_concept_code.concept_code,+ description=blueprint.description,+ structure_class_name=structure_class_name,+ refines=current_refine,+ )diff --git a/pipelex/core/concepts/concept_representation_generator.py b/pipelex/core/concepts/concept_representation_generator.pynew file mode 100644index 000000000..857ec6a6e--- /dev/null+++ b/pipelex/core/concepts/concept_representation_generator.py@@ -0,0 +1,379 @@+"""Generator for concept representation values in JSON and Python formats.++This module provides recursive generation of example representations for concept structures.+It supports two output formats: JSON (dict) and Python (class instantiation strings).+"""++import inspect+from typing import Any, cast, get_args, get_origin++from pydantic import BaseModel++from pipelex.core.stuffs.stuff_content import StuffContent+from pipelex.types import StrEnum+++class ConceptRepresentationFormat(StrEnum):+ """Output format for concept representations."""++ JSON = "json"+ PYTHON = "python"+++class ConceptRepresentationGenerator:+ """Generates representation values for concepts in different formats.++ Supports two output formats (JSON and Python).+ The generation is recursive: nested StuffContent classes are fully expanded.++ JSON format:+ {"concept": "domain.ConceptCode", "content": {"field1": "value1", ...}}++ Python format:+ {"concept": "domain.ConceptCode", "content": "MyClass(field1='value1', ...)"}+ """++ def __init__(self, output_format: ConceptRepresentationFormat):+ self.output_format = output_format+ self._imports_needed: set[str] = set()++ @property+ def imports_needed(self) -> set[str]:+ """Returns the set of class names that need to be imported."""+ return self._imports_needed++ def generate_representation(+ self,+ concept_string: str,+ structure_class: type[StuffContent],+ include_optional: bool = True,+ ) -> dict[str, Any]:+ """Generate a representation value for a concept.++ This is the main entry point. It wraps the generated content with concept.++ Args:+ concept_string: The concept string (e.g., "domain.ConceptCode")+ structure_class: The StuffContent class to generate representation for+ include_optional: If False, exclude fields with default values (optional fields)++ Returns:+ Dict with concept and content keys+ """+ self._imports_needed.clear()++ content = self.generate_class_representation(structure_class, include_optional=include_optional)++ return {"concept": concept_string, "content": content}++ def generate_class_representation(+ self,+ content_class: type[StuffContent],+ include_optional: bool = True,+ ) -> dict[str, Any] | str:+ """Generate representation for a StuffContent class (recursive).++ For JSON format: returns a dict with all fields+ For Python format: returns a string like "ClassName(field1=..., field2=...)"++ Args:+ content_class: The StuffContent class to generate representation for+ include_optional: If False, exclude fields with default values (optional fields)++ Returns:+ Dict (JSON) or string (Python) representing the class+ """+ class_name = content_class.__name__+ self._imports_needed.add(class_name)++ fields_dict = self._generate_fields_dict(content_class, include_optional=include_optional)++ match self.output_format:+ case ConceptRepresentationFormat.JSON:+ return fields_dict+ case ConceptRepresentationFormat.PYTHON:+ return self._format_as_python(class_name, fields_dict)++ def _generate_fields_dict(+ self,+ content_class: type[StuffContent],+ include_optional: bool = True,+ ) -> dict[str, Any]:+ """Generate a dict with field values for a class.++ Args:+ content_class: The class to generate fields for+ include_optional: If False, exclude fields with default values (optional fields)++ Returns:+ Dict mapping field names to their generated values+ """+ fields_dict: dict[str, Any] = {}++ for field_name, field_info in content_class.model_fields.items():+ # Skip optional fields if include_optional is False+ if not include_optional and not field_info.is_required():+ continue++ field_type = field_info.annotation+ field_value = self.generate_field_value(field_type, field_name)+ fields_dict[field_name] = field_value++ return fields_dict++ def generate_field_value(self, field_type: Any, field_name: str) -> Any:+ """Generate a representation value for a field based on its type (recursive).++ Handles:+ - Basic types (str, int, float, bool)+ - StrEnum types (returns first enum value)+ - list[T] types (recursively generates item)+ - dict types (generates placeholder)+ - Nested StuffContent/BaseModel classes (recursive call)++ Args:+ field_type: The type annotation of the field+ field_name: Name of the field (used for generating placeholder values)++ Returns:+ Generated value appropriate for the field type+ """+ actual_type = self._unwrap_optional(field_type)+ origin = get_origin(actual_type)+ args = get_args(actual_type)++ # Handle list types+ if origin is list:+ return self._generate_list_value(args, field_name)++ # Handle dict types+ if origin is dict:+ return self._generate_dict_value(field_name)++ # Handle StrEnum types+ if inspect.isclass(actual_type) and issubclass(actual_type, StrEnum):+ return self._generate_enum_value(actual_type, field_name)++ # Handle nested StuffContent (recursive)+ if inspect.isclass(actual_type) and issubclass(actual_type, StuffContent):+ return self.generate_class_representation(actual_type)++ # Handle nested BaseModel that is NOT StuffContent+ if inspect.isclass(actual_type) and issubclass(actual_type, BaseModel):+ return self._generate_basemodel_representation(actual_type)++ # Handle basic types+ return self._generate_basic_value(actual_type, field_name)++ def _unwrap_optional(self, field_type: Any) -> Any:+ """Unwrap Optional[T] to get T.++ Args:+ field_type: The type to unwrap++ Returns:+ The inner type if Optional, otherwise the original type+ """+ origin = get_origin(field_type)+ args = get_args(field_type)++ if origin is type(None) or (args and type(None) in args):+ return next((arg for arg in args if arg is not type(None)), field_type) if args else field_type+ return field_type++ def _generate_list_value(self, args: tuple[Any, ...], field_name: str) -> list[Any]:+ """Generate a list value with one example item.++ Args:+ args: Type arguments of the list (e.g., (str,) for list[str])+ field_name: Name of the field++ Returns:+ List with one generated item+ """+ if not args:+ return [f"{field_name}_item"]++ item_type = args[0]++ # Handle list of StuffContent (recursive)+ if inspect.isclass(item_type) and issubclass(item_type, StuffContent):+ item_repr = self.generate_class_representation(item_type)+ return [item_repr]++ # Handle list of BaseModel (not StuffContent)+ if inspect.isclass(item_type) and issubclass(item_type, BaseModel):+ item_repr = self._generate_basemodel_representation(item_type)+ return [item_repr]++ # Handle list of basic types+ return [self._generate_basic_value(item_type, f"{field_name}_item")]++ def _generate_dict_value(self, field_name: str) -> dict[str, str]:+ """Generate a placeholder dict value.++ Args:+ field_name: Name of the field++ Returns:+ Dict with placeholder key/value+ """+ return {f"{field_name}_key": f"{field_name}_value"}++ def _generate_enum_value(self, enum_type: type[StrEnum], field_name: str) -> str:+ """Generate a value from a StrEnum type.++ Args:+ enum_type: The StrEnum class+ field_name: Name of the field (used as fallback)++ Returns:+ First enum value or placeholder if empty+ """+ enum_values = list(enum_type)+ return enum_values[0].value if enum_values else f"{field_name}_enum"++ def _generate_basemodel_representation(self, model_class: type[BaseModel]) -> dict[str, Any] | str:+ """Generate representation for a BaseModel that is not StuffContent.++ Args:+ model_class: The BaseModel class++ Returns:+ Dict (JSON) or string (Python)+ """+ class_name = model_class.__name__+ self._imports_needed.add(class_name)++ fields_dict: dict[str, Any] = {}+ for field_name, field_info in model_class.model_fields.items():+ field_type = field_info.annotation+ field_value = self.generate_field_value(field_type, field_name)+ fields_dict[field_name] = field_value++ match self.output_format:+ case ConceptRepresentationFormat.JSON:+ return fields_dict+ case ConceptRepresentationFormat.PYTHON:+ return self._format_as_python(class_name, fields_dict)++ def _generate_basic_value(self, actual_type: Any, field_name: str) -> Any:+ """Generate a value for basic Python types.++ Args:+ actual_type: The type (str, int, float, bool, or unknown)+ field_name: Name of the field (used for generating placeholder)++ Returns:+ Appropriate placeholder value for the type+ """+ if actual_type is str:+ return f"{field_name}_value"+ elif actual_type is int:+ return 0+ elif actual_type is float:+ return 0.0+ elif actual_type is bool:+ return False+ else:+ type_name = getattr(actual_type, "__name__", str(actual_type))+ return f"{field_name}_{type_name}"++ def _format_as_python(self, class_name: str, fields: dict[str, Any]) -> str:+ """Format a class instantiation as Python code string.++ Args:+ class_name: Name of the class+ fields: Dict of field names to values++ Returns:+ String like "ClassName(field1=..., field2=...)"+ """+ args_parts: list[str] = []+ for field_name, value in fields.items():+ formatted_value = self._format_python_value(value)+ args_parts.append(f"{field_name}={formatted_value}")++ args = ", ".join(args_parts)+ return f"{class_name}({args})"++ def _format_python_value(self, value: Any) -> str:+ """Format a value for Python code representation.++ Args:+ value: The value to format++ Returns:+ Python code string representation++ Examples:+ >>> self._format_python_value("hello")+ '"hello"'+ >>> self._format_python_value("MyClass(arg=value)") # Already a Python expression+ 'MyClass(arg=value)'+ >>> self._format_python_value(True)+ 'True'+ >>> self._format_python_value(42)+ '42'+ >>> self._format_python_value(3.14)+ '3.14'+ >>> self._format_python_value({"key": "val"})+ '{"key": "val"}'+ >>> self._format_python_value([1, 2, 3])+ '[1, 2, 3]'+ """+ if isinstance(value, str):+ # Check if it's already a Python expression (class instantiation)+ if "(" in value and ")" in value and "=" in value:+ return value+ return f'"{value}"'+ elif isinstance(value, (bool, int, float)):+ return str(value)+ elif isinstance(value, dict):+ dict_value = cast("dict[str, Any]", value)+ items = ", ".join(f"{self._format_python_value(key)}: {self._format_python_value(val)}" for key, val in dict_value.items())+ return "{" + items + "}"+ elif isinstance(value, list):+ list_value = cast("list[Any]", value)+ items = ", ".join(self._format_python_value(item) for item in list_value)+ return "[" + items + "]"+ else:+ return str(value)+++# Convenience functions+++def generate_json_representation(+ concept_string: str,+ structure_class: type[StuffContent],+) -> dict[str, Any]:+ """Convenience function to generate a JSON format representation.++ Args:+ concept_string: The concept string (e.g., "domain.ConceptCode")+ structure_class: The StuffContent class++ Returns:+ Dict with concept and content+ """+ generator = ConceptRepresentationGenerator(ConceptRepresentationFormat.JSON)+ return generator.generate_representation(concept_string, structure_class)+++def generate_python_representation(+ concept_string: str,+ structure_class: type[StuffContent],+) -> tuple[dict[str, Any], set[str]]:+ """Convenience function to generate a Python format representation.++ Args:+ concept_string: The concept string (e.g., "domain.ConceptCode")+ structure_class: The StuffContent class++ Returns:+ Tuple of (representation dict, imports_needed set)+ """+ generator = ConceptRepresentationGenerator(ConceptRepresentationFormat.PYTHON)+ representation = generator.generate_representation(concept_string, structure_class)+ return representation, generator.imports_neededdiff --git a/pipelex/core/concepts/helpers.py b/pipelex/core/concepts/helpers.pyindex 3945fb5c8..a7f48a0ef 100644--- a/pipelex/core/concepts/helpers.py+++ b/pipelex/core/concepts/helpers.py@@ -1,3 +1,6 @@+from pipelex.core.concepts.concept_structure_blueprint import ConceptStructureBlueprint, ConceptStructureBlueprintFieldType++def strip_multiplicity_from_concept_string_or_code(concept_string_or_code: str) -> str:"""Strip multiplicity from a concept string or code.@@ -10,3 +13,26 @@ def strip_multiplicity_from_concept_string_or_code(concept_string_or_code: str)if "[" in concept_string_or_code:return concept_string_or_code.split("[", maxsplit=1)[0]return concept_string_or_code+++def normalize_structure_blueprint(structure_dict: dict[str, str | ConceptStructureBlueprint]) -> dict[str, ConceptStructureBlueprint]:+ """Convert a mixed structure dictionary to a proper ConceptStructureBlueprint dictionary.++ Args:+ structure_dict: Dictionary that may contain strings or ConceptStructureBlueprint objects++ Returns:+ Dictionary with all values as ConceptStructureBlueprint objects+ """+ normalized: dict[str, ConceptStructureBlueprint] = {}++ for field_name, field_value in structure_dict.items():+ if isinstance(field_value, str):+ normalized[field_name] = ConceptStructureBlueprint(+ description=field_value,+ type=ConceptStructureBlueprintFieldType.TEXT,+ )+ else:+ normalized[field_name] = field_value++ return normalizeddiff --git a/pipelex/core/concepts/native/concept_native.py b/pipelex/core/concepts/native/concept_native.pyindex 1de58771f..960af642e 100644--- a/pipelex/core/concepts/native/concept_native.py+++ b/pipelex/core/concepts/native/concept_native.py@@ -1,6 +1,14 @@from pipelex.core.concepts.native.exceptions import NativeConceptDefinitionErrorfrom pipelex.core.concepts.validation import is_concept_string_or_code_validfrom pipelex.core.domains.domain import SpecialDomain+from pipelex.core.stuffs.dynamic_content import DynamicContent+from pipelex.core.stuffs.image_content import ImageContent+from pipelex.core.stuffs.json_content import JSONContent+from pipelex.core.stuffs.number_content import NumberContent+from pipelex.core.stuffs.page_content import PageContent+from pipelex.core.stuffs.pdf_content import PDFContent+from pipelex.core.stuffs.text_and_images_content import TextAndImagesContent+from pipelex.core.stuffs.text_content import TextContentfrom pipelex.types import StrEnum@@ -28,6 +36,64 @@ class NativeConceptCode(StrEnum):def structure_class_name(self) -> str:return f"{self.value}Content"+ @property+ def structure_class(self) -> type | None:+ """Get the structure class for this native concept.++ Returns:+ The structure class, or None if this concept doesn't have a dedicated content class+ """+ match self:+ case NativeConceptCode.DYNAMIC:+ return DynamicContent+ case NativeConceptCode.TEXT:+ return TextContent+ case NativeConceptCode.IMAGE:+ return ImageContent+ case NativeConceptCode.PDF:+ return PDFContent+ case NativeConceptCode.TEXT_AND_IMAGES:+ return TextAndImagesContent+ case NativeConceptCode.NUMBER:+ return NumberContent+ case NativeConceptCode.PAGE:+ return PageContent+ case NativeConceptCode.JSON:+ return JSONContent+ case NativeConceptCode.IMG_GEN_PROMPT | NativeConceptCode.ANYTHING:+ # These don't have dedicated content classes+ return None++ @classmethod+ def is_native_structure_class(cls, class_name: str) -> bool:+ """Check if a class name is a native structure class.++ Args:+ class_name: The class name to check++ Returns:+ True if it's a native structure class+ """+ return class_name in cls.native_concept_class_names()++ @classmethod+ def get_native_structure_class(cls, class_name: str) -> type | None:+ """Get the native structure class by name.++ Args:+ class_name: The class name to look up++ Returns:+ The structure class, or None if not found+ """+ if not cls.is_native_structure_class(class_name):+ msg = f"Class name '{class_name}' is not a native structure class"+ raise NativeConceptDefinitionError(msg)+ for native_code in cls:+ if native_code.structure_class_name == class_name:+ return native_code.structure_class+ return None+@classmethoddef is_text_concept(cls, concept_code: str) -> bool:try:@@ -92,6 +158,24 @@ class NativeConceptCode(StrEnum):return SpecialDomain.is_native(domain=domain_code) and concept_code in cls.values_list()return concept_string_or_code in cls.values_list()+ @classmethod+ def is_valid_native_concept_string(cls, concept_string: str) -> bool:+ """Check if the string is a valid native concept string (e.g., native.Text, native.Image).++ Unlike is_native_concept_string_or_code, this method requires the full concept string+ with the native domain prefix (e.g., "native.Text" is valid, but "Text" alone is not).++ Args:+ concept_string: The concept string to validate++ Returns:+ True if the string is a valid native concept string with domain prefix+ """+ if "." not in concept_string:+ return False+ domain_code, concept_code = concept_string.split(".", 1)+ return SpecialDomain.is_native(domain=domain_code) and concept_code in cls.values_list()+@classmethoddef validate_native_concept_string_or_code(cls, concept_string_or_code: str) -> None:if not cls.is_native_concept_string_or_code(concept_string_or_code=concept_string_or_code):diff --git a/pipelex/core/concepts/structure_generation/generator.py b/pipelex/core/concepts/structure_generation/generator.pyindex 3abcf24b3..5429cb63c 100644--- a/pipelex/core/concepts/structure_generation/generator.py+++ b/pipelex/core/concepts/structure_generation/generator.py@@ -6,8 +6,9 @@ from typing import Any, Literal, Optionalfrom pydantic import Fieldfrom pipelex.core.concepts.concept_structure_blueprint import ConceptStructureBlueprint, ConceptStructureBlueprintFieldType+from pipelex.core.concepts.native.concept_native import NativeConceptCodefrom pipelex.core.concepts.structure_generation.exceptions import ConceptStructureGeneratorError, ConceptStructureValidationError, SyntaxErrorData-from pipelex.core.stuffs.structured_content import StructuredContent+from pipelex.core.stuffs.stuff_content import StuffContentclass StructureGenerator:@@ -23,12 +24,18 @@ class StructureGenerator:}self.enum_definitions: dict[str, dict[str, Any]] = {} # Store enum definitions- def generate_from_structure_blueprint(self, class_name: str, structure_blueprint: dict[str, ConceptStructureBlueprint]) -> tuple[str, type]:+ def generate_from_structure_blueprint(+ self,+ class_name: str,+ structure_blueprint: dict[str, ConceptStructureBlueprint],+ base_class_name: str | None = None,+ ) -> tuple[str, type]:"""Generate Python module content from structure blueprint.Args:class_name: Name of the class to generatestructure_blueprint: Dictionary mapping field names to their ConceptStructureBlueprint definitions+ base_class_name: Optional base class name to inherit from (defaults to StructuredContent)Returns:Generated Python module content, the generated class@@ -38,17 +45,33 @@ class StructureGenerator:"""# Generate the class- class_code = self._generate_class_source_code_from_blueprint(class_name, structure_blueprint)+ class_code = self._generate_class_source_code_from_blueprint(class_name, structure_blueprint, base_class_name)- # Generate the complete module+ # Generate the complete module with a header commentimports_section = "\n".join(sorted(self.imports))- generated_code = f"{imports_section}\n\n\n{class_code}\n"+ autogen_header = (+ '"""\n'+ "AUTOGENERATED CODE - DO NOT EDIT\n"+ "\n"+ "If you want to customize this structure:\n"+ " 1. Copy this file to your own module\n"+ " 2. Remove the 'structure' or 'refines' declaration from the concept in the PLX file\n"+ " and declare it in inline mode (see https://docs.pipelex.com/home/6-build-reliable-ai-workflows/concepts/define_your_concepts/#basic-concept-definition)\n"+ " 3. Make sure your custom class is importable and registered\n"+ "\n"+ "To regenerate: pipelex build structures <target_directory>\n"+ '"""\n\n'+ )++ generated_code = f"{autogen_header}{imports_section}\n\n\n{class_code}\n"# Validate the generated codetry:the_class = self.validate_generated_code(- python_code=generated_code, expected_class_name=class_name, required_base_class=StructuredContent+ python_code=generated_code,+ expected_class_name=class_name,+ base_class_name=base_class_name,)except SyntaxError as syntax_error:msg = f"Error validating generated code: {syntax_error}"@@ -62,13 +85,18 @@ class StructureGenerator:return generated_code, the_class- def validate_generated_code(self, python_code: str, expected_class_name: str, required_base_class: type) -> type:+ def validate_generated_code(+ self,+ python_code: str,+ expected_class_name: str,+ base_class_name: str | None = None,+ ) -> type:"""Validate that the generated Python code is syntactically correct and executable.Args:python_code: The generated Python code to validateexpected_class_name: The name of the class that should be created- required_base_class: The base class that the generated class should inherit from+ base_class_name: Optional name of a custom base class to use"""ast.parse(python_code)@@ -78,7 +106,7 @@ class StructureGenerator:return self._validate_execution(python_code=python_code,expected_class_name=expected_class_name,- required_base_class=required_base_class,+ base_class_name=base_class_name,)############################################################@@ -123,19 +151,36 @@ class StructureGenerator:return self._escape_string_for_python(value)return repr(value)- def _generate_class_source_code_from_blueprint(self, class_name: str, structure_blueprint: dict[str, ConceptStructureBlueprint]) -> str:+ def _generate_class_source_code_from_blueprint(+ self,+ class_name: str,+ structure_blueprint: dict[str, ConceptStructureBlueprint],+ base_class_name: str | None = None,+ ) -> str:"""Generate a class definition from ConceptStructureBlueprint.Args:class_name: Name of the classstructure_blueprint: Dictionary mapping field names to their ConceptStructureBlueprint definitions+ base_class_name: Optional base class name to inherit from (defaults to StructuredContent)Returns:- Generated class code+ Generated class code with markers for regeneration"""- # Generate class header- class_header = f'class {class_name}(StructuredContent):\n """Generated {class_name} class"""\n'+ # Determine base class+ base_class = base_class_name or "StructuredContent"++ # Add import for the base class if it's a native content class (other than StructuredContent which is already imported)+ if NativeConceptCode.is_native_structure_class(base_class):+ cls = NativeConceptCode.get_native_structure_class(base_class)+ if not cls:+ msg = f"Native structure class '{base_class}' not found"+ raise ConceptStructureGeneratorError(msg)+ self.imports.add(f"from {cls.__module__} import {cls.__name__}")++ # Generate class header with docstring+ class_header = f'class {class_name}({base_class}):\n """Generated {class_name} class"""\n'# Generate fieldsfield_definitions: list[str] = []@@ -144,8 +189,8 @@ class StructureGenerator:field_definitions.append(field_code)if not field_definitions:- # Empty class with just pass- return class_header + "\n pass"+ # Empty class - docstring is sufficient, no need for pass+ return class_headerfields_code = "\n".join(field_definitions)return class_header + "\n" + fields_code@@ -366,14 +411,23 @@ class StructureGenerator:# Unknown FieldType, assume it's a custom typereturn str(field_type)- def _validate_execution(self, python_code: str, expected_class_name: str, required_base_class: type) -> type:+ def _validate_execution(+ self,+ python_code: str,+ expected_class_name: str,+ base_class_name: str | None = None,+ ) -> type:"""Validate that the code executes and creates the expected class."""# Import necessary modules for the execution contextfrom typing import Any # noqa: PLC0415- # Provide necessary imports in the execution context- exec_globals = {+ from kajson.kajson_manager import KajsonManager # noqa: PLC0415++ # exec_globals with basic types but let native class imports execute naturally+ # This ensures we use the same class objects for inheritance checks+ exec_globals: dict[str, Any] = {"__builtins__": __builtins__,+ # Provide these directly as they're used in generated code but not imported from pipelex"datetime": datetime,"Enum": Enum,"Optional": Optional,@@ -382,8 +436,18 @@ class StructureGenerator:"Any": Any,"Literal": Literal,"Field": Field,- "StructuredContent": StructuredContent,}++ # If a custom base class is specified that's not a native class, get it from the registry+ if base_class_name:+ if not NativeConceptCode.is_native_structure_class(base_class_name):+ # Not a native class, provide it from registry+ custom_base_class = KajsonManager.get_class_registry().get_class(name=base_class_name)+ if custom_base_class is None:+ msg = f"Base class '{base_class_name}' not found in native classes or class registry"+ raise ConceptStructureValidationError(msg)+ exec_globals[base_class_name] = custom_base_class # type: ignore[assignment]+exec_locals: dict[str, Any] = {}exec(python_code, exec_globals, exec_locals)@@ -400,8 +464,12 @@ class StructureGenerator:raise ConceptStructureValidationError(msg)# Verify it inherits from the required base class- if not issubclass(the_class, required_base_class):- msg = f"'{expected_class_name}' does not inherit from {required_base_class.__name__}"+ # Check the MRO (Method Resolution Order) by class name to handle cases+ # where the class object references might differ due to exec() context+ mro_class_names = [cls.__name__ for cls in the_class.__mro__]++ if StuffContent.__name__ not in mro_class_names:+ msg = f"'{expected_class_name}' does not inherit from StuffContent. MRO: {mro_class_names}"raise ConceptStructureValidationError(msg)return the_classdiff --git a/pipelex/core/pipes/inputs/input_requirements.py b/pipelex/core/pipes/inputs/input_requirements.pyindex 5f4eb61f5..f80950be2 100644--- a/pipelex/core/pipes/inputs/input_requirements.py+++ b/pipelex/core/pipes/inputs/input_requirements.py@@ -1,9 +1,12 @@+import jsonfrom collections.abc import Callable+from typing import Anyfrom pydantic import BaseModel, Field, RootModel, field_validatorfrom pipelex import logfrom pipelex.core.concepts.concept import Concept+from pipelex.core.concepts.concept_representation_generator import ConceptRepresentationFormatfrom pipelex.core.pipes.inputs.exceptions import PipeInputNotFoundErrorfrom pipelex.core.pipes.variable_multiplicity import VariableMultiplicityfrom pipelex.core.stuffs.stuff_content import StuffContent@@ -125,3 +128,40 @@ class InputRequirements(RootModel[InputRequirementsRoot]):@propertydef nb_inputs(self) -> int:return len(self.root)++ def _is_multiple(self, multiplicity: VariableMultiplicity | None) -> bool:+ """Check if multiplicity indicates multiple items."""+ if multiplicity is None:+ return False+ if isinstance(multiplicity, bool):+ return multiplicity+ return multiplicity > 1++ def generate_json_representation(self) -> dict[str, Any]:+ """Generate a JSON representation for all inputs.++ Returns:+ Dictionary with JSON representations for each input+ """+ json_inputs: dict[str, Any] = {}+ for var_name, input_req in self.root.items():+ is_multiple = self._is_multiple(input_req.multiplicity)+ json_value, _ = input_req.concept.generate_input_representation(+ output_format=ConceptRepresentationFormat.JSON,+ is_multiple=is_multiple,+ )+ json_inputs[var_name] = json_value++ return json_inputs++ def generate_json_string(self, indent: int = 2) -> str:+ """Generate a JSON representation for all inputs as a formatted string.++ Args:+ indent: Number of spaces for indentation (default: 2)++ Returns:+ Formatted JSON string+ """+ json_inputs = self.generate_json_representation()+ return json.dumps(json_inputs, indent=indent, ensure_ascii=False)diff --git a/pipelex/core/stuffs/stuff.py b/pipelex/core/stuffs/stuff.pyindex 4d9ab5681..132dd163d 100644--- a/pipelex/core/stuffs/stuff.py+++ b/pipelex/core/stuffs/stuff.py@@ -148,11 +148,12 @@ Forbidden fields are: 'stuff_name', 'content_class', 'concept_code', 'stuff_code"""list_content = cast("ListContent[StuffContentType]", self.content_as(content_type=ListContent))- # Validate all items are of the expected type+ converted_items: list[StuffContentType] = []for item in list_content.items:- self.verify_content_type(item, item_type)+ converted_item = self.verify_content_type(item, item_type)+ converted_items.append(converted_item)- return list_content+ return ListContent[StuffContentType](items=converted_items)@propertydef as_text(self) -> TextContent:diff --git a/pipelex/core/stuffs/stuff_factory.py b/pipelex/core/stuffs/stuff_factory.pyindex 084cf1572..491922fa0 100644--- a/pipelex/core/stuffs/stuff_factory.py+++ b/pipelex/core/stuffs/stuff_factory.py@@ -10,7 +10,6 @@ from pipelex.core.concepts.concept_factory import ConceptFactoryfrom pipelex.core.concepts.native.concept_native import NativeConceptCodefrom pipelex.core.concepts.validation import validate_concept_stringfrom pipelex.core.stuffs.list_content import ListContent-from pipelex.core.stuffs.structured_content import StructuredContentfrom pipelex.core.stuffs.stuff import DictStuff, Stufffrom pipelex.core.stuffs.stuff_content import StuffContentfrom pipelex.core.stuffs.text_content import TextContent@@ -376,8 +375,8 @@ class StuffFactory:)raise StuffFactoryError(msg)- # Case 2.3: content is a StructuredContent object- if isinstance(content, StructuredContent):+ # Case 2.3: content is a StuffContent object (includes both native and StructuredContent)+ if isinstance(content, StuffContent):if concept.structure_class_name != content.__class__.__name__:msg = (f"Trying to create a Stuff '{name}' in the inputs of your pipe, from a dict that should represent a StuffContentOrData "@@ -441,12 +440,12 @@ class StuffFactory:msg = f"Concept '{concept_string}' is not compatible with list of text content"raise StuffFactoryError(msg)- # Case 2.4: list[StructuredContent]- if isinstance(first_item, StructuredContent):+ # Case 2.4: list[StuffContent] (includes both native and StructuredContent)+ if isinstance(first_item, StuffContent):for item in list_content_2:if not isinstance(item, type(first_item)):msg = (- f"Trying to create a Stuff '{name}' in the inputs of your pipe, from a list of StructuredContent "+ f"Trying to create a Stuff '{name}' in the inputs of your pipe, from a list of StuffContent ""but the items are not of the same type. Every items of the list should be a identical type. "f"If its a '{type(first_item).__name__}', everything should be a '{type(first_item).__name__}'.")@@ -454,7 +453,7 @@ class StuffFactory:return cls.make_stuff(concept=concept,- content=ListContent(items=cast("list[StructuredContent]", list_content_2)),+ content=ListContent(items=cast("list[StuffContent]", list_content_2)),name=name,code=code,)diff --git a/pipelex/libraries/concept/concept_library.py b/pipelex/libraries/concept/concept_library.pyindex ffeba20fd..0e40ef885 100644--- a/pipelex/libraries/concept/concept_library.py+++ b/pipelex/libraries/concept/concept_library.py@@ -49,8 +49,9 @@ class ConceptLibrary(RootModel[ConceptLibraryRoot], ConceptLibraryAbstract):def make_empty_with_native_concepts(cls) -> Self:library = cls(root={})library.setup()- all_native_concepts = ConceptFactory.make_all_native_concepts()- library.add_concepts(concepts=all_native_concepts)+ library.add_concepts(+ concepts=[ConceptFactory.make_native_concept(native_concept_code=native_concept) for native_concept in NativeConceptCode.values_list()]+ )return library@overridediff --git a/pipelex/libraries/library_manager.py b/pipelex/libraries/library_manager.pyindex 4474ad12a..afc91753d 100644--- a/pipelex/libraries/library_manager.py+++ b/pipelex/libraries/library_manager.py@@ -113,7 +113,7 @@ class LibraryManager(LibraryManagerAbstract):library_id: str,library_dirs: list[Path] | None = None,library_file_paths: list[Path] | None = None,- ) -> None:+ ) -> list[PipeAbstract]:# Ensure libraries exist for this library_idif library_id not in self._libraries:msg = f"Trying to load a library that does not exist: '{library_id}'"@@ -167,7 +167,7 @@ class LibraryManager(LibraryManagerAbstract):# Load PLX files into the specific library- self._load_plx_files_into_library(library_id=library_id, valid_plx_paths=valid_plx_paths)+ return self._load_plx_files_into_library(library_id=library_id, valid_plx_paths=valid_plx_paths)@overridedef load_from_blueprints(self, library_id: str, blueprints: list[PipelexBundleBlueprint]) -> list[PipeAbstract]:@@ -202,11 +202,11 @@ class LibraryManager(LibraryManagerAbstract):for blueprint in blueprints:if blueprint.concept is not None:concepts: list[Concept] = []- for concept_code, concept_blueprint_or_description in blueprint.concept.items():- concept = ConceptFactory.make_from_blueprint_or_description(+ for concept_code, concept_blueprint in blueprint.concept.items():+ concept = ConceptFactory.make_from_blueprint(domain=blueprint.domain,concept_code=concept_code,- concept_blueprint_or_description=concept_blueprint_or_description,+ blueprint_or_string_description=concept_blueprint,)concepts.append(concept)all_concepts.extend(concepts)@@ -235,7 +235,7 @@ class LibraryManager(LibraryManagerAbstract):# Private helper methods############################################################- def _load_plx_files_into_library(self, library_id: str, valid_plx_paths: list[Path]) -> None:+ def _load_plx_files_into_library(self, library_id: str, valid_plx_paths: list[Path]) -> list[PipeAbstract]:"""Load PLX files into a specific library.This method:@@ -264,7 +264,7 @@ class LibraryManager(LibraryManagerAbstract):blueprints.append(blueprint)self.loaded_plx_paths.extend([str(plx_file_path) for plx_file_path in valid_plx_paths])- self.load_from_blueprints(library_id=library_id, blueprints=blueprints)+ return self.load_from_blueprints(library_id=library_id, blueprints=blueprints)def _remove_pipes_from_blueprint(self, blueprint: PipelexBundleBlueprint) -> None:library = self.get_current_library()diff --git a/pipelex/libraries/library_manager_abstract.py b/pipelex/libraries/library_manager_abstract.pyindex a6e2061ae..9cf2c4d2e 100644--- a/pipelex/libraries/library_manager_abstract.py+++ b/pipelex/libraries/library_manager_abstract.py@@ -52,5 +52,5 @@ class LibraryManagerAbstract(ABC):library_id: str,library_dirs: list[Path] | None = None,library_file_paths: list[Path] | None = None,- ) -> None:+ ) -> list[PipeAbstract]:passdiff --git a/pipelex/pipe_operators/compose/pipe_compose.py b/pipelex/pipe_operators/compose/pipe_compose.pyindex 2171fc4bd..e5c4c9c30 100644--- a/pipelex/pipe_operators/compose/pipe_compose.py+++ b/pipelex/pipe_operators/compose/pipe_compose.py@@ -13,12 +13,13 @@ from pipelex.core.concepts.concept import Conceptfrom pipelex.core.concepts.concept_factory import ConceptFactoryfrom pipelex.core.concepts.native.concept_native import NativeConceptCodefrom pipelex.core.memory.working_memory import WorkingMemory+from pipelex.core.pipes.exceptions import PipeValidationError, PipeValidationErrorTypefrom pipelex.core.pipes.inputs.input_requirements import InputRequirementsfrom pipelex.core.pipes.inputs.input_requirements_factory import InputRequirementsFactoryfrom pipelex.core.pipes.pipe_output import PipeOutput+from pipelex.core.stuffs.stuff_content import StuffContentfrom pipelex.core.stuffs.stuff_factory import StuffFactory-from pipelex.core.stuffs.text_content import TextContent-from pipelex.hub import get_content_generator+from pipelex.hub import get_class_registry, get_concept_library, get_content_generator, get_native_conceptfrom pipelex.pipe_operators.pipe_operator import PipeOperatorfrom pipelex.pipe_run.exceptions import PipeRunParamsErrorfrom pipelex.pipe_run.pipe_run_mode import PipeRunMode@@ -88,7 +89,24 @@ class PipeCompose(PipeOperator[PipeComposeOutput]):@overridedef validate_output_with_library(self):- pass+ if not get_concept_library().is_compatible(+ tested_concept=self.output,+ wanted_concept=get_native_concept(native_concept=NativeConceptCode.TEXT),+ strict=True,+ ):+ msg = (+ f"The output of a PipeCompose must be strictly compatible with the Text concept. "+ f"In the pipe '{self.code}' the output is '{self.output.concept_string}'. "+ "Make sure this concept refines the native Text concept."+ )+ raise PipeValidationError(+ message=msg,+ error_type=PipeValidationErrorType.INADEQUATE_OUTPUT_CONCEPT,+ domain=self.domain,+ pipe_code=self.code,+ provided_concept_code=self.output.concept_string,+ required_concept_codes=[NativeConceptCode.TEXT.concept_string],+ )@overrideasync def _run_operator_pipe(@@ -118,7 +136,13 @@ class PipeCompose(PipeOperator[PipeComposeOutput]):)log.verbose(f"Jinja2 rendered text:\n{jinja2_text}")assert isinstance(jinja2_text, str)- the_content = TextContent(text=jinja2_text)++ # Get the structure class from the registry (might be a subclass of TextContent)+ structure_class = get_class_registry().get_required_subclass(+ name=self.output.structure_class_name,+ base_class=StuffContent,+ )+ the_content = structure_class(text=jinja2_text)output_stuff = StuffFactory.make_stuff(concept=self.output, content=the_content, name=output_name)diff --git a/pipelex/pipe_operators/img_gen/pipe_img_gen.py b/pipelex/pipe_operators/img_gen/pipe_img_gen.pyindex 129e3eb0a..a50c19345 100644--- a/pipelex/pipe_operators/img_gen/pipe_img_gen.py+++ b/pipelex/pipe_operators/img_gen/pipe_img_gen.py@@ -1,4 +1,4 @@-from typing import TYPE_CHECKING, Literal+from typing import Literalfrom pydantic import Field, model_validatorfrom typing_extensions import Self, override@@ -23,8 +23,9 @@ from pipelex.core.pipes.variable_multiplicity import VariableMultiplicityfrom pipelex.core.stuffs.exceptions import StuffContentTypeErrorfrom pipelex.core.stuffs.image_content import ImageContentfrom pipelex.core.stuffs.list_content import ListContent+from pipelex.core.stuffs.stuff_content import StuffContentfrom pipelex.core.stuffs.stuff_factory import StuffFactory-from pipelex.hub import get_concept_library, get_content_generator, get_model_deck, get_native_concept+from pipelex.hub import get_class_registry, get_concept_library, get_content_generator, get_model_deck, get_native_conceptfrom pipelex.pipe_operators.img_gen.exceptions import PipeImgGenRunErrorfrom pipelex.pipe_operators.pipe_operator import PipeOperatorfrom pipelex.pipe_run.exceptions import PipeRunParamsError@@ -33,9 +34,6 @@ from pipelex.pipe_run.pipe_run_params import PipeRunParams, output_multiplicity_from pipelex.pipe_run.pipe_run_params_factory import PipeRunParamsFactoryfrom pipelex.pipeline.job_metadata import JobMetadata-if TYPE_CHECKING:- from pipelex.core.stuffs.stuff_content import StuffContent-class PipeImgGenOutput(PipeOutput):@property@@ -149,7 +147,7 @@ class PipeImgGen(PipeOperator[PipeImgGenOutput]):strict=True,):msg = (- f"The output of a ImgGen pipe must be compatible with the Image concept. "+ f"The output of a PipeImgGen must be compatible with the Image concept. "f"In the pipe '{self.code}' the output is '{self.output.concept_string}'")raise PipeValidationError(@@ -258,6 +256,12 @@ class PipeImgGen(PipeOperator[PipeImgGenOutput]):else:nb_images = 1+ # Get the structure class from the registry (might be a subclass of ImageContent)+ structure_class = get_class_registry().get_required_subclass(+ name=self.output.structure_class_name,+ base_class=StuffContent,+ )+if nb_images > 1:generated_image_list = await content_generator.make_image_list(job_metadata=job_metadata,@@ -272,7 +276,7 @@ class PipeImgGen(PipeOperator[PipeImgGenOutput]):image_content_items: list[StuffContent] = []for generated_image in generated_image_list:image_content_items.append(- ImageContent(+ structure_class(url=generated_image.url,source_prompt=img_gen_prompt_text,),@@ -296,7 +300,7 @@ class PipeImgGen(PipeOperator[PipeImgGenOutput]):generated_image_url = generated_image.urlimage_urls = [generated_image_url]- the_content = ImageContent(+ the_content = structure_class(url=generated_image_url,source_prompt=img_gen_prompt_text,)diff --git a/pipelex/pipe_operators/llm/pipe_llm.py b/pipelex/pipe_operators/llm/pipe_llm.pyindex 3591505a9..316c46169 100644--- a/pipelex/pipe_operators/llm/pipe_llm.py+++ b/pipelex/pipe_operators/llm/pipe_llm.py@@ -14,6 +14,7 @@ from pipelex.cogt.llm.llm_setting import LLMModelChoice, LLMSetting, LLMSettingCfrom pipelex.cogt.models.model_deck_check import check_llm_choice_with_deckfrom pipelex.cogt.templating.template_category import TemplateCategoryfrom pipelex.config import get_config+from pipelex.core.concepts.concept import Conceptfrom pipelex.core.concepts.concept_factory import ConceptFactoryfrom pipelex.core.concepts.native.concept_native import NativeConceptCodefrom pipelex.core.domains.domain import SpecialDomain@@ -26,7 +27,6 @@ from pipelex.core.pipes.variable_multiplicity import VariableMultiplicityfrom pipelex.core.stuffs.list_content import ListContentfrom pipelex.core.stuffs.stuff_content import StuffContentfrom pipelex.core.stuffs.stuff_factory import StuffFactory-from pipelex.core.stuffs.text_content import TextContentfrom pipelex.hub import (get_class_registry,get_concept_library,@@ -208,7 +208,11 @@ class PipeLLM(PipeOperator[PipeLLMOutput]):# we acknowledge the code here with llm_prompt_1 and llm_prompt_2 is overly complex and should be refactored.the_content: StuffContent- if output_concept.structure_class_name == "TextContent" and not is_multiple_output:++ if (+ Concept.are_concept_compatible(concept_1=output_concept, concept_2=get_native_concept(NativeConceptCode.TEXT), strict=True)+ and not is_multiple_output+ ):llm_prompt_1_for_text = await self.llm_prompt_spec.make_llm_prompt(output_concept_string=output_concept.concept_string,context_provider=working_memory,@@ -227,8 +231,13 @@ class PipeLLM(PipeOperator[PipeLLMOutput]):msg = f"Error generating text with LLM {location}: {error_details}"raise PipeRunError(message=msg, run_mode=pipe_run_params.run_mode) from exc+ structure_class = get_class_registry().get_required_subclass(+ name=output_concept.structure_class_name,+ base_class=StuffContent,+ )+try:- the_content = TextContent(+ the_content = structure_class(text=generated_text,)except ValidationError as exc:diff --git a/pipelex/pipeline/validate_bundle.py b/pipelex/pipeline/validate_bundle.pyindex aa0ddbd1f..7d7bc17cc 100644--- a/pipelex/pipeline/validate_bundle.py+++ b/pipelex/pipeline/validate_bundle.py@@ -1,3 +1,5 @@+from pathlib import Path+from pydantic import BaseModel, ValidationErrorfrom pipelex.base_exceptions import PipelexError@@ -14,6 +16,7 @@ from pipelex.core.pipes.handle_pipe_errors import (from pipelex.core.pipes.pipe_abstract import PipeAbstractfrom pipelex.core.validation import report_validation_errorfrom pipelex.hub import get_library_manager, set_current_library+from pipelex.libraries.library_utils import get_pipelex_plx_files_from_dirsfrom pipelex.pipe_run.dry_run import DryRunError, DryRunOutput, dry_run_pipes@@ -129,3 +132,45 @@ async def validate_bundle() from dry_run_errorreturn ValidateBundleResult(blueprints=loaded_blueprints, pipes=loaded_pipes, dry_run_result=dry_run_results)+++async def validate_bundles_from_directory(directory: Path) -> ValidateBundleResult:+ plx_files = get_pipelex_plx_files_from_dirs(dirs={directory})+ all_blueprints: list[PipelexBundleBlueprint] = []++ library_manager = get_library_manager()+ library_id, _ = library_manager.open_library()+ set_current_library(library_id=library_id)+ try:+ for plx_file in plx_files:+ blueprint = PipelexInterpreter.make_pipelex_bundle_blueprint(bundle_path=str(plx_file))+ all_blueprints.append(blueprint)++ loaded_pipes = library_manager.load_libraries(library_id=library_id, library_dirs=[Path(directory)])+ dry_run_results = await dry_run_pipes(pipes=loaded_pipes, raise_on_failure=True)+ except PipelexInterpreterError as interpreter_error:+ raise ValidateBundleError(+ message=interpreter_error.message,+ pipelex_bundle_blueprint_validation_errors=interpreter_error.validation_errors,+ ) from interpreter_error+ except PipeValidationError as pipe_error:+ pipe_error_data = categorize_pipe_validation_with_libraries_error(pipe_error=pipe_error)+ raise ValidateBundleError(+ message=f"Pipe validation failed: {pipe_error}",+ pipe_validation_errors=[pipe_error_data],+ ) from pipe_error+ except ValidationError as validation_error:+ pipe_validation_errors = categorize_pipe_validation_error(validation_error=validation_error)+ validation_error_msg = report_validation_error(category="plx", validation_error=validation_error)+ msg = f"Could not load blueprints because of: {validation_error_msg}"+ raise ValidateBundleError(+ message=msg,+ pipe_validation_errors=pipe_validation_errors,+ ) from validation_error+ except DryRunError as dry_run_error:+ raise ValidateBundleError(+ message=dry_run_error.message,+ dry_run_error_message=dry_run_error.message,+ ) from dry_run_error++ return ValidateBundleResult(blueprints=all_blueprints, pipes=loaded_pipes, dry_run_result=dry_run_results)diff --git a/pipelex/system/registries/class_registry_utils.py b/pipelex/system/registries/class_registry_utils.pyindex 26c61135f..cc62f280e 100644--- a/pipelex/system/registries/class_registry_utils.py+++ b/pipelex/system/registries/class_registry_utils.py@@ -43,6 +43,7 @@ class ClassRegistryUtils:base_class: type[Any] | None = None,is_recursive: bool = True,is_include_imported: bool = False,+ force_exclude_dirs: list[str] | None = None,) -> None:"""Registers all classes in Python files within folders that are subclasses of base_class.If base_class is None, registers all classes.@@ -52,13 +53,14 @@ class ClassRegistryUtils:base_class: Optional base class to filter registerable classesis_recursive: Whether to search recursively in subdirectoriesis_include_imported: Whether to include classes imported from other modules+ force_exclude_dirs: List of absolute directory paths to exclude from scanning"""python_files = find_files_in_dir(dir_path=folder_path,pattern="*.py",is_recursive=is_recursive,- excluded_dirs=list(get_config().pipelex.scan_config.excluded_dirs),+ excluded_dirs=list(get_config().pipelex.scan_config.excluded_dirs) + (force_exclude_dirs or []),)for python_file in python_files:diff --git a/pipelex/tools/misc/file_utils.py b/pipelex/tools/misc/file_utils.pyindex 7e67ffb11..0301d74ef 100644--- a/pipelex/tools/misc/file_utils.py+++ b/pipelex/tools/misc/file_utils.py@@ -375,7 +375,25 @@ def find_files_in_dir(files = list(path.glob(pattern))for file in files:- is_excluded = excluded_dirs is not None and any(excluded_dir in file.parts for excluded_dir in excluded_dirs)+ # Check if file is under any excluded directory+ is_excluded = False+ if excluded_dirs is not None:+ for excluded_dir in excluded_dirs:+ excluded_path = Path(excluded_dir)+ # If excluded_dir is an absolute path, check if file is under it+ if excluded_path.is_absolute():+ try:+ # Resolve both paths to handle symlinks (e.g., /var -> /private/var on macOS)+ file.resolve().relative_to(excluded_path.resolve())+ is_excluded = True+ break+ except ValueError:+ # file is not relative to excluded_path+ continue+ # If excluded_dir is just a directory name, check if it's in the file path parts+ elif excluded_dir in file.parts:+ is_excluded = True+ break# Check if file is in a force include directory (forced inclusion despite exclusions)# Force include dirs can be full paths or directory namesdiff --git a/pipelex/urls.py b/pipelex/urls.pyindex 09674bdc9..b917cddcb 100644--- a/pipelex/urls.py+++ b/pipelex/urls.py@@ -6,3 +6,4 @@ class URLs:discord = "https://go.pipelex.com/discord"pipe_func_docs = "https://docs.pipelex.com/home/6-build-reliable-ai-workflows/pipes/pipe-operators/PipeFunc/"backend_provider_docs = "https://docs.pipelex.com/home/5-setup/configure-ai-providers/"+ native_concepts_docs = "https://docs.pipelex.com/home/6-build-reliable-ai-workflows/concepts/native-concepts/"diff --git a/tests/integration/pipelex/core/concepts/test_concept_representation_generator.py b/tests/integration/pipelex/core/concepts/test_concept_representation_generator.pynew file mode 100644index 000000000..734f6e73c--- /dev/null+++ b/tests/integration/pipelex/core/concepts/test_concept_representation_generator.py@@ -0,0 +1,260 @@+"""Integration tests for ConceptRepresentationGenerator with complex nested structures."""++from pydantic import Field++from pipelex.core.concepts.concept_representation_generator import (+ ConceptRepresentationFormat,+ ConceptRepresentationGenerator,+ generate_json_representation,+ generate_python_representation,+)+from pipelex.core.stuffs.structured_content import StructuredContent+from pipelex.tools.typing.pydantic_utils import empty_list_factory_of++# =============================================================================+# Complex nested structures for integration testing+# =============================================================================+++class Attachment(StructuredContent):+ """Represents a message attachment."""++ name: str = Field(..., description="Name of the attachment file")+ url: str = Field(..., description="URL of the attachment")+++class Embed(StructuredContent):+ """Represents a message embed."""++ title: str = Field(..., description="Title of the embed")+ description: str = Field(..., description="Description of the embed content")+ embed_type: str = Field(..., description="Type of the embed")+++class Message(StructuredContent):+ """Represents a message with nested attachments and embeds."""++ author: str = Field(..., description="Author of the message")+ content: str = Field(..., description="Content of the message")+ attachments: list[Attachment] = Field(default_factory=empty_list_factory_of(Attachment), description="List of attachments")+ embeds: list[Embed] = Field(default_factory=empty_list_factory_of(Embed), description="List of embeds")+ link: str = Field(..., description="Link to the message")+++class Channel(StructuredContent):+ """Represents a channel with messages."""++ name: str = Field(..., description="Name of the channel")+ position: int = Field(..., description="Position of the channel")+ messages: list[Message] = Field(default_factory=empty_list_factory_of(Message), description="List of messages")+++class ChannelSummary(StructuredContent):+ """Represents a summarized channel."""++ channel_name: str = Field(..., description="Name of the channel")+ summary_items: list[str] = Field(..., description="Summaries of the channel's activity")+++class Newsletter(StructuredContent):+ """Represents a newsletter with multiple sections."""++ weekly_summary: str = Field(..., description="Summary of weekly content")+ new_members: list[str] = Field(default_factory=list, description="New member introductions")+ channel_sections: list[ChannelSummary] = Field(default_factory=empty_list_factory_of(ChannelSummary), description="Channel summaries")+++# =============================================================================+# Integration tests with exact expected results+# =============================================================================+++class TestComplexNestedStructuresJson:+ """Integration tests for complex nested structures with JSON output."""++ def test_attachment_json(self) -> None:+ """Test simple Attachment generates correct JSON."""+ generator = ConceptRepresentationGenerator(ConceptRepresentationFormat.JSON)+ result = generator.generate_representation("test.Attachment", Attachment)+ expected = {+ "concept": "test.Attachment",+ "content": {+ "name": "name_value",+ "url": "url_value",+ },+ }+ assert result == expected++ def test_embed_json(self) -> None:+ """Test Embed with multiple string fields generates correct JSON."""+ generator = ConceptRepresentationGenerator(ConceptRepresentationFormat.JSON)+ result = generator.generate_representation("test.Embed", Embed)+ expected = {+ "concept": "test.Embed",+ "content": {+ "title": "title_value",+ "description": "description_value",+ "embed_type": "embed_type_value",+ },+ }+ assert result == expected++ def test_message_json_recursive(self) -> None:+ """Test Message with nested Attachment and Embed lists generates correct recursive JSON."""+ generator = ConceptRepresentationGenerator(ConceptRepresentationFormat.JSON)+ result = generator.generate_representation("test.Message", Message)+ expected = {+ "concept": "test.Message",+ "content": {+ "author": "author_value",+ "content": "content_value",+ "attachments": [{"name": "name_value", "url": "url_value"}],+ "embeds": [+ {+ "title": "title_value",+ "description": "description_value",+ "embed_type": "embed_type_value",+ }+ ],+ "link": "link_value",+ },+ }+ assert result == expected++ def test_channel_json_deeply_nested(self) -> None:+ """Test Channel with deeply nested messages generates correct JSON."""+ generator = ConceptRepresentationGenerator(ConceptRepresentationFormat.JSON)+ result = generator.generate_representation("test.Channel", Channel)+ expected = {+ "concept": "test.Channel",+ "content": {+ "name": "name_value",+ "position": 0,+ "messages": [+ {+ "author": "author_value",+ "content": "content_value",+ "attachments": [{"name": "name_value", "url": "url_value"}],+ "embeds": [+ {+ "title": "title_value",+ "description": "description_value",+ "embed_type": "embed_type_value",+ }+ ],+ "link": "link_value",+ }+ ],+ },+ }+ assert result == expected++ def test_newsletter_json_multiple_lists(self) -> None:+ """Test Newsletter with multiple list fields generates correct JSON."""+ generator = ConceptRepresentationGenerator(ConceptRepresentationFormat.JSON)+ result = generator.generate_representation("test.Newsletter", Newsletter)+ expected = {+ "concept": "test.Newsletter",+ "content": {+ "weekly_summary": "weekly_summary_value",+ "new_members": ["new_members_item_value"],+ "channel_sections": [+ {+ "channel_name": "channel_name_value",+ "summary_items": ["summary_items_item_value"],+ }+ ],+ },+ }+ assert result == expected+++class TestComplexNestedStructuresPython:+ """Integration tests for complex nested structures with Python output."""++ def test_attachment_python(self) -> None:+ """Test simple Attachment generates correct Python instantiation."""+ generator = ConceptRepresentationGenerator(ConceptRepresentationFormat.PYTHON)+ result = generator.generate_representation("test.Attachment", Attachment)+ expected = {+ "concept": "test.Attachment",+ "content": 'Attachment(name="name_value", url="url_value")',+ }+ assert result == expected++ def test_message_python_recursive(self) -> None:+ """Test Message with nested structures generates correct Python."""+ generator = ConceptRepresentationGenerator(ConceptRepresentationFormat.PYTHON)+ result = generator.generate_representation("test.Message", Message)+ expected_content = (+ 'Message(author="author_value", content="content_value", '+ 'attachments=[Attachment(name="name_value", url="url_value")], '+ 'embeds=[Embed(title="title_value", description="description_value", embed_type="embed_type_value")], '+ 'link="link_value")'+ )+ expected = {+ "concept": "test.Message",+ "content": expected_content,+ }+ assert result == expected++ def test_channel_python_deeply_nested(self) -> None:+ """Test Channel with deeply nested structures generates correct Python."""+ generator = ConceptRepresentationGenerator(ConceptRepresentationFormat.PYTHON)+ result = generator.generate_representation("test.Channel", Channel)++ # Build expected content string+ message_content = (+ 'Message(author="author_value", content="content_value", '+ 'attachments=[Attachment(name="name_value", url="url_value")], '+ 'embeds=[Embed(title="title_value", description="description_value", embed_type="embed_type_value")], '+ 'link="link_value")'+ )+ expected_content = f'Channel(name="name_value", position=0, messages=[{message_content}])'++ expected = {+ "concept": "test.Channel",+ "content": expected_content,+ }+ assert result == expected+++class TestImportsTrackingIntegration:+ """Integration tests for import tracking with complex structures."""++ def test_channel_tracks_all_nested_imports(self) -> None:+ """Test that Channel tracks imports for all nested classes."""+ generator = ConceptRepresentationGenerator(ConceptRepresentationFormat.JSON)+ generator.generate_representation("test.Channel", Channel)++ expected_imports = {"Channel", "Message", "Attachment", "Embed"}+ assert generator.imports_needed == expected_imports++ def test_newsletter_tracks_all_imports(self) -> None:+ """Test that Newsletter tracks imports for all nested classes."""+ generator = ConceptRepresentationGenerator(ConceptRepresentationFormat.JSON)+ generator.generate_representation("test.Newsletter", Newsletter)++ expected_imports = {"Newsletter", "ChannelSummary"}+ assert generator.imports_needed == expected_imports+++class TestConvenienceFunctionsIntegration:+ """Integration tests for convenience functions with complex structures."""++ def test_generate_json_representation_complex(self) -> None:+ """Test generate_json_representation with complex structure."""+ result = generate_json_representation("test.Message", Message)+ assert result["concept"] == "test.Message"+ assert "attachments" in result["content"]+ assert result["content"]["attachments"] == [{"name": "name_value", "url": "url_value"}]++ def test_generate_python_representation_complex(self) -> None:+ """Test generate_python_representation with complex structure."""+ result, imports = generate_python_representation("test.Message", Message)+ assert result["concept"] == "test.Message"+ assert "Message(" in result["content"]+ assert "Attachment(" in result["content"]+ assert "Message" in imports+ assert "Attachment" in imports+ assert "Embed" in importsdiff --git a/tests/integration/pipelex/pipes/controller/pipe_batch/test_pipe_batch_simple.py b/tests/integration/pipelex/pipes/controller/pipe_batch/test_pipe_batch_simple.pyindex 68179b903..8c1d8f942 100644--- a/tests/integration/pipelex/pipes/controller/pipe_batch/test_pipe_batch_simple.py+++ b/tests/integration/pipelex/pipes/controller/pipe_batch/test_pipe_batch_simple.py@@ -39,12 +39,12 @@ class TestPipeBatchSimple:concept_1 = ConceptFactory.make_from_blueprint(concept_code="TestConcept1",domain=domain,- blueprint=ConceptBlueprint(description="Lorem Ipsum"),+ blueprint_or_string_description=ConceptBlueprint(description="Lorem Ipsum"),)concept_2 = ConceptFactory.make_from_blueprint(concept_code="TestConcept2",domain=domain,- blueprint=ConceptBlueprint(description="Lorem Ipsum"),+ blueprint_or_string_description=ConceptBlueprint(description="Lorem Ipsum"),)concept_library = get_concept_library()concept_library.add_concepts([concept_1, concept_2])diff --git a/tests/integration/pipelex/pipes/controller/pipe_condition/test_pipe_condition_validation.py b/tests/integration/pipelex/pipes/controller/pipe_condition/test_pipe_condition_validation.pyindex 57463e50d..1b0db25a2 100644--- a/tests/integration/pipelex/pipes/controller/pipe_condition/test_pipe_condition_validation.py+++ b/tests/integration/pipelex/pipes/controller/pipe_condition/test_pipe_condition_validation.py@@ -19,12 +19,12 @@ class TestPipeConditionValidation:concept_1 = ConceptFactory.make_from_blueprint(concept_code="TestConcept",domain=domain,- blueprint=ConceptBlueprint(description="Lorem Ipsum"),+ blueprint_or_string_description=ConceptBlueprint(description="Lorem Ipsum"),)concept_2 = ConceptFactory.make_from_blueprint(concept_code="Result",domain=domain,- blueprint=ConceptBlueprint(description="Lorem Ipsum"),+ blueprint_or_string_description=ConceptBlueprint(description="Lorem Ipsum"),)concept_library = get_concept_library()concept_library.add_concepts([concept_1, concept_2])@@ -61,12 +61,12 @@ class TestPipeConditionValidation:concept_1 = ConceptFactory.make_from_blueprint(concept_code="TestConcept",domain=domain,- blueprint=ConceptBlueprint(description="Lorem Ipsum"),+ blueprint_or_string_description=ConceptBlueprint(description="Lorem Ipsum"),)concept_2 = ConceptFactory.make_from_blueprint(concept_code="Result",domain=domain,- blueprint=ConceptBlueprint(description="Lorem Ipsum"),+ blueprint_or_string_description=ConceptBlueprint(description="Lorem Ipsum"),)concept_library.add_concepts([concept_1, concept_2])diff --git a/tests/integration/pipelex/pipes/controller/pipe_parallel/test_pipe_parallel_validation.py b/tests/integration/pipelex/pipes/controller/pipe_parallel/test_pipe_parallel_validation.pyindex fad2c1f6e..97ba2a7a1 100644--- a/tests/integration/pipelex/pipes/controller/pipe_parallel/test_pipe_parallel_validation.py+++ b/tests/integration/pipelex/pipes/controller/pipe_parallel/test_pipe_parallel_validation.py@@ -24,17 +24,17 @@ class TestPipeParallelValidation:concept_1 = ConceptFactory.make_from_blueprint(domain=domain,concept_code="Document",- blueprint=concept_blueprint,+ blueprint_or_string_description=concept_blueprint,)concept_2 = ConceptFactory.make_from_blueprint(domain=domain,concept_code="Context",- blueprint=concept_blueprint,+ blueprint_or_string_description=concept_blueprint,)concept_3 = ConceptFactory.make_from_blueprint(domain=domain,concept_code="Analysis",- blueprint=concept_blueprint,+ blueprint_or_string_description=concept_blueprint,)concept_library.add_concepts(concepts=[concept_1, concept_2, concept_3])@@ -105,17 +105,17 @@ class TestPipeParallelValidation:concept_1 = ConceptFactory.make_from_blueprint(domain=domain,concept_code="Document",- blueprint=concept_blueprint,+ blueprint_or_string_description=concept_blueprint,)concept_2 = ConceptFactory.make_from_blueprint(domain=domain,concept_code="Context",- blueprint=concept_blueprint,+ blueprint_or_string_description=concept_blueprint,)concept_3 = ConceptFactory.make_from_blueprint(domain=domain,concept_code="ProcessedAnalysis",- blueprint=concept_blueprint,+ blueprint_or_string_description=concept_blueprint,)concept_library.add_concepts(concepts=[concept_1, concept_2, concept_3])@@ -156,17 +156,17 @@ class TestPipeParallelValidation:concept_1 = ConceptFactory.make_from_blueprint(domain=domain,concept_code="Document",- blueprint=concept_blueprint,+ blueprint_or_string_description=concept_blueprint,)concept_2 = ConceptFactory.make_from_blueprint(domain=domain,concept_code="Context",- blueprint=concept_blueprint,+ blueprint_or_string_description=concept_blueprint,)concept_3 = ConceptFactory.make_from_blueprint(domain=domain,concept_code="ProcessedAnalysis",- blueprint=concept_blueprint,+ blueprint_or_string_description=concept_blueprint,)concept_library.add_concepts(concepts=[concept_1, concept_2, concept_3])diff --git a/tests/integration/pipelex/pipes/controller/pipe_sequence/test_pipe_sequence_validation.py b/tests/integration/pipelex/pipes/controller/pipe_sequence/test_pipe_sequence_creation.pysimilarity index 52%rename from tests/integration/pipelex/pipes/controller/pipe_sequence/test_pipe_sequence_validation.pyrename to tests/integration/pipelex/pipes/controller/pipe_sequence/test_pipe_sequence_creation.pyindex a98605fe3..59b0917fe 100644--- a/tests/integration/pipelex/pipes/controller/pipe_sequence/test_pipe_sequence_validation.py+++ b/tests/integration/pipelex/pipes/controller/pipe_sequence/test_pipe_sequence_creation.py@@ -9,8 +9,8 @@ from pipelex.pipe_controllers.sequence.pipe_sequence_blueprint import PipeSequenfrom pipelex.pipe_controllers.sub_pipe_factory import SubPipeBlueprint-class TestPipeSequenceValidation:- """Tests for PipeSequence validate_inputs method"""+class TestPipeSequenceCreation:+ """Tests for basic PipeSequence creation"""def test_pipe_sequence_creation(self, load_test_library: Callable[[list[Path]], None]):"""Test basic PipeSequence creation"""@@ -19,12 +19,12 @@ class TestPipeSequenceValidation:concept_1 = ConceptFactory.make_from_blueprint(concept_code="TestConcept",domain=domain,- blueprint=ConceptBlueprint(description="Lorem Ipsum"),+ blueprint_or_string_description=ConceptBlueprint(description="Lorem Ipsum"),)concept_2 = ConceptFactory.make_from_blueprint(concept_code="ProcessedText",domain=domain,- blueprint=ConceptBlueprint(description="Lorem Ipsum"),+ blueprint_or_string_description=ConceptBlueprint(description="Lorem Ipsum"),)concept_library = get_concept_library()concept_library.add_concepts([concept_1, concept_2])@@ -47,41 +47,4 @@ class TestPipeSequenceValidation:assert len(pipe_sequence.sequential_sub_pipes) == 1assert pipe_sequence.sequential_sub_pipes[0].pipe_code == "test_pipe_1"assert pipe_sequence.sequential_sub_pipes[0].output_name == "intermediate_result"-- concept_library.teardown()-- def test_pipe_sequence_multiple_sub_pipes(self, load_test_library: Callable[[list[Path]], None]):- """Test PipeSequence with multiple sequential sub-pipes"""- load_test_library([Path("tests/integration/pipelex/pipes/controller/pipe_sequence")])- domain = "test_domain"- concept_1 = ConceptFactory.make_from_blueprint(- concept_code="TestConcept",- domain=domain,- blueprint=ConceptBlueprint(description="Lorem Ipsum"),- )- concept_2 = ConceptFactory.make_from_blueprint(- concept_code="ProcessedText",- domain=domain,- blueprint=ConceptBlueprint(description="Lorem Ipsum"),- )- concept_library = get_concept_library()- concept_library.add_concepts([concept_1, concept_2])-- pipe_sequence_blueprint = PipeSequenceBlueprint(- description="Test sequence with multiple steps",- inputs={"initial_input": concept_1.concept_string},- output=concept_2.concept_string,- steps=[SubPipeBlueprint(pipe="step_1", result="intermediate"), SubPipeBlueprint(pipe="step_2", result="final_output")],- )-- pipe_sequence = PipeFactory[PipeSequence].make_from_blueprint(- domain_code=domain,- pipe_code="test_sequence",- blueprint=pipe_sequence_blueprint,- )-- assert pipe_sequence.code == "test_sequence"- assert len(pipe_sequence.sequential_sub_pipes) == 2- assert pipe_sequence.inputs.root["initial_input"].concept.code == concept_1.code-concept_library.teardown()diff --git a/tests/integration/pipelex/pipes/controller/pipe_sequence/test_pipe_sequence_dry_run.py b/tests/integration/pipelex/pipes/controller/pipe_sequence/test_pipe_sequence_dry_run.pyindex 2cadd4715..06299143e 100644--- a/tests/integration/pipelex/pipes/controller/pipe_sequence/test_pipe_sequence_dry_run.py+++ b/tests/integration/pipelex/pipes/controller/pipe_sequence/test_pipe_sequence_dry_run.py@@ -8,6 +8,7 @@ from pipelex import log, pretty_printfrom pipelex.core.concepts.concept_factory import ConceptFactoryfrom pipelex.core.memory.working_memory_factory import WorkingMemoryFactoryfrom pipelex.core.stuffs.list_content import ListContent+from pipelex.core.stuffs.stuff import Stufffrom pipelex.core.stuffs.stuff_factory import StuffFactoryfrom pipelex.hub import get_pipe_router, get_required_pipefrom pipelex.pipe_run.pipe_job_factory import PipeJobFactory@@ -19,7 +20,6 @@ from tests.integration.pipelex.pipes.controller.pipe_sequence.discord_newsletterif TYPE_CHECKING:from pipelex.core.memory.working_memory import WorkingMemory- from pipelex.core.stuffs.stuff import Stuff@pytest.mark.dry_runnable@@ -104,14 +104,20 @@ class TestPipeSequenceDryRun:assert channel_summaries_stuff is not None, "channel_summaries should be in working memory"assert channel_summaries_stuff.concept.code == "ChannelSummary"assert channel_summaries_stuff.concept.domain == "discord_newsletter"+ assert isinstance(channel_summaries_stuff, Stuff), "ChannelSurrmary Stuff is not a Stuff"# Verify channel_summaries is also a ListContent with multiple ChannelSummary itemschannel_summaries_list: ListContent[ChannelSummary] = channel_summaries_stuff.as_list_of_fixed_content_type(item_type=ChannelSummary)++ pretty_print(channel_summaries_stuff, "jiodisjdqosj")+ pretty_print(channel_summaries_list, title="Channel Summaries List")assert isinstance(channel_summaries_list, ListContent)assert len(channel_summaries_list.items) > 1, "Should have multiple ChannelSummary items from batch processing"# Verify each summary item (these should be proper ChannelSummary objects from the LLM mock)for i, summary in enumerate(channel_summaries_list.items):+ print(summary)+ print(type(summary))assert isinstance(summary, ChannelSummary), f"Summary {i} should be ChannelSummary"assert len(summary.channel_name) > 0, f"Summary {i} should have a non-empty channel name"assert isinstance(summary.summary_items, list), f"Summary {i} should have a list of summary items"diff --git a/tests/integration/pipelex/pipes/controller/pipe_sequence/test_pipe_sequence_multiple_steps.py b/tests/integration/pipelex/pipes/controller/pipe_sequence/test_pipe_sequence_multiple_steps.pynew file mode 100644index 000000000..31bf09bba--- /dev/null+++ b/tests/integration/pipelex/pipes/controller/pipe_sequence/test_pipe_sequence_multiple_steps.py@@ -0,0 +1,49 @@+from pathlib import Path+from typing import Callable++from pipelex.core.concepts.concept_factory import ConceptBlueprint, ConceptFactory+from pipelex.core.pipes.pipe_factory import PipeFactory+from pipelex.hub import get_concept_library+from pipelex.pipe_controllers.sequence.pipe_sequence import PipeSequence+from pipelex.pipe_controllers.sequence.pipe_sequence_blueprint import PipeSequenceBlueprint+from pipelex.pipe_controllers.sub_pipe_factory import SubPipeBlueprint+++class TestPipeSequenceMultipleSteps:+ """Tests for PipeSequence with multiple steps"""++ def test_pipe_sequence_multiple_sub_pipes(self, load_test_library: Callable[[list[Path]], None]):+ """Test PipeSequence with multiple sequential sub-pipes"""+ load_test_library([Path("tests/integration/pipelex/pipes/controller/pipe_sequence")])+ domain = "test_domain"+ concept_1 = ConceptFactory.make_from_blueprint(+ concept_code="TestConcept",+ domain=domain,+ blueprint_or_string_description=ConceptBlueprint(description="Lorem Ipsum"),+ )+ concept_2 = ConceptFactory.make_from_blueprint(+ concept_code="ProcessedText",+ domain=domain,+ blueprint_or_string_description=ConceptBlueprint(description="Lorem Ipsum"),+ )+ concept_library = get_concept_library()+ concept_library.add_concepts([concept_1, concept_2])++ pipe_sequence_blueprint = PipeSequenceBlueprint(+ description="Test sequence with multiple steps",+ inputs={"initial_input": concept_1.concept_string},+ output=concept_2.concept_string,+ steps=[SubPipeBlueprint(pipe="step_1", result="intermediate"), SubPipeBlueprint(pipe="step_2", result="final_output")],+ )++ pipe_sequence = PipeFactory[PipeSequence].make_from_blueprint(+ domain_code=domain,+ pipe_code="test_sequence",+ blueprint=pipe_sequence_blueprint,+ )++ assert pipe_sequence.code == "test_sequence"+ assert len(pipe_sequence.sequential_sub_pipes) == 2+ assert pipe_sequence.inputs.root["initial_input"].concept.code == concept_1.code++ concept_library.teardown()diff --git a/tests/integration/pipelex/pipes/controller/pipe_sequence/test_pipe_sequence_simple.py b/tests/integration/pipelex/pipes/controller/pipe_sequence/test_pipe_sequence_simple.pyindex 96b847208..59f2640d1 100644--- a/tests/integration/pipelex/pipes/controller/pipe_sequence/test_pipe_sequence_simple.py+++ b/tests/integration/pipelex/pipes/controller/pipe_sequence/test_pipe_sequence_simple.py@@ -36,7 +36,7 @@ class TestPipeSequenceSimple:concept_1 = ConceptFactory.make_from_blueprint(concept_code="TestConcept1",domain=domain,- blueprint=ConceptBlueprint(description="Lorem Ipsum"),+ blueprint_or_string_description=ConceptBlueprint(description="Lorem Ipsum"),)concept_library = get_concept_library()concept_library.add_concepts([concept_1])diff --git a/tests/integration/pipelex/pipes/operator/pipe_extract/test_pipe_extract.py b/tests/integration/pipelex/pipes/operator/pipe_extract/test_pipe_extract.pyindex d5c5cd4b6..bbea9f6cf 100644--- a/tests/integration/pipelex/pipes/operator/pipe_extract/test_pipe_extract.py+++ b/tests/integration/pipelex/pipes/operator/pipe_extract/test_pipe_extract.py@@ -36,7 +36,7 @@ class TestPipeExtract:concept_1 = ConceptFactory.make_from_blueprint(concept_code="PageScan",domain="extract",- blueprint=ConceptBlueprint(description="Lorem Ipsum"),+ blueprint_or_string_description=ConceptBlueprint(description="Lorem Ipsum"),)concept_library.add_new_concept(concept=concept_1)diff --git a/tests/integration/pipelex/pipes/operator/pipe_llm/test_pipe_llm.py b/tests/integration/pipelex/pipes/operator/pipe_llm/test_pipe_llm.pyindex 68e4e1253..f23bf78a1 100644--- a/tests/integration/pipelex/pipes/operator/pipe_llm/test_pipe_llm.py+++ b/tests/integration/pipelex/pipes/operator/pipe_llm/test_pipe_llm.py@@ -89,7 +89,6 @@ class TestPipeLLMBasic:):load_test_library([Path("tests/integration/pipelex/pipes/operator/pipe_llm")])# TODO: Add assertion on generated objects vs expected results- pretty_print(data, title="data")working_memory = WorkingMemoryFactory.make_from_single_stuff(stuff=StuffFactory.make_stuff(concept=get_native_concept(NativeConceptCode.TEXT),@@ -109,6 +108,7 @@ class TestPipeLLMBasic:structuring_method=structuring_method,)+ pretty_print(pipe_llm_blueprint, title="pipe_llm_blueprint")pipe_code = f"extract_{topic}_{concept}_{structuring_method}"pipe_code = pipe_code.lower().replace(" ", "_")pipe = PipeFactory[PipeLLM].make_from_blueprint(@@ -124,8 +124,11 @@ class TestPipeLLMBasic:working_memory=working_memory,pipe_run_params=PipeRunParamsFactory.make_run_params(pipe_run_mode=pipe_run_mode),)+pipe_llm_output = await get_pipe_router().run(pipe_job=pipe_job)+ pretty_print(pipe_llm_output, title="pipe_llm_dzqdzqoutput")+# Log test informationlog.verbose(f"Testing {topic} with {structuring_method} method", title="Test Case")diff --git a/tests/integration/pipelex/pipes/test_bracket_notation_controllers.py b/tests/integration/pipelex/pipes/test_bracket_notation_controllers.pyindex 32b2536c2..46fdf5f55 100644--- a/tests/integration/pipelex/pipes/test_bracket_notation_controllers.py+++ b/tests/integration/pipelex/pipes/test_bracket_notation_controllers.py@@ -25,13 +25,13 @@ class TestBracketNotationInControllers:concept_data_item = ConceptFactory.make_from_blueprint(concept_code="DataItem",domain=domain,- blueprint=ConceptBlueprint(description="Data item"),+ blueprint_or_string_description=ConceptBlueprint(description="Data item"),)concept_library.add_concepts([concept_data_item])concept_processed_data = ConceptFactory.make_from_blueprint(concept_code="ProcessedData",domain=domain,- blueprint=ConceptBlueprint(description="Processed data"),+ blueprint_or_string_description=ConceptBlueprint(description="Processed data"),)concept_library.add_concepts([concept_processed_data])@@ -64,12 +64,12 @@ class TestBracketNotationInControllers:concept_1 = ConceptFactory.make_from_blueprint(concept_code="Category",domain=domain,- blueprint=ConceptBlueprint(description="Category"),+ blueprint_or_string_description=ConceptBlueprint(description="Category"),)concept_2 = ConceptFactory.make_from_blueprint(concept_code="Result",domain=domain,- blueprint=ConceptBlueprint(description="Result"),+ blueprint_or_string_description=ConceptBlueprint(description="Result"),)concept_library.add_concepts([concept_1, concept_2])@@ -103,12 +103,12 @@ class TestBracketNotationInControllers:concept_item = ConceptFactory.make_from_blueprint(concept_code="Item",domain=domain,- blueprint=ConceptBlueprint(description="Item"),+ blueprint_or_string_description=ConceptBlueprint(description="Item"),)concept_processed_item = ConceptFactory.make_from_blueprint(concept_code="ProcessedItem",domain=domain,- blueprint=ConceptBlueprint(description="Processed item"),+ blueprint_or_string_description=ConceptBlueprint(description="Processed item"),)concept_library.add_concepts([concept_item, concept_processed_item])diff --git a/tests/integration/pipelex/pipes/test_flow_factory/discord_newsletter.py b/tests/integration/pipelex/pipes/test_flow_factory/discord_newsletter.pyindex c11ac4010..5730daff5 100644--- a/tests/integration/pipelex/pipes/test_flow_factory/discord_newsletter.py+++ b/tests/integration/pipelex/pipes/test_flow_factory/discord_newsletter.py@@ -42,16 +42,3 @@ class ChannelSummaryFlow(StructuredContent):channel_name: str = Field(..., description="Name of the Discord channel")summary_items: list[str] = Field(..., description="Well-written summaries of the channel's activity")---class HtmlNewsletterFlow(StructuredContent):- """Represents the final newsletter content"""-- weekly_summary: str = Field(..., description="200 character summary of weekly Share channel content")- new_members: list[str] = Field(default_factory=empty_list_factory_of(str), description="New member introductions in bullet points")- channel_sections: list[ChannelSummaryFlow] = Field(- default_factory=empty_list_factory_of(ChannelSummaryFlow), description="Ordered channel summaries"- )- geographic_hubs: list[ChannelSummaryFlow] = Field(- default_factory=empty_list_factory_of(ChannelSummaryFlow), description="Geographic hub channels grouped at end"- )diff --git a/tests/integration/pipelex/test_inline_structure_concepts.py b/tests/integration/pipelex/test_inline_structure_concepts.pyindex f15d2806d..fa2ecc59e 100644--- a/tests/integration/pipelex/test_inline_structure_concepts.py+++ b/tests/integration/pipelex/test_inline_structure_concepts.py@@ -50,7 +50,7 @@ class TestInlineStructureConcepts:concept = ConceptFactory.make_from_blueprint(domain="test_domain",concept_code="TestFeatureAnalysis",- blueprint=blueprint,+ blueprint_or_string_description=blueprint,)# Verify concept properties@@ -90,7 +90,7 @@ class TestInlineStructureConcepts:concept = ConceptFactory.make_from_blueprint(domain="test_domain",concept_code="TestStringRef",- blueprint=blueprint,+ blueprint_or_string_description=blueprint,)# Verify concept properties@@ -108,11 +108,12 @@ class TestInlineStructureConcepts:concept = ConceptFactory.make_from_blueprint(domain="test_domain",concept_code="TestAutoDetect",- blueprint=blueprint,+ blueprint_or_string_description=blueprint,)# Should default to TextContent since TestAutoDetect is not a registered class- assert concept.structure_class_name == "TextContent"+ assert concept.structure_class_name == "TestAutoDetect"+ assert concept.refines == "native.Text"def test_inline_structure_with_complex_types(self):"""Test inline structure with complex field types."""@@ -145,7 +146,7 @@ class TestInlineStructureConcepts:concept = ConceptFactory.make_from_blueprint(domain="test_domain",concept_code="ComplexDocument",- blueprint=blueprint,+ blueprint_or_string_description=blueprint,)# Verify concept creation@@ -183,7 +184,7 @@ not a registered subclass of StuffContent",_ = ConceptFactory.make_from_blueprint(domain="test_domain",concept_code="TestInvalidRef",- blueprint=ConceptBlueprint(description="Test invalid reference", structure="NonExistentClass"),+ blueprint_or_string_description=ConceptBlueprint(description="Test invalid reference", structure="NonExistentClass"),)def test_multiple_inline_structures_do_not_conflict(self):@@ -198,7 +199,7 @@ not a registered subclass of StuffContent",concept1 = ConceptFactory.make_from_blueprint(domain="test_domain",concept_code="Person",- blueprint=blueprint1,+ blueprint_or_string_description=blueprint1,)# Second structure with same field names but different context@@ -211,7 +212,7 @@ not a registered subclass of StuffContent",concept2 = ConceptFactory.make_from_blueprint(domain="test_domain",concept_code="Product",- blueprint=blueprint2,+ blueprint_or_string_description=blueprint2,)# Both should be created successfully@@ -261,7 +262,7 @@ not a registered subclass of StuffContent",concept = ConceptFactory.make_from_blueprint(domain="test_domain",concept_code="AnimalInfo",- blueprint=blueprint,+ blueprint_or_string_description=blueprint,)# Verify concept propertiesdiff --git a/tests/unit/pipelex/builder/test_runner_generator.py b/tests/unit/pipelex/builder/test_runner_generator.pyindex fb97efb0f..dfc01f031 100644--- a/tests/unit/pipelex/builder/test_runner_generator.py+++ b/tests/unit/pipelex/builder/test_runner_generator.py@@ -1,101 +1,80 @@-"""Unit tests for runner_generator module."""+"""Unit tests for runner_code module."""from __future__ import annotations-from pipelex.builder.runner_code import generate_compact_memory_entry, value_to_python_code-from pipelex.core.concepts.concept_factory import ConceptFactory-from pipelex.core.concepts.native.concept_native import NativeConceptCode+from typing import Any, cast+from unittest.mock import MagicMock+import pytest-class TestValueToPythonCode:- """Test value_to_python_code function."""-- def test_simple_string(self) -> None:- """Test conversion of a simple string."""- result = value_to_python_code("hello")- assert result == '"hello"'-- def test_simple_number(self) -> None:- """Test conversion of a number."""- result = value_to_python_code(42)- assert result == "42"-- def test_simple_boolean(self) -> None:- """Test conversion of a boolean."""- result = value_to_python_code(True)- assert result == "True"-- def test_dict_with_class_image_content(self) -> None:- """Test conversion of ImageContent dict."""- value = {"_class": "ImageContent", "url": "test_url"}- result = value_to_python_code(value)- assert result == 'ImageContent(url="test_url")'-- def test_dict_with_class_pdf_content(self) -> None:- """Test conversion of PDFContent dict."""- value = {"_class": "PDFContent", "url": "test_url"}- result = value_to_python_code(value)- assert result == 'PDFContent(url="test_url")'-- def test_dict_with_concept_code_and_content_simple(self) -> None:- """Test conversion of refined concept with simple content."""- value = {- "concept_code": "test_domain.Question",- "content": "question_text",- }- result = value_to_python_code(value, indent_level=3)- expected = '{\n "concept": "test_domain.Question",\n "content": "question_text",\n }'- assert result == expected-- def test_dict_with_concept_code_and_content_image(self) -> None:- """Test conversion of refined Image concept."""- value = {- "concept_code": "tables.TableScreenshot",- "content": {"_class": "ImageContent", "url": "table_screenshot_url"},- }- result = value_to_python_code(value, indent_level=3)- expected = (- "{\n"- ' "concept": "tables.TableScreenshot",\n'- ' "content": ImageContent(url="table_screenshot_url"),\n'- " }"- )- assert result == expected+from pipelex.builder.runner_code import generate_runner_code+from pipelex.core.concepts.concept_factory import ConceptFactory+from pipelex.core.concepts.concept_representation_generator import ConceptRepresentationFormat+from pipelex.core.concepts.native.concept_native import NativeConceptCode+from pipelex.core.pipes.inputs.input_requirements import InputRequirement, InputRequirements-class TestGenerateCompactMemoryEntry:- """Test generate_compact_memory_entry function."""+class TestConceptGenerateInputRepresentationJson:+ """Test Concept.generate_input_representation for JSON format."""- def test_generate_entry_for_native_text(self) -> None:- """Test generating entry for native Text concept."""+ def test_native_text_json(self) -> None:+ """Test JSON representation for native Text concept."""concept = ConceptFactory.make_native_concept(NativeConceptCode.TEXT)- result = generate_compact_memory_entry("message", concept)- assert result == ' "message": "message_text",'+ result, _ = concept.generate_input_representation(ConceptRepresentationFormat.JSON)+ assert result["concept"] == "native.Text"+ assert "content" in result+ assert "text" in result["content"]- def test_generate_entry_for_native_image(self) -> None:- """Test generating entry for native Image concept."""+ def test_native_image_json(self) -> None:+ """Test JSON representation for native Image concept."""concept = ConceptFactory.make_native_concept(NativeConceptCode.IMAGE)- result = generate_compact_memory_entry("photo", concept)- assert result == ' "photo": ImageContent(url="photo_url"),'+ result, _ = concept.generate_input_representation(ConceptRepresentationFormat.JSON)+ assert result["concept"] == "native.Image"+ assert "content" in result+ assert "url" in result["content"]++ def test_native_number_json(self) -> None:+ """Test JSON representation for native Number concept."""+ concept = ConceptFactory.make_native_concept(NativeConceptCode.NUMBER)+ result, _ = concept.generate_input_representation(ConceptRepresentationFormat.JSON)+ assert result["concept"] == "native.Number"+ assert "content" in result+ assert "number" in result["content"]++ def test_native_pdf_json_with_multiplicity(self) -> None:+ """Test JSON representation for PDF with multiplicity - content should be a list."""+ concept = ConceptFactory.make_native_concept(NativeConceptCode.PDF)+ result, _ = concept.generate_input_representation(ConceptRepresentationFormat.JSON, is_multiple=True)+ assert result["concept"] == "native.PDF"+ assert "content" in result+ # Content should be a list+ content = cast("list[dict[str, Any]]", result["content"])+ assert isinstance(content, list)+ assert len(content) == 1+ assert "url" in content[0]- def test_generate_entry_for_refined_image(self) -> None:- """Test generating entry for a concept that refines Image."""- concept = ConceptFactory.make(- domain="tables",- concept_code="TableScreenshot",- description="A screenshot of a table",- structure_class_name="ImageContent",- refines="native.Image",- )- result = generate_compact_memory_entry("table_screenshot", concept)- # Should generate the full format with concept and content- assert '"concept": "tables.TableScreenshot"' in result- assert 'ImageContent(url="table_screenshot_url")' in result- assert "table_screenshot" in result+class TestConceptGenerateInputRepresentationPython:+ """Test Concept.generate_input_representation for Python format."""- def test_generate_entry_for_refined_text(self) -> None:- """Test generating entry for a concept that refines Text."""+ def test_native_text_python(self) -> None:+ """Test Python representation for native Text concept."""+ concept = ConceptFactory.make_native_concept(NativeConceptCode.TEXT)+ result, imports = concept.generate_input_representation(ConceptRepresentationFormat.PYTHON)+ assert result["concept"] == "native.Text"+ assert "TextContent" in result["content"]+ assert "TextContent" in imports++ def test_native_image_python(self) -> None:+ """Test Python representation for native Image concept."""+ concept = ConceptFactory.make_native_concept(NativeConceptCode.IMAGE)+ result, imports = concept.generate_input_representation(ConceptRepresentationFormat.PYTHON)+ assert result["concept"] == "native.Image"+ assert "ImageContent" in result["content"]+ assert "ImageContent" in imports++ def test_refined_text_python(self) -> None:+ """Test Python representation for a concept that refines Text."""concept = ConceptFactory.make(domain="test_domain",concept_code="Question",@@ -103,9 +82,204 @@ class TestGenerateCompactMemoryEntry:structure_class_name="TextContent",refines="native.Text",)- result = generate_compact_memory_entry("question", concept)+ result, imports = concept.generate_input_representation(ConceptRepresentationFormat.PYTHON)+ assert result["concept"] == "test_domain.Question"+ assert "TextContent" in result["content"]+ assert "TextContent" in imports++ def test_native_pdf_python_with_multiplicity(self) -> None:+ """Test Python representation for PDF with multiplicity - for Python format, wrapping is handled by caller."""+ concept = ConceptFactory.make_native_concept(NativeConceptCode.PDF)+ result, imports = concept.generate_input_representation(ConceptRepresentationFormat.PYTHON, is_multiple=True)+ # For Python format, is_multiple doesn't wrap content (caller handles it)+ assert result["concept"] == "native.PDF"+ assert "PDFContent" in result["content"]+ assert "PDFContent" in imports+++class TestInputRequirementsGenerateJsonRepresentation:+ """Test InputRequirements.generate_json_representation method."""++ def test_empty_inputs(self) -> None:+ """Test JSON generation with no inputs."""+ inputs = InputRequirements(root={})+ result = inputs.generate_json_representation()+ assert result == {}++ def test_single_input(self) -> None:+ """Test JSON generation with a single input."""+ concept = ConceptFactory.make_native_concept(NativeConceptCode.TEXT)+ inputs = InputRequirements(root={"message": InputRequirement(concept=concept)})+ result = inputs.generate_json_representation()+ assert "message" in result+ assert result["message"]["concept"] == "native.Text"++ def test_multiple_inputs(self) -> None:+ """Test JSON generation with multiple inputs."""+ text_concept = ConceptFactory.make_native_concept(NativeConceptCode.TEXT)+ image_concept = ConceptFactory.make_native_concept(NativeConceptCode.IMAGE)+ inputs = InputRequirements(+ root={+ "text_input": InputRequirement(concept=text_concept),+ "image_input": InputRequirement(concept=image_concept),+ }+ )+ result = inputs.generate_json_representation()+ assert "text_input" in result+ assert "image_input" in result+ assert result["text_input"]["concept"] == "native.Text"+ assert result["image_input"]["concept"] == "native.Image"+++class TestInputRequirementsJsonWithMultiplicity:+ """Test InputRequirements.generate_json_representation with multiplicity - content is wrapped in list."""++ def test_single_item_no_multiplicity(self) -> None:+ """Test that single item (no multiplicity) has content as dict, not list."""+ concept = ConceptFactory.make_native_concept(NativeConceptCode.PDF)+ inputs = InputRequirements(root={"document": InputRequirement(concept=concept, multiplicity=None)})+ result = inputs.generate_json_representation()+ assert "document" in result+ # concept should be present+ assert result["document"]["concept"] == "native.PDF"+ # content should NOT be a list+ content = result["document"]["content"]+ assert isinstance(content, dict)+ assert "url" in content++ def test_multiple_items_true_multiplicity(self) -> None:+ """Test that multiplicity=True wraps content in list."""+ concept = ConceptFactory.make_native_concept(NativeConceptCode.PDF)+ inputs = InputRequirements(root={"documents": InputRequirement(concept=concept, multiplicity=True)})+ result = inputs.generate_json_representation()+ assert "documents" in result+ # concept should be at the top level+ assert result["documents"]["concept"] == "native.PDF"+ # content should be a list+ content = cast("list[dict[str, Any]]", result["documents"]["content"])+ assert isinstance(content, list)+ assert len(content) == 1+ assert "url" in content[0]++ def test_multiple_items_int_multiplicity(self) -> None:+ """Test that multiplicity=N (int > 1) wraps content in list."""+ concept = ConceptFactory.make_native_concept(NativeConceptCode.IMAGE)+ inputs = InputRequirements(root={"images": InputRequirement(concept=concept, multiplicity=3)})+ result = inputs.generate_json_representation()+ assert "images" in result+ # concept should be at the top level+ assert result["images"]["concept"] == "native.Image"+ # content should be a list+ content = cast("list[dict[str, Any]]", result["images"]["content"])+ assert isinstance(content, list)+ assert len(content) == 1+ assert "url" in content[0]++ def test_mixed_multiplicity(self) -> None:+ """Test mixed single and multiple inputs."""+ pdf_concept = ConceptFactory.make_native_concept(NativeConceptCode.PDF)+ inputs = InputRequirements(+ root={+ "single_pdf": InputRequirement(concept=pdf_concept, multiplicity=None),+ "multiple_pdfs": InputRequirement(concept=pdf_concept, multiplicity=True),+ }+ )+ result = inputs.generate_json_representation()+ # single_pdf content should be a dict+ assert isinstance(result["single_pdf"]["content"], dict)+ # multiple_pdfs content should be a list+ assert isinstance(result["multiple_pdfs"]["content"], list)+++class TestGenerateRunnerCode:+ """Test generate_runner_code function."""++ @pytest.fixture+ def mock_pipe_single_output(self) -> MagicMock:+ """Create a mock pipe with a single output."""+ text_concept = ConceptFactory.make_native_concept(NativeConceptCode.TEXT)+ pdf_concept = ConceptFactory.make_native_concept(NativeConceptCode.PDF)++ mock_pipe = MagicMock()+ mock_pipe.code = "test_pipe"+ mock_pipe.output = text_concept+ mock_pipe.inputs = InputRequirements(root={"document": InputRequirement(concept=pdf_concept)})+ return mock_pipe++ @pytest.fixture+ def mock_pipe_list_output(self) -> MagicMock:+ """Create a mock pipe with a list output."""+ text_concept = ConceptFactory.make_native_concept(NativeConceptCode.TEXT)+ pdf_concept = ConceptFactory.make_native_concept(NativeConceptCode.PDF)++ mock_pipe = MagicMock()+ mock_pipe.code = "test_pipe_list"+ mock_pipe.output = text_concept+ mock_pipe.inputs = InputRequirements(root={"documents": InputRequirement(concept=pdf_concept, multiplicity=True)})+ return mock_pipe++ def test_runner_code_includes_imports(self, mock_pipe_single_output: MagicMock) -> None:+ """Test that generated runner code includes necessary imports."""+ runner_code = generate_runner_code(mock_pipe_single_output)+ assert "import asyncio" in runner_code+ assert "from pipelex.pipelex import Pipelex" in runner_code+ assert "from pipelex.pipeline.execute import execute_pipeline" in runner_code++ def test_runner_code_includes_structure_imports(self, mock_pipe_single_output: MagicMock) -> None:+ """Test that generated runner code includes structure class imports."""+ runner_code = generate_runner_code(mock_pipe_single_output)+ assert "from pipelex.core.stuffs.pdf_content import PDFContent" in runner_code+ assert "from pipelex.core.stuffs.text_content import TextContent" in runner_code++ def test_runner_code_single_output_return_type(self, mock_pipe_single_output: MagicMock) -> None:+ """Test that generated runner code has correct return type for single output."""+ runner_code = generate_runner_code(mock_pipe_single_output, output_multiplicity=False)+ assert "async def run_test_pipe() -> TextContent:" in runner_code+ assert "pipe_output.main_stuff_as(content_type=TextContent)" in runner_code++ def test_runner_code_list_output_return_type(self, mock_pipe_list_output: MagicMock) -> None:+ """Test that generated runner code has correct return type for list output."""+ runner_code = generate_runner_code(mock_pipe_list_output, output_multiplicity=True)+ assert "async def run_test_pipe_list() -> list[TextContent]:" in runner_code+ assert "pipe_output.main_stuff_as_items(item_type=TextContent)" in runner_code++ def test_runner_code_includes_inputs(self, mock_pipe_single_output: MagicMock) -> None:+ """Test that generated runner code includes input values."""+ runner_code = generate_runner_code(mock_pipe_single_output)+ assert '"document":' in runner_code+ assert '"concept": "native.PDF"' in runner_code+ assert "PDFContent(" in runner_code++ def test_runner_code_includes_main_block(self, mock_pipe_single_output: MagicMock) -> None:+ """Test that generated runner code includes main block."""+ runner_code = generate_runner_code(mock_pipe_single_output)+ assert 'if __name__ == "__main__":' in runner_code+ assert "Pipelex.make()" in runner_code+ assert "asyncio.run(run_test_pipe())" in runner_code++ def test_runner_code_multiplicity_input(self, mock_pipe_list_output: MagicMock) -> None:+ """Test that generated runner code handles input multiplicity correctly."""+ runner_code = generate_runner_code(mock_pipe_list_output)+ # Should have list-wrapped content for multiplicity input+ assert "[PDFContent(" in runner_code++ def test_runner_code_custom_class_import_format(self) -> None:+ """Test that custom class imports use domain_conceptCode format for standalone scripts."""+ # Create a custom concept (non-native)+ custom_concept = ConceptFactory.make(+ domain="test_domain",+ concept_code="CustomOutput",+ description="A custom output",+ structure_class_name="CustomOutput",+ )+ pdf_concept = ConceptFactory.make_native_concept(NativeConceptCode.PDF)++ mock_pipe = MagicMock()+ mock_pipe.code = "custom_pipe"+ mock_pipe.output = custom_concept+ mock_pipe.inputs = InputRequirements(root={"document": InputRequirement(concept=pdf_concept)})- # Should generate the full format with concept and content- assert '"concept": "test_domain.Question"' in result- assert '"question_text"' in result- assert "question" in result+ runner_code = generate_runner_code(mock_pipe)+ # Custom imports should NOT use relative import (no leading dot) for standalone scripts+ assert "from structures.test_domain_CustomOutput import CustomOutput" in runner_code+ assert "from .structures" not in runner_codediff --git a/tests/unit/pipelex/core/concepts/concept_factory/data.py b/tests/unit/pipelex/core/concepts/concept_factory/data.pyindex c51ac573e..084bb59ff 100644--- a/tests/unit/pipelex/core/concepts/concept_factory/data.py+++ b/tests/unit/pipelex/core/concepts/concept_factory/data.py@@ -73,7 +73,7 @@ class TestCases:code="Text",description="Native text concept",structure_class_name=NativeConceptCode.TEXT.structure_class_name,- refines=None,+ refines=NativeConceptCode.TEXT.concept_string,),),# Test case 6: Concept code from same domain@@ -86,8 +86,8 @@ class TestCases:domain="my_domain",code="DomainConcept",description="A concept from same domain",- structure_class_name=NativeConceptCode.TEXT.structure_class_name,- refines=None,+ structure_class_name="DomainConcept",+ refines="native.Text",),),# Test case 9: Blueprint with dict structure (same domain)@@ -126,7 +126,7 @@ class TestCases:domain="my_domain",code="ConceptWithRefines",description="A concept with refines",- structure_class_name=NativeConceptCode.IMAGE.structure_class_name,+ structure_class_name="ConceptWithRefines",refines="native.Image",),),diff --git a/tests/unit/pipelex/core/concepts/concept_factory/test_concept_factory.py b/tests/unit/pipelex/core/concepts/concept_factory/test_concept_factory.pyindex d42ba6d8c..785e37ad9 100644--- a/tests/unit/pipelex/core/concepts/concept_factory/test_concept_factory.py+++ b/tests/unit/pipelex/core/concepts/concept_factory/test_concept_factory.py@@ -1,5 +1,6 @@import pytest+from pipelex import pretty_printfrom pipelex.core.concepts.concept import Conceptfrom pipelex.core.concepts.concept_blueprint import (ConceptBlueprint,@@ -8,6 +9,7 @@ from pipelex.core.concepts.concept_blueprint import (from pipelex.core.concepts.concept_factory import ConceptFactoryfrom pipelex.core.concepts.concept_structure_blueprint import ConceptStructureBlueprintFieldTypefrom pipelex.core.concepts.exceptions import ConceptFactoryError+from pipelex.core.concepts.helpers import normalize_structure_blueprintfrom tests.unit.pipelex.core.concepts.concept_factory.data import TestCases@@ -54,7 +56,7 @@ class TestConceptFactory:),}- assert ConceptFactory.normalize_structure_blueprint(mixed_structure_blueprint) == expected_structure+ assert normalize_structure_blueprint(mixed_structure_blueprint) == expected_structuremixed_structure_blueprint2: dict[str, str | ConceptStructureBlueprint] = {"name": ConceptStructureBlueprint(description="The name of the person", type=ConceptStructureBlueprintFieldType.TEXT, required=True),@@ -78,7 +80,7 @@ class TestConceptFactory:),}- assert ConceptFactory.normalize_structure_blueprint(mixed_structure_blueprint2) == expected_structure2+ assert normalize_structure_blueprint(mixed_structure_blueprint2) == expected_structure2@pytest.mark.parametrize(("domain", "concept_string_or_code", "expected_result"),@@ -112,8 +114,10 @@ class TestConceptFactory:result = ConceptFactory.make_from_blueprint(domain=domain,concept_code=concept_code,- blueprint=blueprint,+ blueprint_or_string_description=blueprint,)+ pretty_print(result)+ pretty_print(expected_concept)assert result == expected_concept, f"Concept mismatch for {test_name}"@@ -125,5 +129,5 @@ class TestConceptFactory:ConceptFactory.make_from_blueprint(domain="my_domain",concept_code="TestConcept",- blueprint=blueprint,+ blueprint_or_string_description=blueprint,)diff --git a/tests/unit/pipelex/core/concepts/concept_library/test_concept_library.py b/tests/unit/pipelex/core/concepts/concept_library/test_concept_library.pyindex a9d3ef239..818c15fe7 100644--- a/tests/unit/pipelex/core/concepts/concept_library/test_concept_library.py+++ b/tests/unit/pipelex/core/concepts/concept_library/test_concept_library.py@@ -15,7 +15,7 @@ class TestConceptLibrary:concept_1 = ConceptFactory.make_from_blueprint(domain="test",concept_code="TestConcept",- blueprint=ConceptBlueprint(+ blueprint_or_string_description=ConceptBlueprint(description="Lorem Ipsum",structure="ImageContent",),@@ -24,7 +24,7 @@ class TestConceptLibrary:concept_2 = ConceptFactory.make_from_blueprint(domain="test",concept_code="TestConcept2",- blueprint=ConceptBlueprint(+ blueprint_or_string_description=ConceptBlueprint(description="Lorem Ipsum",refines="native.Image",),@@ -32,7 +32,7 @@ class TestConceptLibrary:concept_3 = ConceptFactory.make_from_blueprint(domain="test",concept_code="TestConcept2",- blueprint=ConceptBlueprint(+ blueprint_or_string_description=ConceptBlueprint(description="Lorem Ipsum",refines="Image",),@@ -41,7 +41,7 @@ class TestConceptLibrary:concept_4 = ConceptFactory.make_from_blueprint(domain="test",concept_code="TestConcept4",- blueprint=ConceptBlueprint(+ blueprint_or_string_description=ConceptBlueprint(description="Lorem Ipsum",structure="ImageContent",),@@ -50,7 +50,7 @@ class TestConceptLibrary:concept_5 = ConceptFactory.make_from_blueprint(domain="test",concept_code="TestConcept5",- blueprint=ConceptBlueprint(+ blueprint_or_string_description=ConceptBlueprint(description="Lorem Ipsum",structure="TextContent",),@@ -59,7 +59,7 @@ class TestConceptLibrary:concept_6 = ConceptFactory.make_from_blueprint(domain="test",concept_code="TestConcept6",- blueprint=ConceptBlueprint(+ blueprint_or_string_description=ConceptBlueprint(description="Lorem Ipsum",structure="PDFContent",),diff --git a/tests/unit/pipelex/builder/test_pipelex_bundle_blueprint.py b/tests/unit/pipelex/core/concepts/structure_generation/__init__.pysimilarity index 100%rename from tests/unit/pipelex/builder/test_pipelex_bundle_blueprint.pyrename to tests/unit/pipelex/core/concepts/structure_generation/__init__.pydiff --git a/tests/unit/pipelex/core/concepts/test_concept.py b/tests/unit/pipelex/core/concepts/test_concept.pyindex 98d37dbb0..6b9d64303 100644--- a/tests/unit/pipelex/core/concepts/test_concept.py+++ b/tests/unit/pipelex/core/concepts/test_concept.py@@ -82,7 +82,7 @@ class TestConcept:ConceptFactory.make_from_blueprint(concept_code=NativeConceptCode.TEXT,domain=valid_domain,- blueprint=ConceptBlueprint(description=valid_definition),+ blueprint_or_string_description=ConceptBlueprint(description=valid_definition),),)is True@@ -92,7 +92,7 @@ class TestConcept:ConceptFactory.make_from_blueprint(concept_code=NativeConceptCode.TEXT,domain=SpecialDomain.NATIVE,- blueprint=ConceptBlueprint(description=valid_definition),+ blueprint_or_string_description=ConceptBlueprint(description=valid_definition),),)is True@@ -102,7 +102,7 @@ class TestConcept:ConceptFactory.make_from_blueprint(concept_code=NativeConceptCode.IMAGE,domain=valid_domain,- blueprint=ConceptBlueprint(description=valid_definition),+ blueprint_or_string_description=ConceptBlueprint(description=valid_definition),),)is True@@ -112,7 +112,7 @@ class TestConcept:ConceptFactory.make_from_blueprint(concept_code=NativeConceptCode.PDF,domain=valid_domain,- blueprint=ConceptBlueprint(description=valid_definition),+ blueprint_or_string_description=ConceptBlueprint(description=valid_definition),),)is True@@ -122,7 +122,7 @@ class TestConcept:ConceptFactory.make_from_blueprint(concept_code=NativeConceptCode.TEXT_AND_IMAGES,domain=valid_domain,- blueprint=ConceptBlueprint(description=valid_definition),+ blueprint_or_string_description=ConceptBlueprint(description=valid_definition),),)is True@@ -132,7 +132,7 @@ class TestConcept:ConceptFactory.make_from_blueprint(concept_code=NativeConceptCode.NUMBER,domain=valid_domain,- blueprint=ConceptBlueprint(description=valid_definition),+ blueprint_or_string_description=ConceptBlueprint(description=valid_definition),),)is True@@ -142,7 +142,7 @@ class TestConcept:ConceptFactory.make_from_blueprint(concept_code=NativeConceptCode.ANYTHING,domain=valid_domain,- blueprint=ConceptBlueprint(description=valid_definition),+ blueprint_or_string_description=ConceptBlueprint(description=valid_definition),),)is True@@ -152,7 +152,7 @@ class TestConcept:ConceptFactory.make_from_blueprint(concept_code="RandomConcept",domain=valid_domain,- blueprint=ConceptBlueprint(description=valid_definition),+ blueprint_or_string_description=ConceptBlueprint(description=valid_definition),),)is False@@ -218,22 +218,22 @@ class TestConcept:concept1 = ConceptFactory.make_from_blueprint(concept_code="Code1",domain="domain1",- blueprint=ConceptBlueprint(description="Lorem Ipsum", refines=NativeConceptCode.TEXT),+ blueprint_or_string_description=ConceptBlueprint(description="Lorem Ipsum", refines=NativeConceptCode.TEXT),)concept2 = ConceptFactory.make_from_blueprint(concept_code="Code2",domain="domain1",- blueprint=ConceptBlueprint(description="Lorem Ipsum", refines=NativeConceptCode.TEXT),+ blueprint_or_string_description=ConceptBlueprint(description="Lorem Ipsum", refines=NativeConceptCode.TEXT),)concept3 = ConceptFactory.make_from_blueprint(concept_code="Code3",domain="domain2",- blueprint=ConceptBlueprint(description="Lorem Ipsum", structure="TextContent"),+ blueprint_or_string_description=ConceptBlueprint(description="Lorem Ipsum", structure="TextContent"),)concept4 = ConceptFactory.make_from_blueprint(concept_code="Code4",domain="domain1",- blueprint=ConceptBlueprint(description="Lorem Ipsum", structure="ImageContent"),+ blueprint_or_string_description=ConceptBlueprint(description="Lorem Ipsum", structure="ImageContent"),)concept_5 = ConceptFactory.make_native_concept(@@ -247,7 +247,7 @@ class TestConcept:concept_7 = ConceptFactory.make_from_blueprint(concept_code="VisualDescription",domain="images",- blueprint=ConceptBlueprint(description="Lorem Ipsum"),+ blueprint_or_string_description=ConceptBlueprint(description="Lorem Ipsum"),)assert Concept.are_concept_compatible(concept_7, concept_6, strict=True) is Falsediff --git a/tests/unit/pipelex/core/concepts/test_concept_compact_memory.py b/tests/unit/pipelex/core/concepts/test_concept_compact_memory.pydeleted file mode 100644index ec1f903a2..000000000--- a/tests/unit/pipelex/core/concepts/test_concept_compact_memory.py+++ /dev/null@@ -1,127 +0,0 @@-"""Unit tests for Concept compact memory generation methods."""--from __future__ import annotations--from pipelex.core.concepts.concept_factory import ConceptFactory-from pipelex.core.concepts.native.concept_native import NativeConceptCode---class TestConceptCompactMemory:- """Test Concept methods for generating compact memory examples."""-- def test_get_compact_memory_example_text(self) -> None:- """Test compact memory example for Text concept."""- concept = ConceptFactory.make_native_concept(NativeConceptCode.TEXT)- result = concept.get_compact_memory_example("message")- assert isinstance(result, str)- assert result == "message_text"-- def test_get_compact_memory_example_number(self) -> None:- """Test compact memory example for Number concept."""- concept = ConceptFactory.make_native_concept(NativeConceptCode.NUMBER)- result = concept.get_compact_memory_example("count")- assert isinstance(result, int)- assert result == 0-- def test_get_compact_memory_example_image(self) -> None:- """Test compact memory example for Image concept."""- concept = ConceptFactory.make_native_concept(NativeConceptCode.IMAGE)- result = concept.get_compact_memory_example("photo")- assert isinstance(result, dict)- assert result["_class"] == "ImageContent"- assert result["url"] == "photo_url"-- def test_get_compact_memory_example_pdf(self) -> None:- """Test compact memory example for PDF concept."""- concept = ConceptFactory.make_native_concept(NativeConceptCode.PDF)- result = concept.get_compact_memory_example("document")- assert isinstance(result, dict)- assert result["_class"] == "PDFContent"- assert result["url"] == "document_url"-- def test_get_compact_memory_example_text_and_images(self) -> None:- """Test compact memory example for TextAndImages concept."""- concept = ConceptFactory.make_native_concept(NativeConceptCode.TEXT_AND_IMAGES)-- result = concept.get_compact_memory_example("content")-- # Should return a dict with concept_code and content- assert isinstance(result, dict)- assert result["concept_code"] == "native.TextAndImages"- assert "content" in result- assert "text" in result["content"]- assert result["content"]["text"] == "text_text" # Generated from field name "text"- assert "images" in result["content"]- assert isinstance(result["content"]["images"], list)-- def test_get_compact_memory_example_page(self) -> None:- """Test compact memory example for Page concept."""- concept = ConceptFactory.make_native_concept(NativeConceptCode.PAGE)-- result = concept.get_compact_memory_example("page")-- # Should return a dict with concept_code and content- assert isinstance(result, dict)- assert result["concept_code"] == "native.Page"- assert "content" in result- assert "text_and_images" in result["content"]- assert isinstance(result["content"]["text_and_images"], dict)-- def test_get_compact_memory_example_custom_structured(self) -> None:- """Test compact memory example for a custom structured concept."""- concept = ConceptFactory.make(- concept_code="Invoice",- domain="accounting",- description="Invoice data",- structure_class_name="Invoice",- refines=None,- )-- result = concept.get_compact_memory_example("invoice")-- # Should return a dict with concept_code and content- assert isinstance(result, dict)- assert result["concept_code"] == "accounting.Invoice"- assert "content" in result-- def test_get_compact_memory_example_for_refined_text_concept(self) -> None:- """Test compact memory example for a concept that refines Text."""- # Create a concept that refines Text using direct make() to control domain- concept = ConceptFactory.make(- domain="test_domain",- concept_code="Question",- description="A question",- structure_class_name="TextContent",- refines="native.Text",- )-- # Test - should return full format with concept_code since it's a refined (non-native) concept- # Even though it refines Text, it's not the native Text concept itself- result = concept.get_compact_memory_example("question")- assert isinstance(result, dict)- assert "concept_code" in result- assert result["concept_code"] == "test_domain.Question"- assert "content" in result- assert result["content"] == "question_text" # Content is still the simple text string-- def test_get_compact_memory_example_for_refined_image_concept(self) -> None:- """Test compact memory example for a concept that refines Image (non-native)."""- # Create a concept that refines Image using direct make() to control domain- concept = ConceptFactory.make(- domain="tables",- concept_code="TableScreenshot",- description="A screenshot of a table",- structure_class_name="ImageContent",- refines="native.Image",- )-- # Test - should return full format with concept_code since it's a refined (non-native) concept- result = concept.get_compact_memory_example("table_screenshot")- assert isinstance(result, dict)- assert "concept_code" in result- assert result["concept_code"] == "tables.TableScreenshot"- assert "content" in result- # Content should be wrapped in ImageContent format- assert isinstance(result["content"], dict)- assert result["content"]["_class"] == "ImageContent"- assert result["content"]["url"] == "table_screenshot_url"diff --git a/tests/unit/pipelex/core/concepts/test_concept_representation_generator.py b/tests/unit/pipelex/core/concepts/test_concept_representation_generator.pynew file mode 100644index 000000000..f8ece2df7--- /dev/null+++ b/tests/unit/pipelex/core/concepts/test_concept_representation_generator.py@@ -0,0 +1,461 @@+"""Unit tests for ConceptRepresentationGenerator."""++from pydantic import Field++from pipelex.core.concepts.concept_representation_generator import (+ ConceptRepresentationFormat,+ ConceptRepresentationGenerator,+ generate_json_representation,+ generate_python_representation,+)+from pipelex.core.stuffs.structured_content import StructuredContent+from pipelex.types import StrEnum++# =============================================================================+# Test Fixtures - Simple classes for unit testing+# =============================================================================+++class SimpleContent(StructuredContent):+ """Simple content with basic types."""++ name: str = Field(..., description="A name")+ count: int = Field(..., description="A count")+ rate: float = Field(..., description="A rate")+ active: bool = Field(..., description="Is active")+++class StatusEnum(StrEnum):+ """Status enum for testing."""++ PENDING = "pending"+ DONE = "done"+++class ContentWithEnum(StructuredContent):+ """Content with an enum field."""++ status: StatusEnum = Field(..., description="Status")+++class NestedChild(StructuredContent):+ """A child content class."""++ value: str = Field(..., description="A value")+++class ContentWithNestedClass(StructuredContent):+ """Content with a nested StuffContent field."""++ child: NestedChild = Field(..., description="Nested child")+++class ContentWithList(StructuredContent):+ """Content with list fields."""++ tags: list[str] = Field(..., description="List of tags")+++class ContentWithNestedList(StructuredContent):+ """Content with a list of nested content."""++ children: list[NestedChild] = Field(..., description="List of children")+++class ContentWithDict(StructuredContent):+ """Content with a dict field."""++ metadata: dict[str, str] = Field(..., description="Metadata dict")+++class ContentWithOptional(StructuredContent):+ """Content with an optional field."""++ maybe_name: str | None = Field(None, description="Optional name")+++class ContentWithRequiredAndOptional(StructuredContent):+ """Content with both required and optional fields (like ImageContent)."""++ url: str = Field(..., description="Required URL")+ source_prompt: str | None = Field(None, description="Optional prompt")+ caption: str | None = Field(None, description="Optional caption")+ base_64: str | None = Field(None, description="Optional base64")+++# =============================================================================+# Tests for ConceptRepresentationFormat enum+# =============================================================================+++class TestConceptRepresentationFormat:+ """Test the ConceptRepresentationFormat enum."""++ def test_enum_values(self) -> None:+ """Test that the enum has the expected values."""+ assert ConceptRepresentationFormat.JSON.value == "json"+ assert ConceptRepresentationFormat.PYTHON.value == "python"++ def test_enum_iteration(self) -> None:+ """Test that we can iterate over enum values."""+ values = list(ConceptRepresentationFormat)+ assert len(values) == 2+++# =============================================================================+# Tests for generate_field_value - basic types+# =============================================================================+++class TestGenerateFieldValueBasicTypes:+ """Test generate_field_value for basic Python types."""++ def test_string_field(self) -> None:+ """String field generates 'fieldname_value'."""+ generator = ConceptRepresentationGenerator(ConceptRepresentationFormat.JSON)+ result = generator.generate_field_value(str, "my_field")+ assert result == "my_field_value"++ def test_int_field(self) -> None:+ """Int field generates 0."""+ generator = ConceptRepresentationGenerator(ConceptRepresentationFormat.JSON)+ result = generator.generate_field_value(int, "count")+ assert result == 0++ def test_float_field(self) -> None:+ """Float field generates 0.0."""+ generator = ConceptRepresentationGenerator(ConceptRepresentationFormat.JSON)+ result = generator.generate_field_value(float, "rate")+ assert result == 0.0++ def test_bool_field(self) -> None:+ """Bool field generates False."""+ generator = ConceptRepresentationGenerator(ConceptRepresentationFormat.JSON)+ result = generator.generate_field_value(bool, "active")+ assert result is False+++# =============================================================================+# Tests for generate_field_value - complex types+# =============================================================================+++class TestGenerateFieldValueComplexTypes:+ """Test generate_field_value for complex types (enums, lists, dicts, nested)."""++ def test_enum_field(self) -> None:+ """Enum field generates first enum value."""+ generator = ConceptRepresentationGenerator(ConceptRepresentationFormat.JSON)+ result = generator.generate_field_value(StatusEnum, "status")+ assert result == "pending"++ def test_list_of_strings(self) -> None:+ """List[str] generates list with one placeholder."""+ generator = ConceptRepresentationGenerator(ConceptRepresentationFormat.JSON)+ result = generator.generate_field_value(list[str], "tags")+ assert result == ["tags_item_value"]++ def test_list_of_nested_content_json(self) -> None:+ """List[StuffContent] generates list with nested dict (JSON)."""+ generator = ConceptRepresentationGenerator(ConceptRepresentationFormat.JSON)+ result = generator.generate_field_value(list[NestedChild], "children")+ expected = [{"value": "value_value"}]+ assert result == expected++ def test_list_of_nested_content_python(self) -> None:+ """List[StuffContent] generates list with instantiation (Python)."""+ generator = ConceptRepresentationGenerator(ConceptRepresentationFormat.PYTHON)+ result = generator.generate_field_value(list[NestedChild], "children")+ expected = ['NestedChild(value="value_value")']+ assert result == expected++ def test_dict_field(self) -> None:+ """Dict field generates placeholder dict."""+ generator = ConceptRepresentationGenerator(ConceptRepresentationFormat.JSON)+ result = generator.generate_field_value(dict[str, str], "metadata")+ expected = {"metadata_key": "metadata_value"}+ assert result == expected++ def test_nested_content_json(self) -> None:+ """Nested StuffContent generates dict (JSON)."""+ generator = ConceptRepresentationGenerator(ConceptRepresentationFormat.JSON)+ result = generator.generate_field_value(NestedChild, "child")+ expected = {"value": "value_value"}+ assert result == expected++ def test_nested_content_python(self) -> None:+ """Nested StuffContent generates instantiation (Python)."""+ generator = ConceptRepresentationGenerator(ConceptRepresentationFormat.PYTHON)+ result = generator.generate_field_value(NestedChild, "child")+ expected = 'NestedChild(value="value_value")'+ assert result == expected++ def test_optional_field_unwrapped(self) -> None:+ """Optional[str] is unwrapped and handled as str."""+ generator = ConceptRepresentationGenerator(ConceptRepresentationFormat.JSON)+ result = generator.generate_field_value(str | None, "name")+ assert result == "name_value"+++# =============================================================================+# Tests for generate_class_representation - JSON format+# =============================================================================+++class TestGenerateClassRepresentationJson:+ """Test generate_class_representation with JSON output."""++ def test_simple_content(self) -> None:+ """SimpleContent generates correct JSON dict."""+ generator = ConceptRepresentationGenerator(ConceptRepresentationFormat.JSON)+ result = generator.generate_class_representation(SimpleContent)+ expected = {+ "name": "name_value",+ "count": 0,+ "rate": 0.0,+ "active": False,+ }+ assert result == expected++ def test_content_with_enum(self) -> None:+ """ContentWithEnum generates correct JSON."""+ generator = ConceptRepresentationGenerator(ConceptRepresentationFormat.JSON)+ result = generator.generate_class_representation(ContentWithEnum)+ expected = {"status": "pending"}+ assert result == expected++ def test_content_with_nested_class(self) -> None:+ """ContentWithNestedClass generates nested dict."""+ generator = ConceptRepresentationGenerator(ConceptRepresentationFormat.JSON)+ result = generator.generate_class_representation(ContentWithNestedClass)+ expected = {"child": {"value": "value_value"}}+ assert result == expected++ def test_content_with_list(self) -> None:+ """ContentWithList generates list field."""+ generator = ConceptRepresentationGenerator(ConceptRepresentationFormat.JSON)+ result = generator.generate_class_representation(ContentWithList)+ expected = {"tags": ["tags_item_value"]}+ assert result == expected++ def test_content_with_nested_list(self) -> None:+ """ContentWithNestedList generates list of nested dicts."""+ generator = ConceptRepresentationGenerator(ConceptRepresentationFormat.JSON)+ result = generator.generate_class_representation(ContentWithNestedList)+ expected = {"children": [{"value": "value_value"}]}+ assert result == expected++ def test_content_with_dict(self) -> None:+ """ContentWithDict generates dict field."""+ generator = ConceptRepresentationGenerator(ConceptRepresentationFormat.JSON)+ result = generator.generate_class_representation(ContentWithDict)+ expected = {"metadata": {"metadata_key": "metadata_value"}}+ assert result == expected++ def test_content_with_optional(self) -> None:+ """ContentWithOptional handles optional field."""+ generator = ConceptRepresentationGenerator(ConceptRepresentationFormat.JSON)+ result = generator.generate_class_representation(ContentWithOptional)+ expected = {"maybe_name": "maybe_name_value"}+ assert result == expected+++# =============================================================================+# Tests for generate_class_representation - Python format+# =============================================================================+++class TestGenerateClassRepresentationPython:+ """Test generate_class_representation with Python output."""++ def test_simple_content(self) -> None:+ """SimpleContent generates correct Python instantiation."""+ generator = ConceptRepresentationGenerator(ConceptRepresentationFormat.PYTHON)+ result = generator.generate_class_representation(SimpleContent)+ expected = 'SimpleContent(name="name_value", count=0, rate=0.0, active=False)'+ assert result == expected++ def test_content_with_nested_class(self) -> None:+ """ContentWithNestedClass generates nested instantiation."""+ generator = ConceptRepresentationGenerator(ConceptRepresentationFormat.PYTHON)+ result = generator.generate_class_representation(ContentWithNestedClass)+ expected = 'ContentWithNestedClass(child=NestedChild(value="value_value"))'+ assert result == expected++ def test_content_with_nested_list(self) -> None:+ """ContentWithNestedList generates list of instantiations."""+ generator = ConceptRepresentationGenerator(ConceptRepresentationFormat.PYTHON)+ result = generator.generate_class_representation(ContentWithNestedList)+ expected = 'ContentWithNestedList(children=[NestedChild(value="value_value")])'+ assert result == expected+++# =============================================================================+# Tests for generate_representation (main entry point)+# =============================================================================+++class TestGenerateRepresentation:+ """Test the main generate_representation method."""++ def test_wraps_with_concept_json(self) -> None:+ """Result is wrapped with concept (JSON)."""+ generator = ConceptRepresentationGenerator(ConceptRepresentationFormat.JSON)+ result = generator.generate_representation("test.SimpleContent", SimpleContent)+ expected = {+ "concept": "test.SimpleContent",+ "content": {+ "name": "name_value",+ "count": 0,+ "rate": 0.0,+ "active": False,+ },+ }+ assert result == expected++ def test_wraps_with_concept_python(self) -> None:+ """Result is wrapped with concept (Python)."""+ generator = ConceptRepresentationGenerator(ConceptRepresentationFormat.PYTHON)+ result = generator.generate_representation("test.SimpleContent", SimpleContent)+ expected = {+ "concept": "test.SimpleContent",+ "content": 'SimpleContent(name="name_value", count=0, rate=0.0, active=False)',+ }+ assert result == expected++ def test_nested_structure_wrapped(self) -> None:+ """Nested structure is wrapped correctly."""+ generator = ConceptRepresentationGenerator(ConceptRepresentationFormat.JSON)+ result = generator.generate_representation("test.Nested", ContentWithNestedClass)+ expected = {+ "concept": "test.Nested",+ "content": {"child": {"value": "value_value"}},+ }+ assert result == expected+++# =============================================================================+# Tests for imports tracking+# =============================================================================+++class TestImportsTracking:+ """Test that imports_needed tracks used classes."""++ def test_tracks_main_class(self) -> None:+ """Tracks the main class."""+ generator = ConceptRepresentationGenerator(ConceptRepresentationFormat.JSON)+ generator.generate_representation("test.Simple", SimpleContent)+ assert "SimpleContent" in generator.imports_needed++ def test_tracks_nested_classes(self) -> None:+ """Tracks nested classes."""+ generator = ConceptRepresentationGenerator(ConceptRepresentationFormat.JSON)+ generator.generate_representation("test.Nested", ContentWithNestedClass)+ assert "ContentWithNestedClass" in generator.imports_needed+ assert "NestedChild" in generator.imports_needed++ def test_clears_on_new_generation(self) -> None:+ """Clears imports on new generation call."""+ generator = ConceptRepresentationGenerator(ConceptRepresentationFormat.JSON)+ generator.generate_representation("test.Nested", ContentWithNestedClass)+ generator.generate_representation("test.Simple", SimpleContent)+ assert "NestedChild" not in generator.imports_needed+ assert "SimpleContent" in generator.imports_needed+++# =============================================================================+# Tests for convenience functions+# =============================================================================+++class TestConvenienceFunctions:+ """Test the convenience functions."""++ def test_generate_json_representation(self) -> None:+ """generate_json_representation returns correct result."""+ result = generate_json_representation("test.Simple", SimpleContent)+ expected = {+ "concept": "test.Simple",+ "content": {+ "name": "name_value",+ "count": 0,+ "rate": 0.0,+ "active": False,+ },+ }+ assert result == expected++ def test_generate_python_representation(self) -> None:+ """generate_python_representation returns correct result and imports."""+ result, imports = generate_python_representation("test.Simple", SimpleContent)+ expected = {+ "concept": "test.Simple",+ "content": 'SimpleContent(name="name_value", count=0, rate=0.0, active=False)',+ }+ assert result == expected+ assert "SimpleContent" in imports+++# =============================================================================+# Tests for include_optional parameter+# =============================================================================+++class TestIncludeOptionalParameter:+ """Test that optional fields can be excluded from generation."""++ def test_excludes_optional_fields_json(self) -> None:+ """With include_optional=False, optional fields are excluded (JSON)."""+ generator = ConceptRepresentationGenerator(ConceptRepresentationFormat.JSON)+ result = generator.generate_class_representation(ContentWithRequiredAndOptional, include_optional=False)+ expected = {"url": "url_value"}+ assert result == expected+ # Verify optional fields are NOT present+ assert "source_prompt" not in result+ assert "caption" not in result+ assert "base_64" not in result++ def test_excludes_optional_fields_python(self) -> None:+ """With include_optional=False, optional fields are excluded (Python)."""+ generator = ConceptRepresentationGenerator(ConceptRepresentationFormat.PYTHON)+ result = generator.generate_class_representation(ContentWithRequiredAndOptional, include_optional=False)+ expected = 'ContentWithRequiredAndOptional(url="url_value")'+ assert result == expected++ def test_includes_optional_fields_by_default(self) -> None:+ """By default, optional fields ARE included (backward compatibility)."""+ generator = ConceptRepresentationGenerator(ConceptRepresentationFormat.JSON)+ result = generator.generate_class_representation(ContentWithRequiredAndOptional)+ # All fields should be present by default+ assert "url" in result+ assert "source_prompt" in result+ assert "caption" in result+ assert "base_64" in result++ def test_all_optional_returns_empty_dict_json(self) -> None:+ """Content with only optional fields returns empty dict when include_optional=False."""+ generator = ConceptRepresentationGenerator(ConceptRepresentationFormat.JSON)+ result = generator.generate_class_representation(ContentWithOptional, include_optional=False)+ expected: dict[str, str] = {}+ assert result == expected++ def test_all_optional_returns_empty_instantiation_python(self) -> None:+ """Content with only optional fields returns empty instantiation when include_optional=False."""+ generator = ConceptRepresentationGenerator(ConceptRepresentationFormat.PYTHON)+ result = generator.generate_class_representation(ContentWithOptional, include_optional=False)+ expected = "ContentWithOptional()"+ assert result == expected++ def test_image_content_only_url_required(self) -> None:+ """ImageContent should only have url when include_optional=False."""+ from pipelex.core.stuffs.image_content import ImageContent # noqa: PLC0415++ generator = ConceptRepresentationGenerator(ConceptRepresentationFormat.JSON)+ result = generator.generate_class_representation(ImageContent, include_optional=False)+ expected = {"url": "url_value"}+ assert result == expected+ # Verify optional fields are NOT present+ assert "source_prompt" not in result+ assert "caption" not in result+ assert "base_64" not in resultdiff --git a/tests/unit/pipelex/core/concepts/test_special_args_concept.py b/tests/unit/pipelex/core/concepts/test_special_args_concept.pyindex 1236579e5..134d29147 100644--- a/tests/unit/pipelex/core/concepts/test_special_args_concept.py+++ b/tests/unit/pipelex/core/concepts/test_special_args_concept.py@@ -48,7 +48,7 @@ class TestSpecialArgsConcept:concept = ConceptFactory.make_from_blueprint(domain=domain,concept_code="ContentConcept",- blueprint=concept_blueprint,+ blueprint_or_string_description=concept_blueprint,)concept_library.add_new_concept(concept)diff --git a/tests/unit/pipelex/core/concepts/test_structured_output_generator.py b/tests/unit/pipelex/core/concepts/test_structured_output_generator.pyindex 8cb417264..57e7ec154 100644--- a/tests/unit/pipelex/core/concepts/test_structured_output_generator.py+++ b/tests/unit/pipelex/core/concepts/test_structured_output_generator.py@@ -1,8 +1,8 @@import pytestfrom pipelex import pretty_print-from pipelex.core.concepts.concept_factory import ConceptFactoryfrom pipelex.core.concepts.concept_structure_blueprint import ConceptStructureBlueprint, ConceptStructureBlueprintFieldType+from pipelex.core.concepts.helpers import normalize_structure_blueprintfrom pipelex.core.concepts.structure_generation.exceptions import ConceptStructureValidationErrorfrom pipelex.core.concepts.structure_generation.generator import StructureGeneratorfrom pipelex.core.stuffs.structured_content import StructuredContent@@ -27,21 +27,13 @@ class TestStructureGenerator:pretty_print(structure_blueprint, title="Source Blueprint")pretty_print(result, title="Generated Result")- # Check the complete generated structure- expected_result = (- "from enum import Enum\n"- "from pipelex.core.stuffs.structured_content import StructuredContent\n"- "from pydantic import Field\n"- "from typing import Optional, List, Dict, Any, Literal\n"- "\n\n"- "class TestModel(StructuredContent):\n"- ' """Generated TestModel class"""\n'- "\n"- ' name: str = Field(..., description="Name field")\n'- ' age: Optional[int] = Field(default=None, description="Age field")\n'- ' active: Optional[bool] = Field(default=True, description="Active status")\n'- )- assert result == expected_result+ # Check that generated code contains required elements+ assert "class TestModel(StructuredContent):" in result+ assert ' name: str = Field(..., description="Name field")' in result+ assert ' age: Optional[int] = Field(default=None, description="Age field")' in result+ assert ' active: Optional[bool] = Field(default=True, description="Active status")' in result+ # Check for autogen header+ assert "AUTOGENERATED CODE - DO NOT EDIT" in resultdef test_complex_types_generation(self):"""Test generation with complex types like lists and dicts."""@@ -72,21 +64,13 @@ class TestStructureGenerator:pretty_print(structure_blueprint, title="Source Blueprint")pretty_print(result, title="Generated Result")- # Check the complete generated structure- expected_result = (- "from enum import Enum\n"- "from pipelex.core.stuffs.structured_content import StructuredContent\n"- "from pydantic import Field\n"- "from typing import Optional, List, Dict, Any, Literal\n"- "\n\n"- "class ComplexModel(StructuredContent):\n"- ' """Generated ComplexModel class"""\n'- "\n"- ' tags: Optional[List[str]] = Field(default=None, description="List of tags")\n'- ' metadata: Optional[Dict[str, str]] = Field(default=None, description="Metadata dictionary")\n'- ' scores: List[float] = Field(..., description="List of scores")\n'- )- assert result == expected_result+ # Check that generated code contains required elements+ assert "class ComplexModel(StructuredContent):" in result+ assert ' tags: Optional[List[str]] = Field(default=None, description="List of tags")' in result+ assert ' metadata: Optional[Dict[str, str]] = Field(default=None, description="Metadata dictionary")' in result+ assert ' scores: List[float] = Field(..., description="List of scores")' in result+ # Check for autogen header+ assert "AUTOGENERATED CODE - DO NOT EDIT" in resultdef test_choices_generation(self):"""Test generation with inline choices (Literal type)."""@@ -103,21 +87,13 @@ class TestStructureGenerator:pretty_print(structure_blueprint, title="Source Blueprint")pretty_print(result, title="Generated Result")- # Check the complete generated structure with Literal types- expected_result = (- "from enum import Enum\n"- "from pipelex.core.stuffs.structured_content import StructuredContent\n"- "from pydantic import Field\n"- "from typing import Optional, List, Dict, Any, Literal\n"- "\n\n"- "class Product(StructuredContent):\n"- ' """Generated Product class"""\n'- "\n"- ' name: Optional[str] = Field(default=None, description="Product name")\n'- " category: Literal['electronics', 'clothing', 'food', 'books'] = Field(..., description=\"Product category\")\n"- " size: Optional[Literal['XS', 'S', 'M', 'L', 'XL']] = Field(default=None, description=\"Size of the product\")\n"- )- assert result == expected_result+ # Check that generated code contains required elements+ assert "class Product(StructuredContent):" in result+ assert ' name: Optional[str] = Field(default=None, description="Product name")' in result+ assert " category: Literal['electronics', 'clothing', 'food', 'books']" in result+ assert " size: Optional[Literal['XS', 'S', 'M', 'L', 'XL']]" in result+ # Check for autogen header+ assert "AUTOGENERATED CODE - DO NOT EDIT" in resultdef test_empty_structure(self):"""Test generation of structure with no fields."""@@ -128,19 +104,12 @@ class TestStructureGenerator:pretty_print(structure_blueprint, title="Source Blueprint")pretty_print(result, title="Generated Result")- # Check the complete generated structure for empty model- expected_result = (- "from enum import Enum\n"- "from pipelex.core.stuffs.structured_content import StructuredContent\n"- "from pydantic import Field\n"- "from typing import Optional, List, Dict, Any, Literal\n"- "\n\n"- "class EmptyModel(StructuredContent):\n"- ' """Generated EmptyModel class"""\n'- "\n"- " pass\n"- )- assert result == expected_result+ # Check that generated code contains required elements+ assert "class EmptyModel(StructuredContent):" in result+ assert '"""Generated EmptyModel class"""' in result+ # No pass needed when there's a docstring+ # Check for autogen header+ assert "AUTOGENERATED CODE - DO NOT EDIT" in resultdef test_concept_get_structure_method(self):"""Test the get_structure method on Concept class."""@@ -154,20 +123,12 @@ class TestStructureGenerator:pretty_print(structure_blueprint, title="Source Blueprint")pretty_print(result, title="Generated Result")- # Check the complete generated structure- expected_result = (- "from enum import Enum\n"- "from pipelex.core.stuffs.structured_content import StructuredContent\n"- "from pydantic import Field\n"- "from typing import Optional, List, Dict, Any, Literal\n"- "\n\n"- "class DocumentInfo(StructuredContent):\n"- ' """Generated DocumentInfo class"""\n'- "\n"- ' title: str = Field(..., description="Document title")\n'- ' page_count: Optional[int] = Field(default=None, description="Number of pages")\n'- )- assert result == expected_result+ # Check that generated code contains required elements+ assert "class DocumentInfo(StructuredContent):" in result+ assert ' title: str = Field(..., description="Document title")' in result+ assert ' page_count: Optional[int] = Field(default=None, description="Number of pages")' in result+ # Check for autogen header+ assert "AUTOGENERATED CODE - DO NOT EDIT" in resultdef test_generate_from_blueprint_dict_function(self):"""Test the convenience function for generating from blueprint dict."""@@ -180,19 +141,11 @@ class TestStructureGenerator:pretty_print(structure_blueprint, title="Source Blueprint")pretty_print(result, title="Generated Result")- # Check the complete generated structure- expected_result = (- "from enum import Enum\n"- "from pipelex.core.stuffs.structured_content import StructuredContent\n"- "from pydantic import Field\n"- "from typing import Optional, List, Dict, Any, Literal\n"- "\n\n"- "class ConvenienceTest(StructuredContent):\n"- ' """Generated ConvenienceTest class"""\n'- "\n"- ' value: str = Field(..., description="Test value")\n'- )- assert result == expected_result+ # Check that generated code contains required elements+ assert "class ConvenienceTest(StructuredContent):" in result+ assert ' value: str = Field(..., description="Test value")' in result+ # Check for autogen header+ assert "AUTOGENERATED CODE - DO NOT EDIT" in resultdef test_all_field_types(self):"""Test that all field types are properly handled."""@@ -221,24 +174,16 @@ class TestStructureGenerator:pretty_print(structure_blueprint, title="Source Blueprint")pretty_print(result, title="Generated Result")- # Check the complete generated structure with all field types- expected_result = (- "from enum import Enum\n"- "from pipelex.core.stuffs.structured_content import StructuredContent\n"- "from pydantic import Field\n"- "from typing import Optional, List, Dict, Any, Literal\n"- "\n\n"- "class TypeMappingTest(StructuredContent):\n"- ' """Generated TypeMappingTest class"""\n'- "\n"- ' text_field: Optional[str] = Field(default=None, description="Text field")\n'- ' number_field: Optional[float] = Field(default=None, description="Number field")\n'- ' integer_field: Optional[int] = Field(default=None, description="Integer field")\n'- ' boolean_field: Optional[bool] = Field(default=None, description="Boolean field")\n'- ' list_field: Optional[List[str]] = Field(default=None, description="List field")\n'- ' dict_field: Optional[Dict[str, int]] = Field(default=None, description="Dict field")\n'- )- assert result == expected_result+ # Check that generated code contains all required field types+ assert "class TypeMappingTest(StructuredContent):" in result+ assert ' text_field: Optional[str] = Field(default=None, description="Text field")' in result+ assert ' number_field: Optional[float] = Field(default=None, description="Number field")' in result+ assert ' integer_field: Optional[int] = Field(default=None, description="Integer field")' in result+ assert ' boolean_field: Optional[bool] = Field(default=None, description="Boolean field")' in result+ assert ' list_field: Optional[List[str]] = Field(default=None, description="List field")' in result+ assert ' dict_field: Optional[Dict[str, int]] = Field(default=None, description="Dict field")' in result+ # Check for autogen header+ assert "AUTOGENERATED CODE - DO NOT EDIT" in resultdef test_required_vs_optional_fields(self):"""Test that fields can be marked as required vs optional."""@@ -252,20 +197,12 @@ class TestStructureGenerator:pretty_print(structure_blueprint, title="Source Blueprint")pretty_print(result, title="Generated Result")- # Check the complete generated structure with required vs optional fields- expected_result = (- "from enum import Enum\n"- "from pipelex.core.stuffs.structured_content import StructuredContent\n"- "from pydantic import Field\n"- "from typing import Optional, List, Dict, Any, Literal\n"- "\n\n"- "class RequiredFieldsModel(StructuredContent):\n"- ' """Generated RequiredFieldsModel class"""\n'- "\n"- ' title: str = Field(..., description="Required title")\n'- ' optional_field: Optional[str] = Field(default=None, description="Optional field")\n'- )- assert result == expected_result+ # Check that generated code contains required elements+ assert "class RequiredFieldsModel(StructuredContent):" in result+ assert ' title: str = Field(..., description="Required title")' in result+ assert ' optional_field: Optional[str] = Field(default=None, description="Optional field")' in result+ # Check for autogen header+ assert "AUTOGENERATED CODE - DO NOT EDIT" in resultdef test_default_values(self):"""Test fields with default values."""@@ -295,21 +232,13 @@ class TestStructureGenerator:pretty_print(structure_blueprint, title="Source Blueprint")pretty_print(result, title="Generated Result")- # Check the complete generated structure with default values- expected_result = (- "from enum import Enum\n"- "from pipelex.core.stuffs.structured_content import StructuredContent\n"- "from pydantic import Field\n"- "from typing import Optional, List, Dict, Any, Literal\n"- "\n\n"- "class PersonWithDefaults(StructuredContent):\n"- ' """Generated PersonWithDefaults class"""\n'- "\n"- ' name: Optional[str] = Field(default="Anonymous", description="Person name")\n'- ' age: Optional[int] = Field(default=0, description="Person age")\n'- ' active: Optional[bool] = Field(default=True, description="Is active")\n'- )- assert result == expected_result+ # Check that generated code contains required elements with default values+ assert "class PersonWithDefaults(StructuredContent):" in result+ assert ' name: Optional[str] = Field(default="Anonymous", description="Person name")' in result+ assert ' age: Optional[int] = Field(default=0, description="Person age")' in result+ assert ' active: Optional[bool] = Field(default=True, description="Is active")' in result+ # Check for autogen header+ assert "AUTOGENERATED CODE - DO NOT EDIT" in resultdef test_nested_list_types(self):"""Test nested list types with different item types."""@@ -339,21 +268,13 @@ class TestStructureGenerator:pretty_print(structure_blueprint, title="Source Blueprint")pretty_print(result, title="Generated Result")- # Check the complete generated structure with nested list types- expected_result = (- "from enum import Enum\n"- "from pipelex.core.stuffs.structured_content import StructuredContent\n"- "from pydantic import Field\n"- "from typing import Optional, List, Dict, Any, Literal\n"- "\n\n"- "class ListTypesModel(StructuredContent):\n"- ' """Generated ListTypesModel class"""\n'- "\n"- ' text_list: Optional[List[str]] = Field(default=None, description="List of text items")\n'- ' number_list: List[float] = Field(..., description="List of numbers")\n'- ' integer_list: Optional[List[int]] = Field(default=None, description="List of integers")\n'- )- assert result == expected_result+ # Check that generated code contains required list types+ assert "class ListTypesModel(StructuredContent):" in result+ assert ' text_list: Optional[List[str]] = Field(default=None, description="List of text items")' in result+ assert ' number_list: List[float] = Field(..., description="List of numbers")' in result+ assert ' integer_list: Optional[List[int]] = Field(default=None, description="List of integers")' in result+ # Check for autogen header+ assert "AUTOGENERATED CODE - DO NOT EDIT" in resultdef test_nested_dict_types(self):"""Test nested dict types with different key/value combinations."""@@ -386,21 +307,13 @@ class TestStructureGenerator:pretty_print(structure_blueprint, title="Source Blueprint")pretty_print(result, title="Generated Result")- # Check the complete generated structure with nested dict types- expected_result = (- "from enum import Enum\n"- "from pipelex.core.stuffs.structured_content import StructuredContent\n"- "from pydantic import Field\n"- "from typing import Optional, List, Dict, Any, Literal\n"- "\n\n"- "class DictTypesModel(StructuredContent):\n"- ' """Generated DictTypesModel class"""\n'- "\n"- ' string_to_string: Optional[Dict[str, str]] = Field(default=None, description="String to string mapping")\n'- ' string_to_number: Dict[str, float] = Field(..., description="String to number mapping")\n'- ' string_to_integer: Optional[Dict[str, int]] = Field(default=None, description="String to integer mapping")\n'- )- assert result == expected_result+ # Check that generated code contains required dict types+ assert "class DictTypesModel(StructuredContent):" in result+ assert ' string_to_string: Optional[Dict[str, str]] = Field(default=None, description="String to string mapping")' in result+ assert ' string_to_number: Dict[str, float] = Field(..., description="String to number mapping")' in result+ assert ' string_to_integer: Optional[Dict[str, int]] = Field(default=None, description="String to integer mapping")' in result+ # Check for autogen header+ assert "AUTOGENERATED CODE - DO NOT EDIT" in resultdef test_mixed_complexity_structure(self):"""Test a structure with mixed complexity - simple and complex types together."""@@ -439,24 +352,16 @@ class TestStructureGenerator:pretty_print(structure_blueprint, title="Source Blueprint")pretty_print(result, title="Generated Result")- # Check the complete generated structure with mixed complexity- expected_result = (- "from enum import Enum\n"- "from pipelex.core.stuffs.structured_content import StructuredContent\n"- "from pydantic import Field\n"- "from typing import Optional, List, Dict, Any, Literal\n"- "\n\n"- "class ComplexItem(StructuredContent):\n"- ' """Generated ComplexItem class"""\n'- "\n"- ' id: int = Field(..., description="Unique identifier")\n'- ' name: str = Field(..., description="Display name")\n'- ' tags: Optional[List[str]] = Field(default=None, description="Associated tags")\n'- ' metadata: Optional[Dict[str, str]] = Field(default=None, description="Additional metadata")\n'- ' active: Optional[bool] = Field(default=True, description="Whether item is active")\n'- " priority: Optional[Literal['low', 'medium', 'high', 'urgent']] = Field(default=\"medium\", description=\"Priority level\")\n"- )- assert result == expected_result+ # Check that generated code contains all required complex elements+ assert "class ComplexItem(StructuredContent):" in result+ assert ' id: int = Field(..., description="Unique identifier")' in result+ assert ' name: str = Field(..., description="Display name")' in result+ assert ' tags: Optional[List[str]] = Field(default=None, description="Associated tags")' in result+ assert ' metadata: Optional[Dict[str, str]] = Field(default=None, description="Additional metadata")' in result+ assert ' active: Optional[bool] = Field(default=True, description="Whether item is active")' in result+ assert "priority: Optional[Literal['low', 'medium', 'high', 'urgent']]" in result+ # Check for autogen header+ assert "AUTOGENERATED CODE - DO NOT EDIT" in resultdef test_mixed_structure_blueprint_normalization(self):"""Test that mixed structure blueprints (strings and ConceptStructureBlueprint objects) are properly normalized."""@@ -471,7 +376,7 @@ class TestStructureGenerator:),}- normalized_structure = ConceptFactory.normalize_structure_blueprint(mixed_structure_blueprint)+ normalized_structure = normalize_structure_blueprint(mixed_structure_blueprint)result, _ = StructureGenerator().generate_from_structure_blueprint("PersonInfo", normalized_structure)@@ -479,22 +384,14 @@ class TestStructureGenerator:pretty_print(normalized_structure, title="Normalized Blueprint")pretty_print(result, title="Generated Result")- # Check the complete generated structure with mixed field types- expected_result = (- "from datetime import datetime\n"- "from enum import Enum\n"- "from pipelex.core.stuffs.structured_content import StructuredContent\n"- "from pydantic import Field\n"- "from typing import Optional, List, Dict, Any, Literal\n"- "\n\n"- "class PersonInfo(StructuredContent):\n"- ' """Generated PersonInfo class"""\n'- "\n"- ' name: str = Field(..., description="The name of the person")\n'- ' age: float = Field(..., description="The age of the person")\n'- ' birthdate: datetime = Field(..., description="The birthdate of the person")\n'- )- assert result == expected_result+ # Check that generated code contains required elements including datetime+ assert "from datetime import datetime" in result+ assert "class PersonInfo(StructuredContent):" in result+ assert ' name: str = Field(..., description="The name of the person")' in result+ assert ' age: float = Field(..., description="The age of the person")' in result+ assert ' birthdate: datetime = Field(..., description="The birthdate of the person")' in result+ # Check for autogen header+ assert "AUTOGENERATED CODE - DO NOT EDIT" in result# Verify that the string was properly converted to ConceptStructureBlueprintassert isinstance(normalized_structure["name"], ConceptStructureBlueprint)@@ -517,7 +414,7 @@ class TestStructureGenerator:assert issubclass(the_class, StructuredContent)# Test validation directly- generator.validate_generated_code(python_code=python_code, expected_class_name="ValidTestModel", required_base_class=StructuredContent)+ generator.validate_generated_code(python_code=python_code, expected_class_name="ValidTestModel")def test_code_validation_syntax_error(self):"""Test that invalid syntax fails validation."""@@ -531,7 +428,7 @@ class InvalidModel(StructuredContent):"""with pytest.raises(SyntaxError):- generator.validate_generated_code(python_code=invalid_code, expected_class_name="InvalidModel", required_base_class=StructuredContent)+ generator.validate_generated_code(python_code=invalid_code, expected_class_name="InvalidModel")def test_code_validation_missing_class(self):"""Test that code without the expected class fails validation."""@@ -545,6 +442,250 @@ class WrongClassName(StructuredContent):"""with pytest.raises(ConceptStructureValidationError):- generator.validate_generated_code(- python_code=code_without_expected_class, expected_class_name="ExpectedClassName", required_base_class=StructuredContent+ generator.validate_generated_code(python_code=code_without_expected_class, expected_class_name="ExpectedClassName")++ def test_inheritance_from_text_content(self):+ """Test generating a class that inherits from TextContent."""+ structure_blueprint = {+ "author": ConceptStructureBlueprint(+ description="The author of the question",+ type=ConceptStructureBlueprintFieldType.TEXT,+ required=False,+ ),+ }++ generator = StructureGenerator()+ generated_code, generated_class = generator.generate_from_structure_blueprint(+ "Question",+ structure_blueprint,+ base_class_name="TextContent",+ )++ pretty_print(structure_blueprint, title="Source Blueprint")+ pretty_print(generated_code, title="Generated Result")++ # Verify the code compiles+ import ast # noqa: PLC0415++ ast.parse(generated_code)+ compile(generated_code, "<generated>", "exec")++ # Verify correct import is present+ assert "from pipelex.core.stuffs.text_content import TextContent" in generated_code++ # Verify class inherits from TextContent+ assert "class Question(TextContent):" in generated_code++ # Verify the generated class is valid+ from pipelex.core.stuffs.text_content import TextContent # noqa: PLC0415++ assert issubclass(generated_class, TextContent)+ from pipelex.core.stuffs.stuff_content import StuffContent # noqa: PLC0415++ assert issubclass(generated_class, StuffContent)++ # Test instantiation (TextContent requires 'text' field)+ instance = generated_class(text="Sample text", author="John Doe") # pyright: ignore[reportCallIssue]+ assert instance.author == "John Doe" # type: ignore[attr-defined] # pyright: ignore[reportAttributeAccessIssue]++ def test_inheritance_from_image_content(self):+ """Test generating a class that inherits from ImageContent."""+ structure_blueprint = {+ "table_name": ConceptStructureBlueprint(+ description="Name of the table in the screenshot",+ type=ConceptStructureBlueprintFieldType.TEXT,+ required=False,+ ),+ "row_count": ConceptStructureBlueprint(+ description="Number of rows in the table",+ type=ConceptStructureBlueprintFieldType.INTEGER,+ required=False,+ ),+ }++ generator = StructureGenerator()+ generated_code, generated_class = generator.generate_from_structure_blueprint(+ "TableScreenshot",+ structure_blueprint,+ base_class_name="ImageContent",+ )++ pretty_print(structure_blueprint, title="Source Blueprint")+ pretty_print(generated_code, title="Generated Result")++ # Verify correct import is present+ assert "from pipelex.core.stuffs.image_content import ImageContent" in generated_code++ # Verify class inherits from ImageContent+ assert "class TableScreenshot(ImageContent):" in generated_code++ # Verify the generated class is valid+ from pipelex.core.stuffs.image_content import ImageContent # noqa: PLC0415++ assert issubclass(generated_class, ImageContent)+ from pipelex.core.stuffs.stuff_content import StuffContent # noqa: PLC0415++ assert issubclass(generated_class, StuffContent)++ def test_inheritance_from_number_content(self):+ """Test generating a class that inherits from NumberContent."""+ structure_blueprint = {+ "unit": ConceptStructureBlueprint(+ description="Unit of measurement",+ type=ConceptStructureBlueprintFieldType.TEXT,+ required=False,+ ),+ }++ generator = StructureGenerator()+ generated_code, generated_class = generator.generate_from_structure_blueprint(+ "Temperature",+ structure_blueprint,+ base_class_name="NumberContent",+ )++ # Verify correct import is present+ assert "from pipelex.core.stuffs.number_content import NumberContent" in generated_code++ # Verify class inherits from NumberContent+ assert "class Temperature(NumberContent):" in generated_code++ # Verify the generated class is valid+ from pipelex.core.stuffs.number_content import NumberContent # noqa: PLC0415++ assert issubclass(generated_class, NumberContent)+ from pipelex.core.stuffs.stuff_content import StuffContent # noqa: PLC0415++ assert issubclass(generated_class, StuffContent)++ def test_inheritance_from_json_content(self):+ """Test generating a class that inherits from JSONContent."""+ structure_blueprint = {+ "schema_version": ConceptStructureBlueprint(+ description="Version of the schema",+ type=ConceptStructureBlueprintFieldType.TEXT,+ required=False,+ ),+ }++ generator = StructureGenerator()+ generated_code, generated_class = generator.generate_from_structure_blueprint(+ "ConfigData",+ structure_blueprint,+ base_class_name="JSONContent",+ )++ # Verify correct import is present+ assert "from pipelex.core.stuffs.json_content import JSONContent" in generated_code++ # Verify class inherits from JSONContent+ assert "class ConfigData(JSONContent):" in generated_code++ # Verify the generated class is valid+ from pipelex.core.stuffs.json_content import JSONContent # noqa: PLC0415++ assert issubclass(generated_class, JSONContent)+ from pipelex.core.stuffs.stuff_content import StuffContent # noqa: PLC0415++ assert issubclass(generated_class, StuffContent)++ def test_inheritance_with_empty_structure(self):+ """Test generating a class with no additional fields that just inherits from a base class."""+ structure_blueprint: dict[str, ConceptStructureBlueprint] = {}++ generator = StructureGenerator()+ generated_code, generated_class = generator.generate_from_structure_blueprint(+ "EnhancedText",+ structure_blueprint,+ base_class_name="TextContent",+ )++ pretty_print(generated_code, title="Generated Result with Empty Structure")++ # Verify correct import is present+ assert "from pipelex.core.stuffs.text_content import TextContent" in generated_code++ # Verify class inherits from TextContent+ assert "class EnhancedText(TextContent):" in generated_code++ # Verify it has docstring (no pass needed)+ assert '"""Generated EnhancedText class"""' in generated_code++ # Verify the generated class is valid+ from pipelex.core.stuffs.text_content import TextContent # noqa: PLC0415++ assert issubclass(generated_class, TextContent)+ from pipelex.core.stuffs.stuff_content import StuffContent # noqa: PLC0415++ assert issubclass(generated_class, StuffContent)++ # Test instantiation works (TextContent requires 'text' field)+ instance = generated_class(text="Sample text") # pyright: ignore[reportCallIssue]+ assert instance is not None++ def test_inheritance_from_pdf_content(self):+ """Test generating a class that inherits from PDFContent."""+ structure_blueprint = {+ "document_type": ConceptStructureBlueprint(+ description="Type of PDF document",+ type=ConceptStructureBlueprintFieldType.TEXT,+ required=False,+ ),+ }++ generator = StructureGenerator()+ generated_code, generated_class = generator.generate_from_structure_blueprint(+ "Invoice",+ structure_blueprint,+ base_class_name="PDFContent",+ )++ # Verify correct import is present+ assert "from pipelex.core.stuffs.pdf_content import PDFContent" in generated_code++ # Verify class inherits from PDFContent+ assert "class Invoice(PDFContent):" in generated_code++ # Verify the generated class is valid+ from pipelex.core.stuffs.pdf_content import PDFContent # noqa: PLC0415++ assert issubclass(generated_class, PDFContent)+ from pipelex.core.stuffs.stuff_content import StuffContent # noqa: PLC0415++ assert issubclass(generated_class, StuffContent)++ def test_multiple_inheritance_scenarios(self):+ """Test multiple classes with different inheritance scenarios."""+ from pipelex.core.stuffs.stuff_content import StuffContent # noqa: PLC0415++ test_cases = [+ ("TextContent", "text_content", "Question", {"source": "The source of the question"}),+ ("ImageContent", "image_content", "Diagram", {"diagram_type": "Type of diagram"}),+ ("NumberContent", "number_content", "Score", {"max_value": "Maximum possible score"}),+ ]++ for base_class_name, module_name, class_name, field_dict in test_cases:+ structure_blueprint = {+ field_name: ConceptStructureBlueprint(+ description=description,+ type=ConceptStructureBlueprintFieldType.TEXT,+ required=False,+ )+ for field_name, description in field_dict.items()+ }++ generator = StructureGenerator()+ generated_code, generated_class = generator.generate_from_structure_blueprint(+ class_name,+ structure_blueprint,+ base_class_name=base_class_name,)++ # Verify correct import is present+ assert f"from pipelex.core.stuffs.{module_name} import {base_class_name}" in generated_code++ # Verify class inherits from correct base+ assert f"class {class_name}({base_class_name}):" in generated_code++ # Verify it's a valid subclass of StuffContent+ assert issubclass(generated_class, StuffContent)diff --git a/tests/unit/pipelex/core/stuffs/test_special_args.py b/tests/unit/pipelex/core/stuffs/test_special_args.pyindex bd3b6339d..bd3fcc572 100644--- a/tests/unit/pipelex/core/stuffs/test_special_args.py+++ b/tests/unit/pipelex/core/stuffs/test_special_args.py@@ -71,7 +71,7 @@ class TestSpecialArgsStuff:concept = ConceptFactory.make_from_blueprint(domain=domain,concept_code=f"Concept{field_name.replace('_', '').title()}",- blueprint=concept_blueprint,+ blueprint_or_string_description=concept_blueprint,)concept_library.add_new_concept(concept)diff --git a/tests/unit/pipelex/tools/misc/test_find_files_in_dir.py b/tests/unit/pipelex/tools/misc/test_find_files_in_dir.pyindex 67725a2c2..65f541114 100644--- a/tests/unit/pipelex/tools/misc/test_find_files_in_dir.py+++ b/tests/unit/pipelex/tools/misc/test_find_files_in_dir.py@@ -447,3 +447,104 @@ class TestFindFilesInDir:# Empty include list should exclude all venv filesassert len(files) == 1assert files[0].name == "file1.py"++ def test_exclude_using_absolute_path_structures_directory(self):+ """Test excluding structures directory using absolute path (build_structures_cmd use case)."""+ with tempfile.TemporaryDirectory() as temp_dir:+ temp_path = Path(temp_dir)++ # Create Python files in library directory+ (temp_path / "concept_a.py").write_text("# concept_a")+ (temp_path / "concept_b.py").write_text("# concept_b")++ # Create structures subdirectory with generated files (should be excluded)+ structures_dir = temp_path / "structures"+ structures_dir.mkdir()+ (structures_dir / "__init__.py").write_text("")+ (structures_dir / "domain_Generated1.py").write_text("# generated")+ (structures_dir / "domain_Generated2.py").write_text("# generated")++ # Exclude structures using its absolute path+ result = find_files_in_dir(+ str(temp_path),+ "*.py",+ is_recursive=True,+ excluded_dirs=[str(structures_dir.resolve())],+ )++ # Should find only the manually-created files, not generated ones+ assert len(result) == 2+ result_names = [f.name for f in result]+ assert "concept_a.py" in result_names+ assert "concept_b.py" in result_names+ # Verify no files from structures/ are included+ assert not any("structures" in str(f) for f in result)++ def test_exclude_absolute_path_with_trailing_slash(self):+ """Test that absolute path exclusion works with or without trailing slash."""+ with tempfile.TemporaryDirectory() as temp_dir:+ temp_path = Path(temp_dir)++ (temp_path / "main.py").write_text("# main")++ structures_dir = temp_path / "structures"+ structures_dir.mkdir()+ (structures_dir / "gen.py").write_text("# gen")++ # Test with trailing slash+ result_with_slash = find_files_in_dir(+ str(temp_path),+ "*.py",+ is_recursive=True,+ excluded_dirs=[str(structures_dir.resolve()) + "/"],+ )++ # Test without trailing slash+ result_without_slash = find_files_in_dir(+ str(temp_path),+ "*.py",+ is_recursive=True,+ excluded_dirs=[str(structures_dir.resolve())],+ )++ # Both should produce same result+ assert len(result_with_slash) == 1+ assert len(result_without_slash) == 1+ assert result_with_slash[0].name == "main.py"+ assert result_without_slash[0].name == "main.py"++ def test_mixed_name_and_absolute_path_exclusions(self):+ """Test mixing directory name and absolute path exclusions."""+ with tempfile.TemporaryDirectory() as temp_dir:+ temp_path = Path(temp_dir)++ (temp_path / "main.py").write_text("# main")++ # Exclude by name+ venv_dir = temp_path / ".venv"+ venv_dir.mkdir()+ (venv_dir / "venv.py").write_text("# venv")++ # Exclude by absolute path+ structures_dir = temp_path / "structures"+ structures_dir.mkdir()+ (structures_dir / "gen.py").write_text("# gen")++ # Regular directory+ src_dir = temp_path / "src"+ src_dir.mkdir()+ (src_dir / "code.py").write_text("# code")++ result = find_files_in_dir(+ str(temp_path),+ "*.py",+ is_recursive=True,+ excluded_dirs=[".venv", str(structures_dir.resolve())],+ )++ assert len(result) == 2+ result_names = [f.name for f in result]+ assert "main.py" in result_names+ assert "code.py" in result_names+ assert "venv.py" not in result_names+ assert "gen.py" not in result_names