Build AI methods with your coding agent, then run them anywhere: as an MCP for chatbots, as a webapp for people, or via API for your software.
Fix/better validation error (#474)
Thomas Hebrard451d46bc3f98
diff --git a/pipelex/builder/builder.plx b/pipelex/builder/builder.plxindex beebf0fbc..458bff8e7 100644--- a/pipelex/builder/builder.plx+++ b/pipelex/builder/builder.plx@@ -202,7 +202,9 @@ Shape of the contract for PipeOperator is:**PipeCondition:**- jinja2_expression_template: Jinja2 expression to evaluate.- outcomes: Mapping dict[str, str] of expression results to pipe codes.-- default_outcome: Fallback pipe_code or "fail"/"continue" if no match.+- default_outcome: Add a default outcome to the outcomes map.+Becareful, the outputs "fail" and "continue" are special outcomes, they are not pipe codes.+-> "continue" is a special outcome that allows to continue the sequence, "fail" is a special outcome that fails the whole pipeline. To use with caution.## More rules:- For each pipe: give a unique snake_case pipe_code, based on a verb, and craft description of what the pipe does.diff --git a/pipelex/builder/builder_errors.py b/pipelex/builder/builder_errors.pyindex 62119f2b9..c862ed7a8 100644--- a/pipelex/builder/builder_errors.py+++ b/pipelex/builder/builder_errors.py@@ -13,9 +13,5 @@ class ConceptSpecError(PipelexError):"""Details of a single concept failure during dry run."""-class PipeSpecError(PipelexError):- """Details of a single pipe failure during dry run."""--class PipelexBundleUnexpectedError(PipelexError):"""Raised when an unexpected error occurs during validation."""diff --git a/pipelex/builder/builder_loop.py b/pipelex/builder/builder_loop.pyindex 3fc78b69b..9b9644f04 100644--- a/pipelex/builder/builder_loop.py+++ b/pipelex/builder/builder_loop.py@@ -6,15 +6,14 @@ from pipelex.builder.builder import (PipeSpecUnion,reconstruct_bundle_with_pipe_fixes,)-from pipelex.builder.builder_errors import PipeBuilderErrorfrom pipelex.builder.pipe.pipe_sequence_spec import PipeSequenceSpecfrom pipelex.client.protocol import PipelineInputsfrom pipelex.config import get_config-from pipelex.core.bundles.exceptions import PipelexBundleBlueprintFixableErrorType, PipeValidationErrorType+from pipelex.core.pipes.exceptions import PipeValidationErrorTypefrom pipelex.core.pipes.pipe_blueprint import PipeCategory-from pipelex.hub import get_console, get_required_pipe+from pipelex.core.pipes.variable_multiplicity import format_concept_with_multiplicity+from pipelex.hub import get_required_pipefrom pipelex.language.plx_factory import PlxFactory-from pipelex.pipeline.exceptions import PipelineExecutionErrorfrom pipelex.pipeline.execute import execute_pipelinefrom pipelex.pipeline.validate_bundle import ValidateBundleError, validate_bundlefrom pipelex.tools.misc.file_utils import get_incremental_file_path, save_text_to_path@@ -30,17 +29,11 @@ class BuilderLoop:is_save_second_iteration_enabled: bool = True,is_save_working_memory_enabled: bool = True,) -> PipelexBundleSpec:- try:- pipe_output = await execute_pipeline(- pipe_code=builder_pipe,- library_path=str(Path(builder.__file__).parent),- inputs=inputs,- )- except PipelineExecutionError as exc:- msg = f"Builder loop: Failed to execute pipeline: {exc}."- console = get_console()- console.print_exception()- raise PipeBuilderError(message=msg) from exc+ pipe_output = await execute_pipeline(+ pipe_code=builder_pipe,+ library_path=str(Path(builder.__file__).parent),+ inputs=inputs,+ )if is_save_working_memory_enabled:working_memory_path = get_incremental_file_path(@@ -95,15 +88,8 @@ class BuilderLoop:pipelex_bundle_spec: PipelexBundleSpec,is_save_second_iteration_enabled: bool,) -> PipelexBundleSpec:- """Fix validation errors in the bundle spec.-- Currently supports fixing:- - MISSING_INPUT_VARIABLE / EXTRANEOUS_INPUT_VARIABLE / INPUT_REQUIREMENT_MISMATCH (for PipeController only)- - PIPE_SEQUENCE_OUTPUT_MISMATCH- """fixed_pipes: list[PipeSpecUnion] = []- # Process pipe validation error data (MISSING_INPUT_VARIABLE / EXTRANEOUS_INPUT_VARIABLE for PipeController)for val_error in bundle_error.pipe_validation_error_data:if not val_error.pipe_code or not pipelex_bundle_spec.pipe:continue@@ -133,23 +119,15 @@ class BuilderLoop:for named_requirement in needed_inputs.named_input_requirements:if named_requirement.variable_name == variable_name:old_value = new_inputs.get(variable_name, "NOT SET")- concept_code = named_requirement.concept.code-- # Add multiplicity brackets if required- if named_requirement.multiplicity is not None:- if named_requirement.multiplicity is True:- # Variable-length list []- concept_code = f"{concept_code}[]"- else:- # Fixed-length list [N] where N is an int- concept_code = f"{concept_code}[{named_requirement.multiplicity}]"- # If multiplicity is None, concept_code stays without brackets- # This effectively removes multiplicity if it was present in old_value-- new_inputs[variable_name] = concept_code+ concept_code_with_multiplicity = format_concept_with_multiplicity(+ concept_code_or_string=named_requirement.concept.code,+ multiplicity=named_requirement.multiplicity,+ )+ new_inputs[variable_name] = concept_code_with_multiplicity+ # TODO: return a structured report of what was done, let the caller decide if they want to print it or act on itlog.info(f"π§ Fixed input requirement mismatch for pipe '{val_error.pipe_code}': input '{variable_name}' \- changed from '{old_value}' β '{concept_code}'"+ changed from '{old_value}' β '{concept_code_with_multiplicity}'")break@@ -166,16 +144,11 @@ class BuilderLoop:old_inputs = dict(pipe_spec.inputs) if pipe_spec.inputs else {}fixed_inputs: dict[str, str] = {}for named_requirement in needed_inputs.named_input_requirements:- concept_code = named_requirement.concept.code- # Preserve multiplicity brackets- if named_requirement.multiplicity is not None:- if named_requirement.multiplicity is True:- # Variable-length list []- concept_code = f"{concept_code}[]"- else:- # Fixed-length list [N] where N is an int- concept_code = f"{concept_code}[{named_requirement.multiplicity}]"- fixed_inputs[named_requirement.variable_name] = concept_code+ concept_code_with_multiplicity = format_concept_with_multiplicity(+ concept_code_or_string=named_requirement.concept.code,+ multiplicity=named_requirement.multiplicity,+ )+ fixed_inputs[named_requirement.variable_name] = concept_code_with_multiplicity# Only apply fix if it actually changes something (avoid infinite loops)if fixed_inputs != old_inputs:@@ -208,50 +181,20 @@ class BuilderLoop:# Set the sequence output to match the last step's outputpipe_spec.output = new_outputfixed_pipes.append(pipe_spec)+ # TODO: return a structured report of what was done, let the caller decide if they want to print it or act on itlog.info(f"π§ Fixed output concept for pipe '{val_error.pipe_code}': output changed from '{old_output}' β \'{new_output}' (matching last step '{last_step_pipe_code}')")- case _:- # Other error types not handled for pipe validation errors+ case (+ PipeValidationErrorType.LLM_OUTPUT_CANNOT_BE_IMAGE+ | PipeValidationErrorType.IMG_GEN_INPUT_NOT_TEXT_COMPATIBLE+ | PipeValidationErrorType.UNKNOWN_VALIDATION_ERROR+ | PipeValidationErrorType.CIRCULAR_DEPENDENCY_ERROR+ ):continue- # Process pipelex bundle blueprint validation errors (PIPE_SEQUENCE_OUTPUT_MISMATCH)- for blueprint_error in bundle_error.pipelex_bundle_blueprint_validation_errors:- if blueprint_error.error_type != PipelexBundleBlueprintFixableErrorType.PIPE_SEQUENCE_OUTPUT_MISMATCH:- continue-- if not blueprint_error.pipe_code or not pipelex_bundle_spec.pipe:- continue-- pipe_spec = pipelex_bundle_spec.pipe.get(blueprint_error.pipe_code)- if not pipe_spec or not isinstance(pipe_spec, PipeSequenceSpec):- continue-- # Get the last step's output- if not pipe_spec.steps:- continue-- last_step = pipe_spec.steps[-1]- last_step_pipe_code = last_step.pipe_code-- # Get the last step's pipe spec to retrieve its output- last_step_pipe_spec = pipelex_bundle_spec.pipe.get(last_step_pipe_code)- if not last_step_pipe_spec:- continue-- old_output = pipe_spec.output- new_output = last_step_pipe_spec.output-- # Set the sequence output to match the last step's output- pipe_spec.output = new_output- fixed_pipes.append(pipe_spec)- log.info(- f"π§ Fixed sequence output mismatch for pipe '{blueprint_error.pipe_code}': output changed from '{old_output}' β \- '{new_output}' (from last step '{last_step_pipe_code}')"- )-# Reconstruct bundle if we made changesif fixed_pipes:pipelex_bundle_spec = reconstruct_bundle_with_pipe_fixes(pipelex_bundle_spec=pipelex_bundle_spec, fixed_pipes=fixed_pipes)diff --git a/pipelex/builder/bundle_spec.py b/pipelex/builder/bundle_spec.pyindex 2b14cdab9..5c983f2e2 100644--- a/pipelex/builder/bundle_spec.py+++ b/pipelex/builder/bundle_spec.py@@ -5,7 +5,7 @@ from rich.text import Textfrom typing_extensions import overridefrom pipelex.builder.concept.concept_spec import ConceptSpec-from pipelex.builder.exceptions import PipelexBundleSpecBlueprintError, PipelexBundleSpecValueError+from pipelex.builder.exceptions import PipelexBundleSpecBlueprintErrorfrom pipelex.builder.pipe.pipe_spec_union import PipeSpecUnionfrom pipelex.core.bundles.pipe_sorter import sort_pipes_by_dependenciesfrom pipelex.core.bundles.pipelex_bundle_blueprint import PipeBlueprintUnion, PipelexBundleBlueprint@@ -65,14 +65,14 @@ class PipelexBundleSpec(StructuredContent):validate_domain_code(code=domain)except DomainCodeError as exc:msg = f"Error when trying to validate pipelex bundle spec: domain '{domain}' is not a valid domain code: {exc}"- raise PipelexBundleSpecValueError(msg) from exc+ raise ValueError(msg) from excreturn domain@model_validator(mode="after")def validate_main_pipe(self) -> "PipelexBundleSpec":if not self.pipe or (self.main_pipe not in self.pipe):msg = f"Main pipe '{self.main_pipe}' could not be found in bundle spec"- raise PipelexBundleSpecValueError(msg)+ raise ValueError(msg)return selfdef to_blueprint(self) -> PipelexBundleBlueprint:diff --git a/pipelex/builder/concept/concept_spec.py b/pipelex/builder/concept/concept_spec.pyindex 37fbc7190..0ebfb827e 100644--- a/pipelex/builder/concept/concept_spec.py+++ b/pipelex/builder/concept/concept_spec.py@@ -11,7 +11,6 @@ from pipelex import logfrom pipelex.base_exceptions import PipelexErrorfrom pipelex.core.concepts.concept_blueprint import ConceptBlueprint, ConceptStructureBlueprintfrom pipelex.core.concepts.concept_structure_blueprint import ConceptStructureBlueprintFieldType-from pipelex.core.concepts.exceptions import ConceptBlueprintValueError, ConceptStructureBlueprintValueErrorfrom pipelex.core.concepts.native.concept_native import NativeConceptCodefrom pipelex.core.stuffs.structured_content import StructuredContentfrom pipelex.tools.misc.pretty import PrettyPrintable@@ -95,7 +94,7 @@ class ConceptStructureSpec(StructuredContent):def _raise_type_mismatch_error(self, expected_type_name: str, actual_type_name: str) -> None:msg = f"default_value type mismatch: expected {expected_type_name} for type '{self.type}', but got {actual_type_name}"- raise ConceptStructureBlueprintValueError(msg)+ raise ValueError(msg)def to_blueprint(self) -> ConceptStructureBlueprint:# Convert the type enum value - self.type is already a ConceptStructureBlueprintFieldType enum@@ -183,7 +182,7 @@ class ConceptSpec(StructuredContent):if refines is not None:if not NativeConceptCode.get_validated_native_concept_string(concept_string_or_code=refines):msg = f"Forbidden to refine a non-native concept: '{refines}'. Refining non-native concepts will come soon."- raise ConceptBlueprintValueError(msg)+ raise ValueError(msg)return refines@model_validator(mode="before")diff --git a/pipelex/builder/exceptions.py b/pipelex/builder/exceptions.pyindex dfb36ec13..a6a416f78 100644--- a/pipelex/builder/exceptions.py+++ b/pipelex/builder/exceptions.py@@ -1,9 +1,5 @@from pipelex.base_exceptions import PipelexError-class PipelexBundleSpecValueError(ValueError):- pass--class PipelexBundleSpecBlueprintError(PipelexError):passdiff --git a/pipelex/builder/flow_factory.py b/pipelex/builder/flow_factory.pyindex a71a84df7..7fb16a37f 100644--- a/pipelex/builder/flow_factory.py+++ b/pipelex/builder/flow_factory.py@@ -7,7 +7,7 @@ from pipelex.builder.builder import PipelexBundleSpecfrom pipelex.builder.flow import Flow, FlowElementfrom pipelex.builder.pipe.pipe_signature import PipeSignaturefrom pipelex.core.bundles.pipelex_bundle_blueprint import PipelexBundleBlueprint-from pipelex.core.interpreter import PipelexInterpreter+from pipelex.core.interpreter.interpreter import PipelexInterpreterfrom pipelex.core.pipes.pipe_blueprint import PipeCategoryfrom pipelex.pipe_controllers.batch.pipe_batch_blueprint import PipeBatchBlueprintfrom pipelex.pipe_controllers.condition.pipe_condition_blueprint import PipeConditionBlueprintdiff --git a/pipelex/builder/pipe/pipe_extract_spec.py b/pipelex/builder/pipe/pipe_extract_spec.pyindex 65c0baafd..a048ed333 100644--- a/pipelex/builder/pipe/pipe_extract_spec.py+++ b/pipelex/builder/pipe/pipe_extract_spec.py@@ -7,7 +7,6 @@ from rich.text import Textfrom typing_extensions import overridefrom pipelex.builder.pipe.pipe_spec import PipeSpec-from pipelex.builder.pipe.pipe_spec_exceptions import PipeExtractSpecErrorfrom pipelex.pipe_operators.extract.pipe_extract_blueprint import PipeExtractBlueprintfrom pipelex.tools.misc.pretty import PrettyPrintablefrom pipelex.types import StrEnum@@ -56,10 +55,10 @@ class PipeExtractSpec(PipeSpec):def validate_extract_inputs(cls, inputs_value: dict[str, str] | None) -> dict[str, str] | None:if inputs_value is None:msg = "PipeExtract must have exactly one input which must be either `Image` or `PDF`."- raise PipeExtractSpecError(msg)+ raise ValueError(msg)if len(inputs_value) != 1:msg = "PipeExtract must have exactly one input which must be either `Image` or `PDF`."- raise PipeExtractSpecError(msg)+ raise ValueError(msg)return inputs_value@overridediff --git a/pipelex/builder/pipe/pipe_parallel_spec.py b/pipelex/builder/pipe/pipe_parallel_spec.pyindex ceab29cdd..235b541fd 100644--- a/pipelex/builder/pipe/pipe_parallel_spec.py+++ b/pipelex/builder/pipe/pipe_parallel_spec.py@@ -7,7 +7,6 @@ from rich.text import Textfrom typing_extensions import overridefrom pipelex.builder.pipe.pipe_spec import PipeSpec-from pipelex.builder.pipe.pipe_spec_exceptions import PipeParallelSpecErrorfrom pipelex.builder.pipe.sub_pipe_spec import SubPipeSpecfrom pipelex.core.concepts.validation import validate_concept_string_or_codefrom pipelex.pipe_controllers.parallel.pipe_parallel_blueprint import PipeParallelBlueprint@@ -50,11 +49,7 @@ class PipeParallelSpec(PipeSpec):"PipeParallel requires either add_each_output to be True or combined_output to be set, ""or both, otherwise the pipe won't output anything")- raise PipeParallelSpecError(- message=msg,- pipe_code=self.pipe_code,- description=self.description,- )+ raise ValueError(msg)return self@overridediff --git a/pipelex/builder/pipe/pipe_signature.py b/pipelex/builder/pipe/pipe_signature.pyindex 9c3956126..5e8b19bec 100644--- a/pipelex/builder/pipe/pipe_signature.py+++ b/pipelex/builder/pipe/pipe_signature.py@@ -7,7 +7,6 @@ from rich.table import Tablefrom rich.text import Textfrom typing_extensions import override-from pipelex.core.pipes.exceptions import PipeBlueprintValueErrorfrom pipelex.core.pipes.pipe_blueprint import PipeCategory, PipeTypefrom pipelex.core.stuffs.structured_content import StructuredContentfrom pipelex.tools.misc.pretty import PrettyPrintable@@ -57,7 +56,7 @@ class PipeSignature(StructuredContent):type_str = values["type"]except TypeError as exc:msg = f"Invalid type for '{values}': could not get subscript, required for 'type'"- raise PipeBlueprintValueError(msg) from exc+ raise ValueError(msg) from exc# we need to convert the type string to the PipeType enum because it arrives as a str implictly converted to enum but not yetthe_type = PipeType(type_str)values["pipe_category"] = the_type.categorydiff --git a/pipelex/builder/pipe/pipe_spec.py b/pipelex/builder/pipe/pipe_spec.pyindex 917eb8ae6..b56e0ef40 100644--- a/pipelex/builder/pipe/pipe_spec.py+++ b/pipelex/builder/pipe/pipe_spec.py@@ -9,7 +9,6 @@ from typing_extensions import overridefrom pipelex import logfrom pipelex.core.concepts.validation import validate_concept_string_or_code-from pipelex.core.pipes.exceptions import PipeBlueprintValueErrorfrom pipelex.core.pipes.pipe_blueprint import PipeBlueprint, PipeCategory, PipeTypefrom pipelex.core.pipes.variable_multiplicity import MUTLIPLICITY_PATTERN, parse_concept_with_multiplicityfrom pipelex.core.stuffs.structured_content import StructuredContent@@ -72,7 +71,7 @@ class PipeSpec(StructuredContent):def validate_pipe_type(cls, value: Any) -> Any:if value not in PipeType.value_list():msg = f"Invalid pipe type '{value}'. Must be one of: {PipeType.value_list()}"- raise PipeBlueprintValueError(msg)+ raise ValueError(msg)return value@field_validator("output", mode="after")@@ -92,7 +91,7 @@ class PipeSpec(StructuredContent):for input_name, concept_spec in inputs.items():if not is_snake_case(input_name):msg = f"Invalid input name syntax '{input_name}'. Must be in snake_case."- raise PipeBlueprintValueError(msg)+ raise ValueError(msg)# Validate the concept spec format with optional multiplicity brackets# Pattern allows: ConceptName, domain.ConceptName, ConceptName[], ConceptName[N]@@ -102,7 +101,7 @@ class PipeSpec(StructuredContent):f"Invalid input syntax for '{input_name}': '{concept_spec}'. "f"Expected format: 'ConceptName', 'ConceptName[]', or 'ConceptName[N]' where N is an integer.")- raise PipeBlueprintValueError(msg)+ raise ValueError(msg)# Extract the concept part (without multiplicity) and validate itconcept_string_or_code = match.group(1)@@ -120,7 +119,7 @@ class PipeSpec(StructuredContent):if not is_snake_case(normalized_pipe_code):msg = f"Invalid pipe code syntax '{normalized_pipe_code}'. Must be in snake_case."- raise PipeBlueprintValueError(msg)+ raise ValueError(msg)return normalized_pipe_codedef to_blueprint(self) -> PipeBlueprint:diff --git a/pipelex/builder/pipe/pipe_spec_exceptions.py b/pipelex/builder/pipe/pipe_spec_exceptions.pydeleted file mode 100644index 5375b8104..000000000--- a/pipelex/builder/pipe/pipe_spec_exceptions.py+++ /dev/null@@ -1,13 +0,0 @@-from pipelex.core.pipe_errors import PipeDefinitionError---class PipeSpecError(PipeDefinitionError):- pass---class PipeExtractSpecError(PipeSpecError):- pass---class PipeParallelSpecError(PipeSpecError):- passdiff --git a/pipelex/cli/commands/build_cmd.py b/pipelex/cli/commands/build_cmd.pyindex 690bf3d05..33cbd3106 100644--- a/pipelex/cli/commands/build_cmd.py+++ b/pipelex/cli/commands/build_cmd.py@@ -20,7 +20,8 @@ from pipelex.cli.error_handlers import ()from pipelex.cogt.exceptions import ModelDeckPresetValidatonErrorfrom pipelex.config import get_config-from pipelex.core.pipes.exceptions import PipeInputError, PipeOperatorModelChoiceError+from pipelex.core.pipes.exceptions import PipeOperatorModelChoiceError+from pipelex.core.pipes.inputs.exceptions import PipeInputErrorfrom pipelex.hub import get_console, get_report_delegate, get_required_pipe, get_telemetry_managerfrom pipelex.language.plx_factory import PlxFactoryfrom pipelex.pipe_operators.exceptions import PipeOperatorModelAvailabilityErrordiff --git a/pipelex/cli/commands/run_cmd.py b/pipelex/cli/commands/run_cmd.pyindex 19e3b9b1d..3789e4277 100644--- a/pipelex/cli/commands/run_cmd.py+++ b/pipelex/cli/commands/run_cmd.py@@ -16,7 +16,8 @@ from pipelex.cli.error_handlers import (handle_model_deck_preset_error,)from pipelex.cogt.exceptions import ModelDeckPresetValidatonError-from pipelex.core.pipes.exceptions import PipeInputError, PipeOperatorModelChoiceError+from pipelex.core.pipes.exceptions import PipeOperatorModelChoiceError+from pipelex.core.pipes.inputs.exceptions import PipeInputErrorfrom pipelex.hub import get_console, get_telemetry_managerfrom pipelex.pipe_operators.exceptions import PipeOperatorModelAvailabilityErrorfrom pipelex.pipelex import Pipelexdiff --git a/pipelex/cli/commands/validate_cmd.py b/pipelex/cli/commands/validate_cmd.pyindex eaf27ac7e..26b29ddaf 100644--- a/pipelex/cli/commands/validate_cmd.py+++ b/pipelex/cli/commands/validate_cmd.py@@ -42,6 +42,7 @@ def do_validate_all_libraries_and_dry_run() -> None:library_id, library = library_manager.open_library()set_current_library(library_id=library_id)library_manager.load_libraries(library_id=library_id, library_dirs=[Path.cwd()])+pipes = library.get_pipe_library().get_pipes()get_telemetry_manager().track_event(EventName.PIPE_DRY_RUN, properties={EventProperty.NB_PIPES: len(pipes)})@@ -141,13 +142,16 @@ def validate_cmd(except ValidateBundleError as bundle_error:handle_validate_bundle_error(bundle_error, bundle_path=bundle_path)elif pipe_code:- # Validate a single pipe by codetyper.echo(f"Validating pipe '{pipe_code}'...")- get_telemetry_manager().track_event(- EventName.PIPE_DRY_RUN, properties={EventProperty.PIPE_TYPE: get_required_pipe(pipe_code=pipe_code).type}- )+ library_manager = get_library_manager()+ library_id, _ = library_manager.open_library()+ set_current_library(library_id=library_id)+ library_manager.load_libraries(library_id=library_id, library_dirs=[Path.cwd()])++ pipe = get_required_pipe(pipe_code=pipe_code)+ get_telemetry_manager().track_event(EventName.PIPE_DRY_RUN, properties={EventProperty.PIPE_TYPE: pipe.type})await dry_run_pipe(- get_required_pipe(pipe_code=pipe_code),+ pipe,raise_on_failure=True,)typer.secho(f"β Successfully validated pipe '{pipe_code}'", fg=typer.colors.GREEN)diff --git a/pipelex/cli/error_handlers.py b/pipelex/cli/error_handlers.pyindex 1548e242d..140f540cb 100644--- a/pipelex/cli/error_handlers.py+++ b/pipelex/cli/error_handlers.py@@ -127,64 +127,6 @@ def handle_validate_bundle_error(exc: "ValidateBundleError", bundle_path: str |if bundle_path:console.print(f"[bold cyan]Bundle:[/bold cyan] [yellow]{bundle_path}[/yellow]\n")-- # Display bundle blueprint validation errors- if exc.pipelex_bundle_blueprint_validation_errors:- console.print("[bold cyan]Bundle Blueprint Validation Errors:[/bold cyan]\n")- for idx, error in enumerate(exc.pipelex_bundle_blueprint_validation_errors, 1):- console.print(f"[bold yellow]{idx}. {error.error_type.replace('_', ' ').title()}[/bold yellow]")-- # Display entity context- if error.pipe_code:- console.print(f" [cyan]Pipe:[/cyan] [yellow]{error.pipe_code}[/yellow]")- if error.concept_code:- console.print(f" [cyan]Concept:[/cyan] [yellow]{error.concept_code}[/yellow]")- if error.domain:- console.print(f" [cyan]Domain:[/cyan] [green]{error.domain}[/green]")-- # Field name if present- if error.field_name:- console.print(f" [cyan]Field:[/cyan] [yellow]{error.field_name}[/yellow]")-- # Display the error message- console.print(f" [cyan]β[/cyan] {error.message}")-- # Display context-specific details grouped by category- # Concept-related- if error.concept_refines or error.provided_concept or error.required_concept:- if error.concept_refines:- console.print(f" [cyan]Refines:[/cyan] [green]{error.concept_refines}[/green]")- if error.provided_concept:- console.print(f" [cyan]Provided:[/cyan] [yellow]{error.provided_concept}[/yellow]")- if error.required_concept:- console.print(f" [cyan]Required:[/cyan] [green]{error.required_concept}[/green]")-- # Pipe sequence-related- if error.last_step_pipe_code or error.last_step_output_concept or error.expected_output_concept:- if error.last_step_pipe_code:- console.print(f" [cyan]Last Step:[/cyan] [yellow]{error.last_step_pipe_code}[/yellow]")- if error.last_step_output_concept:- console.print(f" [cyan]Last Step Output:[/cyan] [green]{error.last_step_output_concept}[/green]")- if error.expected_output_concept:- console.print(f" [cyan]Expected Output:[/cyan] [green]{error.expected_output_concept}[/green]")-- # Variable-related- if error.variable_name:- console.print(f" [cyan]Variable:[/cyan] [yellow]{error.variable_name}[/yellow]")-- # Type-related- if error.expected_type or error.actual_type:- if error.expected_type:- console.print(f" [cyan]Expected Type:[/cyan] [green]{error.expected_type}[/green]")- if error.actual_type:- console.print(f" [cyan]Actual Type:[/cyan] [yellow]{error.actual_type}[/yellow]")-- # Field path as secondary info- if error.field_path:- console.print(f" [dim]ββ Path: {error.field_path}[/dim]")-- console.print()-# Display pipe validation errorsif exc.pipe_validation_error_data:console.print("[bold cyan]Pipe Validation Errors:[/bold cyan]\n")@@ -194,29 +136,26 @@ def handle_validate_bundle_error(exc: "ValidateBundleError", bundle_path: str |# Display key identification infoif pipe_error.pipe_code:console.print(f" [cyan]Pipe:[/cyan] [yellow]{pipe_error.pipe_code}[/yellow]")+ if pipe_error.concept_code:+ console.print(f" [cyan]Concept:[/cyan] [yellow]{pipe_error.concept_code}[/yellow]")if pipe_error.domain:console.print(f" [cyan]Domain:[/cyan] [green]{pipe_error.domain}[/green]")+ # Field name if present+ if pipe_error.field_name:+ console.print(f" [cyan]Field:[/cyan] [yellow]{pipe_error.field_name}[/yellow]")+# Variablesif pipe_error.variable_names:variables_str = ", ".join([f"[yellow]{v}[/yellow]" for v in pipe_error.variable_names])console.print(f" [cyan]Variables:[/cyan] {variables_str}")- # Concept information- if pipe_error.required_concept_codes or pipe_error.provided_concept_code:- if pipe_error.required_concept_codes:- required_str = ", ".join([f"[green]{c}[/green]" for c in pipe_error.required_concept_codes])- console.print(f" [cyan]Required:[/cyan] {required_str}")- if pipe_error.provided_concept_code:- console.print(f" [cyan]Provided:[/cyan] [yellow]{pipe_error.provided_concept_code}[/yellow]")-- # Explanation with better formatting- if pipe_error.explanation:- console.print(f" [cyan]β[/cyan] {pipe_error.explanation}")-- # File path as secondary info- if pipe_error.file_path:- console.print(f" [dim]ββ Path: {pipe_error.file_path}[/dim]")+ # Error message+ console.print(f" [cyan]β[/cyan] {pipe_error.message}")++ # Field path as secondary info+ if pipe_error.field_path:+ console.print(f" [dim]ββ Path: {pipe_error.field_path}[/dim]")console.print()diff --git a/pipelex/core/bundles/exceptions.py b/pipelex/core/bundles/exceptions.pyindex 2c3579932..b3b9f57e0 100644--- a/pipelex/core/bundles/exceptions.py+++ b/pipelex/core/bundles/exceptions.py@@ -1,125 +1,19 @@-from pydantic import BaseModel, Field+from pydantic import BaseModelfrom pipelex.types import StrEnum-class PipeValidationErrorType(StrEnum):- """Types of pipe validation errors.-- These error types are raised during pipe validation (validate_inputs_with_library, validate_output_with_library).- """-- MISSING_INPUT_VARIABLE = "missing_input_variable"- EXTRANEOUS_INPUT_VARIABLE = "extraneous_input_variable"- INPUT_REQUIREMENT_MISMATCH = "input_requirement_mismatch"- INADEQUATE_INPUT_CONCEPT = "inadequate_input_concept"- TOO_MANY_CANDIDATE_INPUTS = "too_many_candidate_inputs"- INADEQUATE_OUTPUT_CONCEPT = "inadequate_output_concept"- LLM_OUTPUT_CANNOT_BE_IMAGE = "llm_output_cannot_be_image"- IMG_GEN_INPUT_NOT_TEXT_COMPATIBLE = "img_gen_input_not_text_compatible"-- # Pydantic validation errors- FIELD_REQUIRED = "field_required"- FIELD_MISSING = "field_missing"- MUTUALLY_EXCLUSIVE_FIELDS = "mutually_exclusive_fields"- MODEL_NOT_IN_DECK = "model_not_in_deck"- FUNCTION_NOT_FOUND = "function_not_found"- INVALID_RETURN_TYPE = "invalid_return_type"- OUTPUT_CONCEPT_INCONSISTENCY = "output_concept_inconsistency"- DUPLICATE_OUTPUT_NAME = "duplicate_output_name"- PIPE_PARALLEL_OUTPUT_CONFIG_ERROR = "pipe_parallel_output_config_error"-- # Generic fallback- UNKNOWN_VALIDATION_ERROR = "unknown_validation_error"--class PipelexBundleBlueprintFixableErrorType(StrEnum):"""Types of fixable validation errors in Pipelex bundle blueprints.- These error types represent validation issues that can potentially be- automatically fixed or for which we can provide specific guidance.+ These error types represent validation issues that are actually fixed+ in the builder loop auto-fix system."""- # Concept-related fixable errors- CONCEPT_REFINES_STRUCTURE_CONFLICT = "concept_refines_structure_conflict" # Has both refines and structure- CONCEPT_REFINES_INVALID = "concept_refines_invalid" # Refines references non-existent concept- CONCEPT_STRUCTURE_INVALID = "concept_structure_invalid" # Structure definition has errors-- # Pipe sequence errors- PIPE_SEQUENCE_OUTPUT_MISMATCH = "pipe_sequence_output_mismatch" # Last step output doesn't match sequence output- PIPE_SEQUENCE_EMPTY_STEPS = "pipe_sequence_empty_steps" # No steps defined-- # Domain errors- DOMAIN_CODE_INVALID = "domain_code_invalid" # Domain code format invalid-- # Bundle-level errors- MAIN_PIPE_NOT_FOUND = "main_pipe_not_found" # Main pipe doesn't exist in bundle-- # Pydantic validation errors (often fixable)- MISSING_REQUIRED_FIELD = "missing_required_field" # Required field not provided- TYPE_MISMATCH = "type_mismatch" # Wrong type for field- EXTRA_FORBIDDEN_FIELD = "extra_forbidden_field" # Extra field not allowed- DISCRIMINATOR_MISSING = "discriminator_missing" # Union discriminator field missing- ENUM_INVALID_VALUE = "enum_invalid_value" # Invalid enum value-- # Fallback- UNKNOWN = "unknown" # Could not categorize error---class PipelexBundleBlueprintValueError(ValueError):- pass-class PipelexBundleBlueprintValidationErrorData(BaseModel):- """Structured validation error data with context for categorization and fixing.+ """Structured validation error data for bundle blueprint validation errors.- This model captures comprehensive information about validation errors to enable:- - Automatic error fixing- - Targeted error messages and suggestions- - Analytics tracking- - Action dispatch based on error type+ This model captures information about validation errors that are actually fixed+ in the builder loop auto-fix system."""-- # === Source Context ===- domain: str | None = Field(None, description="Domain where error occurred")- source: str | None = Field(None, description="Source file path")-- # === Entity Context (what failed) ===- pipe_code: str | None = Field(None, description="Pipe code if error is in a pipe")- concept_code: str | None = Field(None, description="Concept code if error is in a concept")- field_name: str | None = Field(None, description="Specific field that failed (e.g., 'refines', 'structure')")-- # === Error Classification ===- error_type: PipelexBundleBlueprintFixableErrorType = Field(- default=PipelexBundleBlueprintFixableErrorType.UNKNOWN,- description="Type of error for dispatch and fixing",- )- error_scope: str | None = Field(None, description="Scope of error: 'concept', 'pipe', 'domain', 'bundle'")-- # === Error Details ===- message: str = Field(description="Human-readable error message")- field_path: str = Field(description="Path to field in dot notation (e.g., 'concept.Invoice.refines')")-- # === Context-specific Data (for fixing) ===- # For concept errors- concept_refines: str | None = Field(None, description="Value of refines field (if applicable)")- concept_structure: dict[str, str] | None = Field(None, description="Value of structure field (if applicable)")-- # For pipe sequence errors- last_step_pipe_code: str | None = Field(None, description="Last step pipe code (for sequence errors)")- last_step_output_concept: str | None = Field(None, description="Output concept of last step")- expected_output_concept: str | None = Field(None, description="Expected output concept")-- # For input/output errors- variable_name: str | None = Field(None, description="Variable name (for input errors)")- provided_concept: str | None = Field(None, description="Provided concept code")- required_concept: str | None = Field(None, description="Required concept code")- candidate_inputs: list[str] | None = Field(None, description="List of candidate inputs")-- # For type/value errors- expected_type: str | None = Field(None, description="Expected type for field")- actual_type: str | None = Field(None, description="Actual type provided")- provided_value: str | None = Field(None, description="String representation of provided value")-- # === Legacy fields (for backwards compatibility) ===- other: str | None = Field(None, description="Other context (legacy)")diff --git a/pipelex/core/bundles/pipe_sorter.py b/pipelex/core/bundles/pipe_sorter.pyindex 6691be067..faf8532eb 100644--- a/pipelex/core/bundles/pipe_sorter.py+++ b/pipelex/core/bundles/pipe_sorter.py@@ -1,7 +1,7 @@"""Topological sorting utilities for pipe dependencies."""from pipelex.core.bundles.pipelex_bundle_blueprint import PipeBlueprintUnion-from pipelex.core.pipe_errors import PipeDefinitionError+from pipelex.core.pipes.exceptions import PipeValidationError, PipeValidationErrorTypedef sort_pipes_by_dependencies(@@ -21,7 +21,7 @@ def sort_pipes_by_dependencies(For PipeSequence, dependencies follow step order. For others, alphabetical order.Raises:- PipeDefinitionError: If circular dependencies are detected among pipes+ PipeValidationError: If circular dependencies are detected among pipesExample:>>> pipes = {@@ -50,7 +50,7 @@ def sort_pipes_by_dependencies(returnif pipe_code in visiting:msg = f"Circular dependency detected involving pipe: {pipe_code}"- raise PipeDefinitionError(message=msg)+ raise PipeValidationError(message=msg, pipe_code=pipe_code, error_type=PipeValidationErrorType.CIRCULAR_DEPENDENCY_ERROR)if pipe_code not in pipes:# Dependency not in this bundle, skip itreturndiff --git a/pipelex/core/bundles/pipelex_bundle_blueprint.py b/pipelex/core/bundles/pipelex_bundle_blueprint.pyindex 06c75ca62..5e046061b 100644--- a/pipelex/core/bundles/pipelex_bundle_blueprint.py+++ b/pipelex/core/bundles/pipelex_bundle_blueprint.py@@ -2,7 +2,6 @@ from typing import Annotatedfrom pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator-from pipelex.core.bundles.exceptions import PipelexBundleBlueprintValueErrorfrom pipelex.core.concepts.concept_blueprint import ConceptBlueprintfrom pipelex.core.domains.exceptions import DomainCodeErrorfrom pipelex.core.domains.validation import validate_domain_code@@ -51,14 +50,14 @@ class PipelexBundleBlueprint(BaseModel):validate_domain_code(code=domain)except DomainCodeError as exc:msg = f"Error when trying to validate the pipelex bundle at domain '{domain}': {exc}"- raise PipelexBundleBlueprintValueError(msg) from exc+ raise ValueError(msg) from excreturn domain@model_validator(mode="after")def validate_main_pipe(self) -> "PipelexBundleBlueprint":if self.main_pipe and (not self.pipe or (self.main_pipe not in self.pipe)):msg = f"Main pipe '{self.main_pipe}' could not be found in pipelex bundle at source '{self.source}' and domain '{self.domain}'"- raise PipelexBundleBlueprintValueError(msg)+ raise ValueError(msg)return self@propertydiff --git a/pipelex/core/concepts/concept.py b/pipelex/core/concepts/concept.pyindex 4f6727bb0..67f26a4bb 100644--- a/pipelex/core/concepts/concept.py+++ b/pipelex/core/concepts/concept.py@@ -28,19 +28,6 @@ class Concept(BaseModel):structure_class_name: strrefines: str | None = None- @property- def concept_string(self) -> str:- return f"{self.domain}.{self.code}"-- @classmethod- def is_implicit_concept(cls, concept_string: str) -> bool:- try:- validate_concept_string(concept_string=concept_string)- except ConceptStringError as exc:- msg = f"Concept string '{concept_string}' is not a valid concept string for concept '{cls.concept_string}'"- raise ConceptValueError(msg) from exc- return concept_string.startswith(SpecialDomain.IMPLICIT)-@field_validator("code")@classmethoddef validate_code(cls, code: str) -> str:@@ -73,6 +60,19 @@ class Concept(BaseModel):raise ConceptValueError(msg) from excreturn refines+ @property+ def concept_string(self) -> str:+ return f"{self.domain}.{self.code}"++ @classmethod+ def is_implicit_concept(cls, concept_string: str) -> bool:+ try:+ validate_concept_string(concept_string=concept_string)+ except ConceptStringError as exc:+ msg = f"Concept string '{concept_string}' is not a valid concept string for concept '{cls.concept_string}'"+ raise ConceptValueError(msg) from exc+ return concept_string.startswith(SpecialDomain.IMPLICIT)+@classmethoddef sentence_from_concept(cls, concept: "Concept") -> str:return pascal_case_to_sentence(name=concept.code)diff --git a/pipelex/core/concepts/concept_blueprint.py b/pipelex/core/concepts/concept_blueprint.pyindex 87cf305db..911358f3b 100644--- a/pipelex/core/concepts/concept_blueprint.py+++ b/pipelex/core/concepts/concept_blueprint.py@@ -3,7 +3,6 @@ from typing import Anyfrom pydantic import BaseModel, ConfigDict, field_validator, model_validatorfrom pipelex.core.concepts.concept_structure_blueprint import ConceptStructureBlueprint-from pipelex.core.concepts.exceptions import ConceptBlueprintValueErrorfrom pipelex.core.concepts.native.concept_native import NativeConceptCodefrom pipelex.core.concepts.native.exceptions import NativeConceptDefinitionError@@ -26,7 +25,7 @@ class ConceptBlueprint(BaseModel):NativeConceptCode.validate_native_concept_string_or_code(concept_string_or_code=refines)except NativeConceptDefinitionError as exc:msg = f"Could not validate refine '{refines}': {exc}"- raise ConceptBlueprintValueError(msg) from exc+ raise ValueError(msg) from excreturn refines@model_validator(mode="before")@@ -37,5 +36,5 @@ class ConceptBlueprint(BaseModel):f"Forbidden to have refines and structure at the same time: `{values.get('refines')}` "f"and `{values.get('structure')}` for concept that has the definition `{values.get('description')}`")- raise ConceptBlueprintValueError(msg)+ raise ValueError(msg)return valuesdiff --git a/pipelex/core/concepts/concept_factory.py b/pipelex/core/concepts/concept_factory.pyindex d0a8c0dae..838ba3e9b 100644--- a/pipelex/core/concepts/concept_factory.py+++ b/pipelex/core/concepts/concept_factory.py@@ -8,10 +8,10 @@ from pipelex.core.concepts.exceptions import (ConceptFactoryError,ConceptRefineError,ConceptStringError,- ConceptStructureGeneratorError,)from pipelex.core.concepts.native.concept_native import NativeConceptCode-from pipelex.core.concepts.structure_generator import StructureGenerator+from pipelex.core.concepts.structure_generation.exceptions import ConceptStructureGeneratorError+from pipelex.core.concepts.structure_generation.generator import StructureGeneratorfrom pipelex.core.concepts.validation import is_concept_code_valid, validate_concept_string_or_codefrom pipelex.core.domains.domain import SpecialDomainfrom pipelex.core.stuffs.text_content import TextContentdiff --git a/pipelex/core/concepts/concept_structure_blueprint.py b/pipelex/core/concepts/concept_structure_blueprint.pyindex ae85b0598..b8f9bfa71 100644--- a/pipelex/core/concepts/concept_structure_blueprint.py+++ b/pipelex/core/concepts/concept_structure_blueprint.py@@ -3,7 +3,6 @@ from typing import Anyfrom pydantic import BaseModel, Field, model_validator-from pipelex.core.concepts.exceptions import ConceptStructureBlueprintValueErrorfrom pipelex.types import Self, StrEnum@@ -35,16 +34,16 @@ class ConceptStructureBlueprint(BaseModel):# If type is None (array), choices must not be Noneif self.type is None and not self.choices:msg = f"When type is None (array), choices must not be empty. Actual type: {self.type}, choices: {self.choices}"- raise ConceptStructureBlueprintValueError(msg)+ raise ValueError(msg)# If type is "dict", key_type and value_type must not be emptyif self.type == ConceptStructureBlueprintFieldType.DICT:if not self.key_type:msg = f"When type is '{ConceptStructureBlueprintFieldType.DICT}', key_type must not be empty. Actual key_type: {self.key_type}"- raise ConceptStructureBlueprintValueError(msg)+ raise ValueError(msg)if not self.value_type:msg = f"When type is '{ConceptStructureBlueprintFieldType.DICT}', value_type must not be empty. Actual value_type: {self.value_type}"- raise ConceptStructureBlueprintValueError(msg)+ raise ValueError(msg)# Check when default_value is not None, type is not None (except for choice fields)if self.default_value is not None and self.type is None and not self.choices:@@ -52,7 +51,7 @@ class ConceptStructureBlueprint(BaseModel):f"When default_value is not None, type must be specified (unless choices are provided). Actual type: {self.type},"f"default_value: {self.default_value}, choices: {self.choices}")- raise ConceptStructureBlueprintValueError(msg)+ raise ValueError(msg)# Check default_value type is the same as typeif self.default_value is not None and self.type is not None:@@ -62,7 +61,7 @@ class ConceptStructureBlueprint(BaseModel):if self.default_value is not None and self.type is None and self.choices:if self.default_value not in self.choices:msg = f"default_value must be one of the valid choices. Got '{self.default_value}', valid choices: {self.choices}"- raise ConceptStructureBlueprintValueError(msg)+ raise ValueError(msg)return self@@ -95,4 +94,4 @@ class ConceptStructureBlueprint(BaseModel):def _raise_type_mismatch_error(self, expected_type_name: str, actual_type_name: str) -> None:msg = f"default_value type mismatch: expected {expected_type_name} for type '{self.type}', but got {actual_type_name}"- raise ConceptStructureBlueprintValueError(msg)+ raise ValueError(msg)diff --git a/pipelex/core/concepts/exceptions.py b/pipelex/core/concepts/exceptions.pyindex 776844e40..e5aba7c60 100644--- a/pipelex/core/concepts/exceptions.py+++ b/pipelex/core/concepts/exceptions.py@@ -1,35 +1,18 @@from pipelex.base_exceptions import PipelexError-from pipelex.core.exceptions import SyntaxErrorDataclass ConceptError(PipelexError):pass-class ConceptBlueprintValueError(ValueError):- pass--class ConceptValueError(ValueError):pass-class ConceptStructureBlueprintValueError(ValueError):- pass---class ConceptStructureValidationError(PipelexError):- pass--class ConceptFactoryError(PipelexError):pass-class StructureClassError(ConceptFactoryError):- pass--class ConceptCodeError(ConceptError):pass@@ -44,10 +27,3 @@ class ConceptRefineError(ConceptError):class ConceptLibraryConceptNotFoundError(PipelexError):pass---class ConceptStructureGeneratorError(PipelexError):- def __init__(self, message: str, structure_class_python_code: str | None = None, syntax_error_data: SyntaxErrorData | None = None):- self.structure_class_python_code = structure_class_python_code- self.syntax_error_data = syntax_error_data- super().__init__(message)diff --git a/pipelex/core/concepts/native/exceptions.py b/pipelex/core/concepts/native/exceptions.pyindex 02d266c91..4d7fe146f 100644--- a/pipelex/core/concepts/native/exceptions.py+++ b/pipelex/core/concepts/native/exceptions.py@@ -3,7 +3,3 @@ from pipelex.base_exceptions import PipelexErrorclass NativeConceptDefinitionError(PipelexError):pass---class NativeConceptEnumError(PipelexError):- passdiff --git a/pipelex/core/concepts/structure_generation/__init__.py b/pipelex/core/concepts/structure_generation/__init__.pynew file mode 100644index 000000000..e69de29bbdiff --git a/pipelex/core/concepts/structure_generation/exceptions.py b/pipelex/core/concepts/structure_generation/exceptions.pynew file mode 100644index 000000000..57d9b16e5--- /dev/null+++ b/pipelex/core/concepts/structure_generation/exceptions.py@@ -0,0 +1,39 @@+from pydantic import BaseModel++from pipelex.base_exceptions import PipelexError+from pipelex.core.concepts.exceptions import ConceptFactoryError+++class SyntaxErrorData(BaseModel):+ message: str+ lineno: int | None = None+ offset: int | None = None+ text: str | None = None+ end_lineno: int | None = None+ end_offset: int | None = None++ @classmethod+ def from_syntax_error(cls, syntax_error: SyntaxError) -> "SyntaxErrorData":+ return cls(+ message=syntax_error.msg,+ lineno=syntax_error.lineno,+ offset=syntax_error.offset,+ text=syntax_error.text,+ end_lineno=syntax_error.end_lineno,+ end_offset=syntax_error.end_offset,+ )+++class StructureClassError(ConceptFactoryError):+ pass+++class ConceptStructureGeneratorError(PipelexError):+ def __init__(self, message: str, structure_class_python_code: str | None = None, syntax_error_data: SyntaxErrorData | None = None):+ self.structure_class_python_code = structure_class_python_code+ self.syntax_error_data = syntax_error_data+ super().__init__(message)+++class ConceptStructureValidationError(PipelexError):+ passdiff --git a/pipelex/core/concepts/structure_generator.py b/pipelex/core/concepts/structure_generation/generator.pysimilarity index 99%rename from pipelex/core/concepts/structure_generator.pyrename to pipelex/core/concepts/structure_generation/generator.pyindex 7311c4701..3abcf24b3 100644--- a/pipelex/core/concepts/structure_generator.py+++ b/pipelex/core/concepts/structure_generation/generator.py@@ -6,7 +6,7 @@ from typing import Any, Literal, Optionalfrom pydantic import Fieldfrom pipelex.core.concepts.concept_structure_blueprint import ConceptStructureBlueprint, ConceptStructureBlueprintFieldType-from pipelex.core.concepts.exceptions import ConceptStructureGeneratorError, ConceptStructureValidationError, SyntaxErrorData+from pipelex.core.concepts.structure_generation.exceptions import ConceptStructureGeneratorError, ConceptStructureValidationError, SyntaxErrorDatafrom pipelex.core.stuffs.structured_content import StructuredContentdiff --git a/pipelex/core/exceptions.py b/pipelex/core/exceptions.pyindex 10f679997..17d7c49e0 100644--- a/pipelex/core/exceptions.py+++ b/pipelex/core/exceptions.py@@ -1,89 +1,36 @@-from pydantic import BaseModel-from typing_extensions import override+from pydantic import BaseModel, Field-from pipelex.base_exceptions import PipelexError-from pipelex.core.bundles.exceptions import PipelexBundleBlueprintValidationErrorData, PipeValidationErrorType-from pipelex.tools.misc.toml_utils import TomlError+from pipelex.core.pipes.exceptions import PipeValidationErrorType-class PipelexConfigurationError(PipelexError):- """Raised when there are configuration issues with the PipelexInterpreter."""+class PipesAndConceptValidationErrorData(BaseModel):+ """Structured validation error data for Pipe/Concept validation errors.+ This model captures validation errors raised by Pipe or Concept classes during+ their validation (NOT blueprint validation errors).-class SyntaxErrorData(BaseModel):- message: str- lineno: int | None = None- offset: int | None = None- text: str | None = None- end_lineno: int | None = None- end_offset: int | None = None+ These errors come from:+ - PipeAbstract and its subclasses (PipeLLM, PipeExtract, etc.)+ - Concept validation+ """- @classmethod- def from_syntax_error(cls, syntax_error: SyntaxError) -> "SyntaxErrorData":- return cls(- message=syntax_error.msg,- lineno=syntax_error.lineno,- offset=syntax_error.offset,- text=syntax_error.text,- end_lineno=syntax_error.end_lineno,- end_offset=syntax_error.end_offset,- )+ # === Source Context ===+ domain: str | None = Field(None, description="Domain where error occurred")+ source: str | None = Field(None, description="Source file path")+ # === Entity Context (what failed) ===+ pipe_code: str | None = Field(None, description="Pipe code if error is in a pipe")+ concept_code: str | None = Field(None, description="Concept code if error is in a concept")+ field_name: str | None = Field(None, description="Specific field that failed")-class PipeValidationError(ValueError):- def __init__(- self,- error_type: PipeValidationErrorType,- domain: str | None = None,- pipe_code: str | None = None,- variable_names: list[str] | None = None,- required_concept_codes: list[str] | None = None,- provided_concept_code: str | None = None,- file_path: str | None = None,- explanation: str | None = None,- ):- self.error_type = error_type- self.domain = domain- self.pipe_code = pipe_code- self.variable_names = variable_names- self.required_concept_codes = required_concept_codes- self.provided_concept_code = provided_concept_code- self.file_path = file_path- self.explanation = explanation- super().__init__()+ # === Error Classification ===+ error_type: PipeValidationErrorType = Field(+ description="Type of pipe/concept validation error",+ )- def desc(self) -> str:- msg = f"{self.error_type} β’ domain='{self.domain}'"- if self.pipe_code:- msg += f" β’ pipe='{self.pipe_code}'"- if self.variable_names:- msg += f" β’ variable='{self.variable_names}'"- if self.required_concept_codes:- msg += f" β’ required_concept_codes='{self.required_concept_codes}'"- if self.provided_concept_code:- msg += f" β’ provided_concept_code='{self.provided_concept_code}'"- if self.file_path:- msg += f" β’ file='{self.file_path}'"- if self.explanation:- msg += f" β’ explanation='{self.explanation}'"- return msg+ # === Error Details ===+ message: str = Field(description="Human-readable error message")+ field_path: str = Field(description="Path to field in dot notation")- @override- def __str__(self) -> str:- return self.desc()---class PLXDecodeError(TomlError):- """Raised when PLX decoding fails."""---class PipelexInterpreterError(PipelexError):- """Raised when PipelexInterpreter fails."""-- def __init__(- self,- message: str,- validation_errors: list[PipelexBundleBlueprintValidationErrorData] | None = None,- ):- self.validation_errors = validation_errors or []- super().__init__(message)+ # === Variable names for input/output errors ===+ variable_names: list[str] | None = Field(None, description="Variable names (for input errors)")diff --git a/pipelex/core/interpreter/__init__.py b/pipelex/core/interpreter/__init__.pynew file mode 100644index 000000000..e69de29bbdiff --git a/pipelex/core/interpreter/exceptions.py b/pipelex/core/interpreter/exceptions.pynew file mode 100644index 000000000..6b5c4125b--- /dev/null+++ b/pipelex/core/interpreter/exceptions.py@@ -0,0 +1,19 @@+from pipelex.base_exceptions import PipelexError+from pipelex.core.bundles.exceptions import PipelexBundleBlueprintValidationErrorData+from pipelex.tools.misc.toml_utils import TomlError+++class PipelexInterpreterError(PipelexError):+ """Raised when PipelexInterpreter fails."""++ def __init__(+ self,+ message: str,+ validation_errors: list[PipelexBundleBlueprintValidationErrorData] | None = None,+ ):+ self.validation_errors = validation_errors or []+ super().__init__(message)+++class PLXDecodeError(TomlError):+ """Raised when PLX decoding fails."""diff --git a/pipelex/core/interpreter/helpers.py b/pipelex/core/interpreter/helpers.pynew file mode 100644index 000000000..04d108aa4--- /dev/null+++ b/pipelex/core/interpreter/helpers.py@@ -0,0 +1,59 @@+from pipelex.types import StrEnum+++class ValidationErrorScope(StrEnum):+ PIPE = "pipe"+ CONCEPT = "concept"+ DOMAIN = "domain"+ MAIN_PIPE = "main_pipe"+ BUNDLE = "bundle"++ @classmethod+ def is_pipe_scope(cls, scope: str) -> bool:+ try:+ scope_enum = cls(scope)+ except ValueError:+ return False+ match scope_enum:+ case ValidationErrorScope.PIPE:+ return True+ case ValidationErrorScope.CONCEPT:+ return False+ case ValidationErrorScope.DOMAIN:+ return False+ case ValidationErrorScope.MAIN_PIPE:+ return False+ case ValidationErrorScope.BUNDLE:+ return False+++def get_error_scope(loc: tuple[int | str, ...]) -> ValidationErrorScope:+ """Get the scope of a validation error from its location tuple.++ Args:+ loc: Location tuple from Pydantic validation error++ Returns:+ ValidationErrorScope - defaults to BUNDLE if scope cannot be determined+ """+ if not loc:+ return ValidationErrorScope.BUNDLE++ scope = str(loc[0])++ try:+ scope_enum = ValidationErrorScope(scope)+ except ValueError:+ return ValidationErrorScope.BUNDLE++ match scope_enum:+ case ValidationErrorScope.PIPE:+ return ValidationErrorScope.PIPE+ case ValidationErrorScope.CONCEPT:+ return ValidationErrorScope.CONCEPT+ case ValidationErrorScope.DOMAIN:+ return ValidationErrorScope.DOMAIN+ case ValidationErrorScope.MAIN_PIPE:+ return ValidationErrorScope.MAIN_PIPE+ case ValidationErrorScope.BUNDLE:+ return ValidationErrorScope.BUNDLEdiff --git a/pipelex/core/interpreter.py b/pipelex/core/interpreter/interpreter.pysimilarity index 70%rename from pipelex/core/interpreter.pyrename to pipelex/core/interpreter/interpreter.pyindex b2b00d747..4f47ec237 100644--- a/pipelex/core/interpreter.py+++ b/pipelex/core/interpreter/interpreter.py@@ -1,12 +1,16 @@from pathlib import Path-from typing import Any+from typing import TYPE_CHECKING, Anyfrom pydantic import BaseModel, ValidationErrorfrom pipelex.core.bundles.pipelex_bundle_blueprint import PipelexBundleBlueprint-from pipelex.core.exceptions import PipelexBundleBlueprintValidationErrorData, PipelexInterpreterError, PLXDecodeError-from pipelex.core.validation_error_categorizer import categorize_and_create_error_data+from pipelex.core.interpreter.exceptions import PipelexInterpreterError, PLXDecodeError+from pipelex.core.interpreter.validation_error_categorizer import categorize_blueprint_validation_errorfrom pipelex.tools.misc.toml_utils import TomlError, load_toml_from_content, load_toml_from_path+from pipelex.tools.typing.pydantic_utils import format_pydantic_validation_error++if TYPE_CHECKING:+ from pipelex.core.bundles.exceptions import PipelexBundleBlueprintValidationErrorDataclass PipelexInterpreter(BaseModel):@@ -71,30 +75,14 @@ class PipelexInterpreter(BaseModel):return PipelexBundleBlueprint.model_validate(blueprint_dict)except ValidationError as exc:# TODO: Move this to the validate_bundle function- # Parse Pydantic validation errors into structured error datablueprint_validation_errors: list[PipelexBundleBlueprintValidationErrorData] = []- # Handle domain and source fields carefully for context- domain_value: str | None = blueprint_dict.get("domain") if blueprint_dict else None- source_value: str | None = blueprint_dict.get("source") if blueprint_dict else bundle_path-for error in exc.errors():- loc = error.get("loc", ())-- # Don't use domain/source from blueprint if they're the fields that failed- error_domain = None if (loc and loc[0] == "domain") else domain_value- error_source = bundle_path if (loc and loc[0] == "source") else source_value-- # Categorize error and create structured error data- val_error = categorize_and_create_error_data(- error=error,- blueprint_dict=blueprint_dict,- domain=error_domain,- source=error_source,- )- blueprint_validation_errors.append(val_error)+ categorized_error = categorize_blueprint_validation_error(blueprint_dict=blueprint_dict, error=error)+ if categorized_error:+ blueprint_validation_errors.append(categorized_error)raise PipelexInterpreterError(- message=str(exc),+ message=format_pydantic_validation_error(exc),validation_errors=blueprint_validation_errors,) from excdiff --git a/pipelex/core/interpreter/validation_error_categorizer.py b/pipelex/core/interpreter/validation_error_categorizer.pynew file mode 100644index 000000000..95a8548b7--- /dev/null+++ b/pipelex/core/interpreter/validation_error_categorizer.py@@ -0,0 +1,43 @@+from typing import Any, cast++from pydantic_core import ErrorDetails++from pipelex import log+from pipelex.core.bundles.exceptions import (+ PipelexBundleBlueprintValidationErrorData,+)+from pipelex.core.interpreter.helpers import get_error_scope+from pipelex.types import StrEnum+++class ErrorCatKey(StrEnum):+ LOC = "loc"+ MSG = "msg"+ TYPE = "type"+++PIPELEX_BUNDLE_BLUEPRINT_DOMAIN_FIELD = "domain"+PIPELEX_BUNDLE_BLUEPRINT_SOURCE_FIELD = "source"+++def categorize_blueprint_validation_error(+ blueprint_dict: dict[str, Any],+ error: ErrorDetails,+) -> PipelexBundleBlueprintValidationErrorData | None:+ """Categorize a BLUEPRINT validation error and create structured error data or return None if the error cannot be categorized.++ Args:+ blueprint_dict: The blueprint dict being validated (for context extraction)+ error: Pydantic error from PipelexBundleBlueprint.model_validate()++ Returns:+ PipelexBundleBlueprintValidationErrorData with all relevant fields populated, or None if error cannot be categorized+ """+ domain = cast("str | None", blueprint_dict.get(PIPELEX_BUNDLE_BLUEPRINT_DOMAIN_FIELD)) if blueprint_dict else None+ source = cast("str | None", blueprint_dict.get(PIPELEX_BUNDLE_BLUEPRINT_SOURCE_FIELD)) if blueprint_dict else None++ loc = error.get(ErrorCatKey.LOC.value, ())+ error_scope = get_error_scope(loc)+ log.warning(f"Pipelex bundle blueprint validation error that is not categorized: {error_scope} - {source} - {domain}")++ return Nonediff --git a/pipelex/core/memory/working_memory_factory.py b/pipelex/core/memory/working_memory_factory.pyindex 6f751867f..37b3c338a 100644--- a/pipelex/core/memory/working_memory_factory.py+++ b/pipelex/core/memory/working_memory_factory.py@@ -160,5 +160,4 @@ class WorkingMemoryFactory(BaseModel):code=shortuuid.uuid()[:5],)working_memory.add_new_stuff(name=requirement.variable_name, stuff=fallback_stuff)-return working_memorydiff --git a/pipelex/core/pipe_errors.py b/pipelex/core/pipe_errors.pydeleted file mode 100644index bc4ac8fb2..000000000--- a/pipelex/core/pipe_errors.py+++ /dev/null@@ -1,37 +0,0 @@-from pipelex.base_exceptions import PipelexError---# TODO: add details from all cases raising this error-class PipeDefinitionError(PipelexError):- def __init__(- self,- message: str,- domain_code: str | None = None,- pipe_code: str | None = None,- description: str | None = None,- source: str | None = None,- ):- self.domain_code = domain_code- self.pipe_code = pipe_code- self.description = description- self.source = source- message = message + " β’ " + self.pipe_details()- super().__init__(message)-- def pipe_details(self) -> str:- if not self.domain_code and not self.pipe_code and not self.description and not self.source:- return "No pipe details provided"- details = "Pipe details:"- if self.domain_code:- details += f" β’ domain='{self.domain_code}'"- if self.pipe_code:- details += f" β’ pipe='{self.pipe_code}'"- if self.description:- details += f" β’ description='{self.description}'"- if self.source:- details += f" β’ source='{self.source}'"- return details---class UnexpectedPipeDefinitionError(PipeDefinitionError):- passdiff --git a/pipelex/core/pipes/exceptions.py b/pipelex/core/pipes/exceptions.pyindex ae0adeba8..9b4d55227 100644--- a/pipelex/core/pipes/exceptions.py+++ b/pipelex/core/pipes/exceptions.py@@ -1,4 +1,3 @@-from pydantic import BaseModel, Fieldfrom typing_extensions import overridefrom pipelex.base_exceptions import PipelexError@@ -6,52 +5,17 @@ from pipelex.cogt.extract.extract_setting import ExtractModelChoicefrom pipelex.cogt.img_gen.img_gen_setting import ImgGenModelChoicefrom pipelex.cogt.llm.llm_setting import LLMModelChoicefrom pipelex.cogt.model_backends.model_type import ModelType-from pipelex.core.bundles.exceptions import PipeValidationErrorType---class PipeBlueprintValueError(ValueError):- pass---class PipeInputNotFoundError(PipelexError):- pass+from pipelex.types import StrEnumclass PipeFactoryError(PipelexError):pass-class PipeAbstractValueError(ValueError):- pass--class PipeVariableMultiplicityError(ValueError):pass-class PipeInputError(PipelexError):- def __init__(self, message: str, pipe_code: str, variable_name: str, concept_code: str | None = None):- self.pipe_code = pipe_code- self.variable_name = variable_name- self.concept_code = concept_code- super().__init__(message)---class PipeRunInputsError(PipelexError):- def __init__(self, message: str, pipe_code: str, missing_inputs: dict[str, str]):- self.pipe_code = pipe_code- self.missing_inputs = missing_inputs- super().__init__(message)---class PipeDefinitionErrorData(BaseModel):- message: str = Field(description="The error message")- domain_code: str | None = Field(None, description="The domain code")- pipe_code: str | None = Field(None, description="The pipe code")- description: str | None = Field(None, description="Description of the pipe")- source: str | None = Field(None, description="Source of the error")--class PipeOperatorModelChoiceError(PipelexError):def __init__(self,@@ -87,14 +51,64 @@ class PipeOperatorModelChoiceError(PipelexError):return self.desc()-class PipeValidationErrorData(BaseModel):- """Structured data for PipeValidationError."""+class PipeValidationErrorType(StrEnum):+ """Types of pipe validation errors.++ These error types are raised during pipe validation from Pipe/Concept classes.+ Only some are auto-fixed in the builder loop (marked below).+ """++ # Errors that are auto-fixed in builder_loop.py+ MISSING_INPUT_VARIABLE = "missing_input_variable" # AUTO-FIXED+ EXTRANEOUS_INPUT_VARIABLE = "extraneous_input_variable" # AUTO-FIXED+ INPUT_REQUIREMENT_MISMATCH = "input_requirement_mismatch" # AUTO-FIXED+ INADEQUATE_OUTPUT_CONCEPT = "inadequate_output_concept" # AUTO-FIXED++ CIRCULAR_DEPENDENCY_ERROR = "circular_dependency_error"- error_type: PipeValidationErrorType = Field(description="The type of pipe validation error")- domain: str | None = Field(None, description="The domain where the error occurred")- pipe_code: str | None = Field(None, description="The pipe code if applicable")- variable_names: list[str] | None = Field(None, description="Variable names involved in the error")- required_concept_codes: list[str] | None = Field(None, description="Required concept codes")- provided_concept_code: str | None = Field(None, description="The provided concept code")- file_path: str | None = Field(None, description="The file path where the error occurred")- explanation: str | None = Field(None, description="Additional explanation of the error")+ # Errors that are raised but NOT auto-fixed (will fail validation)+ LLM_OUTPUT_CANNOT_BE_IMAGE = "llm_output_cannot_be_image"+ IMG_GEN_INPUT_NOT_TEXT_COMPATIBLE = "img_gen_input_not_text_compatible"++ # Generic fallback for unexpected validation errors+ UNKNOWN_VALIDATION_ERROR = "unknown_validation_error"+++class PipeValidationError(ValueError):+ def __init__(+ self,+ message: str,+ error_type: PipeValidationErrorType | None = None,+ domain: str | None = None,+ pipe_code: str | None = None,+ variable_names: list[str] | None = None,+ required_concept_codes: list[str] | None = None,+ provided_concept_code: str | None = None,+ file_path: str | None = None,+ explanation: str | None = None,+ ):+ self.error_type = error_type+ self.domain = domain+ self.pipe_code = pipe_code+ self.variable_names = variable_names+ self.required_concept_codes = required_concept_codes+ self.provided_concept_code = provided_concept_code+ self.file_path = file_path+ self.explanation = explanation+ super().__init__(message)++ def desc(self) -> str:+ msg = f"{self.error_type} β’ domain='{self.domain}'"+ if self.pipe_code:+ msg += f" β’ pipe='{self.pipe_code}'"+ if self.variable_names:+ msg += f" β’ variable='{self.variable_names}'"+ if self.required_concept_codes:+ msg += f" β’ required_concept_codes='{self.required_concept_codes}'"+ if self.provided_concept_code:+ msg += f" β’ provided_concept_code='{self.provided_concept_code}'"+ if self.file_path:+ msg += f" β’ file='{self.file_path}'"+ if self.explanation:+ msg += f" β’ explanation='{self.explanation}'"+ return msgdiff --git a/pipelex/core/pipes/handle_pipe_errors.py b/pipelex/core/pipes/handle_pipe_errors.pynew file mode 100644index 000000000..a80979231--- /dev/null+++ b/pipelex/core/pipes/handle_pipe_errors.py@@ -0,0 +1,165 @@+from enum import Enum++from pydantic import ValidationError+from pydantic_core import ErrorDetails++from pipelex.core.exceptions import PipesAndConceptValidationErrorData+from pipelex.core.interpreter.validation_error_categorizer import ErrorCatKey+from pipelex.core.pipes.exceptions import PipeValidationError, PipeValidationErrorType+++class ModelScope(str, Enum):+ """Indicates which type of model is being validated."""++ PIPE = "pipe"+ CONCEPT = "concept"+++def categorize_pipe_validation_error(+ validation_error: ValidationError,+) -> list[PipesAndConceptValidationErrorData]:+ """Categorize all errors from a ValidationError for Pipe instantiation.++ This function determines the model scope (PIPE or CONCEPT) from the ValidationError+ and processes all errors accordingly.++ Args:+ validation_error: Pydantic ValidationError from Pipe model validation++ Returns:+ List of PipesAndConceptValidationErrorData for all errors+ """+ # Determine model scope by examining field names in errors+ # Pipe-specific fields: type, inputs, output, pipe_category+ # Concept-specific fields: structure_class_name, refines+ errors = validation_error.errors()++ model_scope = ModelScope.PIPE+ if errors:+ # Check first error's field name to determine model type+ first_error = errors[0]+ loc = first_error.get(ErrorCatKey.LOC.value, ())+ # TODO: Refactor how to determine model scope (use error codes)+ if loc:+ field_name = str(loc[0])+ # Concept-specific fields+ if field_name in ("structure_class_name", "refines"):+ model_scope = ModelScope.CONCEPT+ # Pipe-specific fields+ elif field_name in ("type", "inputs", "output", "pipe_category"):+ model_scope = ModelScope.PIPE+ # Ambiguous fields - try to infer from error message+ else:+ error_msg = str(validation_error).lower()+ if "concept" in error_msg and "pipe" not in error_msg:+ model_scope = ModelScope.CONCEPT++ # Process all errors+ categorized_errors: list[PipesAndConceptValidationErrorData] = []+ for error in errors:+ if model_scope == ModelScope.PIPE:+ categorized_error = _handle_pipe_errors(+ error=error,+ pipe_code=None, # We don't have the code at this point+ )+ else:+ loc = error.get(ErrorCatKey.LOC.value, ())+ message = error.get(ErrorCatKey.MSG.value, "Unknown validation error")+ field_path_str = " β ".join(str(item) for item in loc)+ unknown_field_name = str(loc[0]) if len(loc) >= 1 else None++ categorized_error = PipesAndConceptValidationErrorData(+ domain=None,+ source=None,+ pipe_code=None,+ concept_code=None, # We don't have the code at this point+ field_name=unknown_field_name,+ error_type=PipeValidationErrorType.UNKNOWN_VALIDATION_ERROR,+ message=message,+ field_path=field_path_str,+ variable_names=None,+ )+ categorized_errors.append(categorized_error)+ return categorized_errors+++def _handle_pipe_errors(+ error: ErrorDetails,+ pipe_code: str | None,+) -> PipesAndConceptValidationErrorData:+ """Handle all PIPE validation errors.++ Extracts all necessary context from error and categorizes the error type.++ Args:+ error: Pydantic error details+ pipe_code: The pipe code being validated (if known)++ Returns:+ PipesAndConceptValidationErrorData with all context populated+ """+ # Extract data from error+ loc = error.get(ErrorCatKey.LOC.value, ())+ message = error.get(ErrorCatKey.MSG.value, "Unknown validation error")+ pydantic_type = error.get(ErrorCatKey.TYPE.value, "")+ field_path = " β ".join(str(item) for item in loc)++ # Extract field_name from loc[0] (direct field on Pipe model)+ field_name = str(loc[0]) if len(loc) >= 1 else None++ # Extract variable names from loc (for input/output errors)+ variable_names = [str(item) for item in loc]++ # Determine error type - default to UNKNOWN_VALIDATION_ERROR+ error_type = PipeValidationErrorType.UNKNOWN_VALIDATION_ERROR++ # Try to categorize based on error message patterns+ message_lower = message.lower()++ if "missing" in message_lower or "required" in pydantic_type:+ # Could be MISSING_INPUT_VARIABLE but we can't tell for sure from generic Pydantic errors+ # The specific error type should come from PipeValidationError exceptions instead+ error_type = PipeValidationErrorType.UNKNOWN_VALIDATION_ERROR+ elif "extra" in message_lower or "forbidden" in pydantic_type:+ error_type = PipeValidationErrorType.UNKNOWN_VALIDATION_ERROR++ return PipesAndConceptValidationErrorData(+ domain=None,+ source=None,+ pipe_code=pipe_code,+ concept_code=None,+ field_name=field_name,+ error_type=error_type,+ message=message,+ field_path=field_path,+ variable_names=variable_names,+ )+++def categorize_pipe_validation_with_libraries_error(+ pipe_error: PipeValidationError,+) -> PipesAndConceptValidationErrorData:+ """Categorize a PipeValidationError with libraries and create structured error data.++ Args:+ pipe_error: PipeValidationError with libraries and structured error information++ Returns:+ PipesAndConceptValidationErrorData with all relevant fields populated+ """+ message = pipe_error.explanation or str(pipe_error)+ if pipe_error.required_concept_codes and pipe_error.provided_concept_code:+ message += f" (required: {pipe_error.required_concept_codes}, provided: {pipe_error.provided_concept_code})"++ error_type = pipe_error.error_type or PipeValidationErrorType.UNKNOWN_VALIDATION_ERROR+ return PipesAndConceptValidationErrorData(+ error_type=error_type,+ domain=pipe_error.domain,+ source=pipe_error.file_path or None,+ pipe_code=pipe_error.pipe_code,+ concept_code=None, # This is a pipe error, not a concept error+ field_name=None, # Field name is not provided in PipeValidationError+ message=message,+ field_path=pipe_error.file_path or "",+ variable_names=pipe_error.variable_names,+ )diff --git a/pipelex/core/pipes/inputs/exceptions.py b/pipelex/core/pipes/inputs/exceptions.pynew file mode 100644index 000000000..ea418c91e--- /dev/null+++ b/pipelex/core/pipes/inputs/exceptions.py@@ -0,0 +1,20 @@+from pipelex.base_exceptions import PipelexError+++class PipeRunInputsError(PipelexError):+ def __init__(self, message: str, pipe_code: str, missing_inputs: dict[str, str]):+ self.pipe_code = pipe_code+ self.missing_inputs = missing_inputs+ super().__init__(message)+++class PipeInputError(PipelexError):+ def __init__(self, message: str, pipe_code: str, variable_name: str, concept_code: str | None = None):+ self.pipe_code = pipe_code+ self.variable_name = variable_name+ self.concept_code = concept_code+ super().__init__(message)+++class PipeInputNotFoundError(PipelexError):+ passdiff --git a/pipelex/core/pipes/inputs/input_requirements.py b/pipelex/core/pipes/inputs/input_requirements.pyindex b733aeaec..5f4eb61f5 100644--- a/pipelex/core/pipes/inputs/input_requirements.py+++ b/pipelex/core/pipes/inputs/input_requirements.py@@ -4,7 +4,7 @@ from pydantic import BaseModel, Field, RootModel, field_validatorfrom pipelex import logfrom pipelex.core.concepts.concept import Concept-from pipelex.core.pipes.exceptions import PipeInputNotFoundError+from pipelex.core.pipes.inputs.exceptions import PipeInputNotFoundErrorfrom pipelex.core.pipes.variable_multiplicity import VariableMultiplicityfrom pipelex.core.stuffs.stuff_content import StuffContent@@ -77,6 +77,9 @@ class InputRequirements(RootModel[InputRequirementsRoot]):raise PipeInputNotFoundError(msg)return requirement+ def is_variable_existing(self, variable_name: str) -> bool:+ return variable_name in self.root+def add_requirement(self, variable_name: str, concept: Concept, multiplicity: VariableMultiplicity | None = None):self.root[variable_name] = InputRequirement(concept=concept, multiplicity=multiplicity)diff --git a/pipelex/core/pipes/pipe_abstract.py b/pipelex/core/pipes/pipe_abstract.pyindex a0431db67..b5eaaad7c 100644--- a/pipelex/core/pipes/pipe_abstract.py+++ b/pipelex/core/pipes/pipe_abstract.py@@ -4,14 +4,11 @@ from typing import Any, finalfrom pydantic import BaseModel, ConfigDict, Field, field_validator, model_validatorfrom pipelex import log-from pipelex.cogt.exceptions import ModelChoiceNotFoundError-from pipelex.core.bundles.exceptions import PipeValidationErrorTypefrom pipelex.core.concepts.concept import Conceptfrom pipelex.core.concepts.native.concept_native import NativeConceptCode-from pipelex.core.exceptions import PipeValidationErrorfrom pipelex.core.memory.working_memory import WorkingMemory-from pipelex.core.pipe_errors import PipeDefinitionError-from pipelex.core.pipes.exceptions import PipeAbstractValueError, PipeRunInputsError+from pipelex.core.pipes.exceptions import PipeValidationError, PipeValidationErrorType+from pipelex.core.pipes.inputs.exceptions import PipeRunInputsErrorfrom pipelex.core.pipes.inputs.input_requirements import InputRequirementsfrom pipelex.core.pipes.pipe_blueprint import PipeCategory, PipeTypefrom pipelex.core.pipes.pipe_output import PipeOutput@@ -43,7 +40,7 @@ class PipeAbstract(ABC, BaseModel):def validate_pipe_code_syntax(cls, code: str) -> str:if not is_snake_case(code):msg = f"Invalid pipe code syntax '{code}'. Must be in snake_case."- raise PipeDefinitionError(msg)+ raise ValueError(msg)return code@field_validator("type", mode="after")@@ -88,12 +85,8 @@ class PipeAbstract(ABC, BaseModel):@finaldef validate_with_libraries(self):- try:- self.generic_validate_inputs_with_library()- self.generic_validate_output_with_library()- except ModelChoiceNotFoundError as exc:- msg = f"Model choice not found for pipe '{self.code}': {exc}"- raise PipeAbstractValueError(msg) from exc+ self.generic_validate_inputs_with_library()+ self.generic_validate_output_with_library()@finaldef generic_validate_inputs_static(self):@@ -108,14 +101,13 @@ class PipeAbstract(ABC, BaseModel):# First validate required variables are in the inputsfor required_variable_name in self.required_variables():if required_variable_name not in self.inputs.variables:+ msg = f"Required variable '{required_variable_name}' is not in the inputs of pipe '{self.code}'. Current inputs: {self.inputs}"raise PipeValidationError(+ message=msg,error_type=PipeValidationErrorType.MISSING_INPUT_VARIABLE,domain=self.domain,pipe_code=self.code,variable_names=[required_variable_name],- explanation=(- f"Required variable '{required_variable_name}' is not in the inputs of pipe '{self.code}'. Current inputs: {self.inputs}"- ),)# Then validate that all inputs are actually needed and match requirements exactly@@ -126,12 +118,13 @@ class PipeAbstract(ABC, BaseModel):var_name = named_input_requirement.variable_nameif var_name not in self.inputs.variables:+ msg = f"Required variable '{var_name}' is not in the inputs of pipe '{self.code}'. Current inputs: {self.inputs}"raise PipeValidationError(+ message=msg,error_type=PipeValidationErrorType.MISSING_INPUT_VARIABLE,domain=self.domain,pipe_code=self.code,variable_names=[var_name],- explanation=(f"Required variable '{var_name}' is not in the inputs of pipe '{self.code}'. Current inputs: {self.inputs}"),)# TODO: add this to the PipeController validation. (This might need to refactor a little bit how we can override the validation)@@ -146,27 +139,40 @@ class PipeAbstract(ABC, BaseModel):needed_requirement.concept.code not in (NativeConceptCode.DYNAMIC, NativeConceptCode.ANYTHING)and declared_requirement != needed_requirement):+ # Identify the specific mismatched field(s)+ mismatch_details: list[str] = []+ if declared_requirement.concept != needed_requirement.concept:+ mismatch_details.append(f"concept: declared='{declared_requirement.concept}' vs required='{needed_requirement.concept}'")+ if declared_requirement.multiplicity != needed_requirement.multiplicity:+ mismatch_details.append(+ f"multiplicity: declared='{declared_requirement.multiplicity}' vs required='{needed_requirement.multiplicity}'"+ )++ mismatch_summary = ", ".join(mismatch_details)+ msg = (+ f"In the pipe '{self.code}', the input variable '{var_name}' has a requirement mismatch.\n"+ f"Mismatched field(s): {mismatch_summary}\n"+ f"Declared: {declared_requirement}\n"+ f"Required: {needed_requirement}"+ )raise PipeValidationError(+ message=msg,error_type=PipeValidationErrorType.INPUT_REQUIREMENT_MISMATCH,domain=self.domain,pipe_code=self.code,variable_names=[var_name],- explanation=(- f"Input variable '{var_name}' requirement mismatch in pipe '{self.code}'.\n"- f"Declared: input requirement {declared_requirement}.\n"- f"Required: input requirement {needed_requirement}"- ),)# Check that all declared inputs are actually neededfor input_name in self.inputs.variables:if input_name not in the_needed_inputs.required_names:+ msg = f"Extraneous input '{input_name}' found in the inputs of pipe {self.code}"raise PipeValidationError(+ message=msg,error_type=PipeValidationErrorType.EXTRANEOUS_INPUT_VARIABLE,domain=self.domain,pipe_code=self.code,variable_names=[input_name],- explanation=f"Extraneous input '{input_name}' found in the inputs of pipe {self.code}",)self.validate_inputs_with_library()@@ -191,7 +197,7 @@ class PipeAbstract(ABC, BaseModel):def validate_output_static(self):pass- def validate_before_run(+ async def validate_before_run(self,job_metadata: JobMetadata,working_memory: WorkingMemory,@@ -199,7 +205,7 @@ class PipeAbstract(ABC, BaseModel):output_name: str | None = None,): ...- def validate_after_run(+ async def validate_after_run(self,job_metadata: JobMetadata,working_memory: WorkingMemory,@@ -294,13 +300,17 @@ class PipeAbstract(ABC, BaseModel):if pipe_run_params.run_mode == PipeRunMode.LIVE:log.info(pipe_run_info)- self.validate_before_run(job_metadata=job_metadata, working_memory=working_memory, pipe_run_params=pipe_run_params, output_name=output_name)+ await self.validate_before_run(+ job_metadata=job_metadata, working_memory=working_memory, pipe_run_params=pipe_run_params, output_name=output_name+ )pipe_output = await self._run_pipe(job_metadata=job_metadata, working_memory=working_memory, pipe_run_params=pipe_run_params, output_name=output_name)- self.validate_after_run(job_metadata=job_metadata, working_memory=working_memory, pipe_run_params=pipe_run_params, output_name=output_name)+ await self.validate_after_run(+ job_metadata=job_metadata, working_memory=working_memory, pipe_run_params=pipe_run_params, output_name=output_name+ )pipe_run_params.pop_pipe_from_stack(pipe_code=self.code)return pipe_outputdiff --git a/pipelex/core/pipes/validation.py b/pipelex/core/pipes/validation.pyindex 64aef7d29..d52c26014 100644--- a/pipelex/core/pipes/validation.py+++ b/pipelex/core/pipes/validation.py@@ -1,4 +1,3 @@-from pipelex.core.pipes.exceptions import PipeBlueprintValueErrorfrom pipelex.tools.misc.string_utils import is_snake_case@@ -59,7 +58,7 @@ def validate_input_name(input_name: str) -> None:input_name: The input name to validateRaises:- PipeBlueprintValueError: If the input name is invalid+ ValueError: If the input name is invalid"""if not is_valid_input_name(input_name):@@ -69,4 +68,4 @@ def validate_input_name(input_name: str) -> None:"Nested field access is allowed using dots (e.g., 'my_input.field_name'), ""where each part must also be in snake_case.")- raise PipeBlueprintValueError(msg)+ raise ValueError(msg)diff --git a/pipelex/core/pipes/variable_multiplicity.py b/pipelex/core/pipes/variable_multiplicity.pyindex 90a8ce190..b9d6ff3e1 100644--- a/pipelex/core/pipes/variable_multiplicity.py+++ b/pipelex/core/pipes/variable_multiplicity.py@@ -107,3 +107,43 @@ def parse_concept_with_multiplicity(concept_spec: str) -> MultiplicityParseResulmultiplicity = int(bracket_content)return MultiplicityParseResult(concept=concept, multiplicity=multiplicity)+++def format_concept_with_multiplicity(concept_code_or_string: str, multiplicity: VariableMultiplicity | None) -> str:+ """Format a concept code or string with multiplicity notation.++ This is the reverse operation of parse_concept_with_multiplicity.++ Args:+ concept_code_or_string: The concept code or string (e.g., "ConceptName" or "domain.ConceptName")+ multiplicity: The multiplicity value:+ - None: single item (no brackets)+ - True: variable-length list (empty brackets [])+ - int: fixed-length list (brackets with number [N])++ Returns:+ Formatted concept specification string with multiplicity notation++ Examples:+ >>> format_concept_with_multiplicity("Text", None)+ "Text"+ >>> format_concept_with_multiplicity("Text", True)+ "Text[]"+ >>> format_concept_with_multiplicity("Text", 3)+ "Text[3]"+ >>> format_concept_with_multiplicity("domain.Text", None)+ "domain.Text"+ >>> format_concept_with_multiplicity("domain.Text", True)+ "domain.Text[]"+ >>> format_concept_with_multiplicity("domain.Text", 5)+ "domain.Text[5]"+ """+ if multiplicity is None:+ # Single item - no brackets+ return concept_code_or_string+ elif multiplicity is True:+ # Variable-length list - empty brackets+ return f"{concept_code_or_string}[]"+ else:+ # Fixed-length list - brackets with number+ return f"{concept_code_or_string}[{multiplicity}]"diff --git a/pipelex/core/validation_error_categorizer.py b/pipelex/core/validation_error_categorizer.pydeleted file mode 100644index ceaffde27..000000000--- a/pipelex/core/validation_error_categorizer.py+++ /dev/null@@ -1,245 +0,0 @@-"""Categorization utilities for validation errors.--This module provides functions to analyze Pydantic ValidationErrors and categorize them-into PipelexBundleBlueprintFixableErrorType with proper context extraction.-"""--from typing import Any--from pydantic_core import ErrorDetails--from pipelex.core.bundles.exceptions import (- PipelexBundleBlueprintFixableErrorType,- PipelexBundleBlueprintValidationErrorData,-)---def categorize_and_create_error_data(- error: ErrorDetails,- blueprint_dict: dict[str, Any] | None,- domain: str | None,- source: str | None,-) -> PipelexBundleBlueprintValidationErrorData:- """Categorize a Pydantic validation error and create structured error data.-- Args:- error: Pydantic error dict with 'loc', 'msg', 'type', etc.- blueprint_dict: The blueprint dict being validated (for context extraction)- domain: Domain name- source: Source file path-- Returns:- PipelexBundleBlueprintValidationErrorData with all relevant fields populated- """- loc = error.get("loc", ())- msg = error.get("msg", "Unknown validation error")- pydantic_type = error.get("type", "")-- # Extract basic context- pipe_code, concept_code, field_name, error_scope = _extract_context_from_loc(loc)- field_path = " β ".join(str(item) for item in loc)-- # Categorize error type and extract specific context- error_type, extra_context = _categorize_error(- loc=loc,- msg=msg,- pydantic_type=pydantic_type,- blueprint_dict=blueprint_dict,- pipe_code=pipe_code,- concept_code=concept_code,- field_name=field_name,- )-- # Build and return base error data- return PipelexBundleBlueprintValidationErrorData(- domain=domain,- source=source,- pipe_code=pipe_code,- concept_code=concept_code,- field_name=field_name,- error_type=error_type,- error_scope=error_scope,- message=msg,- field_path=field_path,- **extra_context, # Add any type-specific context- )---def _extract_context_from_loc(loc: tuple[int | str, ...]) -> tuple[str | None, str | None, str | None, str | None]:- """Extract pipe_code, concept_code, field_name, and error_scope from location tuple.-- Returns:- (pipe_code, concept_code, field_name, error_scope)- """- if not loc:- return None, None, None, None-- first = str(loc[0])-- # Pipe errors: ('pipe', 'my_pipe', 'field_name', ...)- if first == "pipe" and len(loc) >= 2:- pipe_code = str(loc[1])- field_name = str(loc[2]) if len(loc) >= 3 else None- return pipe_code, None, field_name, "pipe"-- # Concept errors: ('concept', 'Invoice', 'field_name', ...)- elif first == "concept" and len(loc) >= 2:- concept_code = str(loc[1])- field_name = str(loc[2]) if len(loc) >= 3 else None- return None, concept_code, field_name, "concept"-- # Domain errors: ('domain',)- elif first == "domain":- return None, None, None, "domain"-- # Main pipe errors: ('main_pipe',)- elif first == "main_pipe":- return None, None, None, "bundle"-- # Other bundle-level errors- else:- return None, None, None, "bundle"---def _categorize_error(- loc: tuple[int | str, ...],- msg: str,- pydantic_type: str,- blueprint_dict: dict[str, Any] | None,- pipe_code: str | None,- concept_code: str | None,- field_name: str | None,-) -> tuple[PipelexBundleBlueprintFixableErrorType, dict[str, Any]]:- """Categorize error and extract type-specific context.-- Returns:- (error_type, extra_context_dict)- """- extra_context: dict[str, Any] = {}-- # Handle Pydantic built-in errors first- if pydantic_type == "missing":- return PipelexBundleBlueprintFixableErrorType.MISSING_REQUIRED_FIELD, extra_context-- elif pydantic_type in ("type_error", "int_type", "float_type", "bool_type", "str_type"):- return PipelexBundleBlueprintFixableErrorType.TYPE_MISMATCH, extra_context-- elif pydantic_type == "extra_forbidden":- return PipelexBundleBlueprintFixableErrorType.EXTRA_FORBIDDEN_FIELD, extra_context-- elif pydantic_type == "union_tag_not_found":- return PipelexBundleBlueprintFixableErrorType.DISCRIMINATOR_MISSING, extra_context-- elif pydantic_type == "enum":- return PipelexBundleBlueprintFixableErrorType.ENUM_INVALID_VALUE, extra_context-- # Handle custom ValueError subclasses (wrapped as 'value_error')- elif pydantic_type == "value_error":- return _categorize_value_error(- loc=loc,- msg=msg,- blueprint_dict=blueprint_dict,- pipe_code=pipe_code,- concept_code=concept_code,- field_name=field_name,- )-- return PipelexBundleBlueprintFixableErrorType.UNKNOWN, extra_context---def _categorize_value_error(- loc: tuple[int | str, ...],- msg: str,- blueprint_dict: dict[str, Any] | None,- pipe_code: str | None,- concept_code: str | None,- field_name: str | None,-) -> tuple[PipelexBundleBlueprintFixableErrorType, dict[str, Any]]:- """Categorize custom ValueError subclasses."""- extra_context: dict[str, Any] = {}-- # Concept errors- if concept_code:- # Check for specific concept error patterns- if "refines and structure" in msg:- # Extract refines and structure values from blueprint- if blueprint_dict:- concepts = blueprint_dict.get("concept", {})- concept_data = concepts.get(concept_code, {})- if isinstance(concept_data, dict):- extra_context["concept_refines"] = concept_data["refines"]- extra_context["concept_structure"] = concept_data["structure"]- return PipelexBundleBlueprintFixableErrorType.CONCEPT_REFINES_STRUCTURE_CONFLICT, extra_context-- elif field_name == "refines" or "Could not validate refine" in msg:- if blueprint_dict:- concepts = blueprint_dict.get("concept", {})- concept_data = concepts.get(concept_code, {})- if isinstance(concept_data, dict):- extra_context["concept_refines"] = concept_data["refines"]- return PipelexBundleBlueprintFixableErrorType.CONCEPT_REFINES_INVALID, extra_context-- elif field_name == "structure":- return PipelexBundleBlueprintFixableErrorType.CONCEPT_STRUCTURE_INVALID, extra_context-- # Pipe errors- elif pipe_code:- # Pipe sequence errors- if "concept mismatch" in msg.lower() or "not compatible with the output concept" in msg:- # Extract pipe sequence context from message- if "last step" in msg:- # Try to parse the error message for context- # Example: "the output concept 'Text' of the last step 'my_pipe'"- extra_context["last_step_pipe_code"] = _extract_last_step_from_msg(msg)- extra_context["last_step_output_concept"] = _extract_last_step_output_from_msg(msg)- extra_context["expected_output_concept"] = _extract_expected_output_from_msg(msg)- return PipelexBundleBlueprintFixableErrorType.PIPE_SEQUENCE_OUTPUT_MISMATCH, extra_context-- # Domain errors- elif "domain" in str(loc).lower() or "Domain code" in msg:- return PipelexBundleBlueprintFixableErrorType.DOMAIN_CODE_INVALID, extra_context-- # Main pipe errors- elif "main_pipe" in msg.lower() or "Main pipe" in msg:- return PipelexBundleBlueprintFixableErrorType.MAIN_PIPE_NOT_FOUND, extra_context-- return PipelexBundleBlueprintFixableErrorType.UNKNOWN, extra_context---def _extract_last_step_from_msg(msg: str) -> str | None:- """Extract last step pipe code from error message."""- # Example: "of the last step 'my_pipe'"- try:- if "last step '" in msg:- start = msg.index("last step '") + len("last step '")- end = msg.index("'", start)- return msg[start:end]- except (ValueError, IndexError):- pass- return None---def _extract_last_step_output_from_msg(msg: str) -> str | None:- """Extract last step output concept from error message."""- # Example: "the output concept 'Text' of the last step"- try:- if "output concept '" in msg:- start = msg.index("output concept '") + len("output concept '")- end = msg.index("'", start)- return msg[start:end]- except (ValueError, IndexError):- pass- return None---def _extract_expected_output_from_msg(msg: str) -> str | None:- """Extract expected output concept from error message."""- # Example: "with the output concept 'Image' of the sequence"- try:- if "with the output concept '" in msg:- start = msg.index("with the output concept '") + len("with the output concept '")- end = msg.index("'", start)- return msg[start:end]- except (ValueError, IndexError):- pass- return Nonediff --git a/pipelex/exceptions.py b/pipelex/exceptions.pyindex 5eb30abd6..2323d8358 100644--- a/pipelex/exceptions.py+++ b/pipelex/exceptions.py@@ -1,20 +1,14 @@from pipelex.client.exceptions import ClientAuthenticationError-from pipelex.core.bundles.exceptions import PipelexBundleBlueprintValueErrorfrom pipelex.core.concepts.exceptions import (- ConceptBlueprintValueError,ConceptCodeError,ConceptError,ConceptFactoryError,ConceptLibraryConceptNotFoundError,ConceptRefineError,ConceptStringError,- ConceptStructureBlueprintValueError,- ConceptStructureGeneratorError,- ConceptStructureValidationError,- StructureClassError,)+from pipelex.core.concepts.structure_generation.exceptions import ConceptStructureGeneratorError, ConceptStructureValidationError, StructureClassErrorfrom pipelex.core.domains.exceptions import DomainCodeError-from pipelex.core.exceptions import PipelexConfigurationError, PipeValidationError, SyntaxErrorDatafrom pipelex.core.memory.exceptions import (WorkingMemoryConsistencyError,WorkingMemoryError,@@ -24,14 +18,11 @@ from pipelex.core.memory.exceptions import (WorkingMemoryVariableError,)from pipelex.core.pipes.exceptions import (- PipeBlueprintValueError,- PipeDefinitionErrorData,PipeFactoryError,- PipeInputError,- PipeInputNotFoundError,PipeOperatorModelChoiceError,- PipeRunInputsError,+ PipeValidationError,)+from pipelex.core.pipes.inputs.exceptions import PipeInputNotFoundErrorfrom pipelex.core.stuffs.exceptions import (StuffArtefactError,StuffArtefactReservedFieldError,@@ -46,14 +37,11 @@ from pipelex.libraries.exceptions import (LibraryLoadingError,PipeLoadingError,)+from pipelex.libraries.pipe.exceptions import PipeNotFoundErrorfrom pipelex.pipe_controllers.exceptions import PipeControllerError, PipeControllerOutputConceptMismatchErrorfrom pipelex.pipe_operators.exceptions import PipeOperatorModelAvailabilityErrorfrom pipelex.pipe_run.exceptions import BatchParamsError, PipeRouterError, PipeRunError, PipeRunParamsErrorfrom pipelex.pipeline.exceptions import (- DryRunError,- DryRunMissingInputsError,- DryRunMissingPipesError,- DryRunTemplatingError,PipeExecutionError,PipelineExecutionError,PipeStackOverflowError,@@ -72,14 +60,10 @@ from pipelex.system.exceptions import (__all__ = [# from pipelex.client.exceptions"ClientAuthenticationError",- # from pipelex.core.bundles.exceptions- "PipelexBundleBlueprintValueError",# from pipelex.core.domains.exceptions"DomainCodeError",# from pipelex.core.concepts.exceptions"ConceptError",- "ConceptBlueprintValueError",- "ConceptStructureBlueprintValueError","ConceptStructureValidationError","ConceptFactoryError","StructureClassError",@@ -94,6 +78,8 @@ __all__ = ["ConceptLoadingError","DomainLoadingError","PipeLoadingError",+ # from pipelex.libraries.pipe.exceptions+ "PipeNotFoundError",# from pipelex.pipe_controllers.exceptions"PipeControllerError","PipeControllerOutputConceptMismatchError",@@ -105,13 +91,9 @@ __all__ = ["PipeRouterError","PipeRunError",# from pipelex.core.pipes.exceptions- "PipeDefinitionErrorData",- "PipeBlueprintValueError","PipeInputNotFoundError","PipeFactoryError","PipeOperatorModelChoiceError",- "PipeInputError",- "PipeRunInputsError",# from pipelex.core.stuffs.exceptions"StuffArtefactError","StuffArtefactReservedFieldError",@@ -119,8 +101,6 @@ __all__ = ["StuffContentTypeError","StuffContentValidationError",# from pipelex.core.exceptions- "PipelexConfigurationError",- "SyntaxErrorData","PipeValidationError",# from pipelex.core.memory.exceptions"WorkingMemoryConsistencyError",@@ -131,10 +111,6 @@ __all__ = ["WorkingMemoryVariableError",# pipelex.pipeline.exceptions"PipeStackOverflowError",- "DryRunTemplatingError",- "DryRunMissingPipesError",- "DryRunMissingInputsError",- "DryRunError","PipelineExecutionError","PipeExecutionError",# from pipelex.pipeline.track.exceptionsdiff --git a/pipelex/libraries/concept/concept_library.py b/pipelex/libraries/concept/concept_library.pyindex 9791cc1d8..4c34b9ee2 100644--- a/pipelex/libraries/concept/concept_library.py+++ b/pipelex/libraries/concept/concept_library.py@@ -38,7 +38,7 @@ class ConceptLibrary(RootModel[ConceptLibraryRoot], ConceptLibraryAbstract):self.teardown()self.setup()- # TODO: Rethink the make_empty of libraries. It makes sense with the setup and teardown methods+ # TODO: Rethink the make_empty of libraries. It doesn't makes sense to call the setup inside the make_empty method.@classmethoddef make_empty(cls) -> Self:library = cls(root={})diff --git a/pipelex/libraries/exceptions.py b/pipelex/libraries/exceptions.pyindex eda1e5814..b1090c632 100644--- a/pipelex/libraries/exceptions.py+++ b/pipelex/libraries/exceptions.py@@ -1,5 +1,6 @@from pipelex.base_exceptions import PipelexError-from pipelex.core.exceptions import PipelexBundleBlueprintValidationErrorData+from pipelex.core.bundles.exceptions import PipelexBundleBlueprintValidationErrorData+from pipelex.core.exceptions import PipesAndConceptValidationErrorDataclass LibraryError(PipelexError):@@ -9,20 +10,35 @@ class LibraryError(PipelexError):class LibraryLoadingError(LibraryError):"""Error raised when loading library components fails.- This error aggregates all validation errors from:+ This error aggregates TWO types of validation errors:+ 1. Blueprint validation errors (from PipelexBundleBlueprint.model_validate())+ - Stored in: blueprint_validation_errors+ - Example: PIPE_SEQUENCE_OUTPUT_MISMATCH++ 2. Pipe/Concept validation errors (from Pipe or Concept class validation)+ - Stored in: pipe_concept_validation_errors+ - Example: MISSING_INPUT_VARIABLE, INPUT_REQUIREMENT_MISMATCH++ Also handles:- Factory errors (Domain, Concept, Pipe)- - Validation errors (Pydantic ValidationError from validators)- Interpreter errors (blueprint parsing)-- All errors are categorized and stored in validation_errors."""def __init__(self,message: str,- validation_errors: list[PipelexBundleBlueprintValidationErrorData] | None = None,+ blueprint_validation_errors: list[PipelexBundleBlueprintValidationErrorData] | None = None,+ pipe_concept_validation_errors: list[PipesAndConceptValidationErrorData] | None = None,):- self.validation_errors = validation_errors or []+ """Initialize LibraryLoadingError.++ Args:+ message: Error message+ blueprint_validation_errors: Blueprint validation errors (from PipelexBundleBlueprint validation)+ pipe_concept_validation_errors: Pipe/Concept validation errors (from Pipe/Concept validation)+ """+ self.blueprint_validation_errors = blueprint_validation_errors or []+ self.pipe_concept_validation_errors = pipe_concept_validation_errors or []super().__init__(message)diff --git a/pipelex/libraries/library_manager.py b/pipelex/libraries/library_manager.pyindex 9c2445eb0..85f2f267e 100644--- a/pipelex/libraries/library_manager.py+++ b/pipelex/libraries/library_manager.py@@ -2,7 +2,6 @@ import uuidfrom pathlib import Pathfrom typing import TYPE_CHECKING-from pydantic import ValidationErrorfrom typing_extensions import overridefrom pipelex import log@@ -11,14 +10,11 @@ from pipelex.core.concepts.concept_factory import ConceptFactoryfrom pipelex.core.domains.domain import Domainfrom pipelex.core.domains.domain_blueprint import DomainBlueprintfrom pipelex.core.domains.domain_factory import DomainFactory-from pipelex.core.exceptions import PipelexBundleBlueprintValidationErrorData, PipelexInterpreterError-from pipelex.core.interpreter import PipelexInterpreter-from pipelex.core.pipe_errors import PipeDefinitionError+from pipelex.core.interpreter.exceptions import PipelexInterpreterError+from pipelex.core.interpreter.interpreter import PipelexInterpreterfrom pipelex.core.pipes.pipe_abstract import PipeAbstractfrom pipelex.core.pipes.pipe_factory import PipeFactoryfrom pipelex.core.stuffs.structured_content import StructuredContent-from pipelex.core.validation import report_validation_error-from pipelex.core.validation_error_categorizer import categorize_and_create_error_datafrom pipelex.hub import get_current_libraryfrom pipelex.libraries.exceptions import (LibraryError,@@ -257,40 +253,16 @@ class LibraryManager(LibraryManagerAbstract):msg = f"Could not find PLX bundle at '{plx_file_path}'"raise LibraryLoadingError(msg) from file_not_found_errorexcept PipelexInterpreterError as interpreter_error:- # Forward categorized validation errors from interpreter+ # Forward BLUEPRINT validation errors from interpretermsg = f"Could not load PLX bundle from '{plx_file_path}' because of: {interpreter_error.message}"raise LibraryLoadingError(message=msg,- validation_errors=interpreter_error.validation_errors,+ blueprint_validation_errors=interpreter_error.validation_errors,) from interpreter_error- except PipeDefinitionError as pipe_def_error:- msg = f"Could not load PLX bundle from '{plx_file_path}' because of: {pipe_def_error}"- raise LibraryLoadingError(msg) from pipe_def_errorblueprints.append(blueprint)self.loaded_plx_paths.extend([str(plx_file_path) for plx_file_path in valid_plx_paths])-- # Load all blueprints into the library- try:- self.load_from_blueprints(library_id=library_id, blueprints=blueprints)- except ValidationError as validation_error:- # Categorize and forward Pydantic validation errors- validation_errors: list[PipelexBundleBlueprintValidationErrorData] = []- for error in validation_error.errors():- val_error = categorize_and_create_error_data(- error=error,- blueprint_dict=None,- domain=None,- source=None,- )- validation_errors.append(val_error)-- validation_error_msg = report_validation_error(category="plx", validation_error=validation_error)- msg = f"Could not load blueprints because of: {validation_error_msg}"- raise LibraryLoadingError(- message=msg,- validation_errors=validation_errors,- ) from validation_error+ self.load_from_blueprints(library_id=library_id, blueprints=blueprints)def _remove_pipes_from_blueprint(self, blueprint: PipelexBundleBlueprint) -> None:library = self.get_current_library()diff --git a/pipelex/libraries/library_utils.py b/pipelex/libraries/library_utils.pyindex 3f019d6fb..1dad303fd 100644--- a/pipelex/libraries/library_utils.py+++ b/pipelex/libraries/library_utils.py@@ -4,7 +4,7 @@ from pathlib import Pathfrom pipelex import logfrom pipelex.config import get_config-from pipelex.core.interpreter import PipelexInterpreter+from pipelex.core.interpreter.interpreter import PipelexInterpreterfrom pipelex.tools.misc.file_utils import find_files_in_dirdiff --git a/pipelex/libraries/pipe/exceptions.py b/pipelex/libraries/pipe/exceptions.pyindex 22d9bab20..b463b893b 100644--- a/pipelex/libraries/pipe/exceptions.py+++ b/pipelex/libraries/pipe/exceptions.py@@ -3,3 +3,7 @@ from pipelex.base_exceptions import PipelexErrorclass PipeLibraryError(PipelexError):pass+++class PipeNotFoundError(PipeLibraryError):+ passdiff --git a/pipelex/libraries/pipe/pipe_library.py b/pipelex/libraries/pipe/pipe_library.pyindex a2d718cc8..44a9fe071 100644--- a/pipelex/libraries/pipe/pipe_library.py+++ b/pipelex/libraries/pipe/pipe_library.py@@ -7,7 +7,7 @@ from typing_extensions import overridefrom pipelex import pretty_printfrom pipelex.core.pipes.pipe_abstract import PipeAbstract-from pipelex.libraries.pipe.exceptions import PipeLibraryError+from pipelex.libraries.pipe.exceptions import PipeLibraryError, PipeNotFoundErrorfrom pipelex.libraries.pipe.pipe_library_abstract import PipeLibraryAbstractfrom pipelex.types import Self@@ -59,7 +59,7 @@ class PipeLibrary(RootModel[PipeLibraryRoot], PipeLibraryAbstract):the_pipe = self.get_optional_pipe(pipe_code=pipe_code)if not the_pipe:msg = f"Pipe '{pipe_code}' not found. Check for typos and make sure it is declared in plx file in an imported package."- raise PipeLibraryError(msg)+ raise PipeNotFoundError(msg)return the_pipe@overridediff --git a/pipelex/pipe_controllers/batch/exceptions.py b/pipelex/pipe_controllers/batch/exceptions.pyindex 2300a2929..b6f63b547 100644--- a/pipelex/pipe_controllers/batch/exceptions.py+++ b/pipelex/pipe_controllers/batch/exceptions.py@@ -1,9 +1,5 @@from pipelex.base_exceptions import PipelexError-class PipeBatchValueError(ValueError):- pass--class PipeBatchFactoryError(PipelexError):passdiff --git a/pipelex/pipe_controllers/batch/pipe_batch.py b/pipelex/pipe_controllers/batch/pipe_batch.pyindex c28201d3c..8469b1344 100644--- a/pipelex/pipe_controllers/batch/pipe_batch.py+++ b/pipelex/pipe_controllers/batch/pipe_batch.py@@ -5,16 +5,14 @@ import shortuuidfrom typing_extensions import overridefrom pipelex.config import get_config-from pipelex.core.memory.exceptions import WorkingMemoryStuffNotFoundErrorfrom pipelex.core.memory.working_memory import MAIN_STUFF_NAME, WorkingMemory-from pipelex.core.pipes.exceptions import PipeInputError, PipeInputNotFoundErrorfrom pipelex.core.pipes.inputs.input_requirements import InputRequirementsfrom pipelex.core.pipes.pipe_output import PipeOutputfrom pipelex.core.stuffs.list_content import ListContentfrom pipelex.core.stuffs.stuff_factory import StuffFactoryfrom pipelex.hub import get_pipeline_tracker, get_required_pipe-from pipelex.pipe_controllers.batch.exceptions import PipeBatchValueErrorfrom pipelex.pipe_controllers.pipe_controller import PipeController+from pipelex.pipe_run.exceptions import PipeRunErrorfrom pipelex.pipe_run.pipe_run_params import BatchParams, PipeRunMode, PipeRunParamsfrom pipelex.pipeline.job_metadata import JobMetadata@@ -68,26 +66,26 @@ class PipeBatch(PipeController):pass@override- def validate_before_run(+ async def validate_before_run(self, job_metadata: JobMetadata, working_memory: WorkingMemory, pipe_run_params: PipeRunParams, output_name: str | None = None) -> None:- try:- required_concept_code = self.inputs.get_required_input_requirement(variable_name=self.batch_params.input_list_stuff_name).concept.code- except PipeInputNotFoundError as exc:+ batch_params = pipe_run_params.batch_params or self.batch_params or BatchParams.make_default()+ input_list_stuff_name = batch_params.input_list_stuff_name+ if not self.inputs.is_variable_existing(variable_name=input_list_stuff_name):+ msg = f"Batch input list named '{input_list_stuff_name}' is not in PipeBatch '{self.code}' input requirements: {self.inputs}"+ raise PipeRunError(message=msg, run_mode=pipe_run_params.run_mode)++ if not working_memory.is_stuff_exists(input_list_stuff_name):+ msg = f"Input list stuff '{input_list_stuff_name}' required by this PipeBatch '{self.code}' not found in working memory"+ raise PipeRunError(message=msg, run_mode=pipe_run_params.run_mode)++ input_stuff = working_memory.get_stuff(input_list_stuff_name)+ if not isinstance(input_stuff.content, ListContent):msg = (- f"Batch input list named '{self.batch_params.input_list_stuff_name}' is not in "- f"PipeBatch '{self.code}' input requirements: {self.inputs}"+ f"Input list stuff '{input_list_stuff_name}' of PipeBatch '{self.code}' must be ListContent, "+ f"got {input_stuff.stuff_name or 'unnamed'} = {type(input_stuff.content)}. stuff: {input_stuff}")- raise PipeBatchValueError(msg) from exc-- required_stuff_name = self.batch_params.input_list_stuff_name- try:- working_memory.get_stuff(required_stuff_name)- except WorkingMemoryStuffNotFoundError as exc:- variable_name: str = exc.variable_name or required_stuff_name- missing_inputs: dict[str, str] = {variable_name: exc.concept_code or required_concept_code}- msg = f"Missing required inputs for pipe '{self.code}': {missing_inputs}"- raise PipeBatchValueError(msg) from exc+ raise PipeRunError(message=msg, run_mode=pipe_run_params.run_mode)async def _run_batch_pipe(self,@@ -100,34 +98,15 @@ class PipeBatch(PipeController):batch_params = pipe_run_params.batch_params or self.batch_params or BatchParams.make_default()input_item_stuff_name = batch_params.input_item_stuff_nameinput_list_stuff_name = batch_params.input_list_stuff_name- try:- input_requirement = self.inputs.get_required_input_requirement(input_list_stuff_name)- except PipeInputNotFoundError as exc:- msg = f"Batch input item list named '{input_list_stuff_name}' is not in this PipeBatch '{self.code}' input requirements: {self.inputs}"- raise PipeInputError(message=msg, pipe_code=self.code, variable_name=input_list_stuff_name, concept_code=None) from excif pipe_run_params.final_stuff_code:method_name = "dry_run_pipe" if pipe_run_params.run_mode == PipeRunMode.DRY else "_run_controller_pipe"pipe_run_params.final_stuff_code = Nonepipe_run_params.push_pipe_layer(pipe_code=self.branch_pipe_code)- try:- input_stuff = working_memory.get_stuff(batch_params.input_list_stuff_name)- except WorkingMemoryStuffNotFoundError as exc:- msg = (- f"Input list stuff '{batch_params.input_list_stuff_name}' required by this PipeBatch '{self.code}' not found in working memory: {exc}"- )- raise PipeInputError(message=msg, pipe_code=self.code, variable_name=batch_params.input_list_stuff_name, concept_code=None) from exc- input_stuff_code = input_stuff.stuff_code- input_content = input_stuff.content- if not isinstance(input_content, ListContent):- msg = (- f"Input of PipeBatch '{self.code}' must be ListContent, "- f"got {input_stuff.stuff_name or 'unnamed'} = {type(input_content)}. stuff: {input_stuff}"- )- raise PipeInputError(message=msg, pipe_code=self.code, variable_name=batch_params.input_list_stuff_name, concept_code=None)- input_content = cast("ListContent[StuffContent]", input_content)+ input_stuff = working_memory.get_stuff(input_list_stuff_name)+ input_content = cast("ListContent[StuffContent]", input_stuff.content)# TODO: Make commented code work when inputing images named "a.b.c"sub_pipe = get_required_pipe(pipe_code=self.branch_pipe_code)@@ -143,10 +122,10 @@ class PipeBatch(PipeController):branch_output_item_codes.append(branch_output_item_code)if nb_history_items_limit and branch_index >= nb_history_items_limit:break- branch_input_item_code = f"{input_stuff_code}-branch-{branch_index}"+ branch_input_item_code = f"{input_stuff.stuff_code}-branch-{branch_index}"item_input_stuff = StuffFactory.make_stuff(code=branch_input_item_code,- concept=input_requirement.concept,+ concept=self.inputs.get_required_input_requirement(input_list_stuff_name).concept,content=item,name=input_item_stuff_name,)@@ -156,7 +135,7 @@ class PipeBatch(PipeController):required_variables = sub_pipe.required_variables()required_stuffs = branch_memory.get_existing_stuffs(names=required_variables)- required_stuffs = [required_stuff for required_stuff in required_stuffs if required_stuff.stuff_code != input_stuff_code]+ required_stuffs = [required_stuff for required_stuff in required_stuffs if required_stuff.stuff_code != input_stuff.stuff_code]required_stuff_lists.append(required_stuffs)branch_pipe_run_params = pipe_run_params.deep_copy_with_final_stuff_code(final_stuff_code=branch_output_item_code)@@ -247,7 +226,6 @@ class PipeBatch(PipeController):pipe_run_params: PipeRunParams,output_name: str | None = None,) -> PipeOutput:- """Run a pipe in batch mode for each item in the input list."""return await self._run_batch_pipe(job_metadata=job_metadata,working_memory=working_memory,@@ -263,7 +241,6 @@ class PipeBatch(PipeController):pipe_run_params: PipeRunParams,output_name: str | None = None,) -> PipeOutput:- """Dry run a pipe in batch mode for each item in the input list."""return await self._run_batch_pipe(job_metadata=job_metadata,working_memory=working_memory,diff --git a/pipelex/pipe_controllers/condition/exceptions.py b/pipelex/pipe_controllers/condition/exceptions.pyindex 84469e445..e1d4f7cf3 100644--- a/pipelex/pipe_controllers/condition/exceptions.py+++ b/pipelex/pipe_controllers/condition/exceptions.py@@ -3,7 +3,3 @@ from pipelex.base_exceptions import PipelexErrorclass PipeConditionFactoryError(PipelexError):pass---class PipeConditionRunError(PipelexError):- passdiff --git a/pipelex/pipe_controllers/condition/pipe_condition.py b/pipelex/pipe_controllers/condition/pipe_condition.pyindex b6a9f6988..056788446 100644--- a/pipelex/pipe_controllers/condition/pipe_condition.py+++ b/pipelex/pipe_controllers/condition/pipe_condition.py@@ -5,24 +5,21 @@ from typing_extensions import overridefrom pipelex import logfrom pipelex.cogt.templating.template_category import TemplateCategory-from pipelex.core.bundles.exceptions import PipeValidationErrorTypefrom pipelex.core.concepts.concept_factory import ConceptFactoryfrom pipelex.core.concepts.native.concept_native import NativeConceptCode-from pipelex.core.exceptions import PipeValidationErrorfrom pipelex.core.memory.exceptions import WorkingMemoryStuffNotFoundErrorfrom pipelex.core.memory.working_memory import WorkingMemory-from pipelex.core.pipes.exceptions import PipeInputError+from pipelex.core.pipes.exceptions import PipeValidationError, PipeValidationErrorTypefrom pipelex.core.pipes.inputs.input_requirements import InputRequirementsfrom pipelex.core.pipes.inputs.input_requirements_factory import InputRequirementsFactoryfrom pipelex.core.pipes.pipe_output import PipeOutputfrom pipelex.hub import get_content_generator, get_optional_pipe, get_pipe_router, get_pipeline_tracker, get_required_pipe-from pipelex.pipe_controllers.condition.exceptions import PipeConditionRunErrorfrom pipelex.pipe_controllers.condition.pipe_condition_details import PipeConditionDetailsfrom pipelex.pipe_controllers.condition.special_outcome import SpecialOutcomefrom pipelex.pipe_controllers.pipe_controller import PipeController+from pipelex.pipe_run.exceptions import PipeRunErrorfrom pipelex.pipe_run.pipe_job_factory import PipeJobFactoryfrom pipelex.pipe_run.pipe_run_params import PipeRunParams-from pipelex.pipeline.exceptions import DryRunMissingInputsError, DryRunMissingPipesError, DryRunTemplatingErrorfrom pipelex.pipeline.job_metadata import JobMetadatafrom pipelex.tools.jinja2.jinja2_errors import Jinja2DetectVariablesErrorfrom pipelex.tools.jinja2.jinja2_required_variables import detect_jinja2_required_variables@@ -112,7 +109,6 @@ class PipeCondition(PipeController):for input_name, requirement in pipe_needed_inputs.items:needed_inputs.add_requirement(variable_name=input_name, concept=requirement.concept)-return needed_inputs@override@@ -139,18 +135,40 @@ class PipeCondition(PipeController):NativeConceptCode.DYNAMIC.concept_string,NativeConceptCode.ANYTHING.concept_string,):+ msg = (+ f"The output concept code '{self.output.concept_string}' of the pipe '{self.code}' is not "+ f"matching the output concept code '{pipe.output.concept_string}' of the pipe '{pipe_code}'"+ )raise PipeValidationError(+ message=msg,error_type=PipeValidationErrorType.INADEQUATE_OUTPUT_CONCEPT,domain=self.domain,pipe_code=self.code,provided_concept_code=pipe.output.concept_string,required_concept_codes=[self.output.concept_string],- explanation=(- f"The output concept code '{self.output.concept_string}' of the pipe '{self.code}' is not "- f"matching the output concept code '{pipe.output.concept_string}' of the pipe '{pipe_code}'"- ),)+ # TODO: Restore this validation. The problem lies with needed_inputs that construct Anything concepts.+ # @override+ # async def validate_before_run(+ # self,+ # job_metadata: JobMetadata,+ # working_memory: WorkingMemory,+ # pipe_run_params: PipeRunParams,+ # output_name: str | None = None,+ # ):+ # evaluated_expression = await get_content_generator().make_templated_text(+ # context=working_memory.generate_context(),+ # template=self.expression,+ # template_category=TemplateCategory.EXPRESSION,+ # )+ # if not evaluated_expression or evaluated_expression == "None":+ # error_msg = f"PipeCondition '{self.code}': Conditional expression returned no result"+ # raise PipeRunError(+ # message=error_msg,+ # run_mode=pipe_run_params.run_mode,+ # )+async def _evaluate_expression(self,working_memory: WorkingMemory,@@ -162,27 +180,13 @@ class PipeCondition(PipeController):Returns:The evaluated expression-- Raises:- PipeConditionError: If expression evaluation fails or no matching pipe is found"""- content_generator = get_content_generator()-- # Evaluate the expression using templating- evaluated_expression = await content_generator.make_templated_text(+ evaluated_expression = await get_content_generator().make_templated_text(context=working_memory.generate_context(),template=self.expression,template_category=TemplateCategory.EXPRESSION,)- # Validate the evaluated expression- if not evaluated_expression or evaluated_expression == "None":- error_msg = f"Conditional expression returned no result in pipe {self.code}:"- error_msg += f"\n\nExpression: {self.expression}"- raise PipeConditionRunError(error_msg)- log.verbose(f"evaluated_expression: '{evaluated_expression}'")-- # Add alias if configuredlog.verbose(f"add_alias: {evaluated_expression} -> {self.add_alias_from_expression_to}")if self.add_alias_from_expression_to:working_memory.add_alias(@@ -217,7 +221,7 @@ class PipeCondition(PipeController):if SpecialOutcome.is_fail(outcome):msg = f"PipeCondition '{self.code}' failed with outcome: {outcome}. Evaluated expression: {evaluated_expression}"- raise PipeConditionRunError(msg)+ raise PipeRunError(message=msg, run_mode=pipe_run_params.run_mode)chosen_pipe = get_required_pipe(pipe_code=outcome)@@ -237,7 +241,7 @@ class PipeCondition(PipeController):pipe_condition_path_str = ".".join(pipe_condition_path)error_details = f"PipeCondition '{pipe_condition_path_str}', required_variables: {required_variables}, missing: '{exc.variable_name}'"msg = f"Some required stuff(s) not found: {error_details}"- raise PipeInputError(message=msg, pipe_code=self.code, variable_name=exc.variable_name, concept_code=None) from exc+ raise PipeRunError(message=msg, run_mode=pipe_run_params.run_mode) from exc# Track condition stepsfor required_stuff in required_stuffs:@@ -282,7 +286,6 @@ class PipeCondition(PipeController):Validates that all required inputs are present, expression is valid, and target pipes exist."""log.verbose(f"PipeCondition: dry run controller pipe: {self.code}")-# 1. Validate that all required inputs are present in the working memoryneeded_inputs = self.needed_inputs()missing_input_names: list[str] = []@@ -293,11 +296,9 @@ class PipeCondition(PipeController):if missing_input_names:log.error(f"Dry run failed: missing required inputs: {missing_input_names}")- raise DryRunMissingInputsError(+ raise PipeRunError(message=f"Dry run failed for pipe '{self.code}' (PipeCondition): missing required inputs: {', '.join(missing_input_names)}",- pipe_type=self.__class__.__name__,- pipe_code=self.code,- missing_inputs=missing_input_names,+ run_mode=pipe_run_params.run_mode,)# 2. Validate that the expression template is valid@@ -313,13 +314,7 @@ class PipeCondition(PipeController):f"Dry run failed for pipe '{self.code}' (PipeCondition): could not detect required variables "f"from expression template: {exc}\nTemplate:\n'{self.expression}'")- raise DryRunTemplatingError(- message=msg,- pipe_type=self.__class__.__name__,- pipe_code=self.code,- template_category=TemplateCategory.EXPRESSION,- template=self.expression,- ) from exc+ raise PipeRunError(message=msg, run_mode=pipe_run_params.run_mode) from exc# 3. Validate that all values in the outcomes map (appart from special outcomes) do exist as pipe codesall_pipe_codes = set(self.outcome_map.values())@@ -334,7 +329,7 @@ class PipeCondition(PipeController):f"Dry run failed for PipeCondition '{self.code}': missing pipes: {', '.join(missing_pipes)}. "f"Pipe map: {self.outcome_map}, default: {self.default_outcome}")- raise DryRunMissingPipesError(message=msg, pipe_type=self.__class__.__name__, pipe_code=self.code, missing_pipes=missing_pipes)+ raise PipeRunError(message=msg, run_mode=pipe_run_params.run_mode)# Here, it should launch the dry run of all the pipes in the outcomes mapfor pipe_code in self.mapped_pipe_codes:diff --git a/pipelex/pipe_controllers/exceptions.py b/pipelex/pipe_controllers/exceptions.pyindex 55feb340c..af29b7448 100644--- a/pipelex/pipe_controllers/exceptions.py+++ b/pipelex/pipe_controllers/exceptions.py@@ -1,5 +1,4 @@from pipelex.base_exceptions import PipelexError-from pipelex.core.pipes.pipe_abstract import PipeAbstractclass PipeControllerError(PipelexError):@@ -11,12 +10,3 @@ class PipeControllerOutputConceptMismatchError(PipeControllerError):self.tested_concept = tested_conceptself.wanted_concept = wanted_conceptsuper().__init__(message)---class SubPipeBlueprintValueError(ValueError):- pass---class PipeValueError(PipelexError):- pipe: PipeAbstract- category: strdiff --git a/pipelex/pipe_controllers/parallel/exceptions.py b/pipelex/pipe_controllers/parallel/exceptions.pyindex ba7406afd..8428cf8b2 100644--- a/pipelex/pipe_controllers/parallel/exceptions.py+++ b/pipelex/pipe_controllers/parallel/exceptions.py@@ -1,9 +1,5 @@from pipelex.base_exceptions import PipelexError-class PipeParallelValueError(ValueError):- pass--class PipeParallelFactoryError(PipelexError):passdiff --git a/pipelex/pipe_controllers/parallel/pipe_parallel.py b/pipelex/pipe_controllers/parallel/pipe_parallel.pyindex 8656bc77a..e31fa1022 100644--- a/pipelex/pipe_controllers/parallel/pipe_parallel.py+++ b/pipelex/pipe_controllers/parallel/pipe_parallel.py@@ -7,20 +7,19 @@ from typing_extensions import overridefrom pipelex import logfrom pipelex.core.concepts.concept import Conceptfrom pipelex.core.memory.working_memory import WorkingMemory-from pipelex.core.pipe_errors import PipeDefinitionError-from pipelex.core.pipes.exceptions import PipeInputNotFoundError+from pipelex.core.pipes.exceptions import PipeValidationError+from pipelex.core.pipes.inputs.exceptions import PipeInputNotFoundErrorfrom pipelex.core.pipes.inputs.input_requirements import InputRequirementsfrom pipelex.core.pipes.inputs.input_requirements_factory import InputRequirementsFactoryfrom pipelex.core.pipes.pipe_output import PipeOutputfrom pipelex.core.stuffs.stuff_factory import StuffFactoryfrom pipelex.hub import get_pipeline_tracker, get_required_pipe-from pipelex.pipe_controllers.parallel.exceptions import PipeParallelValueError+from pipelex.libraries.pipe.exceptions import PipeNotFoundErrorfrom pipelex.pipe_controllers.pipe_controller import PipeControllerfrom pipelex.pipe_controllers.sub_pipe import SubPipe-from pipelex.pipe_run.exceptions import PipeRunParamsError+from pipelex.pipe_run.exceptions import PipeRunErrorfrom pipelex.pipe_run.pipe_run_mode import PipeRunModefrom pipelex.pipe_run.pipe_run_params import PipeRunParams-from pipelex.pipeline.exceptions import DryRunMissingInputsErrorfrom pipelex.pipeline.job_metadata import JobMetadatafrom pipelex.types import Self@@ -85,7 +84,7 @@ class PipeParallel(PipeController):f"Batch input item named '{sub_pipe.batch_params.input_item_stuff_name}' is not "f"in this Parallel Pipe '{self.code}' input requirements: {pipe_needed_inputs}")- raise PipeParallelValueError(msg) from exc+ raise PipeValidationError(message=msg) from excneeded_inputs.add_requirement(variable_name=sub_pipe.batch_params.input_list_stuff_name,concept=requirement.concept,@@ -136,10 +135,6 @@ class PipeParallel(PipeController):pipe_run_params: PipeRunParams,output_name: str | None = None,) -> PipeOutput:- """Run a list of pipes in parallel."""- if not self.add_each_output and not self.combined_output:- msg = "PipeParallel requires either add_each_output or combined_output to be set"- raise PipeDefinitionError(msg)if pipe_run_params.final_stuff_code:log.verbose(f"PipeBatch.run_pipe() final_stuff_code: {pipe_run_params.final_stuff_code}")pipe_run_params.final_stuff_code = None@@ -168,19 +163,19 @@ class PipeParallel(PipeController):sub_pipe_output_name = self.parallel_sub_pipes[output_index].output_nameif not sub_pipe_output_name:msg = "PipeParallel requires a result specified for each parallel sub pipe"- raise PipeDefinitionError(msg)+ raise PipeRunError(message=msg, run_mode=pipe_run_params.run_mode)if self.add_each_output:working_memory.add_new_stuff(name=sub_pipe_output_name, stuff=output_stuff)output_stuff_content_items.append(output_stuff.content)if sub_pipe_output_name in output_stuffs:# TODO: check that at the blueprint / factory levelmsg = f"PipeParallel requires unique output names for each parallel sub pipe, but {sub_pipe_output_name} is already used"- raise PipeDefinitionError(msg)+ raise PipeRunError(message=msg, run_mode=pipe_run_params.run_mode)output_stuffs[sub_pipe_output_name] = output_stuffif sub_pipe_output_name in output_stuff_contents:# TODO: check that at the blueprint / factory levelmsg = f"PipeParallel requires unique output names for each parallel sub pipe, but {sub_pipe_output_name} is already used"- raise PipeDefinitionError(msg)+ raise PipeRunError(message=msg, run_mode=pipe_run_params.run_mode)output_stuff_contents[sub_pipe_output_name] = output_stuff.contentlog.verbose(f"PipeParallel '{self.code}': output_stuff_contents[{sub_pipe_output_name}]: {output_stuff_contents[sub_pipe_output_name]}")@@ -220,7 +215,7 @@ class PipeParallel(PipeController):log.verbose(f"PipeParallel: dry run controller pipe: {self.code}")if pipe_run_params.run_mode != PipeRunMode.DRY:msg = f"PipeSequence._dry_run_controller_pipe() called with run_mode = {pipe_run_params.run_mode} in pipe {self.code}"- raise PipeRunParamsError(msg)+ raise PipeRunError(message=msg, run_mode=pipe_run_params.run_mode)# 1. Validate that all required inputs are present in the working memoryneeded_inputs = self.needed_inputs()@@ -230,22 +225,16 @@ class PipeParallel(PipeController):missing_input_names.append(named_input_requirement.variable_name)if missing_input_names:- msg = f"Dry run failed: missing required inputs: {missing_input_names}"- log.error(f"Dry run failed: missing required inputs: {missing_input_names}")- raise DryRunMissingInputsError(- message=msg,- pipe_type=self.__class__.__name__,- pipe_code=self.code,- missing_inputs=missing_input_names,- )+ msg = f"Dry run failed for pipe '{self.code}' (PipeParallel): missing required inputs: {missing_input_names}"+ raise PipeRunError(message=msg, run_mode=pipe_run_params.run_mode)# 2. Validate that all sub-pipes existfor sub_pipe in self.parallel_sub_pipes:try:get_required_pipe(pipe_code=sub_pipe.pipe_code)- except Exception as exc:- msg = f"PipeParallel'{self.code}'sub-pipe '{sub_pipe.pipe_code}' not found"- raise PipeDefinitionError(msg) from exc+ except PipeNotFoundError as exc:+ msg = f"Dry run failed for pipe '{self.code}' (PipeParallel): sub-pipe '{sub_pipe.pipe_code}' not found"+ raise PipeRunError(message=msg, run_mode=pipe_run_params.run_mode) from exc# 3. Run all sub-pipes in dry modetasks: list[Coroutine[Any, Any, PipeOutput]] = []@@ -272,7 +261,7 @@ class PipeParallel(PipeController):if not sub_pipe_output_name:sub_pipe_code = self.parallel_sub_pipes[output_index].pipe_codemsg = f"Dry run failed for pipe '{self.code}' (PipeParallel): sub-pipe '{sub_pipe_code}' output name not specified"- raise PipeDefinitionError(msg)+ raise PipeRunError(message=msg, run_mode=pipe_run_params.run_mode)if self.add_each_output:working_memory.add_new_stuff(name=sub_pipe_output_name, stuff=output_stuff)@@ -282,7 +271,7 @@ class PipeParallel(PipeController):msg = (f"Dry run failed for pipe '{self.code}' (PipeParallel): sub-pipe '{sub_pipe_code}' duplicate output name '{sub_pipe_output_name}'")- raise PipeDefinitionError(msg)+ raise PipeRunError(message=msg, run_mode=pipe_run_params.run_mode)output_stuffs[sub_pipe_output_name] = output_stuffoutput_stuff_contents[sub_pipe_output_name] = output_stuff.contentdiff --git a/pipelex/pipe_controllers/sequence/exceptions.py b/pipelex/pipe_controllers/sequence/exceptions.pyindex d78c4478b..12d1c8f57 100644--- a/pipelex/pipe_controllers/sequence/exceptions.py+++ b/pipelex/pipe_controllers/sequence/exceptions.py@@ -1,10 +1,6 @@from pipelex.base_exceptions import PipelexError-class PipeSequenceBlueprintValueError(ValueError):- pass--class PipeSequenceValueError(ValueError):passdiff --git a/pipelex/pipe_controllers/sequence/pipe_sequence.py b/pipelex/pipe_controllers/sequence/pipe_sequence.pyindex d411b2ab1..677ee0844 100644--- a/pipelex/pipe_controllers/sequence/pipe_sequence.py+++ b/pipelex/pipe_controllers/sequence/pipe_sequence.py@@ -1,11 +1,11 @@from typing import Literal+from pydantic import field_validatorfrom typing_extensions import override-from pipelex.core.bundles.exceptions import PipeValidationErrorType-from pipelex.core.exceptions import PipeValidationErrorfrom pipelex.core.memory.working_memory import WorkingMemory-from pipelex.core.pipes.exceptions import PipeInputNotFoundError+from pipelex.core.pipes.exceptions import PipeValidationError, PipeValidationErrorType+from pipelex.core.pipes.inputs.exceptions import PipeInputNotFoundErrorfrom pipelex.core.pipes.inputs.input_requirements import InputRequirementsfrom pipelex.core.pipes.inputs.input_requirements_factory import InputRequirementsFactoryfrom pipelex.core.pipes.pipe_output import PipeOutput@@ -15,7 +15,7 @@ from pipelex.pipe_controllers.parallel.pipe_parallel import PipeParallelfrom pipelex.pipe_controllers.pipe_controller import PipeControllerfrom pipelex.pipe_controllers.sequence.exceptions import PipeSequenceValueErrorfrom pipelex.pipe_controllers.sub_pipe import SubPipe-from pipelex.pipe_run.exceptions import PipeRunParamsError+from pipelex.pipe_run.exceptions import PipeRunError, PipeRunParamsErrorfrom pipelex.pipe_run.pipe_run_params import PipeRunParamsfrom pipelex.pipeline.job_metadata import JobMetadata@@ -28,6 +28,14 @@ class PipeSequence(PipeController):def required_variables(self) -> set[str]:return set()+ @field_validator("sequential_sub_pipes", mode="after")+ @classmethod+ def validate_sequential_sub_pipes(cls, value: list[SubPipe]) -> list[SubPipe]:+ if not value:+ msg = f"PipeSequence '{cls.code}' requires at least one sub-pipe"+ raise ValueError(msg)+ return value+@overridedef validate_inputs_static(self):pass@@ -45,20 +53,22 @@ class PipeSequence(PipeController):"""Validate the output for the pipe sequence.The output of the pipe sequence should match the output of the last step."""- last_step_output_pipe = get_required_pipe(pipe_code=self.sequential_sub_pipes[-1].pipe_code)- concept_of_last_step = last_step_output_pipe.output- if not get_concept_library().is_compatible(tested_concept=concept_of_last_step, wanted_concept=self.output):+ last_step_pipe_code = self.sequential_sub_pipes[-1].pipe_code+ last_step_pipe = get_required_pipe(pipe_code=last_step_pipe_code)+ last_step_output_concept = last_step_pipe.output+ if not get_concept_library().is_compatible(tested_concept=last_step_output_concept, wanted_concept=self.output):+ msg = (+ f"PipeSequence concept mismatch: the output concept '{last_step_output_concept.concept_string}' "+ f"of the last step '{last_step_pipe_code}' of sequence pipe '{self.code}' "+ f"is not compatible with the output concept '{self.output.concept_string}' of the sequence."+ )raise PipeValidationError(+ message=msg,error_type=PipeValidationErrorType.INADEQUATE_OUTPUT_CONCEPT,domain=self.domain,pipe_code=self.code,- provided_concept_code=concept_of_last_step.concept_string,+ provided_concept_code=last_step_output_concept.concept_string,required_concept_codes=[self.output.concept_string],- explanation=(- f"PipeSequence concept mismatch: the output concept '{concept_of_last_step.concept_string}' "- f"of the last step '{self.sequential_sub_pipes[-1].pipe_code}' of sequence pipe '{self.code}' "- f"is not compatible with the output concept '{self.output.concept_string}' of the sequence."- ),)@override@@ -163,7 +173,7 @@ class PipeSequence(PipeController):) -> PipeOutput:if not pipe_run_params.run_mode.is_dry:msg = f"PipeSequence._dry_run_controller_pipe() called with run_mode = {pipe_run_params.run_mode} in pipe {self.code}"- raise PipeRunParamsError(msg)+ raise PipeRunError(message=msg, run_mode=pipe_run_params.run_mode)# Verify the output of this pipe is matching the output of the last step.concept_of_last_step = get_required_pipe(pipe_code=self.sequential_sub_pipes[-1].pipe_code).output# if self.output.concept_string != concept_string_of_last_step:diff --git a/pipelex/pipe_controllers/sub_pipe.py b/pipelex/pipe_controllers/sub_pipe.pyindex d86159da8..20d26dea4 100644--- a/pipelex/pipe_controllers/sub_pipe.py+++ b/pipelex/pipe_controllers/sub_pipe.py@@ -3,7 +3,7 @@ from pydantic import BaseModelfrom pipelex import logfrom pipelex.core.memory.exceptions import WorkingMemoryStuffNotFoundErrorfrom pipelex.core.memory.working_memory import WorkingMemory-from pipelex.core.pipes.exceptions import PipeInputError, PipeInputNotFoundError+from pipelex.core.pipes.inputs.exceptions import PipeInputError, PipeInputNotFoundErrorfrom pipelex.core.pipes.pipe_output import PipeOutputfrom pipelex.core.pipes.variable_multiplicity import VariableMultiplicityfrom pipelex.core.stuffs.list_content import ListContentdiff --git a/pipelex/pipe_operators/compose/exceptions.py b/pipelex/pipe_operators/compose/exceptions.pyindex 5b84213bd..a97b10757 100644--- a/pipelex/pipe_operators/compose/exceptions.py+++ b/pipelex/pipe_operators/compose/exceptions.py@@ -1,13 +1,5 @@from pipelex.base_exceptions import PipelexError-class PipeComposeBlueprintError(PipelexError):- pass--class PipeComposeFactoryError(PipelexError):pass---class PipeComposeRunError(PipelexError):- passdiff --git a/pipelex/pipe_operators/extract/exceptions.py b/pipelex/pipe_operators/extract/exceptions.pyindex 0b45d9937..4a0e9d670 100644--- a/pipelex/pipe_operators/extract/exceptions.py+++ b/pipelex/pipe_operators/extract/exceptions.py@@ -1,13 +1,5 @@from pipelex.base_exceptions import PipelexError-class PipeExtractBlueprintValueError(ValueError):- pass---class PipeExtractValueError(ValueError):- pass--class PipeExtractFactoryError(PipelexError):passdiff --git a/pipelex/pipe_operators/extract/pipe_extract.py b/pipelex/pipe_operators/extract/pipe_extract.pyindex 335ed041c..db570dc35 100644--- a/pipelex/pipe_operators/extract/pipe_extract.py+++ b/pipelex/pipe_operators/extract/pipe_extract.py@@ -11,11 +11,9 @@ from pipelex.cogt.extract.extract_input import ExtractInputfrom pipelex.cogt.extract.extract_job_components import ExtractJobConfig, ExtractJobParamsfrom pipelex.cogt.extract.extract_setting import ExtractModelChoice, ExtractSettingfrom pipelex.cogt.models.model_deck_check import check_extract_choice_with_deck-from pipelex.core.bundles.exceptions import PipeValidationErrorTypefrom pipelex.core.concepts.native.concept_native import NativeConceptCode-from pipelex.core.exceptions import PipeValidationErrorfrom pipelex.core.memory.working_memory import WorkingMemory-from pipelex.core.pipe_errors import PipeDefinitionError+from pipelex.core.pipes.exceptions import PipeValidationError, PipeValidationErrorTypefrom pipelex.core.pipes.inputs.input_requirements import InputRequirementsfrom pipelex.core.pipes.pipe_output import PipeOutputfrom pipelex.core.stuffs.image_content import ImageContent@@ -30,6 +28,7 @@ from pipelex.hub import (get_native_concept,)from pipelex.pipe_operators.pipe_operator import PipeOperator+from pipelex.pipe_run.exceptions import PipeRunErrorfrom pipelex.pipe_run.pipe_run_mode import PipeRunModefrom pipelex.pipe_run.pipe_run_params import PipeRunParamsfrom pipelex.pipeline.job_metadata import JobMetadata@@ -86,13 +85,14 @@ class PipeExtract(PipeOperator[PipeExtractOutput]):@overridedef validate_output_with_library(self):if self.output != get_native_concept(native_concept=NativeConceptCode.PAGE):+ msg = f"PipeExtract output should be a Page concept, but is {self.output.concept_string}"raise PipeValidationError(+ message=msg,error_type=PipeValidationErrorType.INADEQUATE_OUTPUT_CONCEPT,domain=self.domain,pipe_code=self.code,provided_concept_code=self.output.concept_string,required_concept_codes=[NativeConceptCode.PAGE.concept_string],- explanation=f"PipeExtract output should be a Page concept, but is {self.output.concept_string}",)@override@@ -208,7 +208,7 @@ class PipeExtract(PipeOperator[PipeExtractOutput]):log.verbose(f"PipeExtract: dry run operator pipe: {self.code}")if pipe_run_params.run_mode != PipeRunMode.DRY:msg = f"Running pipe '{self.code}' (PipeExtract) _dry_run_operator_pipe() in non-dry mode is not allowed."- raise PipeDefinitionError(msg)+ raise PipeRunError(message=msg, run_mode=pipe_run_params.run_mode)content_generator_dry = ContentGeneratorDry()return await self._run_operator_pipe(diff --git a/pipelex/pipe_operators/func/exceptions.py b/pipelex/pipe_operators/func/exceptions.pydeleted file mode 100644index c727cfe4e..000000000--- a/pipelex/pipe_operators/func/exceptions.py+++ /dev/null@@ -1,2 +0,0 @@-class PipeFuncValueError(ValueError):- passdiff --git a/pipelex/pipe_operators/func/pipe_func.py b/pipelex/pipe_operators/func/pipe_func.pyindex 9c82a1fea..8446b7501 100644--- a/pipelex/pipe_operators/func/pipe_func.py+++ b/pipelex/pipe_operators/func/pipe_func.py@@ -8,7 +8,6 @@ from pipelex import logfrom pipelex.core.concepts.concept_factory import ConceptFactoryfrom pipelex.core.memory.working_memory import WorkingMemoryfrom pipelex.core.memory.working_memory_factory import WorkingMemoryFactory-from pipelex.core.pipe_errors import PipeDefinitionErrorfrom pipelex.core.pipes.inputs.input_requirements import InputRequirements, TypedNamedInputRequirementfrom pipelex.core.pipes.pipe_output import PipeOutputfrom pipelex.core.stuffs.list_content import ListContent@@ -16,8 +15,8 @@ from pipelex.core.stuffs.stuff_content import StuffContentfrom pipelex.core.stuffs.stuff_factory import StuffFactoryfrom pipelex.core.stuffs.text_content import TextContentfrom pipelex.pipe_operators.pipe_operator import PipeOperator+from pipelex.pipe_run.exceptions import PipeRunErrorfrom pipelex.pipe_run.pipe_run_params import PipeRunParams-from pipelex.pipeline.exceptions import DryRunMissingInputsErrorfrom pipelex.pipeline.job_metadata import JobMetadatafrom pipelex.system.registries.func_registry import func_registry@@ -137,23 +136,20 @@ class PipeFunc(PipeOperator[PipeFuncOutput]):if not working_memory.get_optional_stuff(named_input_requirement.variable_name):missing_input_names.append(named_input_requirement.variable_name)if missing_input_names:- msg = f"Dry run failed for PipeFunc (function '{self.function_name}'): missing required inputs: {missing_input_names}"- log.error(f"Dry run failed: missing required inputs: {missing_input_names}")- raise DryRunMissingInputsError(- message=msg,- pipe_type=self.__class__.__name__,- pipe_code=self.code,- missing_inputs=missing_input_names,- )+ msg = f"Dry run failed for pipe '{self.code}' (PipeFunc): missing required inputs: {missing_input_names}"+ raise PipeRunError(message=msg, run_mode=pipe_run_params.run_mode)return_type = get_type_hints(function).get("return")if return_type is None:- msg = f"Function '{self.function_name}' has no return type annotation"- raise PipeDefinitionError(msg)+ msg = f"Dry run failed for pipe '{self.code}' (PipeFunc): function '{self.function_name}' has no return type annotation"+ raise PipeRunError(message=msg, run_mode=pipe_run_params.run_mode)if not issubclass(return_type, StuffContent):- msg = f"Function '{self.function_name}' return type {return_type} is not a subclass of StuffContent"- raise PipeDefinitionError(msg)+ msg = (+ f"Dry run failed for pipe '{self.code}' (PipeFunc): "+ f"function '{self.function_name}' return type {return_type} is not a subclass of StuffContent"+ )+ raise PipeRunError(message=msg, run_mode=pipe_run_params.run_mode)# TODO: Support PipeFunc returning with multiplicity. Create an equivalent of TypedNamedInputRequirement for outputs.requirement = TypedNamedInputRequirement(diff --git a/pipelex/pipe_operators/img_gen/exceptions.py b/pipelex/pipe_operators/img_gen/exceptions.pyindex 6c1faf784..e6656ab3c 100644--- a/pipelex/pipe_operators/img_gen/exceptions.py+++ b/pipelex/pipe_operators/img_gen/exceptions.py@@ -1,17 +1,5 @@from pipelex.base_exceptions import PipelexError-class PipeImgGenBlueprintValueError(ValueError):- pass---class PipeImgGenValueError(ValueError):- pass---class PipeImgGenFactoryError(PipelexError):- pass--class PipeImgGenRunError(PipelexError):passdiff --git a/pipelex/pipe_operators/img_gen/pipe_img_gen.py b/pipelex/pipe_operators/img_gen/pipe_img_gen.pyindex 5399905a0..129e3eb0a 100644--- a/pipelex/pipe_operators/img_gen/pipe_img_gen.py+++ b/pipelex/pipe_operators/img_gen/pipe_img_gen.py@@ -11,14 +11,11 @@ from pipelex.cogt.img_gen.img_gen_prompt import ImgGenPromptfrom pipelex.cogt.img_gen.img_gen_setting import ImgGenModelChoice, ImgGenSettingfrom pipelex.cogt.models.model_deck_check import check_img_gen_choice_with_deckfrom pipelex.config import get_config-from pipelex.core.bundles.exceptions import PipeValidationErrorTypefrom pipelex.core.concepts.concept_factory import ConceptFactoryfrom pipelex.core.concepts.native.concept_native import NativeConceptCode-from pipelex.core.exceptions import PipeValidationErrorfrom pipelex.core.memory.exceptions import WorkingMemoryStuffNotFoundErrorfrom pipelex.core.memory.working_memory import WorkingMemory-from pipelex.core.pipe_errors import UnexpectedPipeDefinitionError-from pipelex.core.pipes.exceptions import PipeInputError+from pipelex.core.pipes.exceptions import PipeValidationError, PipeValidationErrorTypefrom pipelex.core.pipes.inputs.input_requirements import InputRequirementsfrom pipelex.core.pipes.inputs.input_requirements_factory import InputRequirementsFactoryfrom pipelex.core.pipes.pipe_output import PipeOutput@@ -28,6 +25,7 @@ from pipelex.core.stuffs.image_content import ImageContentfrom pipelex.core.stuffs.list_content import ListContentfrom pipelex.core.stuffs.stuff_factory import StuffFactoryfrom pipelex.hub import get_concept_library, get_content_generator, get_model_deck, get_native_concept+from pipelex.pipe_operators.img_gen.exceptions import PipeImgGenRunErrorfrom pipelex.pipe_operators.pipe_operator import PipeOperatorfrom pipelex.pipe_run.exceptions import PipeRunParamsErrorfrom pipelex.pipe_run.pipe_run_mode import PipeRunMode@@ -125,18 +123,18 @@ class PipeImgGen(PipeOperator[PipeImgGenOutput]):wanted_concept=ConceptFactory.make_native_concept(native_concept_code=NativeConceptCode.TEXT),strict=True,):- explanation = (+ msg = (f"The input of a PipeImgGen must be compatible with the Text concept (or refine it). "f"Input '{input_name}' has concept '{input_requirement.concept.concept_string}'. ""Image generation requires a text prompt. Use native.Text or a concept that refines it.")raise PipeValidationError(+ message=msg,error_type=PipeValidationErrorType.IMG_GEN_INPUT_NOT_TEXT_COMPATIBLE,pipe_code=self.code,variable_names=[input_name],required_concept_codes=["native.Text"],provided_concept_code=input_requirement.concept.concept_string,- explanation=explanation,)@override@@ -150,16 +148,17 @@ class PipeImgGen(PipeOperator[PipeImgGenOutput]):wanted_concept=get_native_concept(native_concept=NativeConceptCode.IMAGE),strict=True,):+ msg = (+ f"The output of a ImgGen pipe must be compatible with the Image concept. "+ f"In the pipe '{self.code}' the output is '{self.output.concept_string}'"+ )raise PipeValidationError(+ message=msg,error_type=PipeValidationErrorType.INADEQUATE_OUTPUT_CONCEPT,domain=self.domain,pipe_code=self.code,provided_concept_code=self.output.concept_string,required_concept_codes=[NativeConceptCode.IMAGE.concept_string],- explanation=(- f"The output of a ImgGen pipe must be compatible with the Image concept. "- f"In the pipe '{self.code}' the output is '{self.output.concept_string}'"- ),)@override@@ -179,6 +178,7 @@ class PipeImgGen(PipeOperator[PipeImgGenOutput]):)applied_output_multiplicity = multiplicity_resolution.resolved_multiplicity+ # TODO: move this to the factorylog.verbose("Getting image generation prompt from context")if self.img_gen_prompt:img_gen_prompt_text = self.img_gen_prompt@@ -186,19 +186,23 @@ class PipeImgGen(PipeOperator[PipeImgGenOutput]):try:img_gen_prompt_text = working_memory.get_stuff_as_str(stuff_name)except WorkingMemoryStuffNotFoundError as stuff_not_found_error:- msg = f"Could not find a valid image generation prompt named '{stuff_name}' in the working_memory: {stuff_not_found_error}"- raise PipeInputError(- message=msg,- pipe_code=self.code,- variable_name=stuff_name,- concept_code=None,- ) from stuff_not_found_error+ msg = (+ f"While runnning the PipeImgGen '{self.code}' an error occurred: "+ f"Could not find a valid image generation prompt named '{stuff_name}' in the working_memory: {stuff_not_found_error}"+ )+ raise PipeImgGenRunError(message=msg) from stuff_not_found_errorexcept StuffContentTypeError as stuff_content_type_error:- msg = f"The image generation prompt named '{stuff_name}' is not of the right type: {stuff_content_type_error}"- raise PipeInputError(message=msg, pipe_code=self.code, variable_name=stuff_name, concept_code=None) from stuff_content_type_error+ msg = (+ f"While runnning the PipeImgGen '{self.code}' an error occurred: "+ f"The image generation prompt named '{stuff_name}' is not of the right type: {stuff_content_type_error}"+ )+ raise PipeImgGenRunError(message=msg) from stuff_content_type_errorelse:- msg = "You must provide an image gen prompt either as attribute of the pipe or as a single text input"- raise UnexpectedPipeDefinitionError(msg)+ msg = (+ f"While runnning the PipeImgGen '{self.code}' an error occurred: "+ "You must provide an image gen prompt either as attribute of the pipe or as a single text input"+ )+ raise PipeImgGenRunError(msg)img_gen_config = get_config().cogt.img_gen_configimg_gen_param_defaults = img_gen_config.img_gen_param_defaultsdiff --git a/pipelex/pipe_operators/llm/exceptions.py b/pipelex/pipe_operators/llm/exceptions.pyindex eb1cd9e7f..039717bc2 100644--- a/pipelex/pipe_operators/llm/exceptions.py+++ b/pipelex/pipe_operators/llm/exceptions.py@@ -5,9 +5,5 @@ class LLMPromptBlueprintValueError(ValueError):pass-class PipeLLMValueError(ValueError):- pass--class PipeLLMFactoryError(PipelexError):passdiff --git a/pipelex/pipe_operators/llm/pipe_llm.py b/pipelex/pipe_operators/llm/pipe_llm.pyindex 06668d5fa..4a1d23e14 100644--- a/pipelex/pipe_operators/llm/pipe_llm.py+++ b/pipelex/pipe_operators/llm/pipe_llm.py@@ -1,6 +1,6 @@from typing import Literal, cast-from pydantic import model_validator+from pydantic import ValidationError, model_validatorfrom typing_extensions import overridefrom pipelex import log@@ -14,12 +14,11 @@ from pipelex.cogt.llm.llm_setting import LLMModelChoice, LLMSetting, LLMSettingCfrom pipelex.cogt.models.model_deck_check import check_llm_choice_with_deckfrom pipelex.cogt.templating.template_category import TemplateCategoryfrom pipelex.config import get_config-from pipelex.core.bundles.exceptions import PipeValidationErrorTypefrom pipelex.core.concepts.concept_factory import ConceptFactoryfrom pipelex.core.concepts.native.concept_native import NativeConceptCodefrom pipelex.core.domains.domain import SpecialDomain-from pipelex.core.exceptions import PipeValidationErrorfrom pipelex.core.memory.working_memory import WorkingMemory+from pipelex.core.pipes.exceptions import PipeValidationError, PipeValidationErrorTypefrom pipelex.core.pipes.inputs.input_requirements import InputRequirementsfrom pipelex.core.pipes.inputs.input_requirements_factory import InputRequirementsFactoryfrom pipelex.core.pipes.pipe_output import PipeOutput@@ -39,13 +38,14 @@ from pipelex.hub import (from pipelex.pipe_operators.llm.llm_prompt_blueprint import LLMPromptBlueprintfrom pipelex.pipe_operators.llm.pipe_llm_blueprint import StructuringMethodfrom pipelex.pipe_operators.pipe_operator import PipeOperator+from pipelex.pipe_run.exceptions import PipeRunErrorfrom pipelex.pipe_run.pipe_run_params import (PipeRunParamKey,PipeRunParams,output_multiplicity_to_apply,)-from pipelex.pipeline.exceptions import PipeRunErrorfrom pipelex.pipeline.job_metadata import JobMetadata+from pipelex.tools.typing.pydantic_utils import format_pydantic_validation_errorfrom pipelex.tools.typing.structure_printer import StructurePrinterfrom pipelex.types import Self@@ -89,20 +89,25 @@ class PipeLLM(PipeOperator[PipeLLMOutput]):def validate_output_with_library(self):# TODO: generalize because there are other concepts PipeLLM can't generate, not just images,# and PipeLLM is not the only one with this kind of constraints++ # Allow Dynamic output concept as it's flexible and can represent anything+ if NativeConceptCode.is_dynamic_concept(concept_code=self.output.code):+ return+if get_concept_library().is_compatible(tested_concept=self.output,wanted_concept=get_native_concept(native_concept=NativeConceptCode.IMAGE),):- explanation = (- f"The output of a LLM pipe cannot be compatible with the Image concept. "- f"Output concept is '{self.output.concept_string}'. "+ msg = (+ f"The output of the PipeLLM '{self.code}' cannot be compatible with the Image concept. "+ f"The output concept is '{self.output.concept_string}'. ""Use a PipeImgGen if you want to generate images. You can use a PipeLLM to generate the prompt for a PipeImgGen.")raise PipeValidationError(+ message=msg,error_type=PipeValidationErrorType.LLM_OUTPUT_CANNOT_BE_IMAGE,pipe_code=self.code,provided_concept_code=self.output.concept_string,- explanation=explanation,)@override@@ -218,8 +223,9 @@ class PipeLLM(PipeOperator[PipeLLMOutput]):)except LLMCompletionError as exc:location = self._format_error_location(pipe_run_params=pipe_run_params)- msg = f"Error generating text with LLM {location}: {exc}"- raise PipeRunError(msg) from exc+ error_details = self._format_llm_error(exc)+ msg = f"Error generating text with LLM {location}: {error_details}"+ raise PipeRunError(message=msg, run_mode=pipe_run_params.run_mode) from excthe_content = TextContent(text=generated_text,@@ -332,8 +338,9 @@ class PipeLLM(PipeOperator[PipeLLMOutput]):)except LLMCompletionError as exc:location = self._format_error_location(pipe_run_params=pipe_run_params)- msg = f"Error generating list of objects with text then object {location}: {exc}"- raise PipeRunError(msg) from exc+ error_details = self._format_llm_error(exc)+ msg = f"Error generating list of objects with text then object {location}: {error_details}"+ raise PipeRunError(message=msg, run_mode=pipe_run_params.run_mode) from excelse:# We're generating a list of objects directlymethod_desc = "object_direct"@@ -348,8 +355,9 @@ class PipeLLM(PipeOperator[PipeLLMOutput]):)except LLMCompletionError as exc:location = self._format_error_location(pipe_run_params=pipe_run_params)- msg = f"Error generating list of objects with direct method {location}: {exc}"- raise PipeRunError(msg) from exc+ error_details = self._format_llm_error(exc)+ msg = f"Error generating list of objects with direct method {location}: {error_details}"+ raise PipeRunError(message=msg, run_mode=pipe_run_params.run_mode) from excthe_content = ListContent(items=generated_objects)else:@@ -372,8 +380,9 @@ class PipeLLM(PipeOperator[PipeLLMOutput]):)except LLMCompletionError as exc:location = self._format_error_location(pipe_run_params=pipe_run_params)- msg = f"Error generating single object with text then object {location}: {exc}"- raise PipeRunError(msg) from exc+ error_details = self._format_llm_error(exc)+ msg = f"Error generating single object with text then object {location}: {error_details}"+ raise PipeRunError(message=msg, run_mode=pipe_run_params.run_mode) from excelse:# We're generating a single object directlymethod_desc = "object_direct"@@ -387,8 +396,9 @@ class PipeLLM(PipeOperator[PipeLLMOutput]):)except LLMCompletionError as exc:location = self._format_error_location(pipe_run_params=pipe_run_params)- msg = f"Error generating single object with direct method {location}: {exc}"- raise PipeRunError(msg) from exc+ error_details = self._format_llm_error(exc)+ msg = f"Error generating single object with direct method {location}: {error_details}"+ raise PipeRunError(message=msg, run_mode=pipe_run_params.run_mode) from excthe_content = generated_objectreturn the_content@@ -396,6 +406,17 @@ class PipeLLM(PipeOperator[PipeLLMOutput]):def _format_error_location(self, pipe_run_params: PipeRunParams) -> str:return f"in pipe '{pipe_run_params.pipe_stack_str}'"+ def _format_llm_error(self, exc: LLMCompletionError) -> str:+ """Format an LLMCompletionError, extracting and formatting any ValidationError in the chain."""+ error_details = str(exc)+ current_exc: BaseException | None = exc+ while current_exc is not None:+ if isinstance(current_exc, ValidationError):+ error_details = format_pydantic_validation_error(current_exc)+ break+ current_exc = current_exc.__cause__+ return error_details+@overrideasync def _dry_run_operator_pipe(self,diff --git a/pipelex/pipe_operators/llm/pipe_llm_blueprint.py b/pipelex/pipe_operators/llm/pipe_llm_blueprint.pyindex 79e6bbd85..9aeef6590 100644--- a/pipelex/pipe_operators/llm/pipe_llm_blueprint.py+++ b/pipelex/pipe_operators/llm/pipe_llm_blueprint.py@@ -6,6 +6,7 @@ from pipelex.cogt.llm.llm_setting import LLMModelChoicefrom pipelex.cogt.templating.template_category import TemplateCategoryfrom pipelex.cogt.templating.template_preprocessor import preprocess_templatefrom pipelex.core.pipes.pipe_blueprint import PipeBlueprint+from pipelex.tools.jinja2.jinja2_errors import Jinja2DetectVariablesErrorfrom pipelex.tools.jinja2.jinja2_required_variables import detect_jinja2_required_variablesfrom pipelex.types import StrEnum@@ -34,22 +35,29 @@ class PipeLLMBlueprint(PipeBlueprint):if self.prompt:preprocessed_template = preprocess_template(self.prompt)- required_variables.update(- detect_jinja2_required_variables(- template_category=TemplateCategory.LLM_PROMPT,- template_source=preprocessed_template,+ try:+ required_variables.update(+ detect_jinja2_required_variables(+ template_category=TemplateCategory.LLM_PROMPT,+ template_source=preprocessed_template,+ ))- )+ except Jinja2DetectVariablesError as exc:+ msg = f"Could not detect required variables in prompt for PipeLLM: {exc}"+ raise ValueError(msg) from excif self.system_prompt:preprocessed_system_template = preprocess_template(self.system_prompt)- required_variables.update(- detect_jinja2_required_variables(- template_category=TemplateCategory.LLM_PROMPT,- template_source=preprocessed_system_template,+ try:+ required_variables.update(+ detect_jinja2_required_variables(+ template_category=TemplateCategory.LLM_PROMPT,+ template_source=preprocessed_system_template,+ ))- )-+ except Jinja2DetectVariablesError as exc:+ msg = f"Could not detect required variables in system prompt for PipeLLM: {exc}"+ raise ValueError(msg) from exc# Filter out internal variables that start with underscore and special variables# TODO: replace magic strings by StrEnum and also, make this check clearer and more readablerequired_variables = {var for var in required_variables if not var.startswith("_") and var not in ("preliminary_text", "place_holder")}diff --git a/pipelex/pipe_run/dry_run.py b/pipelex/pipe_run/dry_run.pyindex beaef0928..e95d272a2 100644--- a/pipelex/pipe_run/dry_run.py+++ b/pipelex/pipe_run/dry_run.py@@ -42,7 +42,6 @@ class DryRunOutput(BaseModel):async def dry_run_pipe(pipe: PipeAbstract, raise_on_failure: bool = False) -> DryRunOutput:- """Dry run a single pipe directly without parallelization."""try:needed_inputs_for_factory = _convert_to_working_memory_format(needed_inputs_spec=pipe.needed_inputs())working_memory = WorkingMemoryFactory.make_for_dry_run(needed_inputs=needed_inputs_for_factory)diff --git a/pipelex/pipe_run/exceptions.py b/pipelex/pipe_run/exceptions.pyindex 689ecf67d..cd1a0dc44 100644--- a/pipelex/pipe_run/exceptions.py+++ b/pipelex/pipe_run/exceptions.py@@ -11,7 +11,9 @@ class BatchParamsError(PipelexError):class PipeRunError(PipelexError):- pass+ def __init__(self, message: str, run_mode: PipeRunMode):+ self.run_mode = run_mode+ super().__init__(message)class PipeRouterError(PipelexError):diff --git a/pipelex/pipe_run/pipe_router_protocol.py b/pipelex/pipe_run/pipe_router_protocol.pyindex 1e21387d1..daf3d8b43 100644--- a/pipelex/pipe_run/pipe_router_protocol.py+++ b/pipelex/pipe_run/pipe_router_protocol.py@@ -5,7 +5,6 @@ from pipelex.core.pipes.pipe_output import PipeOutputfrom pipelex.observer.observer_protocol import ObserverProtocol, PayloadKey, PayloadTypefrom pipelex.pipe_run.exceptions import PipeRouterError, PipeRunErrorfrom pipelex.pipe_run.pipe_job import PipeJob-from pipelex.pipeline.exceptions import DryRunMissingInputsErrorclass PipeRouterProtocol(Protocol):@@ -53,16 +52,6 @@ class PipeRouterProtocol(Protocol):try:pipe_output = await self._run_pipe_job(pipe_job)- except DryRunMissingInputsError as exc:- await self._after_failing_run(pipe_job, exc)- raise PipeRouterError(- message=exc.message,- run_mode=pipe_job.pipe_run_params.run_mode,- pipe_code=pipe_job.pipe.code,- output_name=pipe_job.output_name,- pipe_stack=pipe_job.pipe_run_params.pipe_stack,- missing_inputs=exc.missing_inputs,- ) from excexcept PipeRunError as exc:await self._after_failing_run(pipe_job, exc)raise PipeRouterError(diff --git a/pipelex/pipeline/exceptions.py b/pipelex/pipeline/exceptions.pyindex 339ab901d..4c300783b 100644--- a/pipelex/pipeline/exceptions.py+++ b/pipelex/pipeline/exceptions.py@@ -1,6 +1,4 @@from pipelex.base_exceptions import PipelexError-from pipelex.cogt.templating.template_category import TemplateCategory-from pipelex.pipe_run.exceptions import PipeRunErrorfrom pipelex.pipe_run.pipe_run_mode import PipeRunMode@@ -24,40 +22,6 @@ class PipelineExecutionError(PipelexError):super().__init__(message)-class DryRunError(PipeRunError):- """Raised when a dry run fails due to missing inputs or other validation issues."""-- def __init__(self, message: str, pipe_type: str, pipe_code: str | None = None):- self.pipe_type = pipe_type- self.pipe_code = pipe_code- super().__init__(message)---class DryRunMissingInputsError(DryRunError):- """Raised when a dry run fails due to missing inputs or other validation issues."""-- def __init__(self, message: str, pipe_type: str, pipe_code: str, missing_inputs: list[str] | None = None):- self.missing_inputs = missing_inputs or []- super().__init__(message, pipe_type, pipe_code)---class DryRunMissingPipesError(DryRunError):- """Raised when a dry run fails due to missing pipes or other validation issues."""-- def __init__(self, message: str, pipe_type: str, pipe_code: str, missing_pipes: list[str] | None = None):- self.missing_pipes = missing_pipes or []- super().__init__(message, pipe_type, pipe_code)---class DryRunTemplatingError(DryRunError):- """Raised when a dry run fails due to templating issues."""-- def __init__(self, message: str, pipe_type: str, pipe_code: str, template_category: TemplateCategory, template: str):- self.template_category = template_category- self.template = template- super().__init__(message, pipe_type, pipe_code)--class PipeStackOverflowError(PipelexError):def __init__(self, message: str, limit: int, pipe_stack: list[str]):self.limit = limitdiff --git a/pipelex/pipeline/validate_bundle.py b/pipelex/pipeline/validate_bundle.pyindex 3b337d9a5..aa0ddbd1f 100644--- a/pipelex/pipeline/validate_bundle.py+++ b/pipelex/pipeline/validate_bundle.py@@ -1,118 +1,68 @@-from typing import Any-from pydantic import BaseModel, ValidationErrorfrom pipelex.base_exceptions import PipelexError-from pipelex.core.bundles.exceptions import PipeValidationErrorType+from pipelex.core.bundles.exceptions import PipelexBundleBlueprintValidationErrorDatafrom pipelex.core.bundles.pipelex_bundle_blueprint import PipelexBundleBlueprint-from pipelex.core.exceptions import PipelexBundleBlueprintValidationErrorData, PipelexInterpreterError, PipeValidationError-from pipelex.core.interpreter import PipelexInterpreter-from pipelex.core.pipes.exceptions import PipeValidationErrorData+from pipelex.core.exceptions import PipesAndConceptValidationErrorData+from pipelex.core.interpreter.exceptions import PipelexInterpreterError+from pipelex.core.interpreter.interpreter import PipelexInterpreter+from pipelex.core.pipes.exceptions import PipeValidationError+from pipelex.core.pipes.handle_pipe_errors import (+ categorize_pipe_validation_error,+ categorize_pipe_validation_with_libraries_error,+)from pipelex.core.pipes.pipe_abstract import PipeAbstract+from pipelex.core.validation import report_validation_errorfrom pipelex.hub import get_library_manager, set_current_libraryfrom pipelex.pipe_run.dry_run import DryRunError, DryRunOutput, dry_run_pipes-def categorize_pipe_validation_error_data(error_dict: dict[str, Any], pipe_code: str | None = None) -> PipeValidationErrorData:- """Categorize a Pydantic ValidationError into structured PipeValidationErrorData.-- Analyzes the error location, type, and message to determine the appropriate error category.-- Args:- error_dict: The error dictionary from Pydantic ValidationError.errors()- pipe_code: Optional pipe code to associate with the error-- Returns:- PipeValidationErrorData with appropriate error_type categorization- """- error_type_str: str = str(error_dict.get("type", ""))- error_msg: str = str(error_dict.get("msg", ""))- error_loc: tuple[Any, ...] = error_dict.get("loc", ())-- # Convert location tuple to field path string- field_path = ".".join(str(loc) for loc in error_loc) if error_loc else None-- # Determine error type based on patterns in the message and error type- categorized_error_type: PipeValidationErrorType = PipeValidationErrorType.UNKNOWN_VALIDATION_ERROR-- # Pattern matching on error messages to categorize- error_msg_lower: str = error_msg.lower()-- # PipeParallel: requires either add_each_output or combined_output- if "requires either add_each_output or combined_output" in error_msg_lower:- categorized_error_type = PipeValidationErrorType.PIPE_PARALLEL_OUTPUT_CONFIG_ERROR-- # PipeParallel: duplicate output names- elif "output name" in error_msg_lower and ("already used" in error_msg_lower or "duplicate" in error_msg_lower):- categorized_error_type = PipeValidationErrorType.DUPLICATE_OUTPUT_NAME-- # PipeExtract: must provide either pdf or image- elif "must provide either" in error_msg_lower and ("pdf" in error_msg_lower or "image" in error_msg_lower):- categorized_error_type = PipeValidationErrorType.FIELD_REQUIRED-- # Model not found in deck (Extract, ImgGen, LLM)- elif "was not found in the model deck" in error_msg_lower or "not found in deck" in error_msg_lower:- categorized_error_type = PipeValidationErrorType.MODEL_NOT_IN_DECK-- # PipeFunc: function not found in registry- elif "not found in registry" in error_msg_lower:- categorized_error_type = PipeValidationErrorType.FUNCTION_NOT_FOUND-- # PipeFunc: invalid return type- elif "has no return type annotation" in error_msg_lower or "is not a subclass of" in error_msg_lower:- categorized_error_type = PipeValidationErrorType.INVALID_RETURN_TYPE-- # PipeLLM: output concept inconsistency with structuring- elif "cannot be structured" in error_msg_lower or ("output concept" in error_msg_lower and "text concept" in error_msg_lower):- categorized_error_type = PipeValidationErrorType.OUTPUT_CONCEPT_INCONSISTENCY-- # PipeImgGen: mutually exclusive fields- elif "either" in error_msg_lower and "or" in error_msg_lower and "but not both" in error_msg_lower:- categorized_error_type = PipeValidationErrorType.MUTUALLY_EXCLUSIVE_FIELDS-- # Pydantic standard error types- elif error_type_str in ("missing", "missing_required"):- categorized_error_type = PipeValidationErrorType.FIELD_MISSING-- elif error_type_str == "value_error" and not categorized_error_type:- # Generic value error - keep as unknown unless we caught it above- categorized_error_type = PipeValidationErrorType.UNKNOWN_VALIDATION_ERROR-- return PipeValidationErrorData(- error_type=categorized_error_type,- domain=None,- pipe_code=pipe_code,- variable_names=[str(loc) for loc in error_loc] if error_loc else None,- required_concept_codes=None,- provided_concept_code=None,- file_path=field_path,- explanation=error_msg,- )--class ValidateBundleError(PipelexError):"""Raised when bundle validation fails.This error aggregates validation errors from different stages:- - Interpreter errors (blueprint validation)- - Library loading errors (factory and validation errors)+ - Blueprint validation errors (from interpreter)+ - Pipe validation errors (from PipeValidationError exceptions)+ - Pipe/Concept instantiation errors (from Pydantic ValidationError during factory instantiation)- Dry run errors- All errors are categorized and stored in validation_errors.+ All errors are categorized and stored in their respective lists."""def __init__(self,message: str,pipelex_bundle_blueprint_validation_errors: list[PipelexBundleBlueprintValidationErrorData] | None = None,- pipe_validation_error_data: list[PipeValidationErrorData] | None = None,+ pipe_validation_errors: list[PipesAndConceptValidationErrorData] | None = None,+ pipe_concept_instantiation_errors: list[PipesAndConceptValidationErrorData] | None = None,dry_run_error_message: str | None = None,):+ # Blueprint validation errors (e.g., PIPE_SEQUENCE_OUTPUT_MISMATCH)self.pipelex_bundle_blueprint_validation_errors = pipelex_bundle_blueprint_validation_errors or []- self.pipe_validation_error_data = pipe_validation_error_data or []++ # Pipe validation errors from PipeValidationError exceptions+ # (e.g., MISSING_INPUT_VARIABLE, EXTRANEOUS_INPUT_VARIABLE, INPUT_REQUIREMENT_MISMATCH, INADEQUATE_OUTPUT_CONCEPT)+ self.pipe_validation_errors = pipe_validation_errors or []++ # Pipe/Concept instantiation errors from Pydantic ValidationError+ # These occur during factory instantiation of Pipe or Concept classes+ # TODO: Currently not caught, but structure is prepared for future implementation+ self.pipe_concept_instantiation_errors = pipe_concept_instantiation_errors or []++ # Dry run errorsself.dry_run_error_message = dry_run_error_message+super().__init__(message)+ @property+ def pipe_validation_error_data(self) -> list[PipesAndConceptValidationErrorData]:+ """Backwards compatibility: combine pipe validation and instantiation errors.++ This property provides the old interface for accessing all pipe/concept validation errors.+ """+ # TODO: refactor so we don't need this anymore?+ return self.pipe_validation_errors + self.pipe_concept_instantiation_errors+class ValidateBundleResult(BaseModel):blueprints: list[PipelexBundleBlueprint]@@ -154,41 +104,25 @@ async def validate_bundle(dry_run_results = await dry_run_pipes(pipes=loaded_pipes, raise_on_failure=True)except PipelexInterpreterError as interpreter_error:- # Forward categorized validation errors from interpreterraise ValidateBundleError(message=interpreter_error.message,pipelex_bundle_blueprint_validation_errors=interpreter_error.validation_errors,) from interpreter_errorexcept PipeValidationError as pipe_error:- # Convert PipeValidationError to PipeValidationErrorData- pipe_error_data = PipeValidationErrorData(- error_type=pipe_error.error_type,- domain=pipe_error.domain,- pipe_code=pipe_error.pipe_code,- variable_names=pipe_error.variable_names,- required_concept_codes=pipe_error.required_concept_codes,- provided_concept_code=pipe_error.provided_concept_code,- file_path=pipe_error.file_path,- explanation=pipe_error.explanation,- )+ pipe_error_data = categorize_pipe_validation_with_libraries_error(pipe_error=pipe_error)raise ValidateBundleError(message=f"Pipe validation failed: {pipe_error}",- pipe_validation_error_data=[pipe_error_data],+ pipe_validation_errors=[pipe_error_data],) from pipe_errorexcept ValidationError as validation_error:- # Convert Pydantic ValidationError to categorized PipeValidationErrorData- pipe_error_data_list: list[PipeValidationErrorData] = []- for error in validation_error.errors():- # Categorize each validation error based on its content and location- # Cast to dict[str, Any] since ErrorDetails is a TypedDict compatible with dict- categorized_error = categorize_pipe_validation_error_data(error_dict=dict(error), pipe_code=None)- pipe_error_data_list.append(categorized_error)+ pipe_validation_errors = categorize_pipe_validation_error(validation_error=validation_error)+ validation_error_msg = report_validation_error(category="plx", validation_error=validation_error)+ msg = f"Could not load blueprints because of: {validation_error_msg}"raise ValidateBundleError(- message=f"Validation failed: {validation_error}",- pipe_validation_error_data=pipe_error_data_list,+ message=msg,+ pipe_validation_errors=pipe_validation_errors,) from validation_errorexcept DryRunError as dry_run_error:- # Forward dry run error messageraise ValidateBundleError(message=dry_run_error.message,dry_run_error_message=dry_run_error.message,diff --git a/pipelex/system/configuration/configs.py b/pipelex/system/configuration/configs.pyindex 2fc2a8eec..b38fec174 100644--- a/pipelex/system/configuration/configs.py+++ b/pipelex/system/configuration/configs.py@@ -7,7 +7,7 @@ from pipelex.base_exceptions import PipelexConfigErrorfrom pipelex.cogt.config_cogt import Cogtfrom pipelex.cogt.model_backends.prompting_target import PromptingTargetfrom pipelex.cogt.templating.templating_style import TemplatingStyle-from pipelex.core.bundles.exceptions import PipeValidationErrorType+from pipelex.core.pipes.exceptions import PipeValidationErrorTypefrom pipelex.language.plx_config import PlxConfigfrom pipelex.pipeline.track.tracker_config import TrackerConfigfrom pipelex.system.configuration.config_model import ConfigModeldiff --git a/tests/e2e/pipelex/pipes/pipe_controller/pipe_condition/test_pipe_condition_simple.py b/tests/e2e/pipelex/pipes/pipe_controller/pipe_condition/test_pipe_condition_simple.pyindex 912ee9c4b..7779c4569 100644--- a/tests/e2e/pipelex/pipes/pipe_controller/pipe_condition/test_pipe_condition_simple.py+++ b/tests/e2e/pipelex/pipes/pipe_controller/pipe_condition/test_pipe_condition_simple.py@@ -10,13 +10,13 @@ from pipelex.core.concepts.concept_factory import ConceptFactoryfrom pipelex.core.concepts.native.concept_native import NativeConceptCodefrom pipelex.core.domains.domain import SpecialDomainfrom pipelex.core.memory.working_memory_factory import WorkingMemoryFactory-from pipelex.core.pipes.exceptions import PipeRunInputsError+from pipelex.core.pipes.inputs.exceptions import PipeRunInputsErrorfrom pipelex.core.pipes.inputs.input_requirements import TypedNamedInputRequirementfrom pipelex.pipe_controllers.condition.pipe_condition_blueprint import PipeConditionBlueprintfrom pipelex.pipe_controllers.condition.pipe_condition_factory import PipeConditionFactory+from pipelex.pipe_run.exceptions import PipeRunErrorfrom pipelex.pipe_run.pipe_run_params import PipeRunModefrom pipelex.pipe_run.pipe_run_params_factory import PipeRunParamsFactory-from pipelex.pipeline.exceptions import DryRunMissingInputsErrorfrom pipelex.pipeline.job_metadata import JobMetadatafrom tests.integration.pipelex.pipes.controller.pipe_condition.pipe_condition import CategoryInput@@ -108,7 +108,7 @@ class TestPipeConditionSimple:log.info("β Direct PipeCondition SUCCEEDED completely!")pretty_print(pipe_output)- except DryRunMissingInputsError as exc:+ except PipeRunError as exc:msg_exc = str(exc)# If it fails, it should NOT be due to missing inputsassert "missing required inputs" not in str(msg_exc)diff --git a/tests/integration/pipelex/pipes/controller/pipe_condition/test_pipe_condition_complex.py b/tests/integration/pipelex/pipes/controller/pipe_condition/test_pipe_condition_complex.pyindex c7949a186..168b19211 100644--- a/tests/integration/pipelex/pipes/controller/pipe_condition/test_pipe_condition_complex.py+++ b/tests/integration/pipelex/pipes/controller/pipe_condition/test_pipe_condition_complex.py@@ -10,7 +10,7 @@ from pytest import FixtureRequestfrom pipelex import pretty_printfrom pipelex.core.concepts.concept_factory import ConceptFactoryfrom pipelex.core.memory.working_memory_factory import WorkingMemoryFactory-from pipelex.core.pipes.exceptions import PipeRunInputsError+from pipelex.core.pipes.inputs.exceptions import PipeRunInputsErrorfrom pipelex.core.stuffs.stuff_factory import StuffFactoryfrom pipelex.core.stuffs.text_content import TextContentfrom pipelex.hub import get_pipe_router, get_required_pipediff --git a/tests/integration/pipelex/pipes/controller/pipe_condition/test_pipe_condition_simple.py b/tests/integration/pipelex/pipes/controller/pipe_condition/test_pipe_condition_simple.pyindex 01e7848d5..f8e75b03e 100644--- a/tests/integration/pipelex/pipes/controller/pipe_condition/test_pipe_condition_simple.py+++ b/tests/integration/pipelex/pipes/controller/pipe_condition/test_pipe_condition_simple.py@@ -11,7 +11,7 @@ from pipelex.core.concepts.concept_factory import ConceptFactoryfrom pipelex.core.concepts.native.concept_native import NativeConceptCodefrom pipelex.core.domains.domain import SpecialDomainfrom pipelex.core.memory.working_memory_factory import WorkingMemoryFactory-from pipelex.core.pipes.exceptions import PipeRunInputsError+from pipelex.core.pipes.inputs.exceptions import PipeRunInputsErrorfrom pipelex.core.stuffs.stuff_factory import StuffFactoryfrom pipelex.core.stuffs.text_content import TextContentfrom pipelex.hub import get_pipe_router, get_required_pipediff --git a/tests/unit/pipelex/core/bundles/test_data_pipe_sorter.py b/tests/unit/pipelex/core/bundles/test_data_pipe_sorter.pyindex bb40da0c9..b6c48bddd 100644--- a/tests/unit/pipelex/core/bundles/test_data_pipe_sorter.py+++ b/tests/unit/pipelex/core/bundles/test_data_pipe_sorter.py@@ -3,7 +3,7 @@from typing import ClassVarfrom pipelex.core.bundles.pipelex_bundle_blueprint import PipeBlueprintUnion-from pipelex.core.pipe_errors import PipeDefinitionError+from pipelex.core.pipes.exceptions import PipeValidationErrorfrom pipelex.pipe_controllers.batch.pipe_batch_blueprint import PipeBatchBlueprintfrom pipelex.pipe_controllers.condition.pipe_condition_blueprint import PipeConditionBlueprintfrom pipelex.pipe_controllers.parallel.pipe_parallel_blueprint import PipeParallelBlueprint@@ -367,7 +367,7 @@ class PipeSorterTestCases:("multiple_chains", MULTIPLE_CHAINS_PIPES, MULTIPLE_CHAINS_EXPECTED, None),("pipe_batch", PIPE_BATCH_PIPES, PIPE_BATCH_EXPECTED, None),("pipe_condition", PIPE_CONDITION_PIPES, PIPE_CONDITION_EXPECTED, None),- ("circular_dependency", CIRCULAR_PIPES, None, PipeDefinitionError),+ ("circular_dependency", CIRCULAR_PIPES, None, PipeValidationError),("missing_dependency", MISSING_DEPENDENCY_PIPES, MISSING_DEPENDENCY_EXPECTED, None),("image_inversion", IMAGE_INVERSION_PIPES, IMAGE_INVERSION_EXPECTED, None),("complex_nested", COMPLEX_NESTED_PIPES, COMPLEX_NESTED_EXPECTED, None),diff --git a/tests/unit/pipelex/core/concepts/test_structure_generator_escaping.py b/tests/unit/pipelex/core/concepts/test_structure_generator_escaping.pyindex 214ba66d5..bc8629558 100644--- a/tests/unit/pipelex/core/concepts/test_structure_generator_escaping.py+++ b/tests/unit/pipelex/core/concepts/test_structure_generator_escaping.py@@ -8,7 +8,7 @@ generated Python code.import astfrom pipelex.core.concepts.concept_structure_blueprint import ConceptStructureBlueprint, ConceptStructureBlueprintFieldType-from pipelex.core.concepts.structure_generator import StructureGenerator+from pipelex.core.concepts.structure_generation.generator import StructureGeneratorfrom pipelex.core.stuffs.structured_content import StructuredContentdiff --git a/tests/unit/pipelex/core/concepts/test_structured_output_generator.py b/tests/unit/pipelex/core/concepts/test_structured_output_generator.pyindex d25b97b06..8cb417264 100644--- a/tests/unit/pipelex/core/concepts/test_structured_output_generator.py+++ b/tests/unit/pipelex/core/concepts/test_structured_output_generator.py@@ -3,7 +3,8 @@ import pytestfrom pipelex import pretty_printfrom pipelex.core.concepts.concept_factory import ConceptFactoryfrom pipelex.core.concepts.concept_structure_blueprint import ConceptStructureBlueprint, ConceptStructureBlueprintFieldType-from pipelex.core.concepts.structure_generator import ConceptStructureValidationError, StructureGenerator+from pipelex.core.concepts.structure_generation.exceptions import ConceptStructureValidationError+from pipelex.core.concepts.structure_generation.generator import StructureGeneratorfrom pipelex.core.stuffs.structured_content import StructuredContentdiff --git a/tests/unit/pipelex/core/interpreter/test_interpreter.py b/tests/unit/pipelex/core/interpreter/test_interpreter.pyindex 305a33180..a351aa42d 100644--- a/tests/unit/pipelex/core/interpreter/test_interpreter.py+++ b/tests/unit/pipelex/core/interpreter/test_interpreter.py@@ -2,7 +2,7 @@ import pytestfrom pipelex import log, pretty_printfrom pipelex.core.bundles.pipelex_bundle_blueprint import PipelexBundleBlueprint-from pipelex.core.interpreter import PipelexInterpreter+from pipelex.core.interpreter.interpreter import PipelexInterpreterfrom tests.unit.pipelex.core.test_data import InterpreterTestCasesdiff --git a/tests/unit/pipelex/core/pipes/test_concept_multiplicity_formatting.py b/tests/unit/pipelex/core/pipes/test_concept_multiplicity_formatting.pynew file mode 100644index 000000000..818dbb3f1--- /dev/null+++ b/tests/unit/pipelex/core/pipes/test_concept_multiplicity_formatting.py@@ -0,0 +1,43 @@+"""Unit tests for concept code formatting with multiplicity notation."""++import pytest++from pipelex.core.pipes.variable_multiplicity import format_concept_with_multiplicity+++class TestFormatConceptWithMultiplicity:+ """Test the format_concept_with_multiplicity helper function."""++ @pytest.mark.parametrize(+ ("concept_code", "multiplicity", "expected"),+ [+ # Simple concepts - no multiplicity+ ("Text", None, "Text"),+ ("ConceptName", None, "ConceptName"),+ # Simple concepts - variable length+ ("Text", True, "Text[]"),+ ("Invoice", True, "Invoice[]"),+ # Simple concepts - fixed length+ ("Text", 3, "Text[3]"),+ ("Image", 1, "Image[1]"),+ ("Document", 10, "Document[10]"),+ # Domain-qualified concepts+ ("domain.Text", None, "domain.Text"),+ ("domain.Text", True, "domain.Text[]"),+ ("domain.Text", 5, "domain.Text[5]"),+ ("expense.Invoice", None, "expense.Invoice"),+ ("expense.Invoice", True, "expense.Invoice[]"),+ ("expense.Invoice", 10, "expense.Invoice[10]"),+ # Edge cases - underscores, numbers+ ("my_concept", None, "my_concept"),+ ("my_concept", True, "my_concept[]"),+ ("my_domain.MyConcept", 5, "my_domain.MyConcept[5]"),+ ("Concept123", 2, "Concept123[2]"),+ # Edge cases - zero and large numbers+ ("Text", 0, "Text[0]"),+ ("Text", 999, "Text[999]"),+ ],+ )+ def test_format_concept_with_multiplicity(self, concept_code: str, multiplicity: int | bool | None, expected: str):+ """Test formatting concept codes with all multiplicity variations."""+ assert format_concept_with_multiplicity(concept_code, multiplicity) == expecteddiff --git a/tests/unit/pipelex/core/pipes/test_validation.py b/tests/unit/pipelex/core/pipes/test_validation.pyindex 1e20169ea..c9142da35 100644--- a/tests/unit/pipelex/core/pipes/test_validation.py+++ b/tests/unit/pipelex/core/pipes/test_validation.py@@ -1,6 +1,5 @@import pytest-from pipelex.core.pipes.exceptions import PipeBlueprintValueErrorfrom pipelex.core.pipes.validation import is_valid_input_name, validate_input_name@@ -90,13 +89,13 @@ class TestInputNameValidation:],)def test_validate_input_name_invalid(self, invalid_input_name: str):- """Test validate_input_name raises PipeBlueprintValueError for invalid input names."""- with pytest.raises(PipeBlueprintValueError, match="Invalid input name syntax"):+ """Test validate_input_name raises ValueError for invalid input names."""+ with pytest.raises(ValueError, match="Invalid input name syntax"):validate_input_name(invalid_input_name)def test_validate_input_name_error_message(self):"""Test that the error message is helpful and descriptive."""- with pytest.raises(PipeBlueprintValueError) as exc_info:+ with pytest.raises(ValueError, match="Invalid input name syntax") as exc_info:validate_input_name("myInput.fieldName")error_message = str(exc_info.value)diff --git a/tests/unit/pipelex/core/test_data/errors/invalid_plx.py b/tests/unit/pipelex/core/test_data/errors/invalid_plx.pyindex b7f06bef1..a1571ec44 100644--- a/tests/unit/pipelex/core/test_data/errors/invalid_plx.py+++ b/tests/unit/pipelex/core/test_data/errors/invalid_plx.py@@ -1,4 +1,4 @@-from pipelex.core.interpreter import PipelexInterpreterError, PLXDecodeError+from pipelex.core.interpreter.interpreter import PipelexInterpreterError, PLXDecodeErrorINVALID_PLX_SYNTAX = ("invalid_plx_syntax",diff --git a/tests/unit/pipelex/pipe_controllers/condition/test_pipe_condition_input.py b/tests/unit/pipelex/pipe_controllers/condition/test_pipe_condition_input.pyindex f3a80bb01..f46102909 100644--- a/tests/unit/pipelex/pipe_controllers/condition/test_pipe_condition_input.py+++ b/tests/unit/pipelex/pipe_controllers/condition/test_pipe_condition_input.py@@ -3,8 +3,7 @@ from typing import Any, Callableimport pytestfrom pipelex import log-from pipelex.core.exceptions import PipeValidationError-from pipelex.core.pipe_errors import PipeDefinitionError+from pipelex.core.pipes.exceptions import PipeValidationErrorfrom pipelex.pipe_controllers.condition.pipe_condition_blueprint import PipeConditionBlueprintfrom pipelex.pipe_controllers.condition.pipe_condition_factory import PipeConditionFactoryfrom tests.unit.pipelex.pipe_controllers.condition.data import PipeConditionInputTestCases@@ -49,7 +48,7 @@ class TestPipeConditionValidateInputs:load_empty_library()log.verbose(f"Testing error case: {test_id}")- with pytest.raises((PipeValidationError, ValueError, PipeDefinitionError)) as exc_info: # noqa: PT012+ with pytest.raises((PipeValidationError, ValueError)) as exc_info: # noqa: PT012# Construct blueprint from dict at test time to trigger validationblueprint = PipeConditionBlueprint.model_validate(blueprint_dict)PipeConditionFactory.make_from_blueprint(diff --git a/tests/unit/pipelex/pipe_controllers/parallel/test_pipe_parallel_input.py b/tests/unit/pipelex/pipe_controllers/parallel/test_pipe_parallel_input.pyindex 680ffb7dc..4f4dd8bf1 100644--- a/tests/unit/pipelex/pipe_controllers/parallel/test_pipe_parallel_input.py+++ b/tests/unit/pipelex/pipe_controllers/parallel/test_pipe_parallel_input.py@@ -3,8 +3,7 @@ from typing import Any, Callableimport pytestfrom pipelex import log-from pipelex.core.exceptions import PipeValidationError-from pipelex.core.pipe_errors import PipeDefinitionError+from pipelex.core.pipes.exceptions import PipeValidationErrorfrom pipelex.pipe_controllers.parallel.pipe_parallel_blueprint import PipeParallelBlueprintfrom pipelex.pipe_controllers.parallel.pipe_parallel_factory import PipeParallelFactoryfrom tests.unit.pipelex.pipe_controllers.parallel.data import PipeParallelInputTestCases@@ -49,7 +48,7 @@ class TestPipeParallelValidateInputs:load_empty_library()log.verbose(f"Testing error case: {test_id}")- with pytest.raises((PipeValidationError, ValueError, PipeDefinitionError)) as exc_info: # noqa: PT012+ with pytest.raises((PipeValidationError, ValueError)) as exc_info: # noqa: PT012# Construct blueprint from dict at test time to trigger validationblueprint = PipeParallelBlueprint.model_validate(blueprint_dict)PipeParallelFactory.make_from_blueprint(diff --git a/tests/unit/pipelex/pipe_controllers/sequence/test_pipe_sequence_input.py b/tests/unit/pipelex/pipe_controllers/sequence/test_pipe_sequence_input.pyindex efc71dee5..77d523fc1 100644--- a/tests/unit/pipelex/pipe_controllers/sequence/test_pipe_sequence_input.py+++ b/tests/unit/pipelex/pipe_controllers/sequence/test_pipe_sequence_input.py@@ -3,8 +3,7 @@ from typing import Any, Callableimport pytestfrom pipelex import log-from pipelex.core.exceptions import PipeValidationError-from pipelex.core.pipe_errors import PipeDefinitionError+from pipelex.core.pipes.exceptions import PipeValidationErrorfrom pipelex.pipe_controllers.sequence.pipe_sequence_blueprint import PipeSequenceBlueprintfrom pipelex.pipe_controllers.sequence.pipe_sequence_factory import PipeSequenceFactoryfrom tests.unit.pipelex.pipe_controllers.sequence.data import PipeSequenceInputTestCases@@ -49,7 +48,7 @@ class TestPipeSequenceValidateInputs:load_empty_library()log.verbose(f"Testing error case: {test_id}")- with pytest.raises((PipeValidationError, ValueError, PipeDefinitionError)) as exc_info: # noqa: PT012+ with pytest.raises((PipeValidationError, ValueError)) as exc_info: # noqa: PT012# Construct blueprint from dict at test time to trigger validationblueprint = PipeSequenceBlueprint.model_validate(blueprint_dict)PipeSequenceFactory.make_from_blueprint(diff --git a/tests/unit/pipelex/pipe_operators/pipe_img_gen/data.py b/tests/unit/pipelex/pipe_operators/pipe_img_gen/data.pyindex e99a665ff..6b558bb2c 100644--- a/tests/unit/pipelex/pipe_operators/pipe_img_gen/data.py+++ b/tests/unit/pipelex/pipe_operators/pipe_img_gen/data.py@@ -1,7 +1,7 @@from typing import ClassVarfrom pipelex.cogt.img_gen.img_gen_job_components import AspectRatio, Background, OutputFormat-from pipelex.core.bundles.exceptions import PipeValidationErrorType+from pipelex.core.pipes.exceptions import PipeValidationErrorTypefrom pipelex.pipe_operators.img_gen.pipe_img_gen_blueprint import PipeImgGenBlueprintdiff --git a/tests/unit/pipelex/pipe_operators/pipe_img_gen/test_pipe_img_gen_input.py b/tests/unit/pipelex/pipe_operators/pipe_img_gen/test_pipe_img_gen_input.pyindex e3104303e..fda6d1ac2 100644--- a/tests/unit/pipelex/pipe_operators/pipe_img_gen/test_pipe_img_gen_input.py+++ b/tests/unit/pipelex/pipe_operators/pipe_img_gen/test_pipe_img_gen_input.py@@ -3,8 +3,7 @@ from typing import Callableimport pytestfrom pipelex import log-from pipelex.core.bundles.exceptions import PipeValidationErrorType-from pipelex.core.exceptions import PipeValidationError+from pipelex.core.pipes.exceptions import PipeValidationError, PipeValidationErrorTypefrom pipelex.pipe_operators.img_gen.pipe_img_gen_blueprint import PipeImgGenBlueprintfrom pipelex.pipe_operators.img_gen.pipe_img_gen_factory import PipeImgGenFactoryfrom tests.unit.pipelex.pipe_operators.pipe_img_gen.data import PipeImgGenInputTestCases