Skip to content

Build AI methods with your coding agent, then run them anywhere: as an MCP for chatbots, as a webapp for people, or via API for your software.

More validation handling but WIP : it doesn't end well

Louis Choquel877cb91edb63
diff --git a/pipelex/libraries/pipelines/builder/builder.plx b/pipelex/libraries/pipelines/builder/builder.plx
index 91a4dc0f7..bc936e37c 100644
--- a/pipelex/libraries/pipelines/builder/builder.plx
+++ b/pipelex/libraries/pipelines/builder/builder.plx
@@ -5,10 +5,10 @@ definition = "Auto-generate a Pipelex bundle (concepts + pipes) from a short use
UserBrief = "A short, natural-language description of what the user wants."
PlanDraft = "Natural-language pipeline plan text describing sequences, inputs, outputs."
ConceptDrafts = "Textual draft of the concepts to create."
-# PipeSignatureDrafts = "Textual draft of the pipe signatures to create."
PipelexBundleSpec = "A Pipelex bundle spec."
-PipeFailure = "Details of a single pipe failure during dry run."
-DryRunResult = "A result of a dry run of a pipelex bundle spec."
+ValidationResult = "Status (success or failure) and details of the validation failure if applicable."
+# PipeFailure = "Details of a single pipe failure during dry run."
+# DryRunResult = "A result of a dry run of a pipelex bundle spec."
DomainInformation = "A domain information object."
# ────────────────────────────────────────────────────────────────────────────────
@@ -18,7 +18,7 @@ DomainInformation = "A domain information object."
[pipe.pipe_builder]
type = "PipeSequence"
description = "This pipe is going to be the entry point for the builder. It will take a UserBrief and return a PipelexBundleSpec."
-inputs = { brief = "UserBrief" }
+inputs = { brief = "UserBrief", failed_concepts = "ConceptFailure", fixed_concepts = "ConceptSpec", failed_pipes = "PipeFailure", fixed_pipes = "PipeSpec" }
output = "PipelexBundleSpec"
steps = [
{ pipe = "draft_the_plan", result = "plan_draft" },
@@ -29,7 +29,8 @@ steps = [
{ pipe = "pipe_builder_domain_information", result = "domain_information" },
{ pipe = "create_pipes_from_signatures", batch_over = "pipe_signatures", batch_as = "pipe_signature", result = "pipe_specs" },
{ pipe = "compile_in_pipelex_bundle_spec", result = "pipelex_bundle_spec" }
- { pipe = "validate_pipelex_bundle_spec" }
+ { pipe = "validate_pipelex_bundle_loading", result = "pipelex_bundle_spec" }
+ { pipe = "validate_pipelex_bundle_dry_run", result = "pipelex_bundle_spec" }
]
[pipe.pipe_builder_domain_information]
@@ -207,51 +208,75 @@ inputs = { pipe_specs = "PipeSpec", concept_specs = "ConceptSpec", domain_inform
output = "PipelexBundleSpec"
function_name = "compile_in_pipelex_bundle_spec"
-[pipe.validate_pipelex_bundle_spec]
+[pipe.validate_pipelex_bundle_loading]
type = "PipeSequence"
description = "Validate the pipelex bundle spec with iterative fixing."
-inputs = { pipelex_bundle_spec = "PipelexBundleSpec" }
+inputs = { pipelex_bundle_spec = "PipelexBundleSpec", failed_concepts = "ConceptFailure", fixed_concepts = "ConceptSpec", fixed_pipes = "PipeSpec", failed_pipes = "PipeFailure" }
output = "PipelexBundleSpec"
steps = [
- { pipe = "validate_dry_run", result = "failed_pipes" },
- { pipe = "check_validation_status", result = "validation_status" },
+ { pipe = "validate_loading", result = "validation_result" }
+ { pipe = "handle_validation_result", result = "pipelex_bundle_spec" }
+]
+
+[pipe.validate_pipelex_bundle_dry_run]
+type = "PipeSequence"
+description = "Validate the pipelex bundle spec with iterative fixing."
+inputs = { pipelex_bundle_spec = "PipelexBundleSpec", failed_concepts = "ConceptFailure", fixed_concepts = "ConceptSpec", fixed_pipes = "PipeSpec", failed_pipes = "PipeFailure" }
+output = "PipelexBundleSpec"
+steps = [
+ { pipe = "validate_dry_run", result = "validation_result" },
{ pipe = "handle_validation_result", result = "pipelex_bundle_spec" }
]
-[pipe.check_validation_status]
-type = "PipeCompose"
-description = "Check if validation failed by examining if failed_pipes list is empty."
-inputs = { failed_pipes = "PipeFailure" }
-output = "Text"
-jinja2 = "{% if failed_pipes.content.items|length > 0 %}FAILURE{% else %}SUCCESS{% endif %}"
[pipe.handle_validation_result]
type = "PipeCondition"
description = "Handle validation result - continue if success or fix failures once."
-inputs = { pipelex_bundle_spec = "PipelexBundleSpec", failed_pipes = "PipeFailure", validation_status = "Text" }
+inputs = { pipelex_bundle_spec = "PipelexBundleSpec", validation_result = "ValidationResult", failed_concepts = "ConceptFailure", fixed_concepts = "ConceptSpec", fixed_pipes = "PipeSpec", failed_pipes = "PipeFailure" }
output = "PipelexBundleSpec"
-expression = "validation_status.text"
+expression_template = "{% if validation_result.failed_concepts %}fix_concepts{% elif validation_result.failed_pipes %}fix_pipes{% else %}continue{% endif %}"
[pipe.handle_validation_result.pipe_map]
-SUCCESS = "continue"
-FAILURE = "fix_failing_pipes_once"
+fix_concepts = "fix_failing_concepts_once"
+fix_pipes = "fix_failing_pipes_once"
+continue = "continue"
+
+[pipe.fix_failing_concepts_once]
+type = "PipeSequence"
+description = "Fix failing concepts once and return the result."
+inputs = { pipelex_bundle_spec = "PipelexBundleSpec", failed_concepts = "ConceptFailure", fixed_pipes = "PipeSpec", failed_pipes = "PipeFailure" }
+output = "PipelexBundleSpec"
+steps = [
+ { pipe = "fix_failing_concept", batch_over = "failed_concepts", batch_as = "failed_concept", result = "fixed_concepts" },
+ { pipe = "reconstruct_bundle_with_all_fixes", result = "pipelex_bundle_spec" },
+ { pipe = "validate_pipelex_bundle_loading", result = "pipelex_bundle_spec" },
+ { pipe = "validate_pipelex_bundle_dry_run", result = "pipelex_bundle_spec" }
+]
[pipe.fix_failing_pipes_once]
type = "PipeSequence"
description = "Fix failing pipes once and return the result."
-inputs = { pipelex_bundle_spec = "PipelexBundleSpec", failed_pipes = "PipeFailure" }
+inputs = { pipelex_bundle_spec = "PipelexBundleSpec", failed_pipes = "PipeFailure", fixed_concepts = "ConceptSpec", failed_concepts = "ConceptFailure" }
output = "PipelexBundleSpec"
steps = [
{ pipe = "fix_failing_pipe", batch_over = "failed_pipes", batch_as = "failed_pipe", result = "fixed_pipes" },
{ pipe = "reconstruct_bundle_with_all_fixes", result = "pipelex_bundle_spec" },
- { pipe = "validate_pipelex_bundle_spec", result = "pipelex_bundle_spec" }
+ { pipe = "validate_pipelex_bundle_loading", result = "pipelex_bundle_spec" },
+ { pipe = "validate_pipelex_bundle_dry_run", result = "pipelex_bundle_spec" }
]
+[pipe.validate_loading]
+type = "PipeFunc"
+description = "Validate loading of the bundle spec and return the first error."
+inputs = { pipelex_bundle_spec = "PipelexBundleSpec" }
+output = "ValidationResult"
+function_name = "validate_loading"
+
[pipe.validate_dry_run]
type = "PipeFunc"
-description = "Validate the pipelex bundle spec and return only failed pipes."
+description = "Validate the dry running the bundle's pipes and report only failed pipes."
inputs = { pipelex_bundle_spec = "PipelexBundleSpec" }
-output = "PipeFailure"
+output = "ValidationResult"
function_name = "validate_dry_run"
[pipe.continue]
@@ -264,7 +289,7 @@ jinja2 = "{{ pipelex_bundle_spec }}"
[pipe.reconstruct_bundle_with_all_fixes]
type = "PipeFunc"
description = "Reconstruct the bundle spec with all the fixed pipes."
-inputs = { pipelex_bundle_spec = "PipelexBundleSpec", fixed_pipes = "Dynamic" }
+inputs = { pipelex_bundle_spec = "PipelexBundleSpec", fixed_pipes = "Dynamic", fixed_concepts = "Dynamic" }
output = "PipelexBundleSpec"
function_name = "reconstruct_bundle_with_all_fixes"
diff --git a/pipelex/libraries/pipelines/builder/builder.py b/pipelex/libraries/pipelines/builder/builder.py
index 3f934808a..4e95c2668 100644
--- a/pipelex/libraries/pipelines/builder/builder.py
+++ b/pipelex/libraries/pipelines/builder/builder.py
@@ -7,6 +7,7 @@ from pipelex.core.concepts.concept_blueprint import ConceptBlueprint
from pipelex.core.domains.domain_blueprint import DomainBlueprint
from pipelex.core.memory.working_memory import WorkingMemory
from pipelex.core.stuffs.stuff_content import ListContent, StructuredContent
+from pipelex.exceptions import LibraryLoadingError, PipelexError, StaticValidationError
from pipelex.hub import get_library_manager
from pipelex.libraries.pipelines.builder.concept.concept_spec import ConceptSpec
from pipelex.libraries.pipelines.builder.pipe.pipe_batch_spec import PipeBatchSpec
@@ -20,7 +21,7 @@ from pipelex.libraries.pipelines.builder.pipe.pipe_parallel_spec import PipePara
from pipelex.libraries.pipelines.builder.pipe.pipe_sequence_spec import PipeSequenceSpec
from pipelex.libraries.pipelines.builder.pipe.pipe_signature import PipeSignature
from pipelex.pipe_works.pipe_dry import dry_run_pipes
-from pipelex.tools.typing.pydantic_utils import empty_list_factory_of, format_pydantic_validation_error
+from pipelex.tools.typing.pydantic_utils import format_pydantic_validation_error
from pipelex.types import StrEnum
@@ -58,6 +59,36 @@ PipeSpecUnion = Annotated[
]
+class PipeFailure(StructuredContent):
+ """Details of a single pipe failure during dry run."""
+
+ pipe: PipeSpecUnion = Field(description="The failing pipe spec blueprint with pipe code")
+ error_message: str = Field(description="The error message for this pipe")
+
+
+class ConceptFailure(StructuredContent):
+ """Details of a single concept failure during dry run."""
+
+ concept: ConceptSpec = Field(description="The failing concept spec blueprint with concept code")
+ error_message: str = Field(description="The error message for this concept")
+
+
+class PipelexBundleError(PipelexError):
+ def __init__(
+ self,
+ message: str,
+ pipe_code: str | None = None,
+ pipe_failure: PipeFailure | None = None,
+ concept_code: str | None = None,
+ concept_failure: ConceptFailure | None = None,
+ ):
+ self.pipe_code = pipe_code
+ self.pipe_failure = pipe_failure
+ self.concept_code = concept_code
+ self.concept_failure = concept_failure
+ super().__init__(message)
+
+
class PipelexBundleSpec(StructuredContent):
"""Complete spec of a Pipelex bundle TOML definition.
@@ -114,17 +145,27 @@ class PipelexBundleSpec(StructuredContent):
if self.concept:
concept = {}
- for concept_code, concept_blueprint in self.concept.items():
- if isinstance(concept_blueprint, ConceptSpec):
- concept[concept_code] = concept_blueprint.to_blueprint()
+ for concept_code, concept_spec_or_name in self.concept.items():
+ if isinstance(concept_spec_or_name, ConceptSpec):
+ try:
+ concept[concept_code] = concept_spec_or_name.to_blueprint()
+ except ValidationError as exc:
+ msg = f"Failed to validate concept spec for concept code {concept_code}: {format_pydantic_validation_error(exc)}"
+ concept_failure = ConceptFailure(concept=concept_spec_or_name, error_message=msg)
+ raise PipelexBundleError(message=msg, concept_code=concept_code, concept_failure=concept_failure) from exc
else:
- concept[concept_code] = ConceptBlueprint(definition=concept_code, structure=concept_blueprint)
+ concept[concept_code] = ConceptBlueprint(definition=concept_code, structure=concept_spec_or_name)
pipe: dict[str, PipeBlueprintUnion] | None = None
if self.pipe:
pipe = {}
- for pipe_code, pipe_blueprint in self.pipe.items():
- pipe[pipe_code] = pipe_blueprint.to_blueprint()
+ for pipe_code, pipe_spec in self.pipe.items():
+ try:
+ pipe[pipe_code] = pipe_spec.to_blueprint()
+ except ValidationError as exc:
+ msg = f"Failed to validate pipe spec for pipe code {pipe_code}: {format_pydantic_validation_error(exc)}"
+ pipe_failure = PipeFailure(pipe=pipe_spec, error_message=msg)
+ raise PipelexBundleError(message=msg, pipe_code=pipe_code, pipe_failure=pipe_failure) from exc
return PipelexBundleBlueprint(
domain=self.domain,
@@ -199,33 +240,59 @@ async def compile_in_pipelex_bundle_spec(working_memory: WorkingMemory) -> Pipel
# TODO: rename or merge with PipeDry.DryRunStatus
-class DryRunStatus(StrEnum):
+# class DryRunStatus(StrEnum):
+# SUCCESS = "SUCCESS"
+# FAILURE = "FAILURE"
+
+
+class Status(StrEnum):
SUCCESS = "SUCCESS"
FAILURE = "FAILURE"
+class ValidationFollowUp(StrEnum):
+ CONTINUE = "continue"
+ FIX_PIPES = "fix_pipes"
+ FIX_CONCEPTS = "fix_concepts"
+
+
class ValidateDryRunError(Exception):
"""Raised when validating the dry run of a pipelex bundle blueprint."""
-class PipeFailure(StructuredContent):
- """Details of a single pipe failure during dry run."""
-
- pipe: PipeSpecUnion = Field(description="The failing pipe spec blueprint with pipe code")
- error_message: str = Field(description="The error message for this pipe")
+class ValidationResult(StructuredContent):
+ status: Status
+ error_message: str | None = Field(default=None, description="The error message for the failure if applicable")
+ failed_pipes: ListContent[PipeFailure] | None = Field(default=None, description="The error details for the failure if applicable")
+ failed_concepts: ListContent[ConceptFailure] | None = Field(default=None, description="The error details for the failure if applicable")
-class DryRunResult(StructuredContent):
- """A result of a dry run of a pipelex bundle blueprint."""
+async def validate_loading(working_memory: WorkingMemory) -> ValidationResult:
+ library_manager = get_library_manager()
+ pipelex_bundle_spec = working_memory.get_stuff_as(name="pipelex_bundle_spec", content_type=PipelexBundleSpec)
+ try:
+ pipelex_bundle_blueprint = pipelex_bundle_spec.to_blueprint()
+ except PipelexBundleError as exc:
+ if exc.concept_failure:
+ failed_concepts = ListContent[ConceptFailure](items=[exc.concept_failure])
+ return ValidationResult(status=Status.FAILURE, error_message=exc.message, failed_concepts=failed_concepts)
+ elif exc.pipe_failure:
+ failed_pipes = ListContent[PipeFailure](items=[exc.pipe_failure])
+ return ValidationResult(status=Status.FAILURE, error_message=exc.message, failed_pipes=failed_pipes)
+ else:
+ return ValidationResult(status=Status.FAILURE, error_message=exc.message)
+ try:
+ library_manager.load_from_blueprint(blueprint=pipelex_bundle_blueprint)
+ except LibraryLoadingError as library_loading_exc:
+ return ValidationResult(status=Status.FAILURE, error_message=str(library_loading_exc))
+ except StaticValidationError as static_validation_exc:
+ return ValidationResult(status=Status.FAILURE, error_message=str(static_validation_exc))
- status: DryRunStatus
- failed_pipes: list[PipeFailure] = Field(
- default_factory=empty_list_factory_of(PipeFailure),
- description="List of pipes that failed during dry run",
- )
+ library_manager.remove_from_blueprint(blueprint=pipelex_bundle_blueprint)
+ return ValidationResult(status=Status.SUCCESS)
-async def validate_dry_run(working_memory: WorkingMemory) -> ListContent[PipeFailure]:
+async def validate_dry_run(working_memory: WorkingMemory) -> ValidationResult:
pipelex_bundle_spec = working_memory.get_stuff_as(name="pipelex_bundle_spec", content_type=PipelexBundleSpec)
pipelex_bundle_blueprint = pipelex_bundle_spec.to_blueprint()
@@ -262,11 +329,14 @@ async def validate_dry_run(working_memory: WorkingMemory) -> ListContent[PipeFai
error_message=dry_run_output.error_message or "",
),
)
-
- return ListContent[PipeFailure](items=failed_pipes)
+ if failed_pipes:
+ failed_pipes_list_content = ListContent[PipeFailure](items=failed_pipes)
+ return ValidationResult(status=Status.FAILURE, failed_pipes=failed_pipes_list_content)
+ else:
+ return ValidationResult(status=Status.SUCCESS)
-async def reconstruct_bundle_with_all_fixes(working_memory: WorkingMemory) -> PipelexBundleSpec:
+async def reconstruct_bundle_with_pipe_fixes(working_memory: WorkingMemory) -> PipelexBundleSpec:
pipelex_bundle_spec = working_memory.get_stuff_as(name="pipelex_bundle_spec", content_type=PipelexBundleSpec)
fixed_pipes_list = cast("ListContent[PipeSpecUnion]", working_memory.get_stuff(name="fixed_pipes").content)
@@ -279,3 +349,30 @@ async def reconstruct_bundle_with_all_fixes(working_memory: WorkingMemory) -> Pi
pipelex_bundle_spec.pipe[pipe_code] = fixed_pipe_blueprint
return pipelex_bundle_spec
+
+
+async def reconstruct_bundle_with_all_fixes(working_memory: WorkingMemory) -> PipelexBundleSpec:
+ pipelex_bundle_spec = working_memory.get_stuff_as(name="pipelex_bundle_spec", content_type=PipelexBundleSpec)
+ if fixed_pipes := working_memory.get_optional_stuff(name="fixed_pipes"):
+ fixed_pipes_list = cast("ListContent[PipeSpecUnion]", fixed_pipes.content)
+
+ if not pipelex_bundle_spec.pipe:
+ msg = "No pipes section found in bundle spec"
+ raise PipeBuilderError(msg)
+
+ for fixed_pipe_blueprint in fixed_pipes_list.items:
+ pipe_code = fixed_pipe_blueprint.the_pipe_code
+ pipelex_bundle_spec.pipe[pipe_code] = fixed_pipe_blueprint
+
+ if fixed_concepts := working_memory.get_optional_stuff(name="fixed_concepts"):
+ fixed_concepts_list = cast("ListContent[ConceptSpec]", fixed_concepts.content)
+
+ if not pipelex_bundle_spec.concept:
+ msg = "No concepts section found in bundle spec"
+ raise PipeBuilderError(msg)
+
+ for fixed_concept_blueprint in fixed_concepts_list.items:
+ concept_code = fixed_concept_blueprint.the_concept_code
+ pipelex_bundle_spec.concept[concept_code] = fixed_concept_blueprint
+
+ return pipelex_bundle_spec
diff --git a/pipelex/libraries/pipelines/builder/concept/concept.plx b/pipelex/libraries/pipelines/builder/concept/concept.plx
index aa81136bd..c6147def6 100644
--- a/pipelex/libraries/pipelines/builder/concept/concept.plx
+++ b/pipelex/libraries/pipelines/builder/concept/concept.plx
@@ -41,3 +41,20 @@ inputs = { concept_spec_draft = "ConceptSpecDraft", concept_spec_structures = "C
output = "ConceptSpec"
function_name = "create_concept_spec"
+
+[pipe.fix_failing_concept]
+type = "PipeLLM"
+description = "Fix a failing ConceptSpec based on its specific error."
+inputs = { pipelex_bundle_spec = "PipelexBundleSpec", failed_concept = "ConceptFailure" }
+output = "ConceptSpec"
+llm = "llm_to_engineer"
+prompt_template = """
+Fix this failing ConceptSpec.
+
+Failing concept:
+@failed_concept.concept
+
+Error message:
+@failed_concept.error_message
+"""
+
diff --git a/pipelex/libraries/pipelines/builder/pipe/pipe.plx b/pipelex/libraries/pipelines/builder/pipe/pipe.plx
index 3a497de97..de717e35d 100644
--- a/pipelex/libraries/pipelines/builder/pipe/pipe.plx
+++ b/pipelex/libraries/pipelines/builder/pipe/pipe.plx
@@ -323,8 +323,7 @@ Error message:
Please provide only the corrected PipeImgGenSpec. Common ImgGen pipe issues to fix:
- Missing or incorrect prompt input (should be text concept)
-- Wrong img_gen_prompt_var_name (should be None or "prompt")
-- Invalid img_gen choice
+- Invalid img_gen
- Missing required inputs for dynamic prompt generation
"""