Build AI methods with your coding agent, then run them anywhere: as an MCP for chatbots, as a webapp for people, or via API for your software.
Feature/build agent (#763)
Thomas Hebrard4cf49cd9e8d9
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 8917fc7c6..cd34c84fc 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -1,5 +1,21 @@# Changelog+## [Unreleased]++### Added++- **Builder Operations module** (`pipelex/builder/operations/`): New standalone operation functions for the build agent — `concept_ops`, `inputs_ops`, `models_ops`, `output_ops`, `pipe_ops`, `runner_code_ops`, `validate_ops`. These decouple build logic from CLI commands so it can be reused by agents and the API.+- **`dry_run_pipeline`** (`pipelex/pipe_run/dry_run_pipeline.py`): New shared entrypoint to dry-run a pipeline from MTHDS contents and produce a `GraphSpec`. Used by both CLI graph commands and the API.++### Changed++- **`mthds_content` → `mthds_contents`**: Unified singular `mthds_content: str | None` into `mthds_contents: list[str] | None` across `PipelexRunner`, `pipeline_run_setup`, `validate_bundle`, CLI commands, and builder operations. Callers now pass a list of bundle content strings.+- **`bundle_uri` → `bundle_uris`**: Renamed `bundle_uri: str | None` to `bundle_uris: list[str] | None` across `PipelexRunner.__init__`, `pipeline_run_setup`, `dry_run_pipeline`, and all CLI call sites. Multiple bundles can now be loaded simultaneously.+- **Bump `mthds` dependency** to `>=0.2.0` (from `>=0.1.1`) to match the updated `RunnerProtocol` interface.+- **Graph rendering refactor**: Extracted dry-run logic from `graph_rendering._dry_run_bundle` into the new shared `dry_run_pipeline` function.+- **Agent CLI inputs**: Refactored `_inputs_core` to delegate to `builder.operations.inputs_ops.build_inputs_for_pipe` instead of duplicating bundle validation and input rendering logic.+- **Agent CLI run**: Added graph generation support with ReactFlow HTML output and side-effect metadata tracking in `_run_core`.+## [v0.22.0] - 2026-03-25### Addeddiff --git a/pipelex/builder/operations/__init__.py b/pipelex/builder/operations/__init__.pynew file mode 100644index 000000000..e69de29bbdiff --git a/pipelex/builder/operations/concept_ops.py b/pipelex/builder/operations/concept_ops.pynew file mode 100644index 000000000..a668caef2--- /dev/null+++ b/pipelex/builder/operations/concept_ops.py@@ -0,0 +1,131 @@+"""Core operations for concept spec parsing and TOML generation."""++# pyright: reportUnknownMemberType=false++from typing import Any++import tomlkit++from pipelex.builder.concept.concept_spec import ConceptSpec, ConceptStructureSpec+++def parse_concept_spec(spec_data: dict[str, Any]) -> ConceptSpec:+ """Parse and validate a ConceptSpec from JSON-like data.++ Accepts common aliases for "the_concept_code" and converts structure fields.++ Args:+ spec_data: Raw data for the concept spec.++ Returns:+ Validated ConceptSpec instance.++ Raises:+ ValidationError: If validation fails.+ """+ # Work on a copy to avoid mutating the caller's dict+ spec_data = dict(spec_data)++ # Accept common aliases for "the_concept_code"+ for alias in ("concept_code", "code", "name", "concept_name"):+ if alias in spec_data:+ if "the_concept_code" not in spec_data:+ spec_data["the_concept_code"] = spec_data.pop(alias)+ else:+ spec_data.pop(alias)++ # Convert structure if present - need to add field names+ if spec_data.get("structure"):+ structure_data = spec_data["structure"]+ converted_structure: dict[str, Any] = {}+ for field_name, field_data in structure_data.items():+ if isinstance(field_data, str):+ # Simple string means just description, default to text type+ converted_structure[field_name] = {+ "the_field_name": field_name,+ "description": field_data,+ "type": "text",+ }+ else:+ # Full field spec — copy to avoid mutating nested caller data+ field_data = dict(field_data)+ field_data["the_field_name"] = field_name+ converted_structure[field_name] = field_data+ spec_data["structure"] = converted_structure++ return ConceptSpec.model_validate(spec_data)+++def structure_field_to_dict(field_spec: ConceptStructureSpec) -> dict[str, Any]:+ """Convert a ConceptStructureSpec to a dictionary for TOML serialization.++ Args:+ field_spec: The field specification to convert.++ Returns:+ Dictionary with field properties.+ """+ result: dict[str, Any] = {}++ # Type is always needed unless it's just a text description+ if not field_spec.type.is_text or field_spec.required or field_spec.default_value is not None:+ result["type"] = field_spec.type++ result["description"] = field_spec.description++ if field_spec.required:+ result["required"] = True++ if field_spec.default_value is not None:+ result["default"] = field_spec.default_value++ if field_spec.concept_ref:+ result["concept_ref"] = field_spec.concept_ref++ if field_spec.choices:+ result["choices"] = field_spec.choices++ return result+++def concept_spec_to_toml(concept_spec: ConceptSpec) -> str:+ """Convert a ConceptSpec to TOML string format.++ Args:+ concept_spec: The validated ConceptSpec to convert.++ Returns:+ TOML string representation of the concept.+ """+ doc = tomlkit.document()++ # Create the [concept.ConceptName] section+ concept_section = tomlkit.table()+ concept_item_table = tomlkit.table()++ # Add description+ concept_item_table.add("description", concept_spec.description)++ # Add refines if present+ if concept_spec.refines:+ concept_item_table.add("refines", concept_spec.refines)++ # Add structure if present+ if concept_spec.structure:+ structure_table = tomlkit.table()+ for field_name, field_spec in concept_spec.structure.items():+ field_dict = structure_field_to_dict(field_spec)+ # If only description is present, use simple string format+ if len(field_dict) == 1 and "description" in field_dict:+ structure_table.add(field_name, field_dict["description"])+ else:+ inline_table = tomlkit.inline_table()+ for key, value in field_dict.items():+ inline_table.append(key, value)+ structure_table.add(field_name, inline_table)+ concept_item_table.add("structure", structure_table)++ # Build the nested structure: [concept.ConceptName]+ concept_section.add(concept_spec.the_concept_code, concept_item_table)+ doc.add("concept", concept_section)+ return tomlkit.dumps(doc)diff --git a/pipelex/builder/operations/inputs_ops.py b/pipelex/builder/operations/inputs_ops.pynew file mode 100644index 000000000..9e1bb2fcb--- /dev/null+++ b/pipelex/builder/operations/inputs_ops.py@@ -0,0 +1,90 @@+"""Core operations for generating input JSON for pipes."""++from __future__ import annotations++import json+from typing import TYPE_CHECKING, Any++from pipelex.core.pipes.inputs.input_renderer import render_inputs+from pipelex.hub import (+ get_library_manager,+ get_required_pipe,+ resolve_library_dirs,+ set_current_library,+)+from pipelex.pipeline.validate_bundle import validate_bundle++if TYPE_CHECKING:+ from pathlib import Path+++async def build_inputs_for_pipe(+ pipe_code: str | None = None,+ mthds_contents: list[str] | None = None,+ bundle_path: Path | None = None,+ library_dirs: list[Path] | None = None,+) -> dict[str, Any]:+ """Generate example input JSON for a pipe.++ Supports loading from either mthds contents, a bundle file path,+ or from already-loaded libraries.++ Args:+ pipe_code: The pipe code to generate inputs for.+ mthds_contents: List of raw .mthds contents to parse and load.+ bundle_path: Path to the bundle file (.mthds).+ library_dirs: List of library directories to search for pipe definitions.++ Returns:+ Dictionary with inputs suitable for JSON serialization.++ Raises:+ ValidateBundleError: If bundle validation fails.+ ValueError: If no pipe code can be determined.+ """+ if mthds_contents:+ # validate_bundle opens a library, loads blueprints, and sets it as current+ validate_bundle_result = await validate_bundle(mthds_contents=mthds_contents)+ blueprints = validate_bundle_result.blueprints+ if not pipe_code:+ # Find the first blueprint that declares a main_pipe, domain-qualified+ main_pipe_code: str | None = None+ for blueprint in blueprints:+ if blueprint.main_pipe:+ main_pipe_code = f"{blueprint.domain}.{blueprint.main_pipe}"+ break+ if not main_pipe_code:+ msg = "Bundle does not declare a main_pipe. Specify a pipe code."+ raise ValueError(msg)+ pipe_code = main_pipe_code+ elif bundle_path:+ validate_bundle_result = await validate_bundle(mthds_file_path=bundle_path, library_dirs=library_dirs)+ bundle_blueprint = validate_bundle_result.blueprints[0]+ if not pipe_code:+ main_pipe_code = bundle_blueprint.main_pipe+ if not main_pipe_code:+ msg = f"Bundle '{bundle_path}' does not declare a main_pipe. Specify a pipe code."+ raise ValueError(msg)+ pipe_code = f"{bundle_blueprint.domain}.{main_pipe_code}"+ else:+ # No bundle - initialize the library manually+ library_manager = get_library_manager()+ library_id, _ = library_manager.open_library()+ set_current_library(library_id=library_id)+ effective_dirs, _ = resolve_library_dirs(library_dirs)+ if effective_dirs:+ library_manager.load_libraries(library_id=library_id, library_dirs=effective_dirs)++ if not pipe_code:+ msg = "No pipe code specified"+ raise ValueError(msg)++ the_pipe = get_required_pipe(pipe_code=pipe_code)+ inputs_json_str = render_inputs(the_pipe, indent=2)+ inputs_dict = json.loads(inputs_json_str)++ return {+ "success": True,+ "pipe_code": pipe_code,+ "inputs": inputs_dict,+ }diff --git a/pipelex/builder/operations/models_ops.py b/pipelex/builder/operations/models_ops.pynew file mode 100644index 000000000..4c0ac17e2--- /dev/null+++ b/pipelex/builder/operations/models_ops.py@@ -0,0 +1,280 @@+"""Core operation for listing model presets, aliases, waterfalls, and talent mappings."""++from typing import Any++from pipelex.cogt.model_backends.model_spec import InferenceModelSpec+from pipelex.cogt.model_backends.model_type import ModelType+from pipelex.cogt.models.model_deck import ModelDeck+from pipelex.config import get_config+from pipelex.hub import get_model_deck+from pipelex.types import StrEnum+++# TODO: get this from somewhere else, not hardcoded here.+class ModelCategory(StrEnum):+ LLM = "llm"+ EXTRACT = "extract"+ IMG_GEN = "img_gen"+ SEARCH = "search"+++CATEGORY_TO_MODEL_TYPE: dict[ModelCategory, ModelType] = {+ ModelCategory.LLM: ModelType.LLM,+ ModelCategory.EXTRACT: ModelType.TEXT_EXTRACTOR,+ ModelCategory.IMG_GEN: ModelType.IMG_GEN,+ ModelCategory.SEARCH: ModelType.SEARCH,+}+++def _should_include(category: ModelCategory, categories: list[ModelCategory] | None) -> bool:+ """Check whether a category should be included given the filter list.++ Returns True when no filter is set (None or empty) or when the category is in the filter list.+ """+ return categories is None or len(categories) == 0 or category in categories+++def _resolve_preset_backend(+ model_deck: ModelDeck,+ model_handle: str,+ model_type: ModelType,+) -> InferenceModelSpec | None:+ """Resolve a preset's model handle to an InferenceModelSpec, returning None if unresolvable."""+ return model_deck.get_optional_inference_model(model_handle=model_handle, model_type=model_type)+++def _filter_presets_by_backend(+ presets_list: list[dict[str, Any]],+ presets_dict: dict[str, Any],+ model_deck: ModelDeck,+ model_type: ModelType,+ backend: str,+) -> list[dict[str, Any]]:+ """Filter a presets list to only include presets whose model resolves to the given backend."""+ filtered: list[dict[str, Any]] = []+ for preset_entry in presets_list:+ preset_name = preset_entry["name"]+ setting = presets_dict.get(preset_name)+ if setting is None:+ continue+ spec = _resolve_preset_backend(model_deck, setting.model, model_type)+ if spec is not None and spec.backend_name == backend:+ filtered.append(preset_entry)+ return filtered+++def _filter_aliases_by_backend(+ aliases: dict[str, str],+ model_deck: ModelDeck,+ model_type: ModelType,+ backend: str,+) -> dict[str, str]:+ """Filter aliases to only include those whose target resolves to the given backend."""+ filtered: dict[str, str] = {}+ for alias_name, alias_target in aliases.items():+ spec = model_deck.get_optional_inference_model(model_handle=alias_target, model_type=model_type)+ if spec is not None and spec.backend_name == backend:+ filtered[alias_name] = alias_target+ return filtered+++def _filter_waterfalls_by_backend(+ waterfalls: dict[str, list[str]],+ model_deck: ModelDeck,+ model_type: ModelType,+ backend: str,+) -> dict[str, list[str]]:+ """Filter waterfalls to only include those where at least one fallback resolves to the given backend."""+ filtered: dict[str, list[str]] = {}+ for waterfall_name, fallback_list in waterfalls.items():+ for fallback in fallback_list:+ spec = model_deck.get_optional_inference_model(model_handle=fallback, model_type=model_type)+ if spec is not None and spec.backend_name == backend:+ filtered[waterfall_name] = fallback_list+ break+ return filtered+++def _strip_sigil(value: str) -> str:+ """Strip sigil prefix ($, @, ~) from a model reference to get the bare name."""+ if value and value[0] in {"$", "@", "~"}:+ return value[1:]+ return value+++def _filter_talent_mappings_by_backend(+ mappings: dict[str, str],+ presets_dict: dict[str, Any],+ model_deck: ModelDeck,+ model_type: ModelType,+ backend: str,+) -> dict[str, str]:+ """Filter talent mappings to only include those whose preset resolves to the given backend."""+ filtered: dict[str, str] = {}+ for talent_name, preset_name in mappings.items():+ bare_name = _strip_sigil(preset_name)+ setting = presets_dict.get(bare_name)+ if setting is None:+ continue+ spec = _resolve_preset_backend(model_deck, setting.model, model_type)+ if spec is not None and spec.backend_name == backend:+ filtered[talent_name] = preset_name+ return filtered+++def _build_presets_for_category(+ model_deck: ModelDeck,+ category: ModelCategory,+ backend: str | None,+) -> list[dict[str, Any]]:+ """Build the presets list for a given category, optionally filtered by backend."""+ presets_dict: dict[str, Any]+ model_type = CATEGORY_TO_MODEL_TYPE[category]+ match category:+ case ModelCategory.LLM:+ presets_dict = model_deck.llm_presets+ case ModelCategory.EXTRACT:+ presets_dict = model_deck.extract_presets+ case ModelCategory.IMG_GEN:+ presets_dict = model_deck.img_gen_presets+ case ModelCategory.SEARCH:+ presets_dict = model_deck.search_presets++ presets_list: list[dict[str, Any]] = []+ for preset_name, setting in presets_dict.items():+ entry: dict[str, Any] = {"name": preset_name}+ if setting.description is not None:+ entry["description"] = setting.description+ presets_list.append(entry)++ if backend is not None:+ presets_list = _filter_presets_by_backend(presets_list, presets_dict, model_deck, model_type, backend)++ return presets_list+++def _build_aliases_for_category(+ model_deck: ModelDeck,+ category: ModelCategory,+ backend: str | None,+) -> dict[str, str]:+ """Build the aliases dict for a given category, optionally filtered by backend."""+ model_type = CATEGORY_TO_MODEL_TYPE[category]+ aliases: dict[str, str]+ match category:+ case ModelCategory.LLM:+ aliases = model_deck.llm_aliases+ case ModelCategory.EXTRACT:+ aliases = model_deck.extract_aliases+ case ModelCategory.IMG_GEN:+ aliases = model_deck.img_gen_aliases+ case ModelCategory.SEARCH:+ aliases = model_deck.search_aliases++ if backend is not None:+ aliases = _filter_aliases_by_backend(aliases, model_deck, model_type, backend)++ return aliases+++def _build_waterfalls_for_category(+ model_deck: ModelDeck,+ category: ModelCategory,+ backend: str | None,+) -> dict[str, list[str]]:+ """Build the waterfalls dict for a given category, optionally filtered by backend."""+ model_type = CATEGORY_TO_MODEL_TYPE[category]+ waterfalls: dict[str, list[str]]+ match category:+ case ModelCategory.LLM:+ waterfalls = model_deck.llm_waterfalls+ case ModelCategory.EXTRACT:+ waterfalls = model_deck.extract_waterfalls+ case ModelCategory.IMG_GEN:+ waterfalls = model_deck.img_gen_waterfalls+ case ModelCategory.SEARCH:+ waterfalls = model_deck.search_waterfalls++ if backend is not None:+ waterfalls = _filter_waterfalls_by_backend(waterfalls, model_deck, model_type, backend)++ return waterfalls+++def _build_talent_mappings_for_category(+ model_deck: ModelDeck,+ category: ModelCategory,+ mappings: dict[str, str],+ backend: str | None,+) -> dict[str, str]:+ """Build the talent mappings dict for a given category, optionally filtered by backend."""+ if backend is None:+ return mappings++ model_type = CATEGORY_TO_MODEL_TYPE[category]+ presets_dict: dict[str, Any]+ match category:+ case ModelCategory.LLM:+ presets_dict = model_deck.llm_presets+ case ModelCategory.EXTRACT:+ presets_dict = model_deck.extract_presets+ case ModelCategory.IMG_GEN:+ presets_dict = model_deck.img_gen_presets+ case ModelCategory.SEARCH:+ presets_dict = model_deck.search_presets++ return _filter_talent_mappings_by_backend(mappings, presets_dict, model_deck, model_type, backend)+++def list_models(+ categories: list[ModelCategory] | None = None,+ backend: str | None = None,+) -> dict[str, Any]:+ """List available model presets, aliases, waterfalls, and talent mappings.++ Args:+ categories: Filter by model category. None or empty means all categories.+ backend: Filter by backend name. None means no filtering.++ Returns:+ Dict with presets, aliases, waterfalls, and talent_mappings keyed by category.+ """+ model_deck = get_model_deck()+ builder_config = get_config().pipelex.builder_config+ talent_mappings = builder_config.talent_preset_mappings++ presets: dict[str, list[dict[str, Any]]] = {}+ aliases: dict[str, dict[str, str]] = {}+ waterfalls: dict[str, dict[str, list[str]]] = {}+ talent: dict[str, dict[str, str]] = {}++ if _should_include(ModelCategory.LLM, categories):+ presets["llm"] = _build_presets_for_category(model_deck, ModelCategory.LLM, backend)+ aliases["llm"] = _build_aliases_for_category(model_deck, ModelCategory.LLM, backend)+ waterfalls["llm"] = _build_waterfalls_for_category(model_deck, ModelCategory.LLM, backend)+ talent["llm"] = _build_talent_mappings_for_category(model_deck, ModelCategory.LLM, talent_mappings.llm, backend)++ if _should_include(ModelCategory.IMG_GEN, categories):+ presets["img_gen"] = _build_presets_for_category(model_deck, ModelCategory.IMG_GEN, backend)+ aliases["img_gen"] = _build_aliases_for_category(model_deck, ModelCategory.IMG_GEN, backend)+ waterfalls["img_gen"] = _build_waterfalls_for_category(model_deck, ModelCategory.IMG_GEN, backend)+ talent["img_gen"] = _build_talent_mappings_for_category(model_deck, ModelCategory.IMG_GEN, talent_mappings.img_gen, backend)++ if _should_include(ModelCategory.EXTRACT, categories):+ presets["extract"] = _build_presets_for_category(model_deck, ModelCategory.EXTRACT, backend)+ aliases["extract"] = _build_aliases_for_category(model_deck, ModelCategory.EXTRACT, backend)+ waterfalls["extract"] = _build_waterfalls_for_category(model_deck, ModelCategory.EXTRACT, backend)+ talent["extract"] = _build_talent_mappings_for_category(model_deck, ModelCategory.EXTRACT, talent_mappings.extract, backend)++ if _should_include(ModelCategory.SEARCH, categories):+ presets["search"] = _build_presets_for_category(model_deck, ModelCategory.SEARCH, backend)+ aliases["search"] = _build_aliases_for_category(model_deck, ModelCategory.SEARCH, backend)+ waterfalls["search"] = _build_waterfalls_for_category(model_deck, ModelCategory.SEARCH, backend)+ talent["search"] = _build_talent_mappings_for_category(model_deck, ModelCategory.SEARCH, talent_mappings.search, backend)++ return {+ "presets": presets,+ "aliases": aliases,+ "waterfalls": waterfalls,+ "talent_mappings": talent,+ }diff --git a/pipelex/builder/operations/output_ops.py b/pipelex/builder/operations/output_ops.pynew file mode 100644index 000000000..7c655a983--- /dev/null+++ b/pipelex/builder/operations/output_ops.py@@ -0,0 +1,42 @@+"""Core operations for generating output JSON for pipes."""++from __future__ import annotations++import json+from typing import Any++from pipelex.core.concepts.concept_representation_generator import ConceptRepresentationFormat+from pipelex.core.pipes.output.output_renderer import render_output+from pipelex.hub import get_required_pipe+from pipelex.pipeline.validate_bundle import validate_bundle+++async def build_output_for_pipe(+ mthds_contents: list[str],+ pipe_code: str,+ output_format: ConceptRepresentationFormat = ConceptRepresentationFormat.SCHEMA,+) -> dict[str, Any]:+ """Generate output JSON representation for a pipe.++ Args:+ mthds_contents: List of raw .mthds contents to parse and load.+ pipe_code: The pipe code to generate output for.+ output_format: Format to generate output in.++ Returns:+ Dictionary with output representation.++ Raises:+ ValidateBundleError: If bundle validation fails.+ """+ # validate_bundle opens a library, loads blueprints, and sets it as current+ await validate_bundle(mthds_contents=mthds_contents)++ the_pipe = get_required_pipe(pipe_code=pipe_code)+ output_str = render_output(the_pipe, output_format=output_format)++ match output_format:+ case ConceptRepresentationFormat.PYTHON:+ return {"output": output_str}+ case ConceptRepresentationFormat.SCHEMA | ConceptRepresentationFormat.JSON:+ return json.loads(output_str) # type: ignore[no-any-return]diff --git a/pipelex/builder/operations/pipe_ops.py b/pipelex/builder/operations/pipe_ops.pynew file mode 100644index 000000000..56fc476b1--- /dev/null+++ b/pipelex/builder/operations/pipe_ops.py@@ -0,0 +1,238 @@+"""Core operations for pipe spec parsing and TOML generation."""++# tomlkit lacks type stubs; pydantic `model` field clashes with BaseModel namespace+# pyright: reportUnknownMemberType=false+# pyright: reportUnknownVariableType=false+# pyright: reportAttributeAccessIssue=false++from typing import Any++import tomlkit+from tomlkit.items import Table++from pipelex.builder.pipe.pipe_batch_spec import PipeBatchSpec+from pipelex.builder.pipe.pipe_compose_spec import PipeComposeSpec+from pipelex.builder.pipe.pipe_condition_spec import PipeConditionSpec+from pipelex.builder.pipe.pipe_extract_spec import PipeExtractSpec+from pipelex.builder.pipe.pipe_func_spec import PipeFuncSpec+from pipelex.builder.pipe.pipe_img_gen_spec import PipeImgGenSpec+from pipelex.builder.pipe.pipe_llm_spec import PipeLLMSpec+from pipelex.builder.pipe.pipe_parallel_spec import PipeParallelSpec+from pipelex.builder.pipe.pipe_search_spec import PipeSearchSpec+from pipelex.builder.pipe.pipe_sequence_spec import PipeSequenceSpec+from pipelex.builder.pipe.pipe_spec import PipeSpec+from pipelex.builder.pipe.pipe_spec_map import pipe_type_to_spec_class+++def parse_pipe_spec(pipe_type: str, spec_data: dict[str, Any]) -> PipeSpec:+ """Parse and validate a PipeSpec from JSON-like data.++ Args:+ pipe_type: The type of pipe (e.g., "PipeLLM", "PipeSequence").+ spec_data: Raw data for the pipe spec.++ Returns:+ Validated PipeSpec instance of the correct type.++ Raises:+ ValueError: If the pipe type is invalid.+ ValidationError: If validation fails.+ """+ if pipe_type not in pipe_type_to_spec_class:+ valid_types = list(pipe_type_to_spec_class.keys())+ msg = f"Invalid pipe type '{pipe_type}'. Must be one of: {valid_types}"+ raise ValueError(msg)++ spec_class = pipe_type_to_spec_class[pipe_type]++ # Work on a copy to avoid mutating the caller's dict+ spec_data = dict(spec_data)++ # Add type to spec_data if not present+ spec_data["type"] = pipe_type++ # Accept common aliases for "pipe_code"+ for alias in ("code", "the_pipe_code", "name"):+ if alias in spec_data:+ if "pipe_code" not in spec_data:+ spec_data["pipe_code"] = spec_data.pop(alias)+ else:+ spec_data.pop(alias)++ # Handle steps/branches conversion - need to convert pipe to pipe_code+ # Deep-copy nested dicts to avoid mutating caller's nested structures+ if "steps" in spec_data:+ converted_steps = []+ for step in spec_data["steps"]:+ step = dict(step)+ if "pipe" in step and "pipe_code" not in step:+ step["pipe_code"] = step.pop("pipe")+ converted_steps.append(step)+ spec_data["steps"] = converted_steps++ if "branches" in spec_data:+ converted_branches = []+ for branch in spec_data["branches"]:+ branch = dict(branch)+ if "pipe" in branch and "pipe_code" not in branch:+ branch["pipe_code"] = branch.pop("pipe")+ converted_branches.append(branch)+ spec_data["branches"] = converted_branches++ # Handle expression -> jinja2_expression_template for PipeCondition+ if pipe_type == "PipeCondition" and "expression" in spec_data:+ if "jinja2_expression_template" not in spec_data:+ spec_data["jinja2_expression_template"] = spec_data.pop("expression")+ else:+ spec_data.pop("expression")++ # Accept output as dict with "type" key → extract the type string+ if "output" in spec_data and isinstance(spec_data["output"], dict):+ if "type" in spec_data["output"]:+ spec_data["output"] = spec_data["output"]["type"]++ return spec_class.model_validate(spec_data)+++def add_type_specific_fields(pipe_spec: PipeSpec, pipe_table: Table) -> None:+ """Add type-specific fields to the pipe TOML table.++ Args:+ pipe_spec: The pipe spec with type-specific fields.+ pipe_table: The TOML table to add fields to.+ """+ if isinstance(pipe_spec, PipeLLMSpec):+ if pipe_spec.model:+ pipe_table.add("model", pipe_spec.model)+ if pipe_spec.system_prompt:+ pipe_table.add("system_prompt", pipe_spec.system_prompt)+ if pipe_spec.prompt:+ pipe_table.add("prompt", pipe_spec.prompt)++ elif isinstance(pipe_spec, PipeComposeSpec):+ if pipe_spec.construct_spec is not None:+ # Construct mode: serialize the construct block as a nested TOML table+ construct_table = tomlkit.table()+ for field_name, field_value in pipe_spec.construct_spec.items():+ if isinstance(field_value, dict):+ field_inline = tomlkit.inline_table()+ inner_dict: dict[str, Any] = field_value+ for key, value in inner_dict.items():+ field_inline.append(key, value)+ construct_table.add(field_name, field_inline)+ else:+ construct_table.add(field_name, field_value)+ pipe_table.add("construct", construct_table)+ else:+ # Template mode — guard optional fields like other pipe types do+ if pipe_spec.target_format is not None:+ pipe_table.add("target_format", str(pipe_spec.target_format))+ if pipe_spec.template is not None:+ pipe_table.add("template", pipe_spec.template)++ elif isinstance(pipe_spec, PipeSequenceSpec):+ steps_array = tomlkit.array()+ for step in pipe_spec.steps:+ step_inline = tomlkit.inline_table()+ step_inline.append("pipe", step.pipe_code)+ step_inline.append("result", step.result)+ if step.batch_over is not None:+ step_inline.append("batch_over", step.batch_over)+ if step.batch_as is not None:+ step_inline.append("batch_as", step.batch_as)+ steps_array.append(step_inline)+ pipe_table.add("steps", steps_array)++ elif isinstance(pipe_spec, PipeParallelSpec):+ pipe_table.add("add_each_output", pipe_spec.add_each_output)+ if pipe_spec.combined_output:+ pipe_table.add("combined_output", pipe_spec.combined_output)+ branches_array = tomlkit.array()+ for branch in pipe_spec.branches:+ branch_inline = tomlkit.inline_table()+ branch_inline.append("pipe", branch.pipe_code)+ branch_inline.append("result", branch.result)+ branches_array.append(branch_inline)+ pipe_table.add("branches", branches_array)++ elif isinstance(pipe_spec, PipeConditionSpec):+ pipe_table.add("expression", pipe_spec.jinja2_expression_template)+ outcomes_table = tomlkit.inline_table()+ for condition, outcome in pipe_spec.outcomes.items():+ outcomes_table.append(condition, outcome)+ pipe_table.add("outcomes", outcomes_table)+ pipe_table.add("default_outcome", pipe_spec.default_outcome)++ elif isinstance(pipe_spec, PipeBatchSpec):+ pipe_table.add("branch_pipe_code", pipe_spec.branch_pipe_code)+ pipe_table.add("input_list_name", pipe_spec.input_list_name)+ pipe_table.add("input_item_name", pipe_spec.input_item_name)++ elif isinstance(pipe_spec, PipeExtractSpec):+ if pipe_spec.model:+ pipe_table.add("model", pipe_spec.model)+ if pipe_spec.max_page_images is not None:+ pipe_table.add("max_page_images", pipe_spec.max_page_images)+ if pipe_spec.page_views is not None:+ pipe_table.add("page_views", pipe_spec.page_views)++ elif isinstance(pipe_spec, PipeSearchSpec):+ if pipe_spec.model:+ pipe_table.add("model", pipe_spec.model)+ pipe_table.add("prompt", pipe_spec.prompt)+ if pipe_spec.from_date is not None:+ pipe_table.add("from_date", pipe_spec.from_date)+ if pipe_spec.to_date is not None:+ pipe_table.add("to_date", pipe_spec.to_date)+ if pipe_spec.include_domains is not None:+ pipe_table.add("include_domains", pipe_spec.include_domains)+ if pipe_spec.exclude_domains is not None:+ pipe_table.add("exclude_domains", pipe_spec.exclude_domains)+ if pipe_spec.max_results is not None:+ pipe_table.add("max_results", pipe_spec.max_results)++ elif isinstance(pipe_spec, PipeImgGenSpec):+ if pipe_spec.model:+ pipe_table.add("model", pipe_spec.model)+ pipe_table.add("prompt", pipe_spec.prompt)++ elif isinstance(pipe_spec, PipeFuncSpec):+ pipe_table.add("function_name", pipe_spec.function_name)+++def pipe_spec_to_toml(pipe_spec: PipeSpec) -> str:+ """Convert a PipeSpec to TOML string format.++ Args:+ pipe_spec: The validated PipeSpec to convert.++ Returns:+ TOML string representation of the pipe.+ """+ doc = tomlkit.document()+ pipe_item_table = tomlkit.table()++ # Add type+ pipe_item_table.add("type", pipe_spec.type)++ # Add description+ pipe_item_table.add("description", pipe_spec.description)++ # Add inputs as inline table+ if pipe_spec.inputs:+ inputs_inline = tomlkit.inline_table()+ for input_name, input_concept in pipe_spec.inputs.items():+ inputs_inline.append(input_name, input_concept)+ pipe_item_table.add("inputs", inputs_inline)++ # Add output+ pipe_item_table.add("output", pipe_spec.output)++ # Add type-specific fields+ add_type_specific_fields(pipe_spec, pipe_item_table)++ # Build the nested structure: [pipe.pipe_code]+ pipe_section = tomlkit.table()+ pipe_section.add(pipe_spec.pipe_code, pipe_item_table)+ doc.add("pipe", pipe_section)+ return tomlkit.dumps(doc)diff --git a/pipelex/builder/operations/runner_code_ops.py b/pipelex/builder/operations/runner_code_ops.pynew file mode 100644index 000000000..37e5c0b59--- /dev/null+++ b/pipelex/builder/operations/runner_code_ops.py@@ -0,0 +1,47 @@+"""Core operations for generating runner code for pipes."""++from __future__ import annotations++from pipelex.builder.runner_code import generate_runner_code+from pipelex.core.interpreter.interpreter import PipelexInterpreter+from pipelex.hub import get_library_manager, get_required_pipe, set_current_library+from pipelex.pipe_run.dry_run import dry_run_pipes+++async def build_runner_code_for_pipe(+ mthds_contents: list[str],+ pipe_code: str,+) -> str:+ """Generate Python runner code for a pipe from .mthds contents.++ Parses and validates the contents, loads pipes, dry-runs them,+ then generates runner code for the specified pipe.++ Args:+ mthds_contents: List of raw .mthds contents to parse and load.+ pipe_code: The pipe code to generate runner code for.++ Returns:+ Generated Python runner code as a string.++ Raises:+ Exception: If parsing, validation, or code generation fails.+ """+ library_manager = get_library_manager()+ library_id, _ = library_manager.open_library()+ set_current_library(library_id)++ # Parse PLX contents into bundle blueprints+ blueprints = [PipelexInterpreter.make_pipelex_bundle_blueprint(mthds_content=content) for content in mthds_contents]++ # Load pipes from the blueprints+ pipes = library_manager.load_from_blueprints(library_id=library_id, blueprints=blueprints)++ # Validate all pipes then dry-run in a single batch+ for pipe in pipes:+ pipe.validate_with_libraries()+ await dry_run_pipes(pipes=pipes, raise_on_failure=True)++ # Get the required pipe and generate runner code+ pipe = get_required_pipe(pipe_code)+ return generate_runner_code(pipe=pipe)diff --git a/pipelex/builder/operations/validate_ops.py b/pipelex/builder/operations/validate_ops.pynew file mode 100644index 000000000..af1f4936f--- /dev/null+++ b/pipelex/builder/operations/validate_ops.py@@ -0,0 +1,194 @@+"""Core operations for validating pipes and bundles."""++from __future__ import annotations++from typing import TYPE_CHECKING, Any++from pipelex.hub import (+ get_library_manager,+ get_required_pipe,+ resolve_library_dirs,+ set_current_library,+)+from pipelex.pipe_run.dry_run import DryRunStatus, dry_run_pipe, dry_run_pipes+from pipelex.pipeline.validate_bundle import validate_bundle++if TYPE_CHECKING:+ from pathlib import Path+++async def validate_all(+ library_dirs: list[Path] | None = None,+) -> dict[str, Any]:+ """Validate all pipes in all libraries.++ Args:+ library_dirs: List of library directories to search for pipe definitions.++ Returns:+ Dictionary with validation results suitable for JSON serialization.++ Raises:+ ValidateBundleError: If validation fails.+ """+ library_manager = get_library_manager()+ library_id, library = library_manager.open_library()+ set_current_library(library_id=library_id)+ effective_dirs, _ = resolve_library_dirs(library_dirs)++ if effective_dirs:+ library_manager.load_libraries(library_id=library_id, library_dirs=effective_dirs)++ pipes = library.pipe_library.get_pipes()+ for the_pipe in pipes:+ the_pipe.validate_with_libraries()++ dry_run_results = await dry_run_pipes(pipes=pipes, raise_on_failure=True)++ validated_pipes: list[dict[str, str]] = []+ for the_pipe in pipes:+ dry_run_output = dry_run_results.get(the_pipe.pipe_ref)+ status: str = dry_run_output.status if dry_run_output else DryRunStatus.SUCCESS+ validated_pipes.append({"pipe_code": the_pipe.code, "status": status})++ return {+ "success": True,+ "validated_pipes": validated_pipes,+ "total_pipes": len(pipes),+ }+++async def validate_bundle_file(+ bundle_path: Path,+ library_dirs: list[Path] | None = None,+) -> dict[str, Any]:+ """Validate a bundle file.++ Args:+ bundle_path: Path to the bundle file.+ library_dirs: List of library directories to search for pipe definitions.++ Returns:+ Dictionary with validation results suitable for JSON serialization.++ Raises:+ ValidateBundleError: If validation fails.+ """+ result = await validate_bundle(mthds_file_path=bundle_path, library_dirs=library_dirs)++ validated_pipes: list[dict[str, str]] = []+ for the_pipe in result.pipes:+ dry_run_output = result.dry_run_result.get(the_pipe.pipe_ref)+ status: str = dry_run_output.status if dry_run_output else DryRunStatus.SUCCESS+ validated_pipes.append({"pipe_code": the_pipe.code, "status": status})++ return {+ "success": True,+ "bundle_path": str(bundle_path),+ "validated_pipes": validated_pipes,+ "total_pipes": len(result.pipes),+ }+++async def validate_bundle_content(+ mthds_contents: list[str],+) -> dict[str, Any]:+ """Validate bundle content provided as strings.++ Args:+ mthds_contents: List of raw .mthds contents to validate.++ Returns:+ Dictionary with validation results including the blueprint.++ Raises:+ ValidateBundleError: If validation fails.+ """+ validate_bundle_result = await validate_bundle(mthds_contents=mthds_contents)+ blueprints = validate_bundle_result.blueprints++ validated_pipes: list[dict[str, str]] = []+ for the_pipe in validate_bundle_result.pipes:+ dry_run_output = validate_bundle_result.dry_run_result.get(the_pipe.pipe_ref)+ status: str = dry_run_output.status if dry_run_output else DryRunStatus.SUCCESS+ validated_pipes.append({"pipe_code": the_pipe.code, "status": status})++ return {+ "success": True,+ "mthds_contents": mthds_contents,+ "pipelex_bundle_blueprint": blueprints[0].model_dump(mode="json")+ if len(blueprints) == 1+ else [b.model_dump(mode="json") for b in blueprints],+ "validated_pipes": validated_pipes,+ "total_pipes": len(validate_bundle_result.pipes),+ }+++async def validate_pipe(+ pipe_code: str,+ library_dirs: list[Path] | None = None,+) -> dict[str, Any]:+ """Validate a single pipe.++ Args:+ pipe_code: The pipe code to validate.+ library_dirs: List of library directories to search for pipe definitions.++ Returns:+ Dictionary with validation results suitable for JSON serialization.++ Raises:+ ValidateBundleError: If validation fails.+ """+ library_manager = get_library_manager()+ library_id, _ = library_manager.open_library()+ set_current_library(library_id=library_id)+ effective_dirs, _ = resolve_library_dirs(library_dirs)++ if effective_dirs:+ library_manager.load_libraries(library_id=library_id, library_dirs=effective_dirs)++ the_pipe = get_required_pipe(pipe_code=pipe_code)+ await dry_run_pipe(the_pipe, raise_on_failure=True)++ return {+ "success": True,+ "validated_pipes": [{"pipe_code": pipe_code, "status": "SUCCESS"}],+ "total_pipes": 1,+ }+++async def validate_pipe_in_bundle(+ bundle_path: Path,+ pipe_code: str,+ library_dirs: list[Path] | None = None,+) -> dict[str, Any]:+ """Validate a single pipe within a bundle.++ This first validates the bundle to load its pipes into the library,+ then validates only the specified pipe.++ Args:+ bundle_path: Path to the bundle file.+ pipe_code: The pipe code to validate within the bundle.+ library_dirs: List of library directories to search for pipe definitions.++ Returns:+ Dictionary with validation results suitable for JSON serialization.++ Raises:+ ValidateBundleError: If validation fails.+ """+ # Validate the bundle to load all its pipes into the library+ await validate_bundle(mthds_file_path=bundle_path, library_dirs=library_dirs)++ # Now get the specific pipe and dry-run only that one+ the_pipe = get_required_pipe(pipe_code=pipe_code)+ await dry_run_pipe(the_pipe, raise_on_failure=True)++ return {+ "success": True,+ "bundle_path": str(bundle_path),+ "validated_pipes": [{"pipe_code": pipe_code, "status": "SUCCESS"}],+ "total_pipes": 1,+ }diff --git a/pipelex/builder/pipe/pipe_spec_map.py b/pipelex/builder/pipe/pipe_spec_map.pyindex a74dc111b..a716bbafb 100644--- a/pipelex/builder/pipe/pipe_spec_map.py+++ b/pipelex/builder/pipe/pipe_spec_map.py@@ -8,8 +8,9 @@ from pipelex.builder.pipe.pipe_llm_spec import PipeLLMSpecfrom pipelex.builder.pipe.pipe_parallel_spec import PipeParallelSpecfrom pipelex.builder.pipe.pipe_search_spec import PipeSearchSpecfrom pipelex.builder.pipe.pipe_sequence_spec import PipeSequenceSpec+from pipelex.builder.pipe.pipe_spec import PipeSpec-pipe_type_to_spec_class: dict[str, type] = {+pipe_type_to_spec_class: dict[str, type[PipeSpec]] = {"PipeFunc": PipeFuncSpec,"PipeImgGen": PipeImgGenSpec,"PipeCompose": PipeComposeSpec,diff --git a/pipelex/cli/agent_cli/commands/agent_output.py b/pipelex/cli/agent_cli/commands/agent_output.pyindex acd33516b..bc54648c0 100644--- a/pipelex/cli/agent_cli/commands/agent_output.py+++ b/pipelex/cli/agent_cli/commands/agent_output.py@@ -53,7 +53,7 @@ AGENT_ERROR_HINTS: dict[str, str] = {),# API runner errors"ClientAuthenticationError": "Run 'pipelex-agent doctor' to check credentials, or set the PIPELEX_API_KEY environment variable",- "PipelineRequestError": "Check that pipe_code or mthds_content is provided",+ "PipelineRequestError": "Check that pipe_code or mthds_contents is provided",# Graph errors"GraphSpecParseError": "Validate graphspec.json structure; ensure it matches the expected GraphSpec schema",# Input/type errorsdiff --git a/pipelex/cli/agent_cli/commands/inputs/_inputs_core.py b/pipelex/cli/agent_cli/commands/inputs/_inputs_core.pyindex c47bbde30..53ec068b1 100644--- a/pipelex/cli/agent_cli/commands/inputs/_inputs_core.py+++ b/pipelex/cli/agent_cli/commands/inputs/_inputs_core.py@@ -2,17 +2,9 @@from __future__ import annotations-import jsonfrom typing import TYPE_CHECKING, Any-from pipelex.core.pipes.inputs.input_renderer import render_inputs-from pipelex.hub import (- get_library_manager,- get_required_pipe,- resolve_library_dirs,- set_current_library,-)-from pipelex.pipeline.validate_bundle import validate_bundle+from pipelex.builder.operations.inputs_ops import build_inputs_for_pipeif TYPE_CHECKING:from pathlib import Path@@ -37,34 +29,8 @@ async def inputs_core(ValidateBundleError: If bundle validation fails.NoInputsRequiredError: If the pipe has no inputs."""- if bundle_path:- validate_bundle_result = await validate_bundle(mthds_file_path=bundle_path, library_dirs=library_dirs)- bundle_blueprint = validate_bundle_result.blueprints[0]- if not pipe_code:- main_pipe_code = bundle_blueprint.main_pipe- if not main_pipe_code:- msg = f"Bundle '{bundle_path}' does not declare a main_pipe. Specify a pipe code with --pipe."- raise ValueError(msg)- pipe_code = main_pipe_code- else:- # No bundle - initialize the library manually- library_manager = get_library_manager()- library_id, _ = library_manager.open_library()- set_current_library(library_id=library_id)- effective_dirs, _ = resolve_library_dirs(library_dirs)- if effective_dirs:- library_manager.load_libraries(library_id=library_id, library_dirs=effective_dirs)-- if not pipe_code:- msg = "No pipe code specified"- raise ValueError(msg)-- the_pipe = get_required_pipe(pipe_code=pipe_code)- inputs_json_str = render_inputs(the_pipe, indent=2)- inputs_dict = json.loads(inputs_json_str)-- return {- "success": True,- "pipe_code": pipe_code,- "inputs": inputs_dict,- }+ return await build_inputs_for_pipe(+ pipe_code=pipe_code,+ bundle_path=bundle_path,+ library_dirs=library_dirs,+ )diff --git a/pipelex/cli/agent_cli/commands/run/_run_core.py b/pipelex/cli/agent_cli/commands/run/_run_core.pyindex 4c1e06ed9..7a666ba6a 100644--- a/pipelex/cli/agent_cli/commands/run/_run_core.py+++ b/pipelex/cli/agent_cli/commands/run/_run_core.py@@ -3,6 +3,7 @@from __future__ import annotationsimport json+import shutilfrom pathlib import Pathfrom typing import Any@@ -16,8 +17,8 @@ from pipelex.tools.misc.json_utils import clean_json_dumpsasync def run_pipeline_core(pipe_code: str,- mthds_content: str | None = None,- bundle_uri: str | None = None,+ mthds_contents: list[str] | None = None,+ bundle_uris: list[str] | None = None,inputs: dict[str, Any] | None = None,dry_run: bool = False,mock_inputs: bool = False,@@ -29,8 +30,8 @@ async def run_pipeline_core(Args:pipe_code: The pipe code to run.- mthds_content: MTHDS content string (optional).- bundle_uri: Bundle file path (optional).+ mthds_contents: List of MTHDS content strings (optional).+ bundle_uris: List of bundle file paths (optional).inputs: Input dictionary for the pipeline.dry_run: Whether to run in dry mode (no actual inference calls).mock_inputs: Whether to generate mock data for missing inputs.@@ -53,14 +54,14 @@ async def run_pipeline_core()runner = PipelexRunner(- bundle_uri=bundle_uri,+ bundle_uris=bundle_uris,pipe_run_mode=pipe_run_mode,execution_config=execution_config,library_dirs=library_dirs,)response = await runner.execute_pipeline(pipe_code=pipe_code,- mthds_content=mthds_content,+ mthds_contents=mthds_contents,inputs=inputs,)pipe_output = response.pipe_output@@ -87,8 +88,8 @@ async def run_pipeline_core(# Determine output directory: next to the bundle, or mthds-wip/ fallbackoutput_dir: Path- if bundle_uri:- output_dir = Path(bundle_uri).parent+ if bundle_uris and not bundle_uris[0].startswith(("http://", "https://")):+ output_dir = Path(bundle_uris[0]).parentelse:output_dir = Path("mthds-wip")output_dir.mkdir(parents=True, exist_ok=True)@@ -134,7 +135,7 @@ async def run_pipeline_core(reactflow_path = saved_files.get("reactflow_html")if reactflow_path:final_path = reactflow_path.parent / graph_filename- reactflow_path.rename(final_path)+ shutil.move(str(reactflow_path), str(final_path))side_effects["graph_files"] = {"graph_html": str(final_path)}# Save output JSON (includes side-effect paths for on-disk reference)diff --git a/pipelex/cli/agent_cli/commands/run/_run_core_api.py b/pipelex/cli/agent_cli/commands/run/_run_core_api.pyindex 92de0557a..6ed931c56 100644--- a/pipelex/cli/agent_cli/commands/run/_run_core_api.py+++ b/pipelex/cli/agent_cli/commands/run/_run_core_api.py@@ -1,5 +1,9 @@"""Core logic for running a pipeline via the remote MTHDS API in the agent CLI."""+# pyright: reportUnknownMemberType=false+# pyright: reportUnknownVariableType=false+# pyright: reportUnknownArgumentType=false+from __future__ import annotationsfrom typing import Any, cast@@ -12,7 +16,7 @@ from pipelex.cli.agent_cli.commands.run._output_helpers import build_run_outputasync def run_pipeline_core_api(pipe_code: str,- mthds_content: str | None = None,+ mthds_contents: list[str] | None = None,inputs: dict[str, Any] | None = None,with_memory: bool = False,) -> dict[str, Any]:@@ -20,7 +24,7 @@ async def run_pipeline_core_api(Args:pipe_code: The pipe code to run.- mthds_content: MTHDS content string (optional).+ mthds_contents: List of MTHDS content strings (optional).inputs: Input dictionary for the pipeline.with_memory: Whether to include full working memory in output (True) orreturn compact concept JSON only (False, default).@@ -35,7 +39,7 @@ async def run_pipeline_core_api(runner = ApiRunner()response = await runner.execute_pipeline(pipe_code=pipe_code,- mthds_content=mthds_content,+ mthds_contents=mthds_contents,inputs=inputs,)diff --git a/pipelex/cli/agent_cli/commands/run/bundle_cmd.py b/pipelex/cli/agent_cli/commands/run/bundle_cmd.pyindex 3e7dab2b2..ad9e27c8e 100644--- a/pipelex/cli/agent_cli/commands/run/bundle_cmd.py+++ b/pipelex/cli/agent_cli/commands/run/bundle_cmd.py@@ -156,7 +156,7 @@ def run_bundle_cmd(result = asyncio.run(run_pipeline_core_api(pipe_code=pipe_code, # type: ignore[arg-type]- mthds_content=mthds_content,+ mthds_contents=[mthds_content] if mthds_content else None,inputs=pipeline_inputs,with_memory=with_memory,)@@ -179,8 +179,8 @@ def run_bundle_cmd(result = asyncio.run(run_pipeline_core(pipe_code=pipe_code, # type: ignore[arg-type]- mthds_content=mthds_content,- bundle_uri=bundle_path,+ mthds_contents=[mthds_content] if mthds_content else None,+ bundle_uris=[bundle_path] if bundle_path else None,inputs=pipeline_inputs,dry_run=dry_run,mock_inputs=mock_inputs,diff --git a/pipelex/cli/agent_cli/commands/run/method_cmd.py b/pipelex/cli/agent_cli/commands/run/method_cmd.pyindex 98d3eaa8d..b7b7b0ac7 100644--- a/pipelex/cli/agent_cli/commands/run/method_cmd.py+++ b/pipelex/cli/agent_cli/commands/run/method_cmd.py@@ -113,7 +113,7 @@ def run_method_cmd(result = asyncio.run(run_pipeline_core_api(pipe_code=pipe_code,- mthds_content=mthds_content,+ mthds_contents=[mthds_content] if mthds_content else None,inputs=pipeline_inputs,with_memory=with_memory,)@@ -136,8 +136,8 @@ def run_method_cmd(result = asyncio.run(run_pipeline_core(pipe_code=pipe_code,- mthds_content=mthds_content,- bundle_uri=bundle_path,+ mthds_contents=[mthds_content] if mthds_content else None,+ bundle_uris=[bundle_path] if bundle_path else None,inputs=pipeline_inputs,dry_run=dry_run,mock_inputs=mock_inputs,diff --git a/pipelex/cli/agent_cli/commands/run/pipe_cmd.py b/pipelex/cli/agent_cli/commands/run/pipe_cmd.pyindex 93ce65dee..8505a1ca1 100644--- a/pipelex/cli/agent_cli/commands/run/pipe_cmd.py+++ b/pipelex/cli/agent_cli/commands/run/pipe_cmd.py@@ -111,7 +111,6 @@ def run_pipe_cmd(result = asyncio.run(run_pipeline_core_api(pipe_code=pipe_code,- mthds_content=None,inputs=pipeline_inputs,with_memory=with_memory,)@@ -134,8 +133,6 @@ def run_pipe_cmd(result = asyncio.run(run_pipeline_core(pipe_code=pipe_code,- mthds_content=None,- bundle_uri=None,inputs=pipeline_inputs,dry_run=dry_run,mock_inputs=mock_inputs,diff --git a/pipelex/cli/commands/build/pipe_cmd.py b/pipelex/cli/commands/build/pipe_cmd.pyindex 07fc59d67..60986ae14 100644--- a/pipelex/cli/commands/build/pipe_cmd.py+++ b/pipelex/cli/commands/build/pipe_cmd.py@@ -305,7 +305,7 @@ def build_pipe_cmd(library_dirs=[],)built_pipe_response = await built_runner.execute_pipeline(- mthds_content=mthds_content,+ mthds_contents=[mthds_content],)built_pipe_output = built_pipe_response.pipe_outputif built_pipe_output.graph_spec:diff --git a/pipelex/cli/commands/run/_run_core.py b/pipelex/cli/commands/run/_run_core.pyindex 027c5115f..95c69232f 100644--- a/pipelex/cli/commands/run/_run_core.py+++ b/pipelex/cli/commands/run/_run_core.py@@ -118,14 +118,14 @@ async def _execute_run(try:runner = PipelexRunner(- bundle_uri=bundle_path,+ bundle_uris=[bundle_path] if bundle_path else None,pipe_run_mode=pipe_run_mode,execution_config=execution_config,library_dirs=library_dir,)response = await runner.execute_pipeline(pipe_code=pipe_code,- mthds_content=mthds_content,+ mthds_contents=[mthds_content] if mthds_content else None,inputs=pipeline_inputs,)pipe_output = response.pipe_outputdiff --git a/pipelex/graph/graph_rendering.py b/pipelex/graph/graph_rendering.pyindex 548923ed4..86b238a72 100644--- a/pipelex/graph/graph_rendering.py+++ b/pipelex/graph/graph_rendering.py@@ -7,15 +7,11 @@ Used by both the regular CLI and the agent CLI.from pathlib import Pathfrom typing import Any-from pipelex.base_exceptions import PipelexErrorfrom pipelex.config import get_config-from pipelex.core.interpreter.exceptions import PipelexInterpreterError-from pipelex.core.interpreter.interpreter import PipelexInterpreterfrom pipelex.graph.graph_config import GraphConfigfrom pipelex.graph.graph_factory import GraphOutputs, generate_graph_outputs, save_graph_outputs_to_dirfrom pipelex.graph.graphspec import GraphSpec-from pipelex.pipe_run.pipe_run_mode import PipeRunMode-from pipelex.pipeline.runner import PipelexRunner+from pipelex.pipe_run.dry_run_pipeline import dry_run_pipelinefrom pipelex.tools.misc.chart_utils import FlowchartDirectionfrom pipelex.types import StrEnum@@ -108,10 +104,10 @@ async def _dry_run_bundle(bundle_path: Path,library_dirs: list[str] | None = None,) -> tuple[GraphSpec, str]:- """Perform a dry-run pipeline execution on a bundle to produce a GraphSpec.+ """Dry-run a bundle file to produce a GraphSpec.- Reads the bundle, parses main_pipe, builds a runner, and executes a dry-run- with graph tracing enabled.+ Reads the file, ensures its parent directory is in library_dirs,+ then delegates to ``dry_run_pipeline``.Args:bundle_path: Path to the .mthds bundle file.@@ -119,24 +115,8 @@ async def _dry_run_bundle(Returns:Tuple of (GraphSpec, pipe_code).-- Raises:- PipelexInterpreterError: If bundle parsing fails or main_pipe is missing.- PipelexError: If pipeline execution does not produce a graph spec.- PipelineExecutionError: If dry-run execution fails."""mthds_content = bundle_path.read_text(encoding="utf-8")- bundle_blueprint = PipelexInterpreter.make_pipelex_bundle_blueprint(mthds_content=mthds_content)- main_pipe_code = bundle_blueprint.main_pipe- if not main_pipe_code:- msg = f"Bundle '{bundle_path}' does not declare a main_pipe, cannot generate graph"- raise PipelexInterpreterError(msg)- pipe_code: str = main_pipe_code-- execution_config = get_config().pipelex.pipeline_execution_config.with_graph_config_overrides(- generate_graph=True,- mock_inputs=True,- )# Ensure the bundle's parent directory is included in library_dirs# so PipelexRunner can resolve sibling dependencies@@ -149,23 +129,11 @@ async def _dry_run_bundle(else:effective_library_dirs = [bundle_parent_dir]- runner = PipelexRunner(- bundle_uri=str(bundle_path),- pipe_run_mode=PipeRunMode.DRY,- execution_config=execution_config,+ return await dry_run_pipeline(+ mthds_contents=[mthds_content],+ bundle_uris=[str(bundle_path)],library_dirs=effective_library_dirs,)- response = await runner.execute_pipeline(- pipe_code=pipe_code,- mthds_content=mthds_content,- )- pipe_output = response.pipe_output-- if not pipe_output.graph_spec:- msg = "Pipeline execution did not produce a graph spec"- raise PipelexError(msg)-- return pipe_output.graph_spec, pipe_codeasync def generate_graph_for_bundle(diff --git a/pipelex/kit/agent_rules/python_standards.md b/pipelex/kit/agent_rules/python_standards.mdindex 46be8ded5..70716420f 100644--- a/pipelex/kit/agent_rules/python_standards.md+++ b/pipelex/kit/agent_rules/python_standards.md@@ -37,7 +37,7 @@ This document outlines the core coding standards, best practices, and quality co### **Imports at the top of the file**- - Import all necessary libraries at the top of the file+ - Avoid as much as possible import statements outside of a module's top-level scope.- Do not import libraries in functions or classes unless in very specific cases, to be discussed with the user, as they would required a `# noqa: ...` comment to pass linting- Do not bother with ordering the imports or removing unused imports, our Ruff linter will handle it for us.- `if TYPE_CHECKING:` blocks must always be the **last** block in the imports section, placed after all regular imports.diff --git a/pipelex/pipe_run/dry_run_pipeline.py b/pipelex/pipe_run/dry_run_pipeline.pynew file mode 100644index 000000000..ff1c21a11--- /dev/null+++ b/pipelex/pipe_run/dry_run_pipeline.py@@ -0,0 +1,85 @@+"""Pipeline-level dry run with graph generation.++Provides a single entrypoint to dry-run an entire pipeline from MTHDS content,+producing a GraphSpec. Used by both the CLI graph commands and the API.+"""++from pipelex.base_exceptions import PipelexError+from pipelex.config import get_config+from pipelex.core.interpreter.exceptions import PipelexInterpreterError+from pipelex.core.interpreter.interpreter import PipelexInterpreter+from pipelex.graph.graphspec import GraphSpec+from pipelex.pipe_run.pipe_run_mode import PipeRunMode+from pipelex.pipeline.runner import PipelexRunner+++async def dry_run_pipeline(+ mthds_contents: list[str] | None = None,+ bundle_uris: list[str] | None = None,+ library_dirs: list[str] | None = None,+) -> tuple[GraphSpec, str]:+ """Dry-run a full pipeline from MTHDS content, producing a GraphSpec.++ Parses the content, identifies the main pipe, and executes the pipeline+ in dry-run mode with graph tracing enabled and mock inputs.++ All contents are parsed into blueprints and loaded together; the main_pipe+ is found from the first blueprint that declares one.++ Args:+ mthds_contents: List of MTHDS bundle contents as strings.+ bundle_uris: Optional list of URIs for the bundles (used by runner for dedup).+ library_dirs: Optional library directories for pipe resolution.++ Returns:+ Tuple of (GraphSpec, pipe_code).++ Raises:+ PipelexInterpreterError: If content parsing fails or main_pipe is missing.+ PipelexError: If pipeline execution does not produce a graph spec.+ PipelineExecutionError: If dry-run execution fails.+ """+ if not mthds_contents:+ msg = "mthds_contents must be provided"+ raise ValueError(msg)++ # Pre-parse contents to extract main_pipe_code.+ # Note: pipeline_run_setup will re-parse these contents. The double-parse is+ # accepted because the runner interface requires pipe_code upfront and does not+ # expose the internally-resolved pipe code in its response.+ main_pipe_code: str | None = None++ for content in mthds_contents:+ bundle_blueprint = PipelexInterpreter.make_pipelex_bundle_blueprint(mthds_content=content)+ if bundle_blueprint.main_pipe and main_pipe_code is None:+ # Domain-qualify to avoid ambiguity across multiple domains+ main_pipe_code = f"{bundle_blueprint.domain}.{bundle_blueprint.main_pipe}"++ if not main_pipe_code:+ msg = "Bundle does not declare a main_pipe, cannot generate graph"+ raise PipelexInterpreterError(msg)++ pipe_code: str = main_pipe_code++ execution_config = get_config().pipelex.pipeline_execution_config.with_graph_config_overrides(+ generate_graph=True,+ mock_inputs=True,+ )++ runner = PipelexRunner(+ bundle_uris=bundle_uris,+ pipe_run_mode=PipeRunMode.DRY,+ execution_config=execution_config,+ library_dirs=library_dirs or [],+ )+ response = await runner.execute_pipeline(+ pipe_code=pipe_code,+ mthds_contents=mthds_contents,+ )+ pipe_output = response.pipe_output++ if not pipe_output.graph_spec:+ msg = "Pipeline execution did not produce a graph spec"+ raise PipelexError(msg)++ return pipe_output.graph_spec, pipe_codediff --git a/pipelex/pipeline/pipeline_run_setup.py b/pipelex/pipeline/pipeline_run_setup.pyindex eeea0d301..a6aa9b212 100644--- a/pipelex/pipeline/pipeline_run_setup.py+++ b/pipelex/pipeline/pipeline_run_setup.py@@ -48,8 +48,8 @@ async def pipeline_run_setup(library_id: str | None = None,library_dirs: list[str] | None = None,pipe_code: str | None = None,- mthds_content: str | None = None,- bundle_uri: str | None = None,+ mthds_contents: list[str] | None = None,+ bundle_uris: list[str] | None = None,inputs: PipelineInputs | WorkingMemory | None = None,output_name: str | None = None,output_multiplicity: VariableMultiplicity | None = None,@@ -76,22 +76,23 @@ async def pipeline_run_setup(library_dirs:List of directory paths to load pipe definitions from. Combined with directoriesfrom the ``PIPELEXPATH`` environment variable (PIPELEXPATH directories are searched- first). When provided alongside ``mthds_content``, definitions from both sources+ first). When provided alongside ``mthds_contents``, definitions from both sourcesare loaded into the library.pipe_code:- Code identifying the pipe to execute. Required when ``mthds_content`` is not- provided. When both ``mthds_content`` and ``pipe_code`` are provided, the- specified pipe from the MTHDS content will be executed (overriding any- ``main_pipe`` defined in the content).- mthds_content:- Complete MTHDS file content as a string. The pipe to execute is determined by- ``pipe_code`` (if provided) or the ``main_pipe`` property in the MTHDS content.- Can be combined with ``library_dirs`` to load additional definitions.- bundle_uri:- URI identifying the bundle. Used to detect if the bundle was already loaded- from library directories (e.g., via PIPELEXPATH) to avoid duplicate domain- registration. If provided and the resolved absolute path is already in the- loaded MTHDS paths, the ``mthds_content`` loading will be skipped.+ Code identifying the pipe to execute. Required when ``mthds_contents`` is not+ provided. When both ``mthds_contents`` and ``pipe_code`` are provided, the+ specified pipe from the MTHDS contents will be executed (overriding any+ ``main_pipe`` defined in the contents).+ mthds_contents:+ List of MTHDS file contents as strings. The pipe to execute is determined by+ ``pipe_code`` (if provided) or the ``main_pipe`` property in the first content+ that declares one. Can be combined with ``library_dirs`` to load additional+ definitions.+ bundle_uris:+ List of URIs identifying the bundles. Used to detect if bundles were already+ loaded from library directories (e.g., via PIPELEXPATH) to avoid duplicate+ domain registration. If all resolved absolute paths are already in the loaded+ MTHDS paths, the ``mthds_contents`` loading will be skipped.inputs:Inputs passed to the pipeline. Can be either a ``PipelineInputs`` dictionaryor a ``WorkingMemory`` instance.@@ -119,8 +120,8 @@ async def pipeline_run_setup("""user_id = user_id or OTelConstants.DEFAULT_USER_ID- if not mthds_content and not pipe_code:- msg = "Either pipe_code or mthds_content must be provided to the pipeline API."+ if not mthds_contents and not pipe_code:+ msg = "Either pipe_code or mthds_contents must be provided to the pipeline API."raise ValueError(msg)pipeline = get_pipeline_manager().add_new_pipeline(pipe_code=pipe_code)@@ -149,45 +150,52 @@ async def pipeline_run_setup(else:log.verbose(f"No library directories to load ({source_label})")- # Then handle MTHDS content or pipe_code- if mthds_content:- blueprint = PipelexInterpreter.make_pipelex_bundle_blueprint(mthds_content=mthds_content)- blueprints_to_load = [blueprint]+ # Then handle MTHDS content(s) or pipe_code+ if mthds_contents:+ all_blueprints = [PipelexInterpreter.make_pipelex_bundle_blueprint(mthds_content=content) for content in mthds_contents]- # Check if this bundle was already loaded from library directories+ # Check if all bundles were already loaded from library directoriesbundle_already_loaded = False- if bundle_uri:- try:- resolved_bundle_uri = Path(bundle_uri).resolve()- except (OSError, RuntimeError):- # Use str(Path(...)) to normalize the path (e.g., "./file.mthds" -> "file.mthds")- # to match the normalization done in library_manager._load_mthds_files_into_library- resolved_bundle_uri = Path(bundle_uri)+ if bundle_uris:current_library = library_manager.get_library(library_id=library_id)- bundle_already_loaded = resolved_bundle_uri in current_library.loaded_mthds_paths+ all_loaded = True+ for uri in bundle_uris:+ try:+ resolved_uri = Path(uri).resolve()+ except (OSError, RuntimeError):+ resolved_uri = Path(uri)+ if resolved_uri not in current_library.loaded_mthds_paths:+ all_loaded = False+ break+ bundle_already_loaded = all_loadedif bundle_already_loaded:- log.verbose(f"Bundle '{bundle_uri}' already loaded from library directories, skipping duplicate load")+ log.verbose(f"All {len(bundle_uris)} bundle(s) already loaded from library directories, skipping duplicate load")if not bundle_already_loaded:- library_manager.load_from_blueprints(library_id=library_id, blueprints=blueprints_to_load)+ library_manager.load_from_blueprints(library_id=library_id, blueprints=all_blueprints)- # For now, we only support one blueprint when given MTHDS content. So blueprints is of length 1.- # blueprint is already set from make_pipelex_bundle_blueprint above+ # Find the pipe to executeif pipe_code:pipe = get_required_pipe(pipe_code=pipe_code)- elif blueprint.main_pipe:+ else:+ # Find main_pipe from the first blueprint that declares one# Qualify main_pipe with domain to avoid ambiguity when multiple domains define pipes with the same code.# Note: main_pipe is validated as snake_case (no dots allowed), so it is always a bare code — never# already domain-qualified. See PipelexBundleBlueprint.validate_main_pipe_syntax.- qualified_main_pipe = f"{blueprint.domain}.{blueprint.main_pipe}"+ qualified_main_pipe: str | None = None+ for blueprint in all_blueprints:+ if blueprint.main_pipe:+ qualified_main_pipe = f"{blueprint.domain}.{blueprint.main_pipe}"+ break+ if not qualified_main_pipe:+ msg = "No pipe_code provided and no main_pipe found in any of the MTHDS contents."+ raise PipeExecutionError(message=msg)pipe = get_required_pipe(pipe_code=qualified_main_pipe)- else:- msg = "No pipe code or main pipe in the MTHDS content provided to the pipeline API."- raise PipeExecutionError(message=msg)+elif pipe_code:pipe = get_required_pipe(pipe_code=pipe_code)else:- msg = "Either provide pipe_code or mthds_content to the pipeline API. 'pipe_code' must be provided when 'mthds_content' is None"+ msg = "Either provide pipe_code or mthds_contents to the pipeline API."raise PipeExecutionError(message=msg)pipe_code = pipe.codediff --git a/pipelex/pipeline/runner.py b/pipelex/pipeline/runner.pyindex f671242cf..09505adc2 100644--- a/pipelex/pipeline/runner.py+++ b/pipelex/pipeline/runner.py@@ -51,7 +51,7 @@ class PipelexRunner(RunnerProtocol["PipeOutput"]):self,library_id: str | None = None,library_dirs: list[str] | None = None,- bundle_uri: str | None = None,+ bundle_uris: list[str] | None = None,pipe_run_mode: PipeRunMode | None = None,search_domain_codes: list[str] | None = None,user_id: str | None = None,@@ -59,7 +59,7 @@ class PipelexRunner(RunnerProtocol["PipeOutput"]):):self.library_id = library_idself.library_dirs = library_dirs- self.bundle_uri = bundle_uri+ self.bundle_uris = bundle_urisself.pipe_run_mode = pipe_run_modeself.search_domain_codes = search_domain_codesself.user_id = user_id@@ -70,7 +70,7 @@ class PipelexRunner(RunnerProtocol["PipeOutput"]):async def execute_pipeline(self,pipe_code: str | None = None,- mthds_content: str | None = None,+ mthds_contents: list[str] | None = None,inputs: PipelineInputs | WorkingMemoryAbstract[Any] | None = None,output_name: str | None = None,output_multiplicity: VariableMultiplicity | None = None,@@ -87,13 +87,13 @@ class PipelexRunner(RunnerProtocol["PipeOutput"]):Parameters----------pipe_code:- Code identifying the pipe to execute. Required when ``mthds_content`` is not- provided. When both ``mthds_content`` and ``pipe_code`` are provided, the- specified pipe from the PLX content will be executed (overriding any- ``main_pipe`` defined in the content).- mthds_content:- Complete PLX file content as a string. The pipe to execute is determined by- ``pipe_code`` (if provided) or the ``main_pipe`` property in the PLX content.+ Code identifying the pipe to execute. Required when ``mthds_contents`` is not+ provided. When both ``mthds_contents`` and ``pipe_code`` are provided, the+ specified pipe from the MTHDS contents will be executed (overriding any+ ``main_pipe`` defined in the contents).+ mthds_contents:+ List of MTHDS bundle contents as strings. The pipe to execute is determined by+ ``pipe_code`` (if provided) or the ``main_pipe`` property in the content.Can be combined with ``library_dirs`` to load additional definitions.inputs:Inputs passed to the pipeline. Can be either a ``PipelineInputs`` dictionary@@ -134,8 +134,8 @@ class PipelexRunner(RunnerProtocol["PipeOutput"]):library_id=self.library_id,library_dirs=self.library_dirs,pipe_code=pipe_code,- mthds_content=mthds_content,- bundle_uri=self.bundle_uri,+ mthds_contents=mthds_contents,+ bundle_uris=self.bundle_uris,inputs=pipelex_inputs,output_name=output_name,output_multiplicity=output_multiplicity,@@ -223,7 +223,7 @@ class PipelexRunner(RunnerProtocol["PipeOutput"]):async def start_pipeline(self,pipe_code: str | None = None,- mthds_content: str | None = None,+ mthds_contents: list[str] | None = None,inputs: PipelineInputs | WorkingMemoryAbstract[Any] | None = None,output_name: str | None = None,output_multiplicity: VariableMultiplicity | None = None,diff --git a/pipelex/pipeline/validate_bundle.py b/pipelex/pipeline/validate_bundle.pyindex 827592516..e0d7feb4d 100644--- a/pipelex/pipeline/validate_bundle.py+++ b/pipelex/pipeline/validate_bundle.py@@ -9,7 +9,7 @@ from pipelex.core.bundles.exceptions import PipelexBundleBlueprintValidationErrofrom pipelex.core.bundles.pipelex_bundle_blueprint import PipelexBundleBlueprintfrom pipelex.core.concepts.concept import Conceptfrom pipelex.core.exceptions import PipeFactoryErrorData, PipesAndConceptValidationErrorData-from pipelex.core.interpreter.exceptions import PipelexInterpreterError+from pipelex.core.interpreter.exceptions import MthdsDecodeError, PipelexInterpreterErrorfrom pipelex.core.interpreter.interpreter import PipelexInterpreterfrom pipelex.core.pipes.exceptions import PipeFactoryError, PipeValidationErrorfrom pipelex.core.pipes.handle_pipe_errors import (@@ -19,6 +19,7 @@ from pipelex.core.pipes.handle_pipe_errors import ()from pipelex.core.pipes.pipe_abstract import PipeAbstractfrom pipelex.core.validation import report_validation_error+from pipelex.graph.graphspec import GraphSpecfrom pipelex.hub import get_library_manager, resolve_library_dirs, set_current_libraryfrom pipelex.libraries.library_utils import get_pipelex_mthds_files_from_dirsfrom pipelex.pipe_run.dry_run import DryRunError, DryRunOutput, dry_run_pipes@@ -84,20 +85,27 @@ class ValidateBundleResult(BaseModel):blueprints: list[PipelexBundleBlueprint]pipes: list[PipeAbstract]dry_run_result: dict[str, DryRunOutput]+ graph_spec: GraphSpec | None = Noneasync def validate_bundle(mthds_file_path: Path | None = None,- mthds_content: str | None = None,+ mthds_contents: list[str] | None = None,blueprints: list[PipelexBundleBlueprint] | None = None,library_dirs: Sequence[Path] | None = None,) -> ValidateBundleResult:- provided_params = sum([blueprints is not None, mthds_content is not None, mthds_file_path is not None])+ provided_params = sum(+ [+ blueprints is not None,+ mthds_contents is not None,+ mthds_file_path is not None,+ ]+ )if provided_params == 0:- msg = "At least one of blueprints, mthds_content, or mthds_file_path must be provided to validate_bundle"+ msg = "At least one of blueprints, mthds_contents, or mthds_file_path must be provided to validate_bundle"raise ValidateBundleError(message=msg)if provided_params > 1:- msg = "Only one of blueprints, mthds_content, or mthds_file_path can be provided to validate_bundle, not multiple"+ msg = "Only one of blueprints, mthds_contents, or mthds_file_path can be provided to validate_bundle, not multiple"raise ValidateBundleError(message=msg)library_manager = get_library_manager()@@ -124,10 +132,9 @@ async def validate_bundle(dry_run_results = await dry_run_pipes(pipes=loaded_pipes, raise_on_failure=True)return ValidateBundleResult(blueprints=loaded_blueprints, pipes=loaded_pipes, dry_run_result=dry_run_results)- elif mthds_content is not None:- blueprint = PipelexInterpreter.make_pipelex_bundle_blueprint(mthds_content=mthds_content)- loaded_blueprints = [blueprint]- loaded_pipes = library_manager.load_from_blueprints(library_id=library_id, blueprints=[blueprint])+ elif mthds_contents is not None:+ loaded_blueprints = [PipelexInterpreter.make_pipelex_bundle_blueprint(mthds_content=content) for content in mthds_contents]+ loaded_pipes = library_manager.load_from_blueprints(library_id=library_id, blueprints=loaded_blueprints)dry_run_results = await dry_run_pipes(pipes=loaded_pipes, raise_on_failure=True)return ValidateBundleResult(blueprints=loaded_blueprints, pipes=loaded_pipes, dry_run_result=dry_run_results)@@ -172,6 +179,9 @@ async def validate_bundle(message=msg,pipe_validation_errors=pipe_validation_errors,) from validation_error+ except MthdsDecodeError as decode_error:+ msg = f"TOML syntax error at line {decode_error.lineno}, column {decode_error.colno}: {decode_error.message}"+ raise ValidateBundleError(message=msg) from decode_errorexcept PipeRunError as pipe_run_error:raise ValidateBundleError(message=pipe_run_error.message,@@ -198,6 +208,9 @@ async def validate_bundles_from_directory(directory: Path) -> ValidateBundleResuloaded_pipes = library_manager.load_libraries(library_id=library_id, library_dirs=[Path(directory)])dry_run_results = await dry_run_pipes(pipes=loaded_pipes, raise_on_failure=True)+ except MthdsDecodeError as decode_error:+ msg = f"TOML syntax error at line {decode_error.lineno}, column {decode_error.colno}: {decode_error.message}"+ raise ValidateBundleError(message=msg) from decode_errorexcept PipelexInterpreterError as interpreter_error:raise ValidateBundleError(message=interpreter_error.message,@@ -245,7 +258,7 @@ class LoadConceptsOnlyResult(BaseModel):def load_concepts_only(mthds_file_path: Path | None = None,- mthds_content: str | None = None,+ mthds_contents: list[str] | None = None,blueprints: list[PipelexBundleBlueprint] | None = None,library_dirs: Sequence[Path] | None = None,) -> LoadConceptsOnlyResult:@@ -257,7 +270,7 @@ def load_concepts_only(Args:mthds_file_path: Path to a single MTHDS file to load (mutually exclusive with others)- mthds_content: MTHDS content string to load (mutually exclusive with others)+ mthds_contents: List of MTHDS content strings to load (mutually exclusive with others)blueprints: Pre-parsed blueprints to load (mutually exclusive with others)library_dirs: Optional directories containing additional MTHDS library files@@ -267,12 +280,12 @@ def load_concepts_only(Raises:ValidateBundleError: If loading fails due to interpreter or validation errors"""- provided_params = sum([blueprints is not None, mthds_content is not None, mthds_file_path is not None])+ provided_params = sum([blueprints is not None, mthds_contents is not None, mthds_file_path is not None])if provided_params == 0:- msg = "At least one of blueprints, mthds_content, or mthds_file_path must be provided to load_concepts_only"+ msg = "At least one of blueprints, mthds_contents, or mthds_file_path must be provided to load_concepts_only"raise ValidateBundleError(message=msg)if provided_params > 1:- msg = "Only one of blueprints, mthds_content, or mthds_file_path can be provided to load_concepts_only, not multiple"+ msg = "Only one of blueprints, mthds_contents, or mthds_file_path can be provided to load_concepts_only, not multiple"raise ValidateBundleError(message=msg)library_manager = get_library_manager()@@ -299,10 +312,9 @@ def load_concepts_only(loaded_concepts = library_manager.load_concepts_only_from_blueprints(library_id=library_id, blueprints=blueprints)return LoadConceptsOnlyResult(blueprints=loaded_blueprints, concepts=loaded_concepts)- elif mthds_content is not None:- blueprint = PipelexInterpreter.make_pipelex_bundle_blueprint(mthds_content=mthds_content)- loaded_blueprints = [blueprint]- loaded_concepts = library_manager.load_concepts_only_from_blueprints(library_id=library_id, blueprints=[blueprint])+ elif mthds_contents is not None:+ loaded_blueprints = [PipelexInterpreter.make_pipelex_bundle_blueprint(mthds_content=content) for content in mthds_contents]+ loaded_concepts = library_manager.load_concepts_only_from_blueprints(library_id=library_id, blueprints=loaded_blueprints)return LoadConceptsOnlyResult(blueprints=loaded_blueprints, concepts=loaded_concepts)else:@@ -320,6 +332,9 @@ def load_concepts_only(return LoadConceptsOnlyResult(blueprints=loaded_blueprints, concepts=loaded_concepts)+ except MthdsDecodeError as decode_error:+ msg = f"TOML syntax error at line {decode_error.lineno}, column {decode_error.colno}: {decode_error.message}"+ raise ValidateBundleError(message=msg) from decode_errorexcept PipelexInterpreterError as interpreter_error:raise ValidateBundleError(message=interpreter_error.message,@@ -363,6 +378,9 @@ def load_concepts_only_from_directory(directory: Path) -> LoadConceptsOnlyResultall_blueprints.append(blueprint)loaded_concepts = library_manager.load_concepts_only_from_blueprints(library_id=library_id, blueprints=all_blueprints)+ except MthdsDecodeError as decode_error:+ msg = f"TOML syntax error at line {decode_error.lineno}, column {decode_error.colno}: {decode_error.message}"+ raise ValidateBundleError(message=msg) from decode_errorexcept PipelexInterpreterError as interpreter_error:raise ValidateBundleError(message=interpreter_error.message,diff --git a/pipelex/tools/misc/http_utils.py b/pipelex/tools/misc/http_utils.pyindex 4cb9a273c..4dcc0e7b2 100644--- a/pipelex/tools/misc/http_utils.py+++ b/pipelex/tools/misc/http_utils.py@@ -22,7 +22,7 @@ def validate_url_resource_exists(url: str) -> None:By the time a URL reaches DocumentContent/ImageContent, it should alreadybe resolved (absolute path or fully qualified URL).- For HTTP/HTTPS URLs: performs a HEAD request to check reachability.+ For HTTP/HTTPS URLs: performs a streaming GET request to check reachability.For local file paths: checks that the file exists on disk.Skips validation for internal URIs (base64 data URLs, pipelex-storage://).@@ -41,8 +41,8 @@ def validate_url_resource_exists(url: str) -> None:def _validate_http_url(url: str) -> None:user_agent = get_user_agent()try:- response = httpx.head(url, timeout=10, follow_redirects=True, headers={"User-Agent": user_agent})- response.raise_for_status()+ with httpx.stream("GET", url, timeout=10, follow_redirects=True, headers={"User-Agent": user_agent}) as response:+ response.raise_for_status()except httpx.HTTPStatusError as exc:msg = f"URL '{url}' returned HTTP {exc.response.status_code}"raise ValueError(msg) from excdiff --git a/pyproject.toml b/pyproject.tomlindex af2f341e3..efa44ac3a 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -28,7 +28,7 @@ dependencies = ["json2html>=1.3.0","kajson==0.3.1","markdown>=3.6",- "mthds>=0.1.1",+ "mthds>=0.2.0","networkx>=3.4.2","openai>=1.108.1","opentelemetry-api",diff --git a/tests/integration/pipelex/pipeline/test_load_concepts_only.py b/tests/integration/pipelex/pipeline/test_load_concepts_only.pyindex e24712437..8d8e79850 100644--- a/tests/integration/pipelex/pipeline/test_load_concepts_only.py+++ b/tests/integration/pipelex/pipeline/test_load_concepts_only.py@@ -252,7 +252,7 @@ description = "An item"name = { type = "text", description = "Item name" }"""- result = load_concepts_only(mthds_content=mthds_content)+ result = load_concepts_only(mthds_contents=[mthds_content])assert len(result.blueprints) == 1assert len(result.concepts) == 1diff --git a/tests/unit/pipelex/builder/pipe/pipe_operator/pipe_compose/test_pipe_compose_to_toml.py b/tests/unit/pipelex/builder/pipe/pipe_operator/pipe_compose/test_pipe_compose_to_toml.pyindex 542685d9e..dcf7a27a1 100644--- a/tests/unit/pipelex/builder/pipe/pipe_operator/pipe_compose/test_pipe_compose_to_toml.py+++ b/tests/unit/pipelex/builder/pipe/pipe_operator/pipe_compose/test_pipe_compose_to_toml.py@@ -1,13 +1,11 @@-"""Unit tests for PipeCompose TOML serialization via _pipe_spec_to_toml."""+"""Unit tests for PipeCompose TOML serialization via pipe_spec_to_toml."""from typing import Any, ClassVarimport pytest+from pipelex.builder.operations.pipe_ops import pipe_spec_to_tomlfrom pipelex.builder.pipe.pipe_compose_spec import PipeComposeSpec-from pipelex.cli.agent_cli.commands.pipe_cmd import (- _pipe_spec_to_toml, # noqa: PLC2701 # pyright: ignore[reportPrivateUsage]-)class _TomlTestCases:@@ -56,7 +54,7 @@ class _TomlTestCases:class TestPipeComposeToToml:- """Tests for _pipe_spec_to_toml with PipeCompose specs."""+ """Tests for pipe_spec_to_toml with PipeCompose specs."""@pytest.mark.parametrize(("_test_name", "spec_data"),@@ -73,7 +71,7 @@ class TestPipeComposeToToml:) -> None:"""All PipeCompose modes should serialize to TOML without errors."""pipe_spec = PipeComposeSpec.model_validate({**spec_data, "type": "PipeCompose"})- result = _pipe_spec_to_toml(pipe_spec)+ result = pipe_spec_to_toml(pipe_spec)assert isinstance(result, str)assert len(result) > 0@@ -81,7 +79,7 @@ class TestPipeComposeToToml:"""Construct mode should produce a [pipe.<code>.construct] section with inline tables."""spec_data = _TomlTestCases.CONSTRUCT_WITH_FROM_MAPPINGS[1]pipe_spec = PipeComposeSpec.model_validate({**spec_data, "type": "PipeCompose"})- result = _pipe_spec_to_toml(pipe_spec)+ result = pipe_spec_to_toml(pipe_spec)assert "[pipe.compose_sheet.construct]" in resultassert 'score = {from = "analysis.overall_score"}' in result@@ -94,7 +92,7 @@ class TestPipeComposeToToml:"""Non-dict values in construct spec should be serialized directly."""spec_data = _TomlTestCases.CONSTRUCT_WITH_STATIC_VALUES[1]pipe_spec = PipeComposeSpec.model_validate({**spec_data, "type": "PipeCompose"})- result = _pipe_spec_to_toml(pipe_spec)+ result = pipe_spec_to_toml(pipe_spec)assert "[pipe.compose_report.construct]" in resultassert 'title = {from = "title"}' in result@@ -105,7 +103,7 @@ class TestPipeComposeToToml:"""Template mode should produce target_format and template fields."""spec_data = _TomlTestCases.TEMPLATE_MODE[1]pipe_spec = PipeComposeSpec.model_validate({**spec_data, "type": "PipeCompose"})- result = _pipe_spec_to_toml(pipe_spec)+ result = pipe_spec_to_toml(pipe_spec)assert 'target_format = "markdown"' in resultassert 'template = "Hello @name!"' in resultdiff --git a/tests/unit/pipelex/cli/test_concept_cmd.py b/tests/unit/pipelex/cli/test_concept_cmd.pyindex dbe00d271..8a3645313 100644--- a/tests/unit/pipelex/cli/test_concept_cmd.py+++ b/tests/unit/pipelex/cli/test_concept_cmd.py@@ -6,15 +6,15 @@ import jsonfrom typing import Any, ClassVarfrom pipelex.builder.concept.concept_spec import ConceptStructureSpec, ConceptStructureSpecFieldType-from pipelex.cli.agent_cli.commands.concept_cmd import (- _concept_spec_to_toml, # noqa: PLC2701 # pyright: ignore[reportPrivateUsage]- _parse_concept_spec_from_json, # noqa: PLC2701 # pyright: ignore[reportPrivateUsage]- _structure_field_to_dict, # noqa: PLC2701 # pyright: ignore[reportPrivateUsage]+from pipelex.builder.operations.concept_ops import (+ concept_spec_to_toml,+ parse_concept_spec,+ structure_field_to_dict,)class TestStructureFieldToDict:- """Tests for _structure_field_to_dict function."""+ """Tests for structure_field_to_dict function."""def test_text_field_with_description_only(self) -> None:"""Text field with only description should produce minimal dict."""@@ -23,7 +23,7 @@ class TestStructureFieldToDict:description="The person's name",type=ConceptStructureSpecFieldType.TEXT,)- result = _structure_field_to_dict(field_spec)+ result = structure_field_to_dict(field_spec)assert result == {"description": "The person's name"}assert "type" not in result@@ -36,7 +36,7 @@ class TestStructureFieldToDict:type=ConceptStructureSpecFieldType.TEXT,required=True,)- result = _structure_field_to_dict(field_spec)+ result = structure_field_to_dict(field_spec)assert result["description"] == "The person's name"assert result["type"] == ConceptStructureSpecFieldType.TEXT@@ -49,7 +49,7 @@ class TestStructureFieldToDict:description="Age in years",type=ConceptStructureSpecFieldType.INTEGER,)- result = _structure_field_to_dict(field_spec)+ result = structure_field_to_dict(field_spec)assert result["type"] == ConceptStructureSpecFieldType.INTEGERassert result["description"] == "Age in years"@@ -62,7 +62,7 @@ class TestStructureFieldToDict:type=ConceptStructureSpecFieldType.INTEGER,default_value=0,)- result = _structure_field_to_dict(field_spec)+ result = structure_field_to_dict(field_spec)assert result["default"] == 0assert result["type"] == ConceptStructureSpecFieldType.INTEGER@@ -75,7 +75,7 @@ class TestStructureFieldToDict:type=ConceptStructureSpecFieldType.CONCEPT,concept_ref="myapp.Customer",)- result = _structure_field_to_dict(field_spec)+ result = structure_field_to_dict(field_spec)assert result["concept_ref"] == "myapp.Customer"assert result["type"] == ConceptStructureSpecFieldType.CONCEPT@@ -91,7 +91,7 @@ class TestStructureFieldToDict:type=ConceptStructureSpecFieldType.TEXT,choices=["pending", "in_progress", "complete"],)- result = _structure_field_to_dict(field_spec)+ result = structure_field_to_dict(field_spec)assert result["choices"] == ["pending", "in_progress", "complete"]assert result["description"] == "Current status"@@ -105,7 +105,7 @@ class TestStructureFieldToDict:choices=["low", "medium", "high"],required=True,)- result = _structure_field_to_dict(field_spec)+ result = structure_field_to_dict(field_spec)assert result["choices"] == ["low", "medium", "high"]assert result["required"] is True@@ -120,14 +120,14 @@ class TestStructureFieldToDict:choices=["active", "inactive", "pending"],default_value="pending",)- result = _structure_field_to_dict(field_spec)+ result = structure_field_to_dict(field_spec)assert result["choices"] == ["active", "inactive", "pending"]assert result["default"] == "pending"class TestConceptCodeAliases:- """Tests for concept_code alias handling in _parse_concept_spec_from_json."""+ """Tests for concept_code alias handling in parse_concept_spec."""_BASE: ClassVar[dict[str, Any]] = {"description": "A test concept",@@ -136,39 +136,39 @@ class TestConceptCodeAliases:def test_canonical_the_concept_code(self) -> None:spec = {**self._BASE, "the_concept_code": "Invoice"}- result = _parse_concept_spec_from_json(spec)+ result = parse_concept_spec(spec)assert result.the_concept_code == "Invoice"def test_alias_concept_code(self) -> None:spec = {**self._BASE, "concept_code": "Invoice"}- result = _parse_concept_spec_from_json(spec)+ result = parse_concept_spec(spec)assert result.the_concept_code == "Invoice"def test_alias_code(self) -> None:spec = {**self._BASE, "code": "Invoice"}- result = _parse_concept_spec_from_json(spec)+ result = parse_concept_spec(spec)assert result.the_concept_code == "Invoice"def test_alias_name(self) -> None:spec = {**self._BASE, "name": "Invoice"}- result = _parse_concept_spec_from_json(spec)+ result = parse_concept_spec(spec)assert result.the_concept_code == "Invoice"def test_alias_concept_name(self) -> None:spec = {**self._BASE, "concept_name": "Invoice"}- result = _parse_concept_spec_from_json(spec)+ result = parse_concept_spec(spec)assert result.the_concept_code == "Invoice"def test_canonical_ignores_alias(self) -> None:"""When the_concept_code is present, alias keys are removed so Pydantic doesn't reject them."""spec = {**self._BASE, "the_concept_code": "Canonical", "name": "Alias"}- result = _parse_concept_spec_from_json(spec)+ result = parse_concept_spec(spec)assert result.the_concept_code == "Canonical"def test_multiple_aliases_all_cleaned_up(self) -> None:"""When the_concept_code and multiple aliases are present, all aliases are removed."""spec = {**self._BASE, "concept_code": "Invoice", "name": "Alt", "code": "Alt2"}- result = _parse_concept_spec_from_json(spec)+ result = parse_concept_spec(spec)assert result.the_concept_code == "Invoice"@@ -183,7 +183,7 @@ class TestStructureFieldStringShorthand:"title": "The title of the item",},}- result = _parse_concept_spec_from_json(spec)+ result = parse_concept_spec(spec)assert result.structure is not Noneassert result.structure["title"].type == ConceptStructureSpecFieldType.TEXTassert result.structure["title"].description == "The title of the item"@@ -197,14 +197,14 @@ class TestStructureFieldStringShorthand:"age": {"type": "integer", "description": "Age in years"},},}- result = _parse_concept_spec_from_json(spec)+ result = parse_concept_spec(spec)assert result.structure is not Noneassert result.structure["name"].type == ConceptStructureSpecFieldType.TEXTassert result.structure["age"].type == ConceptStructureSpecFieldType.INTEGERclass TestParseConceptSpecFromJson:- """Tests for _parse_concept_spec_from_json function."""+ """Tests for parse_concept_spec function."""def test_parse_simple_concept_with_refines(self) -> None:"""Parse a simple concept that refines Text."""@@ -213,7 +213,7 @@ class TestParseConceptSpecFromJson:"description": "A commercial invoice","refines": "Text",}- result = _parse_concept_spec_from_json(spec_data)+ result = parse_concept_spec(spec_data)assert result.the_concept_code == "Invoice"assert result.description == "A commercial invoice"@@ -229,7 +229,7 @@ class TestParseConceptSpecFromJson:"age": {"type": "integer", "description": "Age in years"},},}- result = _parse_concept_spec_from_json(spec_data)+ result = parse_concept_spec(spec_data)assert result.the_concept_code == "Person"assert result.structure is not None@@ -255,14 +255,14 @@ class TestParseConceptSpecFromJson:}},}- result = _parse_concept_spec_from_json(spec_data)+ result = parse_concept_spec(spec_data)assert result.structure is not Noneassert result.structure["status"].choices == ["pending", "in_progress", "complete"]class TestConceptSpecToToml:- """Tests for _concept_spec_to_toml function."""+ """Tests for concept_spec_to_toml function."""def test_toml_output_includes_choices(self) -> None:"""TOML output should include choices when present.@@ -281,8 +281,8 @@ class TestConceptSpecToToml:}},}- concept_spec = _parse_concept_spec_from_json(spec_data)- toml_output = _concept_spec_to_toml(concept_spec)+ concept_spec = parse_concept_spec(spec_data)+ toml_output = concept_spec_to_toml(concept_spec)assert 'choices = ["pending", "in_progress", "complete"]' in toml_output@@ -293,8 +293,8 @@ class TestConceptSpecToToml:"description": "A commercial invoice","refines": "Text",}- concept_spec = _parse_concept_spec_from_json(spec_data)- toml_output = _concept_spec_to_toml(concept_spec)+ concept_spec = parse_concept_spec(spec_data)+ toml_output = concept_spec_to_toml(concept_spec)assert "[concept.Invoice]" in toml_outputassert 'description = "A commercial invoice"' in toml_output@@ -315,8 +315,8 @@ class TestConceptSpecToToml:},},}- concept_spec = _parse_concept_spec_from_json(spec_data)- toml_output = _concept_spec_to_toml(concept_spec)+ concept_spec = parse_concept_spec(spec_data)+ toml_output = concept_spec_to_toml(concept_spec)assert "[concept.Order]" in toml_outputassert "order_id" in toml_output@@ -378,8 +378,8 @@ class TestConceptStructureSpecChoices:})spec_data = json.loads(input_json)- concept_spec = _parse_concept_spec_from_json(spec_data)- toml_output = _concept_spec_to_toml(concept_spec)+ concept_spec = parse_concept_spec(spec_data)+ toml_output = concept_spec_to_toml(concept_spec)assert 'choices = ["low", "medium", "high", "critical"]' in toml_outputassert "required = true" in toml_outputdiff --git a/uv.lock b/uv.lockindex 57b584bb4..aef707837 100644--- a/uv.lock+++ b/uv.lock@@ -2448,7 +2448,7 @@ wheels = [[[package]]name = "mthds"-version = "0.1.1"+version = "0.2.0"source = { registry = "https://pypi.org/simple" }dependencies = [{ name = "backports-strenum", marker = "python_full_version < '3.11'" },@@ -2459,9 +2459,9 @@ dependencies = [{ name = "tomlkit" },{ name = "typing-extensions" },]-sdist = { url = "https://files.pythonhosted.org/packages/88/f3/dad8e15a09a336e79255420f00d8a1bafbd9e449b4f193534c696f33c1d2/mthds-0.1.1.tar.gz", hash = "sha256:409d744b13987c2890e9fcdb005e35e0b8497b8288ecbfbdda2e26d5401483b9", size = 112278, upload-time = "2026-03-12T18:08:16.805Z" }+sdist = { url = "https://files.pythonhosted.org/packages/c6/e8/69d60b665f10a28c5efede63ccf95bdf75954701023e38138dc97da5862a/mthds-0.2.0.tar.gz", hash = "sha256:95952b489e0a6336667086a3eddb0f761f81d4af29f32de28b4182b33d265fc9", size = 112564, upload-time = "2026-03-24T13:33:51.927Z" }wheels = [- { url = "https://files.pythonhosted.org/packages/96/5a/8ebd5ec801bd37323c2a0cf8b7b57bc09775fa768ab8c67c475824521ea4/mthds-0.1.1-py3-none-any.whl", hash = "sha256:2742a0d0d886315de4a60866fb7995fcef682b1f56df37026f36c24ebdc86ca0", size = 49146, upload-time = "2026-03-12T18:08:15.338Z" },+ { url = "https://files.pythonhosted.org/packages/cc/64/3fe70f33f0f9c6237336f368cf0a24d28fce60fc1d486f0df308b36d3beb/mthds-0.2.0-py3-none-any.whl", hash = "sha256:72f6765ef48caff24211c0fc3523f61a7118ae829bd54caf1b9eba7fb4e4f281", size = 49346, upload-time = "2026-03-24T13:33:50.479Z" },][[package]]@@ -3588,7 +3588,7 @@ requires-dist = [{ name = "mkdocs-meta-manager", marker = "extra == 'docs'", specifier = ">=1.1.0" },{ name = "mkdocs-redirects", marker = "extra == 'docs'", specifier = "==1.2.2" },{ name = "moto", extras = ["s3"], marker = "extra == 'dev'", specifier = ">=5.0.0" },- { name = "mthds", specifier = ">=0.1.1" },+ { name = "mthds", specifier = ">=0.2.0" },{ name = "mypy", marker = "extra == 'dev'", specifier = "==1.19.1" },{ name = "networkx", specifier = ">=3.4.2" },{ name = "openai", specifier = ">=1.108.1" },