Skip to content

Build AI methods with your coding agent, then run them anywhere: as an MCP for chatbots, as a webapp for people, or via API for your software.

Refactor pipe namespace: pipe_ref consistency and duplicate detection (#780)

Louis Choquel628e414d971d
diff --git a/pipelex/cli/agent_cli/commands/validate/_validate_core.py b/pipelex/cli/agent_cli/commands/validate/_validate_core.py
index e0a0d9956..6855d6759 100644
--- a/pipelex/cli/agent_cli/commands/validate/_validate_core.py
+++ b/pipelex/cli/agent_cli/commands/validate/_validate_core.py
@@ -47,9 +47,9 @@ async def validate_all_core(
validated_pipes: list[dict[str, str]] = []
for the_pipe in pipes:
- dry_run_output = dry_run_results.get(the_pipe.code)
+ dry_run_output = dry_run_results.get(the_pipe.pipe_ref)
status: str = dry_run_output.status if dry_run_output else DryRunStatus.SUCCESS
- validated_pipes.append({"pipe_code": the_pipe.code, "status": status})
+ validated_pipes.append({"pipe_code": the_pipe.pipe_ref, "status": status})
return {
"success": True,
@@ -76,7 +76,7 @@ async def validate_bundle_core(
"""
result = await validate_bundle(mthds_file_path=bundle_path, library_dirs=library_dirs)
- validated_pipes = [{"pipe_code": the_pipe.code, "status": "SUCCESS"} for the_pipe in result.pipes]
+ validated_pipes = [{"pipe_code": the_pipe.pipe_ref, "status": "SUCCESS"} for the_pipe in result.pipes]
return {
"success": True,
diff --git a/pipelex/core/pipes/pipe_abstract.py b/pipelex/core/pipes/pipe_abstract.py
index 20ffe9293..d6143f488 100644
--- a/pipelex/core/pipes/pipe_abstract.py
+++ b/pipelex/core/pipes/pipe_abstract.py
@@ -53,6 +53,11 @@ class PipeAbstract(ABC, BaseModel):
inputs: InputStuffSpecs = Field(default_factory=InputStuffSpecs)
output: StuffSpec
+ @property
+ def pipe_ref(self) -> str:
+ """Domain-qualified pipe reference, e.g. 'scoring.compute_score'."""
+ return f"{self.domain_code}.{self.code}"
+
@property
def pipe_type(self) -> str:
return self.__class__.__name__
diff --git a/pipelex/libraries/domain/domain_library.py b/pipelex/libraries/domain/domain_library.py
index a1c74d583..f4eee4c74 100644
--- a/pipelex/libraries/domain/domain_library.py
+++ b/pipelex/libraries/domain/domain_library.py
@@ -1,6 +1,7 @@
from pydantic import RootModel
from typing_extensions import override
+from pipelex import log
from pipelex.core.domains.domain import Domain
from pipelex.libraries.domain.domain_library_abstract import DomainLibraryAbstract
from pipelex.libraries.domain.exceptions import DomainLibraryError
@@ -30,10 +31,22 @@ class DomainLibrary(RootModel[DomainLibraryRoot], DomainLibraryAbstract):
return self.root.get(domain_code)
def add_domain(self, domain: Domain):
+ # TODO: resolve domain metadata conflicts properly — currently first-write-wins with a warning.
+ # The system_prompt field is used as a PipeLLM fallback; it should be inlined at pipe factory time
+ # before LibraryCrate construction so domain-level system_prompt becomes unnecessary.
domain_code = domain.code
if domain_code in self.root:
- msg = f"Trying to add domain '{domain_code}' to domain library but it already exists"
- raise DomainLibraryError(msg)
+ existing = self.root[domain_code]
+ if existing.description != domain.description:
+ log.warning(
+ f"Domain '{domain_code}' declared with different descriptions: "
+ f"'{existing.description}' vs '{domain.description}'. Keeping the first.",
+ )
+ if existing.system_prompt != domain.system_prompt:
+ log.warning(
+ f"Domain '{domain_code}' declared with different system_prompts. Keeping the first.",
+ )
+ return
self.root[domain_code] = domain
def add_domains(self, domains: list[Domain]):
diff --git a/pipelex/libraries/library_manager.py b/pipelex/libraries/library_manager.py
index 8f2bc217a..1108ed89a 100644
--- a/pipelex/libraries/library_manager.py
+++ b/pipelex/libraries/library_manager.py
@@ -28,6 +28,7 @@ from pipelex.core.qualified_ref import QualifiedRef
from pipelex.core.stuffs.structured_content import StructuredContent
from pipelex.core.validation import report_validation_error
from pipelex.hub import get_current_library
+from pipelex.libraries.concept.exceptions import ConceptLibraryError
from pipelex.libraries.exceptions import (
LibraryError,
LibraryLoadingError,
@@ -91,7 +92,7 @@ class LibraryManager(LibraryManagerAbstract):
def __init__(self):
# UNTITLED library is the fallback library for all others
self._libraries: dict[str, Library] = {}
- self._pipe_source_map: dict[str, Path] = {} # pipe_code -> source .mthds file
+ self._pipe_source_map: dict[str, Path] = {} # pipe_ref (domain.pipe_code) -> source .mthds file
############################################################
# Manager lifecycle
@@ -111,8 +112,8 @@ class LibraryManager(LibraryManagerAbstract):
raise LibraryError(msg)
library = self._libraries[library_id]
# Remove source map entries for pipes in this library
- for pipe_code in library.pipe_library.root:
- self._pipe_source_map.pop(pipe_code, None)
+ for pipe_ref in library.pipe_library.root:
+ self._pipe_source_map.pop(pipe_ref, None)
library.teardown()
del self._libraries[library_id]
return
@@ -164,12 +165,22 @@ class LibraryManager(LibraryManagerAbstract):
"""Get the source file path for a pipe.
Args:
- pipe_code: The pipe code to look up.
+ pipe_code: The pipe code or pipe_ref (domain.code) to look up.
Returns:
Path to the .mthds file the pipe was loaded from, or None if unknown.
"""
- return self._pipe_source_map.get(pipe_code)
+ # Direct lookup by pipe_ref
+ result = self._pipe_source_map.get(pipe_code)
+ if result is not None:
+ return result
+ # Bare code fallback: search for a key ending with .pipe_code
+ if "." not in pipe_code:
+ suffix = f".{pipe_code}"
+ matches = [path for key, path in self._pipe_source_map.items() if key.endswith(suffix)]
+ if len(matches) == 1:
+ return matches[0]
+ return None
############################################################
# Private methods
@@ -374,22 +385,23 @@ class LibraryManager(LibraryManagerAbstract):
new_source = Path(blueprint.source) if blueprint.source else None
if blueprint.pipe is not None:
for pipe_code, pipe_blueprint in blueprint.pipe.items():
+ pipe_ref = f"{blueprint.domain}.{pipe_code}"
# Detect duplicate pipe declarations across different bundles in the same library
- if pipe_code in pipe_source_in_this_load:
- existing_source = pipe_source_in_this_load[pipe_code]
+ if pipe_ref in pipe_source_in_this_load:
+ existing_source = pipe_source_in_this_load[pipe_ref]
if existing_source == new_source:
msg = (
- f"Pipe '{pipe_code}' is declared twice in the same bundle file: '{existing_source}'. "
+ f"Pipe '{pipe_ref}' is declared twice in the same bundle file: '{existing_source}'. "
"Please remove the duplicate declaration."
)
else:
msg = (
- f"Pipe '{pipe_code}' is declared in two different bundle files: "
+ f"Pipe '{pipe_ref}' is declared in two different bundle files: "
f"'{existing_source}' and '{new_source}'. "
"Please remove one of the declarations or rename one of the pipes."
)
raise PipeLibraryError(msg)
- pipe_source_in_this_load[pipe_code] = new_source
+ pipe_source_in_this_load[pipe_ref] = new_source
pipe = PipeFactory[PipeAbstract].make_from_blueprint(
domain_code=blueprint.domain,
pipe_code=pipe_code,
@@ -399,7 +411,7 @@ class LibraryManager(LibraryManagerAbstract):
pipes.append(pipe)
# Track source file for this pipe (used by get_pipe_source)
if new_source:
- self._pipe_source_map[pipe_code] = new_source
+ self._pipe_source_map[pipe_ref] = new_source
all_pipes.extend(pipes)
library.pipe_library.add_pipes(pipes=all_pipes)
@@ -474,16 +486,34 @@ class LibraryManager(LibraryManagerAbstract):
Returns:
List of loaded concepts
"""
- # Step 1: Collect all concept entries with metadata
+ # Step 1: Collect all concept entries with metadata, detecting duplicates
+ concept_source_in_this_load: dict[str, Path | None] = {}
ref_to_entry: dict[str, tuple[str, str, ConceptBlueprint | str]] = {}
for blueprint in blueprints:
if blueprint.concept is not None:
+ new_source = Path(blueprint.source) if blueprint.source else None
for concept_code, concept_blueprint in blueprint.concept.items():
concept_ref = ConceptFactory.make_concept_ref_with_domain(
domain_code=blueprint.domain,
concept_code=concept_code,
)
+ # Detect duplicate concept declarations across different bundles in the same library
+ if concept_ref in concept_source_in_this_load:
+ existing_source = concept_source_in_this_load[concept_ref]
+ if existing_source == new_source:
+ msg = (
+ f"Concept '{concept_ref}' is declared twice in the same bundle file: '{existing_source}'. "
+ "Please remove the duplicate declaration."
+ )
+ else:
+ msg = (
+ f"Concept '{concept_ref}' is declared in two different bundle files: "
+ f"'{existing_source}' and '{new_source}'. "
+ "Please remove one of the declarations or rename one of the concepts."
+ )
+ raise ConceptLibraryError(msg)
+ concept_source_in_this_load[concept_ref] = new_source
ref_to_entry[concept_ref] = (
blueprint.domain,
concept_code,
@@ -967,7 +997,8 @@ class LibraryManager(LibraryManagerAbstract):
def _remove_pipes_from_blueprint(self, blueprint: PipelexBundleBlueprint) -> None:
library = self.get_current_library()
if blueprint.pipe is not None:
- library.pipe_library.remove_pipes_by_codes(pipe_codes=list(blueprint.pipe.keys()))
+ pipe_refs_to_remove = [f"{blueprint.domain}.{pipe_code}" for pipe_code in blueprint.pipe]
+ library.pipe_library.remove_pipes_by_refs(pipe_refs=pipe_refs_to_remove)
def _remove_concepts_from_blueprint(self, blueprint: PipelexBundleBlueprint) -> None:
library = self.get_current_library()
diff --git a/pipelex/libraries/pipe/pipe_library.py b/pipelex/libraries/pipe/pipe_library.py
index 5e7a4f0fa..db2e7e307 100644
--- a/pipelex/libraries/pipe/pipe_library.py
+++ b/pipelex/libraries/pipe/pipe_library.py
@@ -7,7 +7,7 @@ from typing_extensions import override
from pipelex import pretty_print
from pipelex.core.pipes.pipe_abstract import PipeAbstract
-from pipelex.core.qualified_ref import QualifiedRef, QualifiedRefError
+from pipelex.core.qualified_ref import QualifiedRef
from pipelex.libraries.pipe.exceptions import PipeLibraryError, PipeNotFoundError
from pipelex.libraries.pipe.pipe_library_abstract import PipeLibraryAbstract
from pipelex.types import Self
@@ -37,15 +37,15 @@ class PipeLibrary(RootModel[PipeLibraryRoot], PipeLibraryAbstract):
@override
def add_new_pipe(self, pipe: PipeAbstract):
- if pipe.code in self.root:
+ if pipe.pipe_ref in self.root:
msg = (
- f"Pipe '{pipe.code}' already exists in the library. "
+ f"Pipe '{pipe.pipe_ref}' already exists in the library. "
"It is likely declared in two different bundle files loaded into the same library. "
"Check your library configuration: PIPELEXPATH environment variable, "
"library_dirs passed to Pipelex.make(), or the --library-dir / -L CLI option."
)
raise PipeLibraryError(msg)
- self.root[pipe.code] = pipe
+ self.root[pipe.pipe_ref] = pipe
@override
def add_pipes(self, pipes: list[PipeAbstract]):
@@ -54,31 +54,44 @@ class PipeLibrary(RootModel[PipeLibraryRoot], PipeLibraryAbstract):
@override
def get_optional_pipe(self, pipe_code: str) -> PipeAbstract | None:
- # Direct lookup first (bare code or exact match)
+ # 1. Direct lookup — handles pipe_ref keys (domain.code) and cross-package keys (alias->domain.code)
pipe = self.root.get(pipe_code)
if pipe is not None:
return pipe
- # Cross-package: "alias->domain.pipe_code" -> lookup "alias->pipe_code"
+
+ # 2. Cross-package refs
if QualifiedRef.has_cross_package_prefix(pipe_code):
alias, remainder = QualifiedRef.split_cross_package_ref(pipe_code)
- try:
- ref = QualifiedRef.parse(remainder)
- except QualifiedRefError:
- return None
- pipe = self.root.get(f"{alias}->{ref.local_code}")
- if pipe is not None and ref.is_qualified and pipe.domain_code != ref.domain_path:
- return None
- return pipe
- # If it's a domain-qualified ref (e.g. "scoring.compute_score"), try the local code
- if "." in pipe_code:
- try:
- ref = QualifiedRef.parse(pipe_code)
- except QualifiedRefError:
- return None
- pipe = self.root.get(ref.local_code)
- if pipe is not None and ref.is_qualified and pipe.domain_code != ref.domain_path:
- return None
- return pipe
+ # Try domain-qualified remainder as direct key
+ aliased_key = f"{alias}->{remainder}"
+ pipe = self.root.get(aliased_key)
+ if pipe is not None:
+ return pipe
+ # Bare code remainder — search aliased entries matching the bare code
+ if "." not in remainder:
+ matches = [val for key, val in self.root.items() if key.startswith(f"{alias}->") and val.code == remainder]
+ if len(matches) == 1:
+ return matches[0]
+ if len(matches) > 1:
+ domains = [match.domain_code for match in matches]
+ msg = f"Ambiguous cross-package pipe code '{pipe_code}' found in domains: {domains}. Use domain-qualified ref."
+ raise PipeLibraryError(msg)
+ return None
+
+ # 3. Bare code fallback — search across domains (excluding cross-package entries)
+ # TODO: add domain_hint parameter so controllers can prefer their own domain when bare code is ambiguous.
+ # Currently, controllers resolve sub-pipes by bare code; if two domains share a pipe code the lookup
+ # raises PipeLibraryError instead of preferring the caller's domain. See PR #779 review.
+ if "." not in pipe_code:
+ matches = [val for key, val in self.root.items() if not QualifiedRef.has_cross_package_prefix(key) and val.code == pipe_code]
+ if len(matches) == 1:
+ return matches[0]
+ if len(matches) > 1:
+ domains = [match.domain_code for match in matches]
+ msg = f"Ambiguous pipe code '{pipe_code}' found in domains: {domains}. Use domain-qualified ref."
+ raise PipeLibraryError(msg)
+ return None
+
return None
def add_dependency_pipe(self, alias: str, pipe: PipeAbstract) -> None:
@@ -88,7 +101,7 @@ class PipeLibrary(RootModel[PipeLibraryRoot], PipeLibraryAbstract):
alias: The dependency alias
pipe: The pipe to add
"""
- key = f"{alias}->{pipe.code}"
+ key = f"{alias}->{pipe.pipe_ref}"
if key in self.root:
msg = f"Dependency pipe '{key}' already exists in the library"
raise PipeLibraryError(msg)
@@ -111,13 +124,13 @@ class PipeLibrary(RootModel[PipeLibraryRoot], PipeLibraryAbstract):
return self.root
@override
- def remove_pipes_by_codes(self, pipe_codes: list[str]) -> None:
+ def remove_pipes_by_refs(self, pipe_refs: list[str]) -> None:
# TODO: We should create a separate library, that copies the original one, and then removes the pipes from it
# Then run the dry run + validation to see if removing those pipe has not broken any other pipe.
# If validated, it should update the real library.
- for pipe_code in pipe_codes:
- if pipe_code in self.root:
- del self.root[pipe_code]
+ for pipe_ref in pipe_refs:
+ if pipe_ref in self.root:
+ del self.root[pipe_ref]
@override
def pretty_list_pipes(self) -> None:
diff --git a/pipelex/libraries/pipe/pipe_library_abstract.py b/pipelex/libraries/pipe/pipe_library_abstract.py
index 12e035400..6764e1339 100644
--- a/pipelex/libraries/pipe/pipe_library_abstract.py
+++ b/pipelex/libraries/pipe/pipe_library_abstract.py
@@ -33,7 +33,7 @@ class PipeLibraryAbstract(ABC):
def get_pipes_dict(self) -> dict[str, PipeAbstract]:
pass
- def remove_pipes_by_codes(self, pipe_codes: list[str]) -> None:
+ def remove_pipes_by_refs(self, pipe_refs: list[str]) -> None:
pass
@abstractmethod
diff --git a/pipelex/pipe_controllers/condition/pipe_condition.py b/pipelex/pipe_controllers/condition/pipe_condition.py
index df0443e50..e88c7d987 100644
--- a/pipelex/pipe_controllers/condition/pipe_condition.py
+++ b/pipelex/pipe_controllers/condition/pipe_condition.py
@@ -62,11 +62,11 @@ class PipeCondition(PipeController):
visited_pipes = set()
# If we've already visited this pipe, stop recursion
- if self.code in visited_pipes:
+ if self.pipe_ref in visited_pipes:
return InputStuffSpecsFactory.make_empty()
# Add this pipe to visited set for recursive calls
- visited_pipes_with_current = visited_pipes | {self.code}
+ visited_pipes_with_current = visited_pipes | {self.pipe_ref}
needed_inputs = InputStuffSpecsFactory.make_empty()
diff --git a/pipelex/pipe_controllers/parallel/pipe_parallel.py b/pipelex/pipe_controllers/parallel/pipe_parallel.py
index 927de4c97..48579c971 100644
--- a/pipelex/pipe_controllers/parallel/pipe_parallel.py
+++ b/pipelex/pipe_controllers/parallel/pipe_parallel.py
@@ -66,11 +66,11 @@ class PipeParallel(PipeController):
visited_pipes = set()
# If we've already visited this pipe, stop recursion
- if self.code in visited_pipes:
+ if self.pipe_ref in visited_pipes:
return InputStuffSpecsFactory.make_empty()
# Add this pipe to visited set for recursive calls
- visited_pipes_with_current = visited_pipes | {self.code}
+ visited_pipes_with_current = visited_pipes | {self.pipe_ref}
needed_inputs = InputStuffSpecsFactory.make_empty()
diff --git a/pipelex/pipe_controllers/sequence/pipe_sequence.py b/pipelex/pipe_controllers/sequence/pipe_sequence.py
index 7e1f4db1f..6b56e590f 100644
--- a/pipelex/pipe_controllers/sequence/pipe_sequence.py
+++ b/pipelex/pipe_controllers/sequence/pipe_sequence.py
@@ -109,11 +109,11 @@ class PipeSequence(PipeController):
visited_pipes = set()
# If we've already visited this pipe, stop recursion
- if self.code in visited_pipes:
+ if self.pipe_ref in visited_pipes:
return InputStuffSpecsFactory.make_empty()
# Add this pipe to visited set for recursive calls
- visited_pipes_with_current = visited_pipes | {self.code}
+ visited_pipes_with_current = visited_pipes | {self.pipe_ref}
needed_inputs = InputStuffSpecsFactory.make_empty()
generated_outputs: set[str] = set()
diff --git a/pipelex/pipe_run/dry_run.py b/pipelex/pipe_run/dry_run.py
index ec3479e1f..b967b99cb 100644
--- a/pipelex/pipe_run/dry_run.py
+++ b/pipelex/pipe_run/dry_run.py
@@ -110,28 +110,30 @@ async def dry_run_pipes(pipes: list[PipeAbstract], raise_on_failure: bool = True
allowed_to_fail_pipes = get_config().pipelex.dry_run_config.allowed_to_fail_pipes
for pipe in pipes:
- results[pipe.code] = await dry_run_pipe(pipe, raise_on_failure=raise_on_failure)
+ results[pipe.pipe_ref] = await dry_run_pipe(pipe, raise_on_failure=raise_on_failure)
successful_pipes: list[str] = []
failed_pipes: list[str] = []
skipped_pipes: list[str] = []
- for pipe_code, dry_run_output in results.items():
+ for pipe_ref, dry_run_output in results.items():
match dry_run_output.status:
case DryRunStatus.SUCCESS:
- successful_pipes.append(pipe_code)
+ successful_pipes.append(pipe_ref)
case DryRunStatus.FAILURE:
- failed_pipes.append(pipe_code)
+ failed_pipes.append(pipe_ref)
case DryRunStatus.SKIPPED:
- skipped_pipes.append(pipe_code)
+ skipped_pipes.append(pipe_ref)
- unexpected_failures = {pipe_code: results[pipe_code] for pipe_code in failed_pipes if pipe_code not in allowed_to_fail_pipes}
+ # TODO: allowed_to_fail_pipes uses bare codes, so one allowed code can silently match pipes from multiple domains.
+ # Consider supporting namespaced pipe_refs (e.g. "domain.pipe_code") in the config to allow precise targeting.
+ unexpected_failures = {pipe_ref: results[pipe_ref] for pipe_ref in failed_pipes if results[pipe_ref].pipe_code not in allowed_to_fail_pipes}
log.verbose(
f"Dry run completed: {len(successful_pipes)} successful, {len(failed_pipes)} failed, "
f"{len(skipped_pipes)} skipped, {len(allowed_to_fail_pipes)} allowed to fail, in {time.time() - start_time:.2f} seconds",
)
if unexpected_failures:
- unexpected_failures_details = "\n".join([f"'{pipe_code}': {results[pipe_code]}" for pipe_code in unexpected_failures])
+ unexpected_failures_details = "\n".join([f"'{pipe_ref}': {results[pipe_ref]}" for pipe_ref in unexpected_failures])
if raise_on_failure:
msg = f"Dry run failed with '{len(unexpected_failures)}' unexpected pipe failures:\n{unexpected_failures_details}"
raise DryRunError(msg)
diff --git a/pipelex/pipeline/pipeline_run_setup.py b/pipelex/pipeline/pipeline_run_setup.py
index 67aeff08c..09c30c86e 100644
--- a/pipelex/pipeline/pipeline_run_setup.py
+++ b/pipelex/pipeline/pipeline_run_setup.py
@@ -176,7 +176,9 @@ async def pipeline_run_setup(
if pipe_code:
pipe = get_required_pipe(pipe_code=pipe_code)
elif blueprint.main_pipe:
- pipe = get_required_pipe(pipe_code=blueprint.main_pipe)
+ # Qualify main_pipe with domain to avoid ambiguity when multiple domains define pipes with the same code
+ qualified_main_pipe = f"{blueprint.domain}.{blueprint.main_pipe}"
+ pipe = get_required_pipe(pipe_code=qualified_main_pipe)
else:
msg = "No pipe code or main pipe in the MTHDS content provided to the pipeline API."
raise PipeExecutionError(message=msg)
diff --git a/tests/integration/pipelex/libraries/test_pipe_namespace.py b/tests/integration/pipelex/libraries/test_pipe_namespace.py
new file mode 100644
index 000000000..26d26cb80
--- /dev/null
+++ b/tests/integration/pipelex/libraries/test_pipe_namespace.py
@@ -0,0 +1,235 @@
+"""Integration tests for pipe namespace (pipe_ref) indexing with real .mthds files."""
+
+import tempfile
+from collections.abc import Callable
+from pathlib import Path
+
+import pytest
+
+from pipelex.hub import get_library_manager
+from pipelex.libraries.concept.exceptions import ConceptLibraryError
+from pipelex.libraries.pipe.exceptions import PipeLibraryError, PipeNotFoundError
+
+SCORING_MTHDS = """\
+domain = "scoring"
+description = "Scoring domain"
+
+[concept]
+ScoreResult = "A scoring result"
+
+[pipe.process]
+type = "PipeLLM"
+description = "Process scoring data"
+inputs = { data = "Text" }
+output = "ScoreResult"
+model = "$quick-reasoning"
+prompt = "Process scoring from @data"
+"""
+
+ANALYTICS_MTHDS = """\
+domain = "analytics"
+description = "Analytics domain"
+
+[concept]
+AnalyticsResult = "An analytics result"
+
+[pipe.process]
+type = "PipeLLM"
+description = "Process analytics data"
+inputs = { data = "Text" }
+output = "AnalyticsResult"
+model = "$quick-reasoning"
+prompt = "Process analytics from @data"
+"""
+
+SCORING_CORE_MTHDS = """\
+domain = "scoring"
+description = "Scoring core"
+
+[pipe.compute_score]
+type = "PipeLLM"
+description = "Compute a score"
+inputs = { data = "Text" }
+output = "Text"
+model = "$quick-reasoning"
+prompt = "Compute score from @data"
+"""
+
+SCORING_ADVANCED_MTHDS = """\
+domain = "scoring"
+description = "Scoring advanced"
+
+[pipe.weighted_score]
+type = "PipeLLM"
+description = "Compute a weighted score"
+inputs = { data = "Text" }
+output = "Text"
+model = "$quick-reasoning"
+prompt = "Compute weighted score from @data"
+"""
+
+SCORING_DUPLICATE_MTHDS = """\
+domain = "scoring"
+description = "Scoring duplicate"
+
+[pipe.compute_score]
+type = "PipeLLM"
+description = "Duplicate compute score"
+inputs = { data = "Text" }
+output = "Text"
+model = "$quick-reasoning"
+prompt = "Duplicate compute score from @data"
+"""
+
+SCORING_CONCEPT_A_MTHDS = """\
+domain = "scoring"
+description = "Scoring with concept A"
+
+[concept]
+ScoreResult = "A scoring result"
+
+[pipe.compute_score]
+type = "PipeLLM"
+description = "Compute a score"
+inputs = { data = "Text" }
+output = "ScoreResult"
+model = "$quick-reasoning"
+prompt = "Compute score from @data"
+"""
+
+SCORING_CONCEPT_B_MTHDS = """\
+domain = "scoring"
+description = "Scoring with concept B"
+
+[concept]
+ScoreResult = "A scoring result (duplicate)"
+
+[pipe.weighted_score]
+type = "PipeLLM"
+description = "Compute a weighted score"
+inputs = { data = "Text" }
+output = "ScoreResult"
+model = "$quick-reasoning"
+prompt = "Compute weighted score from @data"
+"""
+
+SINGLE_DOMAIN_MTHDS = """\
+domain = "scoring"
+description = "Scoring domain"
+
+[concept]
+ScoreResult = "A scoring result"
+
+[pipe.compute_score]
+type = "PipeLLM"
+description = "Compute a score"
+inputs = { data = "Text" }
+output = "ScoreResult"
+model = "$quick-reasoning"
+prompt = "Compute score from @data"
+"""
+
+
+class TestPipeNamespace:
+ """Integration tests for pipe_ref-based namespace indexing."""
+
+ def test_same_pipe_code_different_domains(self, load_test_library: Callable[[list[Path]], None]):
+ """Two domains with the same pipe code coexist — would have collided under old bare-code indexing."""
+ with tempfile.TemporaryDirectory() as tmp_dir:
+ (Path(tmp_dir) / "scoring.mthds").write_text(SCORING_MTHDS, encoding="utf-8")
+ (Path(tmp_dir) / "analytics.mthds").write_text(ANALYTICS_MTHDS, encoding="utf-8")
+
+ load_test_library([Path(tmp_dir)])
+
+ library = get_library_manager().get_current_library()
+
+ # Both pipes retrievable by domain-qualified pipe_ref
+ scoring_pipe = library.pipe_library.get_required_pipe("scoring.process")
+ analytics_pipe = library.pipe_library.get_required_pipe("analytics.process")
+
+ assert scoring_pipe.code == "process"
+ assert scoring_pipe.domain_code == "scoring"
+ assert analytics_pipe.code == "process"
+ assert analytics_pipe.domain_code == "analytics"
+
+ # Bare code is ambiguous
+ with pytest.raises(PipeLibraryError, match="Ambiguous pipe code"):
+ library.pipe_library.get_optional_pipe("process")
+
+ def test_multi_bundle_same_domain(self, load_test_library: Callable[[list[Path]], None]):
+ """Multiple .mthds files contributing to the same domain load successfully."""
+ with tempfile.TemporaryDirectory() as tmp_dir:
+ (Path(tmp_dir) / "scoring_core.mthds").write_text(SCORING_CORE_MTHDS, encoding="utf-8")
+ (Path(tmp_dir) / "scoring_advanced.mthds").write_text(SCORING_ADVANCED_MTHDS, encoding="utf-8")
+
+ load_test_library([Path(tmp_dir)])
+
+ library = get_library_manager().get_current_library()
+
+ # Both pipes from the same domain exist
+ compute_pipe = library.pipe_library.get_required_pipe("scoring.compute_score")
+ weighted_pipe = library.pipe_library.get_required_pipe("scoring.weighted_score")
+
+ assert compute_pipe.domain_code == "scoring"
+ assert weighted_pipe.domain_code == "scoring"
+
+ # Domain exists once
+ assert library.domain_library.get_domain("scoring") is not None
+
+ def test_duplicate_pipe_same_domain_raises(self, load_empty_library: Callable[[], str]):
+ """Two .mthds files with the same domain and same pipe code raises on load."""
+ with tempfile.TemporaryDirectory() as tmp_dir:
+ (Path(tmp_dir) / "scoring_core.mthds").write_text(SCORING_CORE_MTHDS, encoding="utf-8")
+ (Path(tmp_dir) / "scoring_duplicate.mthds").write_text(SCORING_DUPLICATE_MTHDS, encoding="utf-8")
+
+ library_id = load_empty_library()
+ library_manager = get_library_manager()
+
+ with pytest.raises(PipeLibraryError):
+ library_manager.load_libraries(
+ library_id=library_id,
+ library_dirs=[Path(tmp_dir)],
+ )
+
+ def test_duplicate_concept_same_domain_raises(self, load_empty_library: Callable[[], str]):
+ """Two .mthds files with the same domain and same concept code raises on load."""
+ with tempfile.TemporaryDirectory() as tmp_dir:
+ (Path(tmp_dir) / "scoring_a.mthds").write_text(SCORING_CONCEPT_A_MTHDS, encoding="utf-8")
+ (Path(tmp_dir) / "scoring_b.mthds").write_text(SCORING_CONCEPT_B_MTHDS, encoding="utf-8")
+
+ library_id = load_empty_library()
+ library_manager = get_library_manager()
+
+ with pytest.raises(ConceptLibraryError, match="declared in two different bundle files"):
+ library_manager.load_libraries(
+ library_id=library_id,
+ library_dirs=[Path(tmp_dir)],
+ )
+
+ def test_bare_code_lookup_from_mthds(self, load_test_library: Callable[[list[Path]], None]):
+ """Bare code lookup works when pipe code is unambiguous (single domain)."""
+ with tempfile.TemporaryDirectory() as tmp_dir:
+ (Path(tmp_dir) / "scoring.mthds").write_text(SINGLE_DOMAIN_MTHDS, encoding="utf-8")
+
+ load_test_library([Path(tmp_dir)])
+
+ library = get_library_manager().get_current_library()
+
+ # Both bare code and pipe_ref lookup work
+ pipe_by_ref = library.pipe_library.get_required_pipe("scoring.compute_score")
+ pipe_by_code = library.pipe_library.get_required_pipe("compute_score")
+
+ assert pipe_by_ref is pipe_by_code
+ assert pipe_by_ref.pipe_ref == "scoring.compute_score"
+
+ def test_wrong_domain_lookup_returns_not_found(self, load_test_library: Callable[[list[Path]], None]):
+ """Looking up a pipe with the wrong domain raises PipeNotFoundError."""
+ with tempfile.TemporaryDirectory() as tmp_dir:
+ (Path(tmp_dir) / "scoring.mthds").write_text(SINGLE_DOMAIN_MTHDS, encoding="utf-8")
+
+ load_test_library([Path(tmp_dir)])
+
+ library = get_library_manager().get_current_library()
+
+ with pytest.raises(PipeNotFoundError):
+ library.pipe_library.get_required_pipe("analytics.compute_score")
diff --git a/tests/unit/pipelex/libraries/test_domain_library.py b/tests/unit/pipelex/libraries/test_domain_library.py
new file mode 100644
index 000000000..09ba510ac
--- /dev/null
+++ b/tests/unit/pipelex/libraries/test_domain_library.py
@@ -0,0 +1,26 @@
+from pipelex.core.domains.domain import Domain
+from pipelex.libraries.domain.domain_library import DomainLibrary
+
+
+class TestDomainLibrary:
+ """Tests for DomainLibrary domain management."""
+
+ def test_add_domain_idempotent(self):
+ """Adding the same domain twice does not raise — idempotent for multi-bundle loading."""
+ library = DomainLibrary.make_empty()
+ domain = Domain(code="scoring", description="Scoring domain")
+ library.add_domain(domain=domain)
+ library.add_domain(domain=domain)
+ assert len(library.root) == 1
+ assert library.get_domain("scoring") is domain
+
+ def test_add_different_domains(self):
+ """Adding two different domains stores both."""
+ library = DomainLibrary.make_empty()
+ domain_scoring = Domain(code="scoring", description="Scoring domain")
+ domain_analytics = Domain(code="analytics", description="Analytics domain")
+ library.add_domain(domain=domain_scoring)
+ library.add_domain(domain=domain_analytics)
+ assert len(library.root) == 2
+ assert library.get_domain("scoring") is domain_scoring
+ assert library.get_domain("analytics") is domain_analytics
diff --git a/tests/unit/pipelex/libraries/test_pipe_library_lookup.py b/tests/unit/pipelex/libraries/test_pipe_library_lookup.py
index 9767e1379..3dde744b8 100644
--- a/tests/unit/pipelex/libraries/test_pipe_library_lookup.py
+++ b/tests/unit/pipelex/libraries/test_pipe_library_lookup.py
@@ -3,47 +3,95 @@ from typing import Any
import pytest
from pytest_mock import MockerFixture
-from pipelex.libraries.pipe.exceptions import PipeNotFoundError
+from pipelex.libraries.pipe.exceptions import PipeLibraryError, PipeNotFoundError
from pipelex.libraries.pipe.pipe_library import PipeLibrary
def _make_stub_pipe(mocker: MockerFixture, code: str, domain_code: str) -> Any:
- """Create a minimal mock pipe with code and domain_code."""
+ """Create a minimal mock pipe with code, domain_code, and pipe_ref."""
mock_pipe = mocker.MagicMock()
mock_pipe.code = code
mock_pipe.domain_code = domain_code
+ mock_pipe.pipe_ref = f"{domain_code}.{code}"
return mock_pipe
class TestPipeLibraryLookup:
- """Tests for PipeLibrary.get_optional_pipe domain enforcement and malformed-ref safety."""
+ """Tests for PipeLibrary lookup with pipe_ref-based indexing."""
- def test_bare_code_lookup(self, mocker: MockerFixture):
- """Bare code lookup still works."""
- library = PipeLibrary.make_empty()
- mock_pipe = _make_stub_pipe(mocker, code="compute_score", domain_code="scoring")
- library.root["compute_score"] = mock_pipe
- result = library.get_optional_pipe("compute_score")
- assert result is mock_pipe
+ # ── Direct pipe_ref lookup ──────────────────────────────────────
- def test_domain_qualified_ref_correct_domain(self, mocker: MockerFixture):
- """Domain-qualified ref resolves when pipe domain matches."""
+ def test_pipe_ref_lookup(self, mocker: MockerFixture):
+ """Domain-qualified pipe_ref lookup works as primary path."""
library = PipeLibrary.make_empty()
mock_pipe = _make_stub_pipe(mocker, code="compute_score", domain_code="scoring")
- library.root["compute_score"] = mock_pipe
+ library.root["scoring.compute_score"] = mock_pipe
result = library.get_optional_pipe("scoring.compute_score")
assert result is mock_pipe
- def test_domain_qualified_ref_wrong_domain(self, mocker: MockerFixture):
- """Domain-qualified ref returns None when pipe domain does not match."""
+ def test_pipe_ref_wrong_domain_returns_none(self, mocker: MockerFixture):
+ """Domain-qualified ref returns None when domain does not exist."""
library = PipeLibrary.make_empty()
mock_pipe = _make_stub_pipe(mocker, code="compute_score", domain_code="scoring")
- library.root["compute_score"] = mock_pipe
+ library.root["scoring.compute_score"] = mock_pipe
result = library.get_optional_pipe("wrong_domain.compute_score")
assert result is None
- def test_cross_package_ref_correct_domain(self, mocker: MockerFixture):
- """Cross-package ref resolves when pipe domain matches."""
+ # ── Bare code fallback ──────────────────────────────────────────
+
+ def test_bare_code_unambiguous(self, mocker: MockerFixture):
+ """Bare code lookup works when only one pipe has that code."""
+ library = PipeLibrary.make_empty()
+ mock_pipe = _make_stub_pipe(mocker, code="compute_score", domain_code="scoring")
+ library.root["scoring.compute_score"] = mock_pipe
+ result = library.get_optional_pipe("compute_score")
+ assert result is mock_pipe
+
+ def test_bare_code_ambiguous_raises(self, mocker: MockerFixture):
+ """Bare code lookup raises PipeLibraryError when multiple domains have the same code."""
+ library = PipeLibrary.make_empty()
+ pipe_scoring = _make_stub_pipe(mocker, code="compute_score", domain_code="scoring")
+ pipe_analytics = _make_stub_pipe(mocker, code="compute_score", domain_code="analytics")
+ library.root["scoring.compute_score"] = pipe_scoring
+ library.root["analytics.compute_score"] = pipe_analytics
+ with pytest.raises(PipeLibraryError, match="Ambiguous pipe code"):
+ library.get_optional_pipe("compute_score")
+
+ def test_bare_code_no_match_returns_none(self):
+ """Bare code lookup returns None when no pipe has that code."""
+ library = PipeLibrary.make_empty()
+ result = library.get_optional_pipe("nonexistent")
+ assert result is None
+
+ # ── Multi-domain coexistence ────────────────────────────────────
+
+ def test_multi_domain_coexistence(self, mocker: MockerFixture):
+ """Two pipes with the same code in different domains coexist and are individually retrievable."""
+ library = PipeLibrary.make_empty()
+ pipe_scoring = _make_stub_pipe(mocker, code="compute_score", domain_code="scoring")
+ pipe_analytics = _make_stub_pipe(mocker, code="compute_score", domain_code="analytics")
+ library.root["scoring.compute_score"] = pipe_scoring
+ library.root["analytics.compute_score"] = pipe_analytics
+
+ assert library.get_optional_pipe("scoring.compute_score") is pipe_scoring
+ assert library.get_optional_pipe("analytics.compute_score") is pipe_analytics
+ assert len(library.root) == 2
+
+ def test_add_new_pipe_multi_domain(self, mocker: MockerFixture):
+ """add_new_pipe stores by pipe_ref so same code in different domains works."""
+ library = PipeLibrary.make_empty()
+ pipe_scoring = _make_stub_pipe(mocker, code="compute_score", domain_code="scoring")
+ pipe_analytics = _make_stub_pipe(mocker, code="compute_score", domain_code="analytics")
+ library.add_new_pipe(pipe=pipe_scoring)
+ library.add_new_pipe(pipe=pipe_analytics)
+
+ assert "scoring.compute_score" in library.root
+ assert "analytics.compute_score" in library.root
+
+ # ── Cross-package refs ──────────────────────────────────────────
+
+ def test_cross_package_ref_with_pipe_ref(self, mocker: MockerFixture):
+ """Cross-package ref with domain-qualified remainder resolves correctly."""
library = PipeLibrary.make_empty()
mock_pipe = _make_stub_pipe(mocker, code="compute_score", domain_code="scoring")
library.add_dependency_pipe(alias="lib", pipe=mock_pipe)
@@ -51,13 +99,23 @@ class TestPipeLibraryLookup:
assert result is mock_pipe
def test_cross_package_ref_wrong_domain(self, mocker: MockerFixture):
- """Cross-package ref returns None when pipe domain does not match."""
+ """Cross-package ref with wrong domain returns None."""
library = PipeLibrary.make_empty()
mock_pipe = _make_stub_pipe(mocker, code="compute_score", domain_code="scoring")
library.add_dependency_pipe(alias="lib", pipe=mock_pipe)
result = library.get_optional_pipe("lib->wrong_domain.compute_score")
assert result is None
+ def test_cross_package_ref_bare_code_unambiguous(self, mocker: MockerFixture):
+ """Cross-package ref with bare code resolves when unambiguous."""
+ library = PipeLibrary.make_empty()
+ mock_pipe = _make_stub_pipe(mocker, code="compute_score", domain_code="scoring")
+ library.add_dependency_pipe(alias="lib", pipe=mock_pipe)
+ result = library.get_optional_pipe("lib->compute_score")
+ assert result is mock_pipe
+
+ # ── Malformed refs ──────────────────────────────────────────────
+
@pytest.mark.parametrize(
"malformed_ref",
[
@@ -86,8 +144,10 @@ class TestPipeLibraryLookup:
result = library.get_optional_pipe(malformed_ref)
assert result is None
+ # ── get_required_pipe error cases ───────────────────────────────
+
def test_get_required_pipe_malformed_raises_not_found(self):
- """Malformed ref through get_required_pipe raises PipeNotFoundError, not QualifiedRefError."""
+ """Malformed ref through get_required_pipe raises PipeNotFoundError."""
library = PipeLibrary.make_empty()
with pytest.raises(PipeNotFoundError):
library.get_required_pipe("foo..bar")
@@ -96,6 +156,42 @@ class TestPipeLibraryLookup:
"""Domain mismatch through get_required_pipe raises PipeNotFoundError."""
library = PipeLibrary.make_empty()
mock_pipe = _make_stub_pipe(mocker, code="compute_score", domain_code="scoring")
- library.root["compute_score"] = mock_pipe
+ library.root["scoring.compute_score"] = mock_pipe
with pytest.raises(PipeNotFoundError):
library.get_required_pipe("wrong_domain.compute_score")
+
+ # ── Collision tests ───────────────────────────────────────────────
+
+ def test_same_code_different_domains_no_collision(self, mocker: MockerFixture):
+ """Same bare code in different domains does NOT collide — this would have failed under old bare-code indexing."""
+ library = PipeLibrary.make_empty()
+ pipe_scoring = _make_stub_pipe(mocker, code="process", domain_code="scoring")
+ pipe_analytics = _make_stub_pipe(mocker, code="process", domain_code="analytics")
+
+ # Both add_new_pipe calls succeed — no collision
+ library.add_new_pipe(pipe=pipe_scoring)
+ library.add_new_pipe(pipe=pipe_analytics)
+
+ # Both retrievable by pipe_ref
+ assert library.get_optional_pipe("scoring.process") is pipe_scoring
+ assert library.get_optional_pipe("analytics.process") is pipe_analytics
+
+ def test_same_pipe_ref_collision_raises(self, mocker: MockerFixture):
+ """Same pipe_ref (same domain + same code) from separate add calls raises PipeLibraryError."""
+ library = PipeLibrary.make_empty()
+ pipe_first = _make_stub_pipe(mocker, code="compute_score", domain_code="scoring")
+ pipe_duplicate = _make_stub_pipe(mocker, code="compute_score", domain_code="scoring")
+
+ library.add_new_pipe(pipe=pipe_first)
+ with pytest.raises(PipeLibraryError, match="already exists"):
+ library.add_new_pipe(pipe=pipe_duplicate)
+
+ # ── remove_pipes_by_refs ────────────────────────────────────────
+
+ def test_remove_pipes_by_refs(self, mocker: MockerFixture):
+ """remove_pipes_by_refs removes pipes by their pipe_ref keys."""
+ library = PipeLibrary.make_empty()
+ mock_pipe = _make_stub_pipe(mocker, code="compute_score", domain_code="scoring")
+ library.root["scoring.compute_score"] = mock_pipe
+ library.remove_pipes_by_refs(pipe_refs=["scoring.compute_score"])
+ assert "scoring.compute_score" not in library.root
diff --git a/tests/unit/pipelex/pipe_run/test_dry_run.py b/tests/unit/pipelex/pipe_run/test_dry_run.py
index 6c31e9355..4aeb9205f 100644
--- a/tests/unit/pipelex/pipe_run/test_dry_run.py
+++ b/tests/unit/pipelex/pipe_run/test_dry_run.py
@@ -38,12 +38,14 @@ class TestDryRun:
"""Skipped pipes must not inflate the success count in dry_run_pipes."""
mock_successful_pipe = mocker.MagicMock()
mock_successful_pipe.code = "successful_pipe"
+ mock_successful_pipe.pipe_ref = "test_domain.successful_pipe"
mock_successful_pipe.needed_inputs.return_value = mocker.MagicMock(named_stuff_specs=[])
mock_successful_pipe.validate_with_libraries.return_value = None
mock_successful_pipe.run_pipe = mocker.AsyncMock(return_value=None)
mock_skipped_pipe = mocker.MagicMock()
mock_skipped_pipe.code = "skipped_pipe"
+ mock_skipped_pipe.pipe_ref = "test_domain.skipped_pipe"
mock_skipped_pipe.needed_inputs.side_effect = PipeNotFoundError("dep->domain.pipe not found")
results = await dry_run_pipes(
@@ -51,8 +53,8 @@ class TestDryRun:
raise_on_failure=False,
)
- assert results["successful_pipe"].status == DryRunStatus.SUCCESS
- assert results["skipped_pipe"].status == DryRunStatus.SKIPPED
+ assert results["test_domain.successful_pipe"].status == DryRunStatus.SUCCESS
+ assert results["test_domain.skipped_pipe"].status == DryRunStatus.SKIPPED
@pytest.mark.asyncio
async def test_dry_run_pipe_skipped_is_not_failure(self, mocker: MockerFixture) -> None: