Skip to content

Build AI methods with your coding agent, then run them anywhere: as an MCP for chatbots, as a webapp for people, or via API for your software.

Generate concept structures in the sandbox for custom PipeFuncs

thomashebrardaa1215619454
diff --git a/pipelex/pipe_operators/func/direct_pipe_func_executor.py b/pipelex/pipe_operators/func/direct_pipe_func_executor.py
index c581baeaa..2ed9c1ced 100644
--- a/pipelex/pipe_operators/func/direct_pipe_func_executor.py
+++ b/pipelex/pipe_operators/func/direct_pipe_func_executor.py
@@ -9,6 +9,8 @@ from typing import cast
from typing_extensions import override
from pipelex import log
+from pipelex.codegen.emitters.python_structures import emit_python_structures
+from pipelex.codegen.resolved_concepts import resolve_concepts_from_crate
from pipelex.core.memory.working_memory import WorkingMemory
from pipelex.core.stuffs.list_content import ListContent
from pipelex.core.stuffs.structured_content import StructuredContent
@@ -109,6 +111,30 @@ class DirectPipeFuncExecutor(PipeFuncExecutorProtocol):
target.parent.mkdir(parents=True, exist_ok=True)
target.write_text(source, encoding="utf-8")
+ # Generate the method's concept structure classes from the crate — the .mthds is the single
+ # source of truth for them (`resolve_concepts_from_crate` reads the crate's ConceptBlueprints,
+ # `emit_python_structures` projects a `structures.py`). So a PipeFunc can
+ # `from structures import <domain>__<Concept>` without the method having to STORE or transport a
+ # hand-written copy that could drift from the .mthds. A file the method actually shipped at the
+ # same path wins (already materialized above) — never clobber customer source.
+ #
+ # Force domain-qualified class names (`atlas_devis__PricedQuote`): the runtime ALWAYS qualifies a
+ # concept's structure class (`concept.structure_class_name`), and that is the exact name a
+ # PipeFunc's return-type validation checks against — so the generated classes must match the
+ # runtime spelling, not the emitter's default collision-only qualification (which would emit a
+ # bare `PricedQuote` the validator rejects).
+ resolved_library = resolve_concepts_from_crate(request.crate)
+ qualified_library = resolved_library.model_copy(
+ update={"concepts": [concept.model_copy(update={"needs_qualification": True}) for concept in resolved_library.concepts]}
+ )
+ for emitted in emit_python_structures(qualified_library):
+ generated_target = workdir / emitted.filename
+ if generated_target.exists():
+ log.verbose(f"Structures file '{emitted.filename}' was shipped by the method; keeping the shipped copy over the generated one.")
+ continue
+ generated_target.parent.mkdir(parents=True, exist_ok=True)
+ generated_target.write_text(emitted.content, encoding="utf-8")
+
# Put the bundle's directories on sys.path so a PipeFunc (or structure class) can import a sibling
# file from the same bundle. This is only reached in-box (transported path): the crate is
# re-materialized into a throwaway workdir that is NOT on sys.path, and each module is imported by
diff --git a/pipelex/system/registries/func_registry_utils.py b/pipelex/system/registries/func_registry_utils.py
index 3de481ed8..b6d7374ef 100644
--- a/pipelex/system/registries/func_registry_utils.py
+++ b/pipelex/system/registries/func_registry_utils.py
@@ -203,10 +203,13 @@ class FuncRegistryUtils:
# Expected: file validation issues (directories with .py extension, etc.)
# log.verbose(f"Skipping file {file_path}: {e}")
pass
- except ImportError:
- # Common: missing dependencies, circular imports, relative imports
- # log.verbose(f"Could not import {file_path}: {e}")
- pass
+ except ImportError as exc:
+ # A module that fails to import (missing sibling module, circular import, bad relative
+ # import) cannot contribute its @pipe_func functions — they never register. Surfacing this
+ # is essential: the only downstream symptom is an opaque "Function '<name>' not found in
+ # registry" raised much later by the PipeFunc validator, with the real ImportError (the
+ # actual cause) otherwise swallowed here and invisible.
+ log.warning(f"Could not import '{file_path}' while registering PipeFuncs; its functions are unavailable: {exc}")
except SyntaxError as exc:
# Potentially problematic: invalid Python syntax may indicate broken code
log.warning(f"Syntax error in {file_path}: {exc}")
diff --git a/tests/unit/pipelex/pipe_operators/pipe_func/test_direct_executor_workdir.py b/tests/unit/pipelex/pipe_operators/pipe_func/test_direct_executor_workdir.py
index 48399fd74..1daf28041 100644
--- a/tests/unit/pipelex/pipe_operators/pipe_func/test_direct_executor_workdir.py
+++ b/tests/unit/pipelex/pipe_operators/pipe_func/test_direct_executor_workdir.py
@@ -1,4 +1,5 @@
import sys
+import tempfile
from pathlib import Path
import pytest
@@ -6,7 +7,9 @@ from pydantic import ValidationError
from pytest_mock import MockerFixture
from pipelex.base_exceptions import PipelexError
-from pipelex.hub import get_library_manager, scoped_current_library
+from pipelex.config import get_config
+from pipelex.core.memory.working_memory_factory import WorkingMemoryFactory
+from pipelex.hub import get_library_manager, scoped_current_library, set_current_library
from pipelex.libraries.library_crate import LibraryCrate
from pipelex.pipe_operators.func import direct_pipe_func_executor
from pipelex.pipe_operators.func.direct_pipe_func_executor import DirectPipeFuncExecutor
@@ -15,6 +18,37 @@ from pipelex.pipe_run.pipe_run_mode import PipeRunMode
from pipelex.pipe_run.pipe_run_params import PipeRunParams
from pipelex.pipeline.job_metadata import JobMetadata
+# A method whose ONLY custom source is the PipeFunc — its output structure (Greeting) is a concept
+# declared in the .mthds, NOT a shipped .py. The box must regenerate `structures.py` from the crate's
+# concepts so the func's `from structures import Greeting` resolves and the func registers.
+_GREET_MTHDS = """\
+domain = "greet_demo"
+description = "Greeting demo for structure regeneration"
+
+[concept.Greeting]
+description = "A greeting"
+[concept.Greeting.structure]
+text = { type = "text", description = "the greeting text", required = true }
+
+[pipe.greet]
+type = "PipeFunc"
+description = "Build a greeting via customer code"
+output = "Greeting"
+function_name = "greet_it"
+"""
+
+_GREET_FUNC = """\
+from pipelex.core.memory.working_memory import WorkingMemory
+from pipelex.system.registries.func_registry import pipe_func
+
+from structures import greet_demo__Greeting
+
+
+@pipe_func(name="greet_it")
+async def greet_it(working_memory: WorkingMemory) -> greet_demo__Greeting:
+ return greet_demo__Greeting(text="hello from generated structures")
+"""
+
class TestDirectExecutorWorkdir:
@pytest.mark.asyncio
@@ -67,3 +101,53 @@ class TestDirectExecutorWorkdir:
stale_entries = [entry for entry in sys.path if Path(entry).is_relative_to(workdir)]
assert stale_entries == []
+
+ @pytest.mark.asyncio
+ async def test_transported_run_generates_concept_structures(self):
+ """A PipeFunc that imports its output concept structure runs even though the method ships no
+ structure .py — the box regenerates `structures.py` from the crate's concepts.
+
+ Regression for the atlas custom-PipeFunc failure: the method's `structures/*.py` never traveled,
+ so `from structures import <Concept>` raised ModuleNotFoundError in the box and the func never
+ registered ("Function not found in registry"). Generating the structures from the .mthds (the
+ single source of truth) makes the import resolve without shipping — or drifting — a copy.
+ """
+ # Build a real crate from the .mthds (hosted mode: the loader tolerates the yet-unregistered
+ # function), then attach ONLY the PipeFunc source — deliberately NO structures/ source.
+ pipe_func_config = get_config().pipelex.pipe_func_config
+ previous_mode = pipe_func_config.execution_mode
+ pipe_func_config.execution_mode = "local_sandbox"
+ library_manager = get_library_manager()
+ try:
+ with tempfile.TemporaryDirectory() as mthds_dir:
+ (Path(mthds_dir) / "greet.mthds").write_text(_GREET_MTHDS, encoding="utf-8")
+ build_library_id, _ = library_manager.open_library()
+ set_current_library(library_id=build_library_id)
+ try:
+ library_manager.load_libraries(library_id=build_library_id, library_dirs=[Path(mthds_dir)])
+ crate = library_manager.get_crate(library_id=build_library_id)
+ assert crate is not None
+ finally:
+ library_manager.teardown(library_id=build_library_id)
+ finally:
+ pipe_func_config.execution_mode = previous_mode
+
+ crate_with_func = crate.model_copy(update={"python_sources": {"funcs.py": _GREET_FUNC}})
+ request = PipeFuncExecutionRequest(
+ crate=crate_with_func,
+ working_memory_raw=WorkingMemoryFactory.make_empty().dump_for_transport(),
+ pipe_code="greet",
+ function_name="greet_it",
+ job_metadata=JobMetadata(user_id="user", pipeline_run_id="run"),
+ pipe_run_params=PipeRunParams(run_mode=PipeRunMode.DRY, pipe_stack_limit=10),
+ )
+
+ with scoped_current_library(library_id=direct_pipe_func_executor._TRANSPORTED_LIBRARY_ID): # noqa: SLF001 # pyright: ignore[reportPrivateUsage]
+ try:
+ # No raise = the func registered = `from structures import Greeting` resolved against the
+ # generated structures.py. Before the fix this raised "Function 'greet_it' not found in registry".
+ response = await DirectPipeFuncExecutor().run_pipe_func_transported(request=request)
+ finally:
+ get_library_manager().teardown(library_id=direct_pipe_func_executor._TRANSPORTED_LIBRARY_ID) # noqa: SLF001 # pyright: ignore[reportPrivateUsage]
+
+ assert response.function_qualname == "greet_it"