Build AI methods with your coding agent, then run them anywhere: as an MCP for chatbots, as a webapp for people, or via API for your software.
Generate concept structures in the sandbox for custom PipeFuncs
thomashebrardaa1215619454
diff --git a/pipelex/pipe_operators/func/direct_pipe_func_executor.py b/pipelex/pipe_operators/func/direct_pipe_func_executor.pyindex c581baeaa..2ed9c1ced 100644--- a/pipelex/pipe_operators/func/direct_pipe_func_executor.py+++ b/pipelex/pipe_operators/func/direct_pipe_func_executor.py@@ -9,6 +9,8 @@ from typing import castfrom typing_extensions import overridefrom pipelex import log+from pipelex.codegen.emitters.python_structures import emit_python_structures+from pipelex.codegen.resolved_concepts import resolve_concepts_from_cratefrom pipelex.core.memory.working_memory import WorkingMemoryfrom pipelex.core.stuffs.list_content import ListContentfrom pipelex.core.stuffs.structured_content import StructuredContent@@ -109,6 +111,30 @@ class DirectPipeFuncExecutor(PipeFuncExecutorProtocol):target.parent.mkdir(parents=True, exist_ok=True)target.write_text(source, encoding="utf-8")+ # Generate the method's concept structure classes from the crate — the .mthds is the single+ # source of truth for them (`resolve_concepts_from_crate` reads the crate's ConceptBlueprints,+ # `emit_python_structures` projects a `structures.py`). So a PipeFunc can+ # `from structures import <domain>__<Concept>` without the method having to STORE or transport a+ # hand-written copy that could drift from the .mthds. A file the method actually shipped at the+ # same path wins (already materialized above) — never clobber customer source.+ #+ # Force domain-qualified class names (`atlas_devis__PricedQuote`): the runtime ALWAYS qualifies a+ # concept's structure class (`concept.structure_class_name`), and that is the exact name a+ # PipeFunc's return-type validation checks against — so the generated classes must match the+ # runtime spelling, not the emitter's default collision-only qualification (which would emit a+ # bare `PricedQuote` the validator rejects).+ resolved_library = resolve_concepts_from_crate(request.crate)+ qualified_library = resolved_library.model_copy(+ update={"concepts": [concept.model_copy(update={"needs_qualification": True}) for concept in resolved_library.concepts]}+ )+ for emitted in emit_python_structures(qualified_library):+ generated_target = workdir / emitted.filename+ if generated_target.exists():+ log.verbose(f"Structures file '{emitted.filename}' was shipped by the method; keeping the shipped copy over the generated one.")+ continue+ generated_target.parent.mkdir(parents=True, exist_ok=True)+ generated_target.write_text(emitted.content, encoding="utf-8")+# Put the bundle's directories on sys.path so a PipeFunc (or structure class) can import a sibling# file from the same bundle. This is only reached in-box (transported path): the crate is# re-materialized into a throwaway workdir that is NOT on sys.path, and each module is imported bydiff --git a/pipelex/system/registries/func_registry_utils.py b/pipelex/system/registries/func_registry_utils.pyindex 3de481ed8..b6d7374ef 100644--- a/pipelex/system/registries/func_registry_utils.py+++ b/pipelex/system/registries/func_registry_utils.py@@ -203,10 +203,13 @@ class FuncRegistryUtils:# Expected: file validation issues (directories with .py extension, etc.)# log.verbose(f"Skipping file {file_path}: {e}")pass- except ImportError:- # Common: missing dependencies, circular imports, relative imports- # log.verbose(f"Could not import {file_path}: {e}")- pass+ except ImportError as exc:+ # A module that fails to import (missing sibling module, circular import, bad relative+ # import) cannot contribute its @pipe_func functions — they never register. Surfacing this+ # is essential: the only downstream symptom is an opaque "Function '<name>' not found in+ # registry" raised much later by the PipeFunc validator, with the real ImportError (the+ # actual cause) otherwise swallowed here and invisible.+ log.warning(f"Could not import '{file_path}' while registering PipeFuncs; its functions are unavailable: {exc}")except SyntaxError as exc:# Potentially problematic: invalid Python syntax may indicate broken codelog.warning(f"Syntax error in {file_path}: {exc}")diff --git a/tests/unit/pipelex/pipe_operators/pipe_func/test_direct_executor_workdir.py b/tests/unit/pipelex/pipe_operators/pipe_func/test_direct_executor_workdir.pyindex 48399fd74..1daf28041 100644--- a/tests/unit/pipelex/pipe_operators/pipe_func/test_direct_executor_workdir.py+++ b/tests/unit/pipelex/pipe_operators/pipe_func/test_direct_executor_workdir.py@@ -1,4 +1,5 @@import sys+import tempfilefrom pathlib import Pathimport pytest@@ -6,7 +7,9 @@ from pydantic import ValidationErrorfrom pytest_mock import MockerFixturefrom pipelex.base_exceptions import PipelexError-from pipelex.hub import get_library_manager, scoped_current_library+from pipelex.config import get_config+from pipelex.core.memory.working_memory_factory import WorkingMemoryFactory+from pipelex.hub import get_library_manager, scoped_current_library, set_current_libraryfrom pipelex.libraries.library_crate import LibraryCratefrom pipelex.pipe_operators.func import direct_pipe_func_executorfrom pipelex.pipe_operators.func.direct_pipe_func_executor import DirectPipeFuncExecutor@@ -15,6 +18,37 @@ from pipelex.pipe_run.pipe_run_mode import PipeRunModefrom pipelex.pipe_run.pipe_run_params import PipeRunParamsfrom pipelex.pipeline.job_metadata import JobMetadata+# A method whose ONLY custom source is the PipeFunc — its output structure (Greeting) is a concept+# declared in the .mthds, NOT a shipped .py. The box must regenerate `structures.py` from the crate's+# concepts so the func's `from structures import Greeting` resolves and the func registers.+_GREET_MTHDS = """\+domain = "greet_demo"+description = "Greeting demo for structure regeneration"++[concept.Greeting]+description = "A greeting"+[concept.Greeting.structure]+text = { type = "text", description = "the greeting text", required = true }++[pipe.greet]+type = "PipeFunc"+description = "Build a greeting via customer code"+output = "Greeting"+function_name = "greet_it"+"""++_GREET_FUNC = """\+from pipelex.core.memory.working_memory import WorkingMemory+from pipelex.system.registries.func_registry import pipe_func++from structures import greet_demo__Greeting+++@pipe_func(name="greet_it")+async def greet_it(working_memory: WorkingMemory) -> greet_demo__Greeting:+ return greet_demo__Greeting(text="hello from generated structures")+"""+class TestDirectExecutorWorkdir:@pytest.mark.asyncio@@ -67,3 +101,53 @@ class TestDirectExecutorWorkdir:stale_entries = [entry for entry in sys.path if Path(entry).is_relative_to(workdir)]assert stale_entries == []++ @pytest.mark.asyncio+ async def test_transported_run_generates_concept_structures(self):+ """A PipeFunc that imports its output concept structure runs even though the method ships no+ structure .py — the box regenerates `structures.py` from the crate's concepts.++ Regression for the atlas custom-PipeFunc failure: the method's `structures/*.py` never traveled,+ so `from structures import <Concept>` raised ModuleNotFoundError in the box and the func never+ registered ("Function not found in registry"). Generating the structures from the .mthds (the+ single source of truth) makes the import resolve without shipping — or drifting — a copy.+ """+ # Build a real crate from the .mthds (hosted mode: the loader tolerates the yet-unregistered+ # function), then attach ONLY the PipeFunc source — deliberately NO structures/ source.+ pipe_func_config = get_config().pipelex.pipe_func_config+ previous_mode = pipe_func_config.execution_mode+ pipe_func_config.execution_mode = "local_sandbox"+ library_manager = get_library_manager()+ try:+ with tempfile.TemporaryDirectory() as mthds_dir:+ (Path(mthds_dir) / "greet.mthds").write_text(_GREET_MTHDS, encoding="utf-8")+ build_library_id, _ = library_manager.open_library()+ set_current_library(library_id=build_library_id)+ try:+ library_manager.load_libraries(library_id=build_library_id, library_dirs=[Path(mthds_dir)])+ crate = library_manager.get_crate(library_id=build_library_id)+ assert crate is not None+ finally:+ library_manager.teardown(library_id=build_library_id)+ finally:+ pipe_func_config.execution_mode = previous_mode++ crate_with_func = crate.model_copy(update={"python_sources": {"funcs.py": _GREET_FUNC}})+ request = PipeFuncExecutionRequest(+ crate=crate_with_func,+ working_memory_raw=WorkingMemoryFactory.make_empty().dump_for_transport(),+ pipe_code="greet",+ function_name="greet_it",+ job_metadata=JobMetadata(user_id="user", pipeline_run_id="run"),+ pipe_run_params=PipeRunParams(run_mode=PipeRunMode.DRY, pipe_stack_limit=10),+ )++ with scoped_current_library(library_id=direct_pipe_func_executor._TRANSPORTED_LIBRARY_ID): # noqa: SLF001 # pyright: ignore[reportPrivateUsage]+ try:+ # No raise = the func registered = `from structures import Greeting` resolved against the+ # generated structures.py. Before the fix this raised "Function 'greet_it' not found in registry".+ response = await DirectPipeFuncExecutor().run_pipe_func_transported(request=request)+ finally:+ get_library_manager().teardown(library_id=direct_pipe_func_executor._TRANSPORTED_LIBRARY_ID) # noqa: SLF001 # pyright: ignore[reportPrivateUsage]++ assert response.function_qualname == "greet_it"