Skip to content

Build AI methods with your coding agent, then run them anywhere: as an MCP for chatbots, as a webapp for people, or via API for your software.

feature/Pipefunc-import-path · L-261015-95d262 (#1434)

Louis Choqueld4a0b2005992
diff --git a/.drift/acks/cli-docs.toml b/.drift/acks/cli-docs.toml
index 2f63444cf..81d69729f 100644
--- a/.drift/acks/cli-docs.toml
+++ b/.drift/acks/cli-docs.toml
@@ -1,14 +1,14 @@
contract = "cli-docs"
-digest = "sha256:36a85341cb88dc54c1e7898ed2acca893097e68c77386b0200b9dce7ca74a397"
-reviewed_by = "Claude Opus 5.5"
-reviewed_at = "2026-10-16T00:08:00Z"
-rationale = "real-catch: docs/tools/cli/run.md now says a hosted folder run sends its main pipe as domain.pipe_code and that output folders are labelled by the bare pipe code on hosted runs too; docs/tools/cli/agent-cli.md and pipelex/cli/agent_cli/CLAUDE.md say pipelex-agent run writes into results/<pipe_code>_output_NN/ with a .gitignore beside its bundle, else under mthds-wip/, with the override cases."
+digest = "sha256:2eed486cdc33f2c309e73dcc98319838bbfe08a441579f76ce52e069a8b51a85"
+reviewed_by = "Louis Choquel"
+reviewed_at = "2026-10-16T03:27:39Z"
+rationale = "real-catch: fix function-names now reports unresolved_markers (a marker kept because a step naming it is unresolved, or whose name is not a string literal) and exits 1 for them; docs/tools/cli/fix.md and agent-cli.md were updated with the change, and pipelex/cli/agent_cli/CLAUDE.md, which still listed the old envelope without unresolved_markers, now names it and the exit rule."
[trigger_files]
"pipelex/cli/__init__.py" = "blob:8b137891791fe96927ad78e64b0aad7bded08bdc"
-"pipelex/cli/_cli.py" = "blob:9eca8e06a004933e5cfa2ba871e4fb1ff6c6bf0c"
+"pipelex/cli/_cli.py" = "blob:86fa74a5c45f33915548dc5ad4ec8ddbaf2b75d6"
"pipelex/cli/agent_cli/__init__.py" = "blob:e69de29bb2d1d6434b8b29ae775ad8c2e48c5391"
-"pipelex/cli/agent_cli/_agent_cli.py" = "blob:5ab17e334cdfd4b5f8e4afdc932f3fca3747ed59"
+"pipelex/cli/agent_cli/_agent_cli.py" = "blob:8146c939ab6558e90f6c62f51d1d50a83c559bf9"
"pipelex/cli/agent_cli/commands/__init__.py" = "blob:e69de29bb2d1d6434b8b29ae775ad8c2e48c5391"
"pipelex/cli/agent_cli/commands/agent_cli_factory.py" = "blob:050ca1344abc4c1ca22d431923e1577681552fe6"
"pipelex/cli/agent_cli/commands/agent_output.py" = "blob:10588465f86eb7ce7d0bed3985304612b154d013"
@@ -20,8 +20,9 @@ rationale = "real-catch: docs/tools/cli/run.md now says a hosted folder run send
"pipelex/cli/agent_cli/commands/codegen/types_cmd.py" = "blob:5c78bd70803a8f32616ccba9c57538ed49ab2d60"
"pipelex/cli/agent_cli/commands/doctor_cmd.py" = "blob:948be63cf4fab165846c83b4bd2517bd4a46be5e"
"pipelex/cli/agent_cli/commands/fix/__init__.py" = "blob:65662c86f17f610ad55ecfc3e27a022b9aa26e8b"
-"pipelex/cli/agent_cli/commands/fix/app.py" = "blob:e6190d9bde6a05755c04bd21806f699461038ac0"
+"pipelex/cli/agent_cli/commands/fix/app.py" = "blob:b11db5417fea85fbb8ea4bbf900663d9a1acc0ab"
"pipelex/cli/agent_cli/commands/fix/bundle_cmd.py" = "blob:cfd9e645986636c44c02a6a04a0fc67ed60d444b"
+"pipelex/cli/agent_cli/commands/fix/function_names_cmd.py" = "blob:4b06e5d112fe9b9dfd3e9a5fdda5ba1bc99cff14"
"pipelex/cli/agent_cli/commands/fmt_cmd.py" = "blob:97d6fb6c328e0b718b1293bd6b73cba03edcb7dc"
"pipelex/cli/agent_cli/commands/init_cmd.py" = "blob:513b1160d8e3815b34b905bbabccd59d6c379842"
"pipelex/cli/agent_cli/commands/inputs/__init__.py" = "blob:e69de29bb2d1d6434b8b29ae775ad8c2e48c5391"
@@ -78,8 +79,9 @@ rationale = "real-catch: docs/tools/cli/run.md now says a hosted folder run send
"pipelex/cli/commands/fix/__init__.py" = "blob:e69de29bb2d1d6434b8b29ae775ad8c2e48c5391"
"pipelex/cli/commands/fix/_diff_sandbox.py" = "blob:d39c0df92b95567f74881deb01f8c1b3119fc8ef"
"pipelex/cli/commands/fix/_fix_core.py" = "blob:aa698305e6b60dfb7caf133698dc1d5c1f5f68f1"
-"pipelex/cli/commands/fix/app.py" = "blob:6f019930e8e3e0043ee46d87493ba4602eb33a76"
+"pipelex/cli/commands/fix/app.py" = "blob:23b28875f874e2687e183c759e45459377256711"
"pipelex/cli/commands/fix/bundle_cmd.py" = "blob:427364cc453957dfe0cec879f019050329c13b3c"
+"pipelex/cli/commands/fix/function_names_cmd.py" = "blob:b645a3d7687f01f18474446d29e23df4f1acc751"
"pipelex/cli/commands/graph_cmd.py" = "blob:d215ad2b66c4421364c90f2d8f875b6312dadfe4"
"pipelex/cli/commands/init/__init__.py" = "blob:e69de29bb2d1d6434b8b29ae775ad8c2e48c5391"
"pipelex/cli/commands/init/backends.py" = "blob:0cc3543c78f832149ebbbf3cc8beafee619e4f63"
diff --git a/.drift/acks/config-docs.toml b/.drift/acks/config-docs.toml
index 286bef0e7..3f597414c 100644
--- a/.drift/acks/config-docs.toml
+++ b/.drift/acks/config-docs.toml
@@ -1,8 +1,8 @@
contract = "config-docs"
-digest = "sha256:1de0bee628b09fb93a93f501aaaca3d5871fa21372874986b66cdc64d9da5a1a"
-reviewed_by = "Claude Opus 5.5"
-reviewed_at = "2026-10-15T23:13:43Z"
-rationale = "real-catch: InterpreterConfig gains method_tree_limits ([interpreter.method_tree_limits], max_files and max_total_bytes required with their defaults in pipelex/pipelex.toml and the kit and project copies, max_file_bytes optional). docs/configuration/index.md now names the tree limits among the interpreter sections and library-config.md documents which files a library directory loads, the limits, and that [interpreter.scan] excluded_dirs now only scopes the Python scan; the section is documented in full in docs/building-methods/method-trees.md. The change is additive with its bounds in annotations, so the regenerated head goldens account for it and no ledger entry is needed."
+digest = "sha256:c97e5b71ae9df6f40b8f3f593ff9ba70748634577f0978d0b9b25b4ef038da3c"
+reviewed_by = "Louis Choquel"
+reviewed_at = "2026-10-16T02:18:45Z"
+rationale = "real-catch: pipe_func_config.py changes only the comment on execution_mode, which now says a non-direct mode captures the method's files onto the crate, proves a PipeFunc's function form from the captured source and imports nothing here; docs/configuration/config-technical/library-config.md was stale on the retired func_registry and now documents PipeFunc functions as import paths from the method tree's root, one tree's Python per process and @pipe_func() without arguments. No config field moved, so the migration ledgers need no entry."
[trigger_files]
"pipelex/cogt/config_cogt.py" = "blob:293ef22ae123266e6a1886d4ffb07f455e5be9e3"
@@ -18,7 +18,7 @@ rationale = "real-catch: InterpreterConfig gains method_tree_limits ([interprete
"pipelex/system/configuration/config_root.py" = "blob:c7d43851270912d17413cfc6a511247d3c7cc160"
"pipelex/system/configuration/config_surface.py" = "blob:49424a8103a5611f5b2885135dd699273bde57c8"
"pipelex/system/configuration/configs.py" = "blob:74175f5e5aac2d50af6f319d3706db0d90dd5eab"
-"pipelex/system/configuration/pipe_func_config.py" = "blob:dc8890e914fbcdb611147e42eaa0cd87b32288a5"
+"pipelex/system/configuration/pipe_func_config.py" = "blob:8e82c67de60238d0a02370b3ac53847f8b17b597"
"pipelex/system/telemetry/telemetry_config.py" = "blob:068a785d97a2d52ee43e82e723cd944b8b6601bc"
"pipelex/tools/aws/aws_config.py" = "blob:0a7c9dbf276e61355b1fa79b9db8caf252450a39"
"pipelex/tools/log/log_config.py" = "blob:0a893b9e62737b7954a924eaed577ba5c8ef9eaf"
diff --git a/.drift/acks/hub-layering-convention.toml b/.drift/acks/hub-layering-convention.toml
index 47f7c9c51..c6f8a9ce2 100644
--- a/.drift/acks/hub-layering-convention.toml
+++ b/.drift/acks/hub-layering-convention.toml
@@ -1,10 +1,10 @@
contract = "hub-layering-convention"
-digest = "sha256:72fac07cc613e7b8eefe3ffa16d368fcfd1c883e9ca67fa42521189cd29509f0"
+digest = "sha256:b43388e7c97d4a39aadb375cde0f3431d0e9bce1c15adab9d9d4c61e958a403a"
reviewed_by = "Louis Choquel"
-reviewed_at = "2026-10-16T01:21:40Z"
-rationale = "clean-pass: interpreter_hub.py gains the PipeFuncPackageOfferRegistry setter and getters and the module accessor get_pipe_func_package_offer_registry; this change already lists it in the pipe func row of the partition table and names the registry's install beside the PipeFuncExecutorRegistry in Pipelex.setup; no import crosses the boundary."
+reviewed_at = "2026-10-16T02:32:28Z"
+rationale = "real-catch: after merging dev, runtime_hub.py drops FuncRegistry, set_func_registry and get_func_registry with the retired registry, and interpreter_hub.py carries dev's PipeFuncPackageOfferRegistry accessors; docs/contribute/hub-layering.md no longer names get_func_registry among the runtime hub's getters and lists get_pipe_func_package_offer_registry in the pipe func row. No import crosses the boundary."
[trigger_files]
"pipelex/cli/dev_cli/commands/hub_layering_guard.py" = "blob:fda2237457fe0b6830ff2ae8f720698735bec4c3"
"pipelex/interpreter_hub.py" = "blob:4efc29ca60c42ca0fe0533c402d7bb246b0bb2c2"
-"pipelex/runtime_hub.py" = "blob:2af9f62d93b2005daa55a7c61e0390601e430c00"
+"pipelex/runtime_hub.py" = "blob:87f051f8ab31244ed34c9c5dfc499d5e309edcaf"
diff --git a/.drift/acks/pipelex-kernel-docs.toml b/.drift/acks/pipelex-kernel-docs.toml
index ce0360753..fc27965dd 100644
--- a/.drift/acks/pipelex-kernel-docs.toml
+++ b/.drift/acks/pipelex-kernel-docs.toml
@@ -1,8 +1,8 @@
contract = "pipelex-kernel-docs"
-digest = "sha256:3ac351addba0c0ef338f06edc502ccbfb51ab30a145a97a2a5c911ad874451d7"
+digest = "sha256:1061433dfa5c3faf0162436f54ac70663d3b0cf518d68d158cdf44c2aaf7b8d1"
reviewed_by = "Louis Choquel"
-reviewed_at = "2026-10-15T23:42:58Z"
-rationale = "clean-pass: the PipelexKernel.make docstring's parenthetical now names the scoped event log beside the runner fallback as stamping the metadata's pipeline_run_id; pipelex-kernel.md already says the same since this branch, so the doc needs no change."
+reviewed_at = "2026-10-16T02:18:50Z"
+rationale = "real-catch: func_ops.py replaces call_registered_function with call_function, which is handed the resolved function, and run_func takes function= instead of function_name=; docs/under-the-hood/pipelex-kernel.md now names call_function and says the kernel is handed the callable while the interpreter resolves a PipeFunc's import path from its method tree. The kernel boot contract test hands run_func the function."
[trigger_files]
"pipelex/kernel/__init__.py" = "blob:c4f1b0fe2ccc17db9110733f71b612f4ac1ba4ce"
@@ -11,7 +11,7 @@ rationale = "clean-pass: the PipelexKernel.make docstring's parenthetical now na
"pipelex/kernel/exceptions.py" = "blob:ec784b58d732cca19b9278c83afd009422792e81"
"pipelex/kernel/extract_ops.py" = "blob:19d556f0254017d7d52c30f4f1a46e2235289042"
"pipelex/kernel/extract_results.py" = "blob:79405a56760f3d4b483660d76441fff95f58c64d"
-"pipelex/kernel/func_ops.py" = "blob:97c68c977ddc260c9b36b7eda5f4c7964e956a27"
+"pipelex/kernel/func_ops.py" = "blob:4a1f8cf0df1a8e493fefeb9313c53dd469565b84"
"pipelex/kernel/func_results.py" = "blob:a7425a0ddc21571f23c53b92f5d7559ffdd08b71"
"pipelex/kernel/img_gen_ops.py" = "blob:b85360f6c118e93be53bf8d1a04504ccb7552469"
"pipelex/kernel/img_gen_prompt.py" = "blob:c653190d40e4323f763b9da511bd55a4a1646888"
@@ -28,4 +28,4 @@ rationale = "clean-pass: the PipelexKernel.make docstring's parenthetical now na
"pipelex/kernel/search_ops.py" = "blob:b7de7ff62c45dde7f7aa8a9f409aecf3b664cce5"
"pipelex/kernel/search_results.py" = "blob:101b70dc96cc5ee1721c9f6b1cb725a5ca78ff70"
"pipelex/kernel/templating_style_ops.py" = "blob:395473806d600dfc4f124c07f2afebb951fc086b"
-"tests/unit/pipelex/kernel/test_kernel_boot_contract.py" = "blob:2b60152e6fa07fb0a1eb242cf065d0106d47bc44"
+"tests/unit/pipelex/kernel/test_kernel_boot_contract.py" = "blob:88ec91a8b7cc9a0819756e3499a5bbb661fe1b01"
diff --git a/.test_durations b/.test_durations
index 9c99da74b..26b37e5ca 100644
--- a/.test_durations
+++ b/.test_durations
@@ -10312,9 +10312,6 @@
"tests/unit/pipelex/libraries/test_library_crate_accumulation.py::TestLibraryCrateAccumulation::test_load_from_crate_idempotent_on_same_fingerprint": 0.0458,
"tests/unit/pipelex/libraries/test_library_crate_accumulation.py::TestLibraryCrateAccumulation::test_multiple_load_from_blueprints_all_included_in_crate": 0.0082,
"tests/unit/pipelex/libraries/test_library_crate_accumulation.py::TestLibraryCrateAccumulation::test_teardown_clears_blueprints_for_library_id": 0.0019,
- "tests/unit/pipelex/libraries/test_library_crate_python_sources.py::TestLibraryCratePythonSources::test_fingerprint_excludes_python_sources": 0.0985,
- "tests/unit/pipelex/libraries/test_library_crate_python_sources.py::TestLibraryCratePythonSources::test_python_sources_default_empty": 0.7176,
- "tests/unit/pipelex/libraries/test_library_crate_python_sources.py::TestLibraryCratePythonSources::test_python_sources_round_trip": 0.0006,
"tests/unit/pipelex/libraries/test_library_isolation.py::TestLibraryIsolation::test_concept_isolation_no_native_key_in_main": 0.0004,
"tests/unit/pipelex/libraries/test_library_isolation.py::TestLibraryIsolation::test_concept_name_collision_two_deps": 0.0005,
"tests/unit/pipelex/libraries/test_library_isolation.py::TestLibraryIsolation::test_cross_package_lookup_via_alias": 0.0007,
@@ -16822,42 +16819,6 @@
"tests/unit/pipelex/tools/test_environment.py::TestEnvironment::test_get_required_env_success": 0.7847,
"tests/unit/pipelex/tools/test_environment.py::TestEnvironment::test_set_env_overwrites_existing": 0.0006,
"tests/unit/pipelex/tools/test_environment.py::TestEnvironment::test_set_env_success": 0.0008,
- "tests/unit/pipelex/tools/test_func_registry.py::TestFuncRegistry::test_function_eligibility_and_registration[another_function-another_function-False]": 0.0011,
- "tests/unit/pipelex/tools/test_func_registry.py::TestFuncRegistry::test_function_eligibility_and_registration[no_params-no_params-False]": 0.001,
- "tests/unit/pipelex/tools/test_func_registry.py::TestFuncRegistry::test_function_eligibility_and_registration[no_type_hints-no_type_hints-False]": 0.0009,
- "tests/unit/pipelex/tools/test_func_registry.py::TestFuncRegistry::test_function_eligibility_and_registration[sample_function-sample_function-False]": 0.0007,
- "tests/unit/pipelex/tools/test_func_registry.py::TestFuncRegistry::test_function_eligibility_and_registration[too_many_params-too_many_params-False]": 0.0012,
- "tests/unit/pipelex/tools/test_func_registry.py::TestFuncRegistry::test_function_eligibility_and_registration[valid_async_function-valid_async_function-True]": 0.0007,
- "tests/unit/pipelex/tools/test_func_registry.py::TestFuncRegistry::test_function_eligibility_and_registration[valid_function-valid_function-True]": 0.8211,
- "tests/unit/pipelex/tools/test_func_registry.py::TestFuncRegistry::test_function_eligibility_and_registration[wrong_param_name-wrong_param_name-False]": 0.0007,
- "tests/unit/pipelex/tools/test_func_registry.py::TestFuncRegistry::test_function_eligibility_and_registration[wrong_param_type-wrong_param_type-False]": 0.0012,
- "tests/unit/pipelex/tools/test_func_registry.py::TestFuncRegistry::test_function_eligibility_and_registration[wrong_return_type-wrong_return_type-False]": 0.0006,
- "tests/unit/pipelex/tools/test_func_registry.py::TestFuncRegistry::test_get_required_function_not_found": 0.0011,
- "tests/unit/pipelex/tools/test_func_registry.py::TestFuncRegistry::test_get_required_function_with_signature": 0.0008,
- "tests/unit/pipelex/tools/test_func_registry.py::TestFuncRegistry::test_get_required_function_with_signature_not_callable": 0.0008,
- "tests/unit/pipelex/tools/test_func_registry.py::TestFuncRegistry::test_get_required_function_with_signature_not_found": 0.0004,
- "tests/unit/pipelex/tools/test_func_registry.py::TestFuncRegistry::test_ineligible_function_does_not_collide": 0.0009,
- "tests/unit/pipelex/tools/test_func_registry.py::TestFuncRegistry::test_register_colliding_function_raises_and_names_both_sources": 0.0004,
- "tests/unit/pipelex/tools/test_func_registry.py::TestFuncRegistry::test_register_existing_function_with_warning": 0.0028,
- "tests/unit/pipelex/tools/test_func_registry.py::TestFuncRegistry::test_register_function_with_custom_name": 0.0004,
- "tests/unit/pipelex/tools/test_func_registry.py::TestFuncRegistry::test_register_functions_dict_with_ineligible_functions": 0.0006,
- "tests/unit/pipelex/tools/test_func_registry.py::TestFuncRegistry::test_register_functions_empty_list": 0.0016,
- "tests/unit/pipelex/tools/test_func_registry.py::TestFuncRegistry::test_register_functions_list_with_ineligible_functions": 0.0009,
- "tests/unit/pipelex/tools/test_func_registry.py::TestFuncRegistry::test_register_same_function_twice_is_idempotent": 0.0011,
- "tests/unit/pipelex/tools/test_func_registry.py::TestFuncRegistry::test_set_logger": 0.0978,
- "tests/unit/pipelex/tools/test_func_registry.py::TestFuncRegistry::test_teardown": 0.0005,
- "tests/unit/pipelex/tools/test_func_registry.py::TestFuncRegistry::test_unregister_function": 0.0008,
- "tests/unit/pipelex/tools/test_func_registry.py::TestFuncRegistry::test_unregister_function_by_name": 0.0015,
- "tests/unit/pipelex/tools/test_func_registry.py::TestFuncRegistry::test_unregister_function_by_name_not_found": 0.0006,
- "tests/unit/pipelex/tools/test_func_registry.py::TestFuncRegistry::test_unregister_function_not_found": 0.0008,
- "tests/unit/pipelex/tools/test_func_registry_utils.py::TestFuncRegistryUtils::test_eligibility_check_directly": 0.0007,
- "tests/unit/pipelex/tools/test_func_registry_utils.py::TestFuncRegistryUtils::test_function_registration_eligibility[invalid_functions-\\n# Wrong parameter name - should be rejected\\ndef invalid_function_wrong_param_name(other_param: WorkingMemory) -> StructuredContent:\\n '''This should not be registered - wrong parameter name.'''\\n pass\\n\\n# Wrong parameter type - should be rejected\\ndef invalid_function_wrong_param_type(working_memory: str) -> StructuredContent:\\n '''This should not be registered - wrong parameter type.'''\\n pass\\n\\n# Wrong return type - should be rejected\\ndef invalid_function_wrong_return_type(working_memory: WorkingMemory) -> str:\\n '''This should not be registered - wrong return type.'''\\n return \"test\"\\n\\n# No return type annotation - should be rejected\\nasync def invalid_function_no_return_type(working_memory: WorkingMemory):\\n '''This should not be registered - no return type annotation.'''\\n return TextContent(text=\"no return type\")\\n\\n# Wrong return type (not StuffContent) - should be rejected\\nasync def invalid_function_returns_str(working_memory: WorkingMemory) -> str:\\n '''This should not be registered - returns str instead of StuffContent.'''\\n return \"wrong type\"\\n\\n# Too many parameters - should be rejected\\ndef invalid_function_too_many_params(working_memory: WorkingMemory, extra_param: str) -> StructuredContent:\\n '''This should not be registered - too many parameters.'''\\n pass\\n\\n# No parameters - should be rejected\\ndef invalid_function_no_params() -> StructuredContent:\\n '''This should not be registered - no parameters.'''\\n pass\\n\\n# Missing type hints - should be rejected\\ndef invalid_function_no_type_hints(working_memory):\\n '''This should not be registered - missing type hints.'''\\n pass\\n-expected_registered2-expected_not_registered2]": 0.008,
- "tests/unit/pipelex/tools/test_func_registry_utils.py::TestFuncRegistryUtils::test_function_registration_eligibility[valid_async_function-\\nfrom pipelex.system.registries.func_registry import pipe_func\\n\\n@pipe_func()\\nasync def read_file_content(working_memory: WorkingMemory) -> ListContent[CodebaseFileContent]:\\n '''Read the content of related codebase files.'''\\n\\n file_paths_list = working_memory.get_stuff_as_list(\"related_file_paths\", item_type=FilePath)\\n\\n codebase_files: List[CodebaseFileContent] = []\\n for file_path in file_paths_list.items:\\n try:\\n with open(file_path.path, \"r\", encoding=\"utf-8\") as file:\\n content = file.read()\\n codebase_files.append(CodebaseFileContent(file_path=file_path.path, file_content=content))\\n except Exception as e:\\n codebase_files.append(\\n CodebaseFileContent(file_path=file_path.path, file_content=f\"# File not found or unreadable: {file_path.path}\\\\n# Error: {str(e)}\")\\n )\\n\\n return ListContent[CodebaseFileContent](items=codebase_files)\\n-expected_registered0-expected_not_registered0]": 0.88,
- "tests/unit/pipelex/tools/test_func_registry_utils.py::TestFuncRegistryUtils::test_function_registration_eligibility[valid_sync_function-\\nfrom pipelex.system.registries.func_registry import pipe_func\\n\\n# Sync function - should be accepted\\n@pipe_func()\\ndef sync_function(working_memory: WorkingMemory) -> StructuredContent:\\n '''This should be registered - sync functions are now eligible.'''\\n pass\\n-expected_registered1-expected_not_registered1]": 0.0084,
- "tests/unit/pipelex/tools/test_func_registry_utils.py::TestFuncRegistryUtils::test_recursive_folder_search": 0.0048,
- "tests/unit/pipelex/tools/test_func_registry_utils.py::TestFuncRegistryUtils::test_register_function_checks_eligibility": 0.1155,
- "tests/unit/pipelex/tools/test_func_registry_utils.py::TestFuncRegistryUtils::test_rescanning_the_same_folder_is_idempotent": 0.0021,
- "tests/unit/pipelex/tools/test_func_registry_utils.py::TestFuncRegistryUtils::test_same_function_name_in_two_folders_raises": 0.004,
"tests/unit/pipelex/tools/test_image_registry.py::TestImageRegistry::test_deduplication_scenarios[all_duplicates-urls3-1]": 0.0672,
"tests/unit/pipelex/tools/test_image_registry.py::TestImageRegistry::test_deduplication_scenarios[single-urls0-1]": 0.0005,
"tests/unit/pipelex/tools/test_image_registry.py::TestImageRegistry::test_deduplication_scenarios[three_with_duplicate-urls2-2]": 0.0004,
@@ -18726,8 +18687,6 @@
"tests/unit/pipelex/tools/test_log_summary_event.py::TestLogSummaryEvent::test_the_outcome_says_whether_the_work_returned_failed_or_was_cancelled[raised]": 0.0011,
"tests/unit/pipelex/tools/test_log_summary_event.py::TestLogSummaryEvent::test_the_outcome_says_whether_the_work_returned_failed_or_was_cancelled[returned]": 0.1105,
"tests/unit/pipelex/tools/test_log_summary_event.py::TestLogSummaryEvent::test_the_outcome_says_whether_the_work_returned_failed_or_was_cancelled[the interpreter exiting]": 0.0028,
- "tests/unit/pipelex/tools/test_read_py_sources.py::TestReadPySources::test_captures_nested_sources_without_importing": 0.6583,
- "tests/unit/pipelex/tools/test_read_py_sources.py::TestReadPySources::test_empty_folder_returns_empty": 0.1142,
"tests/unit/pipelex/tools/test_template_content_rendering.py::TestTemplateContentRendering::test_format_json_renders_json": 0.0028,
"tests/unit/pipelex/tools/test_template_content_rendering.py::TestTemplateContentRendering::test_format_markdown_renders_markdown": 0.0966,
"tests/unit/pipelex/tools/test_template_content_rendering.py::TestTemplateContentRendering::test_multiple_artefacts_in_template": 0.0026,
diff --git a/CHANGELOG.md b/CHANGELOG.md
index e0ca5fae1..a98477543 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -5,10 +5,18 @@
### Added
- **`PluginRegistrar.add_pipe_func_package_offer`**: the plugin providing an out-of-process PipeFunc execution mode states, keyed by the same mode, the Python packages and the marker environment its sandbox offers (`PipeFuncPackageOffer`), which a method's `requirements.txt` is checked against; a mode stating none refuses every method declaring requirements with `PipeFuncPackageSetUnstatedError`, a configuration fault. `MethodRequirementsFileError`, `MethodRequirementsNotMetError`, `HostLibraryRequirementsError`, `PipeFuncPackageSetUnstatedError` and `DuplicatePipeFuncPackageOfferError` are new `error_type`s.
+- **`pipelex fix function-names` and `pipelex-agent fix function-names`**: rewrite every flat `PipeFunc` `function_name` of a method folder as the function's import path and drop the argument of every `@pipe_func(name=...)` marker, reading the files as syntax trees without importing them or booting Pipelex, and keeping every comment and alignment; a flat name no module of the method's tree registers, or that two register, is left as it is and reported with the markers registering it, which keep their argument, and so is a marker whose name is not a string literal, the agent command's `is_complete` being false and both commands exiting `1`, with `--dry-run` to preview.
+- **`pipe_func_function_refused` and the method-tree Python errors**: a `PipeFunc` whose function cannot be found or proven is refused at validation with the `pipe_func_function_refused` validation error type, located on the step's `function_name`, and the new `PipeFuncFunctionError`, `TreePythonError`, `TreePythonHeldError`, `TreeModuleShadowingError` and `FunctionNamesMigrationError` name the failures of a function, of a tree's Python and of the migration.
### Changed
- **A method's `requirements.txt` is checked when it loads (Breaking)**: a method tree holding Python and a `requirements.txt` at its root is refused before any of its Python is imported, naming each requirement, when the environment its PipeFunc steps run in does not meet it, or when the file holds a line the MTHDS standard refuses (a URL, an option such as `-r` or `-e`, a comment after a requirement, a continued line). A package a method calls by its address is checked the same way, named by its address. Locally the environment is the interpreter Pipelex runs in and the refusal gives the `uv pip install` command that fixes it; on a sandbox-hosted deployment it is the package set the deployment states for its sandbox, a run is refused with a `422` and a validation by `method_ref` with an `is_valid: false` verdict located on the line, naming no path on the server. A method of a server's own library directories whose requirements are not met is the server's fault, a `500` `HostLibraryRequirementsError` whose message never reaches the caller. Pipelex never installs anything. A project root loaded as a library directory is such a tree, so hand over the folder holding the methods instead when the root's `requirements.txt` is an application's.
+- **A `PipeFunc` names its function by import path (Breaking)**: `function_name` is the function's import path from the root of the step's method tree, such as `funcs.totals.compute_totals` for the function `compute_totals` of `funcs/totals.py`, and a flat name is refused as the bundle is read with the advice to run `pipelex fix function-names`. Validation locates the module among the tree's files and proves, without running any Python, that the function is a top-level `def` or `async def` the module defines carrying the `@pipe_func()` marker; the module is imported under its real name, the tree's root first on the import path, when the step first runs, where its signature and return type are checked. A bundle passed as content imports locally from the process's import path, and on a sandbox-hosted deployment from the Python submitted with it. The runner's OpenAPI document gives the `PipeFunc` blueprint's `function_name` its import path pattern and `PipeValidationErrorType` its `pipe_func_function_refused` value.
+- **One method tree's Python per process (Breaking)**: a run holds the Python of the tree it imports from until its library is torn down, and a step of another tree that would load its Python meanwhile is refused with `TreePythonHeldError` naming both trees by a bundle each holds and the start of its digest, never by a path of the host, so a run whose own steps and a dependency package's steps both call Python is refused locally; once no run holds a tree, its root leaves `sys.path` and the next tree evicts every module imported from a file of it, whatever imported it. A top-level module of a tree named like a standard-library module or like a module importable outside the tree is refused at validation, `structures` excepted, and on a sandbox-hosted deployment, whose sandbox holds its own packages, only a standard-library name is, and while a tree is held, importing a file its membership rule keeps out, such as a `.gitignore`d module, is refused naming the rule, and no bytecode is written into the tree.
+- **The function registry is retired and `@pipe_func()` takes no argument (Breaking)**: `FuncRegistry`, `func_registry`, `FuncRegistryUtils`, `FuncRegistryError`, `RuntimeHub.get_func_registry` and the `name=` argument of `@pipe_func` are removed, no function is discovered or imported while a library loads, and `is_marked_pipe_func(func=...)` tells a function carrying the marker.
+- **Kernel func ops take the function (Breaking)**: `pipelex.kernel.func_ops.call_registered_function` becomes `call_function(function=..., memory=...)`, and `run_func` takes the `function` itself in place of `function_name`.
+- **`LibraryCrate.pipe_func_files` replaces `python_sources` (Breaking)**: a sandbox-hosted crate carries the files a `PipeFunc` call receives, keyed by Python root, the key of a method tree holding a `PipeFunc` step or `submitted` for the bundles passed as content, then by tree path: every file of the tree except its `.mthds` files and a generated `structures` module, a file that is not UTF-8 text in Base64. They are read as the library loads, so a crate built later reads no file. The `submitted` root holds the files of the library directories whose own tree holds no bundle, such as the directory a server writes a request's Python into; a directory holding a bundle submits nothing, so two methods' `README.md` or `requirements.txt` never clash, and two directories submitting different files at one path refuse the load with `LibraryError`. A structure class is refused in every tree the load lists, a package or repository nested in a library directory included.
+- **A `PipeFunc` call carries its tree's files (Breaking)**: `PipeFuncExecutionRequest.files` holds the files of the called step's Python root at their tree paths, and `build_pipe_func_execution_request` moves them there from the crate, which no longer carries `pipe_func_files`. The direct executor's transported run writes them into a temporary directory, generates `structures` unless the method shipped one, imports the function by its import path under the process's hold on that directory and unloads its modules before removing it; `DirectPipeFuncExecutor.run_pipe_func_transported_at` and an optional third argument of the transported entrypoint take a directory a sandbox client filled itself.
## [v0.86.1] - 2026-10-16
diff --git a/api/pipelex_api/routes/pipelex/pipeline.py b/api/pipelex_api/routes/pipelex/pipeline.py
index 21034b0d9..32a1501d8 100644
--- a/api/pipelex_api/routes/pipelex/pipeline.py
+++ b/api/pipelex_api/routes/pipelex/pipeline.py
@@ -1027,7 +1027,7 @@ async def start(
run_request, extras = parsed
scopes = _resolve_run_scopes(request, extras=extras)
# The run source is materialized only for the synchronous setup phase: `start` builds the
- # PipeJob (crate carrying the captured `python_sources`) before it enqueues, so the temp dir
+ # PipeJob (crate carrying the captured `pipe_func_files`) before it enqueues, so the temp dir
# (a bundle's, or a fetched package's) is no longer needed once `start` returns — cleanup on
# context exit is safe for the async path.
with _run_source(run_request) as source:
diff --git a/api/tests/unit/test_bundle_materialize.py b/api/tests/unit/test_bundle_materialize.py
index 6dcffb843..a94c79877 100644
--- a/api/tests/unit/test_bundle_materialize.py
+++ b/api/tests/unit/test_bundle_materialize.py
@@ -27,7 +27,7 @@ type = "PipeFunc"
description = "e"
inputs = { text = "Text" }
output = "Text"
-function_name = "echo"
+function_name = "pipe_func.echo"
"""
_PIPE_FUNC_PY = "def echo(working_memory):\n return 'hi'\n"
diff --git a/api/tests/unit/test_bundle_structures_refusal.py b/api/tests/unit/test_bundle_structures_refusal.py
index 18dd7dfc6..b5090a843 100644
--- a/api/tests/unit/test_bundle_structures_refusal.py
+++ b/api/tests/unit/test_bundle_structures_refusal.py
@@ -36,7 +36,7 @@ type = "PipeFunc"
description = "Crunch numbers in caller code"
inputs = { data = "Text" }
output = "Text"
-function_name = "crunch"
+function_name = "crunch_funcs.pipe_func.crunch"
"""
_PIPE_FUNC_PY = """\
@@ -45,7 +45,7 @@ from pipelex.core.stuffs.text_content import TextContent
from pipelex.system.registries.func_registry import pipe_func
-@pipe_func(name="crunch")
+@pipe_func()
async def crunch(working_memory: WorkingMemory) -> TextContent:
return TextContent(text="crunched")
"""
@@ -137,7 +137,7 @@ class TestBundleStructuresRefusal:
orchestrator = _RecordingOrchestrator()
client = _build_client(mocker, orchestrator=orchestrator)
- files = {"main.mthds": _PIPE_FUNC_MTHDS, "funcs/pipe_func.py": _PIPE_FUNC_PY}
+ files = {"main.mthds": _PIPE_FUNC_MTHDS, "crunch_funcs/pipe_func.py": _PIPE_FUNC_PY}
response = client.post("/v1/start", json={"files": files, "inputs": {"data": "1,2,3"}})
assert response.status_code in {200, 202}, response.text
diff --git a/api/tests/unit/test_method_requirements_refusal.py b/api/tests/unit/test_method_requirements_refusal.py
index 77f07e712..a55dfb638 100644
--- a/api/tests/unit/test_method_requirements_refusal.py
+++ b/api/tests/unit/test_method_requirements_refusal.py
@@ -60,7 +60,7 @@ type = "PipeFunc"
description = "Crunch numbers in caller code"
inputs = { data = "Text" }
output = "Text"
-function_name = "crunch"
+function_name = "funcs.pipe_func.crunch"
"""
_PIPE_FUNC_PY = """\
@@ -69,7 +69,7 @@ from pipelex.core.stuffs.text_content import TextContent
from pipelex.system.registries.func_registry import pipe_func
-@pipe_func(name="crunch")
+@pipe_func()
async def crunch(working_memory: WorkingMemory) -> TextContent:
return TextContent(text="crunched")
"""
diff --git a/docs/api-server/openapi/pipelex-api.openapi.yaml b/docs/api-server/openapi/pipelex-api.openapi.yaml
index 0d14273e6..9482970ae 100644
--- a/docs/api-server/openapi/pipelex-api.openapi.yaml
+++ b/docs/api-server/openapi/pipelex-api.openapi.yaml
@@ -4945,8 +4945,11 @@ components:
title: Output
function_name:
type: string
+ pattern: ^[^.]+(\.[^.]+)+$
title: Function Name
- description: The name of the function to call.
+ description: 'The import path of the function to call, from the root of the method tree: the module''s dotted path,
+ then the function''s name, such as ''funcs.totals.compute_totals'' for the function compute_totals of funcs/totals.py.
+ The function must be a top-level def or async def of that module, decorated with @pipe_func().'
additionalProperties: false
type: object
required:
@@ -5968,6 +5971,7 @@ components:
- unresolved_package_dependency
- unexported_pipe_dependency
- unknown_model
+ - pipe_func_function_refused
- unknown_validation_error
title: PipeValidationErrorType
description: 'Types of pipe validation errors.
diff --git a/docs/api-server/pipe-validate.md b/docs/api-server/pipe-validate.md
index 64ad706e3..d83930f84 100644
--- a/docs/api-server/pipe-validate.md
+++ b/docs/api-server/pipe-validate.md
@@ -106,7 +106,7 @@ The 200 body is one of two arms, discriminated on the mandatory `is_valid` field
**Response Fields (invalid arm):**
- `is_valid` (`false`): the discriminant of the invalid arm
-- `validation_errors` (list): the structured per-error diagnostics a client maps to per-line problems — built by pipelex's one shared builder, so they are byte-for-byte the same items the agent CLI emits. Each item carries a `category` (the closed set `blueprint_validation` | `pipe_factory` | `pipe_validation` | `dry_run`), a `message`, and the locators the runtime can attribute (`error_type`, `pipe_code` / `concept_code` / `domain_code`, `field_path` / `field_name`, and `source`). Absent locators are omitted, not null. An item may also carry a **`suggested_fix`** — a structured, deterministic repair (`fix_code`, `description`, `safety: safe|unsafe`, and the `ops[]` of semantic TOML patch operations to apply). It is present only when the fix planner derived one from the typed error data; a client that ignores it behaves exactly as before. See [Error Responses → Structured validation errors](error-responses.md#structured-validation-errors) and [→ Suggested fixes](error-responses.md#suggested-fixes) for every field. The array is **never empty on an invalid verdict** (the structured-info invariant is total): a failing dry run becomes one `dry_run` item per failing pipe, located at the innermost pipe that failed (its `pipe_code`, `domain_code` and, when known, `source`), a TOML syntax error carries the `line` and `column` the parser stopped at, and a parse-level failure with no attributable locator (an empty blueprint, an elaborator failure) becomes one `blueprint_validation` residual item carrying the message (no `source`). Every refusal of the bundle while it loads is an item of the verdict, never a no-verdict `5xx`: an unknown model is an `unknown_model` item on the pipe's `model` field (a reference the deck does not define, or a bare handle it serves only as another model type, such as an LLM named in a `PipeImgGen`; [`GET /v1/models/check`](models.md#check-a-model-reference) answers the same question for one reference), a model setting the model it resolves to refuses (a reasoning setting on a model that does not reason, for instance) is an `llm_setting_refused_by_model` item on the field naming the setting, and a call into a method package this runtime cannot resolve (not installed and not fetchable, or a fetch that failed) is an `unresolved_package_dependency` item on the first pipe naming the package, whose `missing_pipe_code` is the reference as written, `<address>->domain.code`
+- `validation_errors` (list): the structured per-error diagnostics a client maps to per-line problems — built by pipelex's one shared builder, so they are byte-for-byte the same items the agent CLI emits. Each item carries a `category` (the closed set `blueprint_validation` | `pipe_factory` | `pipe_validation` | `dry_run`), a `message`, and the locators the runtime can attribute (`error_type`, `pipe_code` / `concept_code` / `domain_code`, `field_path` / `field_name`, and `source`). Absent locators are omitted, not null. An item may also carry a **`suggested_fix`** — a structured, deterministic repair (`fix_code`, `description`, `safety: safe|unsafe`, and the `ops[]` of semantic TOML patch operations to apply). It is present only when the fix planner derived one from the typed error data; a client that ignores it behaves exactly as before. See [Error Responses → Structured validation errors](error-responses.md#structured-validation-errors) and [→ Suggested fixes](error-responses.md#suggested-fixes) for every field. The array is **never empty on an invalid verdict** (the structured-info invariant is total): a failing dry run becomes one `dry_run` item per failing pipe, located at the innermost pipe that failed (its `pipe_code`, `domain_code` and, when known, `source`), a TOML syntax error carries the `line` and `column` the parser stopped at, and a parse-level failure with no attributable locator (an empty blueprint, an elaborator failure) becomes one `blueprint_validation` residual item carrying the message (no `source`). Every refusal of the bundle while it loads is an item of the verdict, never a no-verdict `5xx`: an unknown model is an `unknown_model` item on the pipe's `model` field (a reference the deck does not define, or a bare handle it serves only as another model type, such as an LLM named in a `PipeImgGen`; [`GET /v1/models/check`](models.md#check-a-model-reference) answers the same question for one reference), a model setting the model it resolves to refuses (a reasoning setting on a model that does not reason, for instance) is an `llm_setting_refused_by_model` item on the field naming the setting, a `PipeFunc` step whose function its method's files do not hold, or do not define as a top-level function marked `@pipe_func()`, is a `pipe_func_function_refused` item on its `function_name` field, and a call into a method package this runtime cannot resolve (not installed and not fetchable, or a fetch that failed) is an `unresolved_package_dependency` item on the first pipe naming the package, whose `missing_pipe_code` is the reference as written, `<address>->domain.code`
- `pending_signatures` (list[str]): best-effort outstanding signatures (empty on the invalid arm, since no library was assembled)
- `is_runnable` (`false`): an invalid bundle is never runnable
- `message` (string): the human-readable verdict summary: the lone item's message when there is one item, `N validation errors (first: <the first item's message>)` when there are several, and the parser's own message for a parse-level failure
diff --git a/docs/building-methods/method-trees.md b/docs/building-methods/method-trees.md
index 58ee64eba..1855d1a59 100644
--- a/docs/building-methods/method-trees.md
+++ b/docs/building-methods/method-trees.md
@@ -32,6 +32,10 @@ The tree's `.gitignore` files are what keeps a run's outputs out of the method:
A file a bundle names, such as a [`PipeDocGen`](pipes/pipe-operators/PipeDocGen.md) step's `template_file`, is a path relative to the bundle's file, written with forward slashes, and it is resolved in the bundle's tree: joined to the bundle's own place in the tree, a `..` may climb out of the bundle's folder to a file elsewhere in the tree, and the result must be a file of the tree. So a bundle in `bundles/` can name `../templates/invoice.docx`, while a path climbing above the tree's root, an absolute path, a file a `.gitignore` keeps out and a file of a nested package are all refused, each refusal naming the files by their paths in the tree. Pipelex looks the path up among the tree's files and never opens a path a bundle wrote.
+## Python modules
+
+A [`PipeFunc`](pipes/pipe-operators/PipeFunc.md) step names its function by its import path from the root of the tree, such as `funcs.totals.compute_totals` for the function `compute_totals` of `funcs/totals.py`, and its module is found among the tree's files. A run imports the tree's modules with the root first on the import path, refuses the import of a file the membership rule keeps out, such as a `.gitignore`d module, and writes no `__pycache__` into the tree. On a sandbox-hosted deployment, every file of the tree except its `.mthds` files travels with each call of the step.
+
## Library directories
A library directory, such as a `-L` directory or one of `PIPELEXPATH`, is a place methods are found rather than one method. Pipelex loads its tree, then the tree of each nested package and nested repository under it, each as a method of its own whose steps read only its own files, so every method in the directory loads. A `.mthds` file kept out by the membership rule, a hidden one, a gitignored one or one inside a virtual environment, is not loaded.
diff --git a/docs/building-methods/pipes/pipe-operators/PipeFunc.md b/docs/building-methods/pipes/pipe-operators/PipeFunc.md
index 3b72a5e4d..d874dfd0b 100644
--- a/docs/building-methods/pipes/pipe-operators/PipeFunc.md
+++ b/docs/building-methods/pipes/pipe-operators/PipeFunc.md
@@ -11,102 +11,107 @@ The `PipeFunc` operator provides an essential escape hatch, allowing you to exec
## How it works
-`PipeFunc` operates by calling a Python function that has been registered with Pipelex's central function registry.
+A `PipeFunc` step names a Python function of its method by the function's **import path from the root of the method's tree**: `function_name = "funcs.totals.compute_totals"` names the function `compute_totals` of the file `funcs/totals.py` in the method's folder. The [method tree](../../method-trees.md) is the set of files the method is made of, so the function travels with the method wherever the method goes.
-1. **Decorator Required**: Functions must be decorated with `@pipe_func()` to be discovered and registered (since v0.12.0).
-2. **Automatic Discovery**: Functions with the `@pipe_func()` decorator are automatically discovered from anywhere in your project when Pipelex starts up.
-3. **Function Signature**: Eligible functions are registered using their function name (or a custom name) as the registry key.
-4. **Execution**: When the `PipeFunc` pipe is executed, it looks up your function by name and calls it, passing in the current `working_memory`.
-5. **Returning Data**: The function returns data, which `PipeFunc` places back into the working memory, associated with the pipe's `output` concept.
+1. **Validation** locates the module in the tree and proves, from its source and without running any Python, that the function is a top-level `def` or `async def` of the module carrying the `@pipe_func()` marker. A function it cannot find or prove is a `pipe_func_function_refused` validation error on the step's `function_name`, saying what is missing.
+2. **The step's first run** imports the module under its real name, with the tree's root first on Python's import path, so the module imports its siblings in the tree as it would anywhere else (`from funcs.rates import VAT_RATE`). The imported function is then checked against the calling convention and the step's output.
+3. **Each run** calls the function with the current `working_memory`, and stores what it returns as the step's output.
-## Function Eligibility Requirements
+Nothing is discovered or imported when Pipelex starts: a module's top-level code runs when a step that names one of its functions first runs.
-For a function to be automatically registered and available to `PipeFunc`, it **must** meet all of the following criteria:
+## Writing a function
-!!! warning "Function Eligibility Requirements"
+A function a `PipeFunc` step can call:
-- **Must be decorated with** `@pipe_func()` (required since v0.12.0)
-- **Can be async or sync**: `async def` functions are awaited directly; plain `def` functions run in a worker thread (via `asyncio.to_thread`) so they don't block the pipeline
-- **Must have exactly 1 parameter** named `working_memory`
-- **Parameter type must be** `WorkingMemory`
-- **Return type must be** a subclass of `StuffContent` (or a generic type like `ListContent[SomeType]`)
-- **Must be discoverable** (not in excluded directories like `.venv`, `__pycache__`, etc.)
+- **carries the `@pipe_func()` marker**, which takes no argument;
+- **is a top-level function of its module**, a `def` or an `async def` the module defines itself: a function the module imports from elsewhere, or assigns to the name, is refused;
+- **takes exactly one parameter**, named `working_memory` and typed `WorkingMemory`;
+- **returns a `StuffContent` subclass**, such as `TextContent`, a `StructuredContent` model or `ListContent[SomeType]`, matching the step's `output`: a list type for an output such as `Text[]`, the concept's own structure class otherwise.
-### Return values
+An `async def` function is awaited; a plain `def` function runs in a worker thread (via `asyncio.to_thread`), so it does not block the pipeline.
-Your Python function can return:
+Here is a method holding its Python beside its bundle:
-- A `StuffContent` object (e.g., `TextContent`, `ImageContent`, or a custom `StructuredContent` model).
-- A `ListContent` containing `StuffContent` objects.
-
-## How to Create a Function
-
-To make a Python function available to `PipeFunc`:
-
-1. Add the `@pipe_func()` decorator to your function
-2. Place the function anywhere in your project (it will be auto-discovered)
-3. Ensure it meets all eligibility requirements
-
-!!! warning "Module Execution During Auto-Discovery"
- When Pipelex discovers functions with `@pipe_func()`, it imports the module containing them. **Any code at the module level (outside functions/classes) will be executed during import.** This can have unintended side effects.
-
- **Best practice:** Keep your `@pipe_func()` functions in dedicated modules with minimal module-level code, or ensure module-level code is safe to execute during discovery.
-
-Here is an example of an eligible function:
+```text
+invoicing/
+├── METHODS.toml
+├── invoicing.mthds
+└── funcs/
+ ├── rates.py
+ └── totals.py
+```
```python
-# in any Python file in your project (e.g., my_project/custom_functions.py)
+# invoicing/funcs/totals.py
-from pipelex.system.registries.func_registry import pipe_func
from pipelex.core.memory.working_memory import WorkingMemory
from pipelex.core.stuffs.text_content import TextContent
+from pipelex.system.registries.func_registry import pipe_func
+from funcs.rates import VAT_RATE
-@pipe_func() # Required decorator for auto-discovery
+
+@pipe_func()
async def concatenate_texts(working_memory: WorkingMemory) -> TextContent:
- """
- Retrieves two text stuffs, concatenates them, and returns a new text stuff.
- """
- # Get data from working memory using the names from previous steps
+ """Retrieve two texts, concatenate them, and return a new text."""
text1 = working_memory.get_stuff_as_str("text_a")
text2 = working_memory.get_stuff_as_str("text_b")
+ return TextContent(text=f"{text1} -- {text2} (VAT {VAT_RATE})")
+```
- concatenated = f"{text1} -- {text2}"
+```toml
+# invoicing/invoicing.mthds
- return TextContent(text=concatenated)
+[pipe.combine_them]
+type = "PipeFunc"
+description = "Combine two text inputs using a custom Python function"
+inputs = { text_a = "Text", text_b = "Text" }
+output = "Text"
+function_name = "funcs.totals.concatenate_texts"
```
-The function will be automatically registered with the name `concatenate_texts` (the function name) when Pipelex starts up.
+A folder of modules, such as `funcs/` here, is a package with or without an `__init__.py`.
-### Custom Registration Name
+## The function name
-You can optionally specify a custom name for registration:
+`function_name` is a dotted import path: the module's path from the tree's root, its folders and its file name without `.py` joined by dots, then the function's name. A `function_name` holding no dot names no module, and is refused as the bundle is read, the message naming the migration command below.
-```python
-@pipe_func(name="custom_concat")
-async def concatenate_texts(working_memory: WorkingMemory) -> TextContent:
- # Implementation...
- pass
-```
+Where the path starts depends on how the bundle reaches Pipelex:
-Then use `function_name = "custom_concat"` in your `.mthds` file.
+- **A bundle of a method tree**, a method folder, a `.mthds` file in it, a library directory or a dependency package, imports from the root of its tree: the folder holding the method's `METHODS.toml`, or the folder handed over when there is none (see [Where a tree is rooted](../../method-trees.md#where-a-tree-is-rooted)).
+- **A bundle passed as content**, with no file of its own, imports locally from Python's own import path, such as a module of your project or an installed package. On a sandbox-hosted deployment, it imports from the root of the Python submitted with it: the files of each library directory loaded beside it whose own tree holds no bundle, such as the directory a server writes a request's Python into. A library directory holding a bundle submits nothing, since its steps import from their own tree, and two directories submitting different files at one path refuse the load, since the sandbox writes them all under one root.
-### Function names share one flat name space
+## One method's Python in a process
-Registration names are unqualified — there is no module or package prefix — and every library directory Pipelex scans registers into the same process-wide registry. Two functions registered under the same name is therefore an error: Pipelex raises a `FuncRegistryError` naming both origins instead of letting scan order decide which one your `.mthds` files actually call.
+A local run imports a method's modules under their real names, with the tree's root first on the import path, so Pipelex keeps the methods of one process from stepping on each other:
-Registering the *same* function again — a directory scanned twice, a module reached by two paths — is a no-op, so overlapping scan roots are fine.
+- **No shadowing.** A top-level module of the tree named like a standard-library module, such as `json.py`, or like a module importable outside the tree, such as an installed package, is refused as the method is validated, the message naming the module, the file and what it would shadow. On a sandbox-hosted deployment, where a function runs in a sandbox holding its own packages, only a standard-library name is refused. The one exception is a `structures` module, which the runtime generates from a method's concepts (`pipelex build structures`), so the method's own copy of it is the intended one.
+- **One tree's Python at a time.** A run holds the Python of the tree it imports from until its library is torn down, and every run of that same tree shares the hold. A step of another tree that would load its Python while it is held is refused, the message naming both trees by a bundle each holds and the start of its digest, never by a path on the machine: two trees could define modules of the same name, and one process has one module of each name. So a run whose own steps and a dependency package's steps both call Python is refused locally.
+- **Eviction before the next tree.** Once no run holds a tree, its root leaves the import path, and the next tree to load its Python first removes every module imported from a file of the earlier tree, however it was imported, a helper your structure module imported while the method loaded included, so a process running several methods one after the other never runs one method's module for another's step. A module of a file the tree keeps out, such as one of a virtual environment inside the folder, is not the tree's and stays.
-If you hit a collision, give one of the two functions a distinct registration name:
+## The local import guard
-```python
-@pipe_func(name="invoices_summarize")
-async def summarize(working_memory: WorkingMemory) -> TextContent:
- # Implementation...
- pass
+While a run holds a tree, Pipelex serves the imports of every module under the tree's root:
+
+- **A file the tree keeps out cannot be imported**, such as a `.gitignore`d module or a module of a virtual environment inside the folder: the import is refused with the membership rule that excluded it. What runs locally is what a sandbox would receive.
+- **No bytecode is written into the tree**: no `__pycache__` folder appears beside your modules, so a method's files, and its tree digest, stay as they were written.
+
+The guard covers imports only. A file your function reads with `open()` is not checked locally: read the files your method ships by their path in the tree, and keep in mind that a sandbox receives only the tree's files.
+
+## Sandbox-hosted execution
+
+On a sandbox-hosted deployment, the runner never imports a method's Python. Validation proves each function's form from the method's files, and each call of a `PipeFunc` step runs in a sandbox that receives every file of the step's tree except its `.mthds` files, written at their paths in the tree, the tree's root first on the import path. The runner reads those files once, as the method loads, and only for a tree holding a `PipeFunc` step, so a run goes on after the folder it read them from is gone. A structure class is refused wherever the load finds one, in the library directory's own tree or in a package or repository nested in it. The sandbox generates the `structures` module from the method's concepts, so a function can `from structures import <domain>__<Concept>` without the method shipping a copy.
+
+## Migrating flat function names
+
+A `PipeFunc` step used to name its function by a flat name, the key under which a process-wide function registry held every `@pipe_func` function of every scanned directory, and `@pipe_func(name="...")` gave a function another key. The registry is retired: a flat name is refused, and `@pipe_func()` takes no argument. One command rewrites a method:
+
+```bash
+pipelex fix function-names invoicing/
+pipelex fix function-names invoicing/ --dry-run
```
-Prefixing names with the domain they serve is the simplest way to keep them distinct as a project grows.
+For each `PipeFunc` step whose `function_name` holds no dot, it finds the marked function of the tree that the flat name registered and writes its import path, then drops the argument of every `@pipe_func(name=...)` marker. It imports nothing and keeps every comment and alignment of your files. A flat name that no module of the tree registers, or that two register, is left as it is and reported with the markers registering it, a marker whose name is not a string literal is reported too, and the command exits `1`. See [Fix Commands](../../../tools/cli/fix.md#fix-function-names).
## Declaring the packages your functions need
@@ -114,30 +119,17 @@ When your functions import a package beyond Pipelex and the standard library, su
## Configuration
-Once the function is registered, you can use it in your `.mthds` file.
-
### MTHDS Parameters
-| Parameter | Type | Description | Required |
-| --------------- | ------ | --------------------------------------------------------------------------- | -------- |
-| `type` | string | The type of the pipe: `PipeFunc` | Yes |
-| `description` | string | A description of the function operation. | Yes |
-| `function_name` | string | The unique name used to register the Python function (e.g., "combine_two_texts"). | Yes |
-| `output` | string | The concept to associate with the function's return value. | Yes |
-
-### Example
-
-This MTHDS snippet shows how to use the `concatenate_texts` function defined above. It assumes two previous pipes have produced outputs named `text_a` and `text_b`.
-
-```toml
-[pipe.combine_them]
-type = "PipeFunc"
-description = "Combine two text inputs using a custom Python function"
-function_name = "concatenate_texts"
-output = "ConcatenatedText"
-```
+| Parameter | Type | Description | Required |
+| --------------- | ------ | ---------------------------------------------------------------------------------------------------------------- | -------- |
+| `type` | string | The type of the pipe: `PipeFunc` | Yes |
+| `description` | string | A description of the function operation. | Yes |
+| `function_name` | string | The function's import path from the root of the method tree, such as `"funcs.totals.compute_totals"`. | Yes |
+| `output` | string | The concept to associate with the function's return value. | Yes |
## Related Documentation
+- [Method Trees](../../method-trees.md) - The files a method is made of, and where its tree is rooted
- [Advanced Customizations](../../../advanced/index.md) - Extend Pipelex with custom logic
- [Working Memory](../working-memory.md) - How pipes access and share data
diff --git a/docs/configuration/config-technical/library-config.md b/docs/configuration/config-technical/library-config.md
index d37752157..772dd8404 100644
--- a/docs/configuration/config-technical/library-config.md
+++ b/docs/configuration/config-technical/library-config.md
@@ -418,17 +418,19 @@ Pipelex loads your pipelines in a specific order to ensure dependencies are reso
- Links pipes with their respective domains
- Resolves input/output concept references
-### 5. Function Registration
+### 5. PipeFunc Functions
-- Discovers functions decorated with `@pipe_func()`
-- Registers them in the function registry
-- Makes them available for `PipeFunc` operators
+- Locates the module each `PipeFunc` step names by its import path in the step's method tree
+- Proves from the module's source, without importing it, that the function is a top-level function carrying the `@pipe_func()` marker
+- Imports nothing: a module is imported when a step naming one of its functions first runs
-## Custom Function Registration
+## Custom Functions
-For custom functions used in `PipeFunc` operators, add the `@pipe_func()` decorator:
+A `PipeFunc` step names its function by its import path from the root of the method tree, and the function carries the `@pipe_func()` marker, which takes no argument:
```python
+# funcs/processing.py, in the method's folder
+
from pipelex.system.registries.func_registry import pipe_func
from pipelex.core.memory.working_memory import WorkingMemory
from pipelex.core.stuffs.text_content import TextContent
@@ -436,21 +438,13 @@ from pipelex.core.stuffs.text_content import TextContent
@pipe_func()
async def my_custom_function(working_memory: WorkingMemory) -> TextContent:
- """
- This function is automatically discovered and registered.
- """
input_data = working_memory.get_stuff("input_name")
# Process data
return TextContent(text=f"Processed: {input_data.content.text}")
-
-
-# Optional: specify a custom name
-@pipe_func(name="custom_processor")
-async def another_function(working_memory: WorkingMemory) -> TextContent:
- # Implementation
- pass
```
+The step then reads `function_name = "funcs.processing.my_custom_function"`. See [PipeFunc](../../building-methods/pipes/pipe-operators/PipeFunc.md) for the rules, and `pipelex fix function-names` to migrate a method still naming its functions by flat names.
+
## Validation
After making changes to your pipelines, validate them:
@@ -489,11 +483,12 @@ pipelex show pipe YOUR_PIPE_CODE
### 3. Custom Functions
-- Always use the `@pipe_func()` decorator
+- Always use the `@pipe_func()` marker, and name each function in the `.mthds` file by its import path from the method's folder
+- Keep the functions' modules inside the method's folder, so they travel with the method
- Use descriptive function names
- Document function parameters and return types
- Keep functions focused and testable
-- **Keep modules clean**: Avoid module-level code that executes on import (Pipelex imports modules during auto-discovery)
+- **Keep modules clean**: A module's top-level code runs when a step naming one of its functions first runs
### 4. Validation
diff --git a/docs/contribute/hub-layering.md b/docs/contribute/hub-layering.md
index 0717af29c..c00b36ac7 100644
--- a/docs/contribute/hub-layering.md
+++ b/docs/contribute/hub-layering.md
@@ -38,7 +38,7 @@ There is deliberately **no** `pipelex.hub`. It was deleted rather than kept as a
| config | `get_required_config`, `get_optional_config` |
| console | `get_console` |
| secrets | `get_secrets_provider`, `get_secret` |
-| system registries | `get_class_registry`, `get_func_registry` |
+| system registries | `get_class_registry` |
| storage | `get_storage_provider` |
| telemetry | `get_telemetry_manager`, `get_otel_tracer` |
| models | `get_models_manager`, `get_model_deck`, `get_sdk_client_manager` |
diff --git a/docs/contribute/mthds-test-corpus.md b/docs/contribute/mthds-test-corpus.md
index 0d0cf92e9..e0ee12cf5 100644
--- a/docs/contribute/mthds-test-corpus.md
+++ b/docs/contribute/mthds-test-corpus.md
@@ -90,7 +90,7 @@ Everything the file says is decided in the generator, `pipelex/cli/dev_cli/comma
An **exclusion** marks a registry code that no standalone focused entry could meaningfully exercise. It stays in the vocabulary and stays usable by an entry; it is simply not owed a focused entry. The exclusion map is keyed on the registry enum rather than on strings, which is what keeps an exclusion from outliving the code it names: a removed code breaks the generator at import, naming itself.
-No native concept is excluded — every one of them turned out to support a real entry. `PipeFunc` is excluded: it names a Python function that the runtime resolves against its function registry *at validation time*, so an entry declaring one fails with `Function '<name>' not found in registry` in every consumer that has not registered that exact function — and a cross-language consumer cannot register a Python function at all. Exercise `PipeFunc` in a consumer's own fixtures, where the function exists.
+No native concept is excluded — every one of them turned out to support a real entry. `PipeFunc` is excluded: it names a Python function by its import path in the method tree, and validation locates that module among the tree's files and reads it, so an entry declaring one is refused wherever its Python module does not travel with it — and a cross-language consumer does not read Python at all. Exercise `PipeFunc` in a consumer's own fixtures, where the function exists.
`error.*` carries the rest of the exclusions, and the distinction they turn on is worth stating: **membership in the registry means a value is reachable on the wire, never that a focused entry can produce it.** A corpus entry is a `.mthds` bundle validated through `validate bundle`, so a fault the validate path cannot reach is excluded here rather than pruned from the runtime's own truth. One is unreachable by construction: `unknown_concept` fires in the pipe factory under exactly the condition the earlier concept-reference resolution has already rejected as `unresolved_concept`. Several have no shape an entry could take: the advisory-only codes — `optional_force_redundant`, `input_presence_vacuous`, the three `hint_*` lints and `template_check_warning` — ride the validation report's `warnings` array and never make a verdict invalid, so the bundle they fire on is valid and has no `expected_error` to carry, and the manifest models no warnings axis. The two `unknown_*` fallbacks are excluded because an entry pinning one would be pinning the absence of a diagnosis, and would have to be rewritten the moment the fault got a code of its own.
diff --git a/docs/errors/authoring-and-language.md b/docs/errors/authoring-and-language.md
index a04617bb0..8a4df2cce 100644
--- a/docs/errors/authoring-and-language.md
+++ b/docs/errors/authoring-and-language.md
@@ -61,6 +61,9 @@ own page. Classes are grouped by subsystem.
- [`StuffFactoryError`](stuff-factory-error.md) — Stuff factory
- [`StuffSpecError`](stuff-spec-error.md) — Stuff spec
- [`StuffSpecFactoryError`](stuff-spec-factory-error.md) — Stuff spec factory
+- [`TreeModuleShadowingError`](tree-module-shadowing-error.md) — Tree module shadowing
+- [`TreePythonError`](tree-python-error.md) — Tree python
+- [`TreePythonHeldError`](tree-python-held-error.md) — Tree python held
- [`UnknownInputNameError`](unknown-input-name-error.md) — Unknown input name
- [`WorkingMemoryConsistencyError`](working-memory-consistency-error.md) — Working memory consistency
- [`WorkingMemoryError`](working-memory-error.md) — Working memory
@@ -83,6 +86,7 @@ own page. Classes are grouped by subsystem.
- [`PipeDocGenUndefinedValueError`](pipe-doc-gen-undefined-value-error.md) — Pipe doc gen undefined value
- [`PipeExtractFactoryError`](pipe-extract-factory-error.md) — Pipe extract factory
- [`PipeFuncExecutionError`](pipe-func-execution-error.md) — Pipe func execution
+- [`PipeFuncFunctionError`](pipe-func-function-error.md) — Pipe func function
- [`PipeFuncTransportError`](pipe-func-transport-error.md) — Pipe func transport
- [`PipeImgGenFactoryError`](pipe-img-gen-factory-error.md) — Pipe img gen factory
- [`PipeImgGenRunError`](pipe-img-gen-run-error.md) — Pipe img gen run
diff --git a/docs/errors/func-registry-error.md b/docs/errors/func-registry-error.md
deleted file mode 100644
index e7fc2527f..000000000
--- a/docs/errors/func-registry-error.md
+++ /dev/null
@@ -1,19 +0,0 @@
----
-title: "Func registry"
-description: "Reference for the `FuncRegistryError` Pipelex error class."
----
-
-<!-- pipelex:generated -->
-
-# Func registry
-
-| Field | Value |
-|---|---|
-| `error_type` | `FuncRegistryError` |
-| `title` | Func registry |
-| `type_uri` | `https://docs.pipelex.com/latest/errors/func-registry-error/` |
-| `error_domain` | _(inherited from parent)_ |
-| Defined in | `pipelex.system.registries.exceptions` |
-| Parent class | [`ToolError`](tool-error.md) |
-
-[Back to Error Reference](index.md)
diff --git a/docs/errors/function-names-migration-error.md b/docs/errors/function-names-migration-error.md
new file mode 100644
index 000000000..be617c4f9
--- /dev/null
+++ b/docs/errors/function-names-migration-error.md
@@ -0,0 +1,22 @@
+---
+title: "Function names migration error"
+description: "Reference for the `FunctionNamesMigrationError` Pipelex error class."
+---
+
+<!-- pipelex:generated -->
+
+# Function names migration error
+
+The function names of a method could not be migrated to import paths.
+
+| Field | Value |
+|---|---|
+| `error_type` | `FunctionNamesMigrationError` |
+| `title` | Function names migration error |
+| `type_uri` | `https://docs.pipelex.com/latest/errors/function-names-migration-error/` |
+| `error_domain` | `input` |
+| Defined in | `pipelex.migration.exceptions` |
+| Parent class | [`PipelexError`](pipelex-error.md) |
+| `retryable` | `false` |
+
+[Back to Error Reference](index.md)
diff --git a/docs/errors/pipe-func-function-error.md b/docs/errors/pipe-func-function-error.md
new file mode 100644
index 000000000..306829cb2
--- /dev/null
+++ b/docs/errors/pipe-func-function-error.md
@@ -0,0 +1,22 @@
+---
+title: "Pipe func function"
+description: "Reference for the `PipeFuncFunctionError` Pipelex error class."
+---
+
+<!-- pipelex:generated -->
+
+# Pipe func function
+
+A `PipeFunc`'s `function_name` names no function the step may call.
+
+| Field | Value |
+|---|---|
+| `error_type` | `PipeFuncFunctionError` |
+| `title` | Pipe func function |
+| `type_uri` | `https://docs.pipelex.com/latest/errors/pipe-func-function-error/` |
+| `error_domain` | `input` |
+| Defined in | `pipelex.pipe_operators.func.exceptions` |
+| Parent class | [`PipelexError`](pipelex-error.md) |
+| `retryable` | `false` |
+
+[Back to Error Reference](index.md)
diff --git a/docs/errors/platform-and-tooling.md b/docs/errors/platform-and-tooling.md
index cc600e69e..437ac9982 100644
--- a/docs/errors/platform-and-tooling.md
+++ b/docs/errors/platform-and-tooling.md
@@ -71,7 +71,6 @@ own page. Classes are grouped by subsystem.
- [`CredentialsError`](credentials-error.md) — Missing or invalid credentials
- [`EnvVarNotFoundError`](env-var-not-found-error.md) — Environment variable not set
- [`FatalError`](fatal-error.md) — Fatal error
-- [`FuncRegistryError`](func-registry-error.md) — Func registry
- [`InvocationPathError`](invocation-path-error.md) — Invocation path not derivable
- [`JobMetadataError`](job-metadata-error.md) — Job metadata
- [`LangfuseCredentialsError`](langfuse-credentials-error.md) — Langfuse credentials
@@ -131,6 +130,7 @@ own page. Classes are grouped by subsystem.
## Migration
- [`FormerReleaseConfigError`](former-release-config-error.md) — Configuration left by a former release
+- [`FunctionNamesMigrationError`](function-names-migration-error.md) — Function names migration error
- [`MigrationError`](migration-error.md) — Migration error
- [`MigrationGoldenError`](migration-golden-error.md) — Migration golden error
- [`MigrationLedgerError`](migration-ledger-error.md) — Migration ledger error
diff --git a/docs/errors/tree-module-shadowing-error.md b/docs/errors/tree-module-shadowing-error.md
new file mode 100644
index 000000000..15671745c
--- /dev/null
+++ b/docs/errors/tree-module-shadowing-error.md
@@ -0,0 +1,22 @@
+---
+title: "Tree module shadowing"
+description: "Reference for the `TreeModuleShadowingError` Pipelex error class."
+---
+
+<!-- pipelex:generated -->
+
+# Tree module shadowing
+
+A top-level module of a method tree shares its name with a standard-library module or a module importable outside the tree.
+
+| Field | Value |
+|---|---|
+| `error_type` | `TreeModuleShadowingError` |
+| `title` | Tree module shadowing |
+| `type_uri` | `https://docs.pipelex.com/latest/errors/tree-module-shadowing-error/` |
+| `error_domain` | `input` |
+| Defined in | `pipelex.core.method_trees.exceptions` |
+| Parent class | [`TreePythonError`](tree-python-error.md) |
+| `retryable` | `false` |
+
+[Back to Error Reference](index.md)
diff --git a/docs/errors/tree-python-error.md b/docs/errors/tree-python-error.md
new file mode 100644
index 000000000..61cafde32
--- /dev/null
+++ b/docs/errors/tree-python-error.md
@@ -0,0 +1,21 @@
+---
+title: "Tree python"
+description: "Reference for the `TreePythonError` Pipelex error class."
+---
+
+<!-- pipelex:generated -->
+
+# Tree python
+
+A method tree's Python cannot be imported in this process.
+
+| Field | Value |
+|---|---|
+| `error_type` | `TreePythonError` |
+| `title` | Tree python |
+| `type_uri` | `https://docs.pipelex.com/latest/errors/tree-python-error/` |
+| `error_domain` | _(inherited from parent)_ |
+| Defined in | `pipelex.core.method_trees.exceptions` |
+| Parent class | [`PipelexError`](pipelex-error.md) |
+
+[Back to Error Reference](index.md)
diff --git a/docs/errors/tree-python-held-error.md b/docs/errors/tree-python-held-error.md
new file mode 100644
index 000000000..1697e12a9
--- /dev/null
+++ b/docs/errors/tree-python-held-error.md
@@ -0,0 +1,21 @@
+---
+title: "Tree python held"
+description: "Reference for the `TreePythonHeldError` Pipelex error class."
+---
+
+<!-- pipelex:generated -->
+
+# Tree python held
+
+Another method tree's Python is held by a run in flight, so this tree's Python cannot load in this process.
+
+| Field | Value |
+|---|---|
+| `error_type` | `TreePythonHeldError` |
+| `title` | Tree python held |
+| `type_uri` | `https://docs.pipelex.com/latest/errors/tree-python-held-error/` |
+| `error_domain` | _(inherited from parent)_ |
+| Defined in | `pipelex.core.method_trees.exceptions` |
+| Parent class | [`TreePythonError`](tree-python-error.md) |
+
+[Back to Error Reference](index.md)
diff --git a/docs/features/pipe-operators.md b/docs/features/pipe-operators.md
index 1da7fb6fd..9438b426b 100644
--- a/docs/features/pipe-operators.md
+++ b/docs/features/pipe-operators.md
@@ -55,6 +55,6 @@ See [PipeCompose reference](../building-methods/pipes/pipe-operators/PipeCompose
## PipeFunc
-Execute custom Python functions within pipelines. Functions are auto-discovered via the `@pipe_func()` decorator and receive the current working memory as their input. Use PipeFunc when you need logic that goes beyond what declarative operators provide.
+Execute custom Python functions within pipelines. A step names its function by its import path in the method's folder, the function carries the `@pipe_func()` marker, and it receives the current working memory as its input. Use PipeFunc when you need logic that goes beyond what declarative operators provide.
See [PipeFunc reference](../building-methods/pipes/pipe-operators/PipeFunc.md).
diff --git a/docs/tools/cli/agent-cli.md b/docs/tools/cli/agent-cli.md
index 25fcf0e76..1f3782586 100644
--- a/docs/tools/cli/agent-cli.md
+++ b/docs/tools/cli/agent-cli.md
@@ -23,7 +23,7 @@ There is no `--log-level` flag: `pipelex-agent` is machine-consumed, so logging
## Commands Overview
-The agent CLI mirrors the main CLI's subcommand structure for `run`, `validate`, and `inputs`, each with `pipe`, `bundle`, and `method` subcommands. It also provides `fix bundle` for deterministic in-place repairs.
+The agent CLI mirrors the main CLI's subcommand structure for `run`, `validate`, and `inputs`, each with `pipe`, `bundle`, and `method` subcommands. It also provides `fix bundle` for deterministic in-place repairs, and `fix function-names` to migrate a method's `PipeFunc` function names.
### Run
@@ -120,6 +120,14 @@ pipelex-agent fix bundle <PATH> [OPTIONS]
The result reports `iterations`, `fixes_applied`, `files_written`, and any `remaining_errors`. A fully valid result exits 0; a completed fix attempt that remains invalid exits 1; argument, setup, or unexpected failures exit 2.
+Rewrite a method's flat `PipeFunc` function names as import paths from its folder, the migration of the retired function registry (see [PipeFunc](../../building-methods/pipes/pipe-operators/PipeFunc.md#migrating-flat-function-names)):
+
+```bash
+pipelex-agent fix function-names <PATH> [--dry-run] [--format markdown|json] [--error-format markdown|json]
+```
+
+The result reports `directory`, `is_dry_run`, `changed_files`, the `rewrites` it made (each with its `bundle`, `pipe_code`, the `flat_name` and the `import_path`), the `marker_removals` (each `@pipe_func(name=...)` marker whose argument it dropped), the `unresolved` flat names it left, each with the `reason` and, when several modules register the name, their import paths in `candidates`, and the `unresolved_markers` whose argument it left in place, each with its `module`, `function`, `registered_name` and `reason`: a marker registering a name some step still holds unresolved, or one whose name is not a string literal. Branch on `is_complete`, which is true when no flat name and no marker was left unresolved. Nothing is imported and Pipelex does not boot. The command exits 1 when a flat name or a marker was left, and 2 when the folder cannot be listed as a method tree or a file cannot be read or written.
+
### Inputs
Generate an example inputs template for a pipe, bundle, or method. By default the template is the **light** signature-driven shape (bare values matching the pipe's declared concepts), which is exactly what `run` accepts; `--explicit` emits the ceremonial `{concept, content}` envelope form instead.
diff --git a/docs/tools/cli/fix.md b/docs/tools/cli/fix.md
index 462f57bec..ee91d0a9b 100644
--- a/docs/tools/cli/fix.md
+++ b/docs/tools/cli/fix.md
@@ -4,10 +4,11 @@ description: "Automatically fix validation errors in your .mthds bundles with de
# Fix Commands
-Apply deterministic, safe fixes to a bundle and re-validate until it is valid.
+Apply deterministic, safe fixes to a bundle and re-validate until it is valid, or migrate a method's `PipeFunc` function names to import paths.
```bash
-pipelex fix bundle ... # Fix a bundle file or directory in place
+pipelex fix bundle ... # Fix a bundle file or directory in place
+pipelex fix function-names ... # Rewrite a method's flat PipeFunc function names as import paths
```
When `pipelex validate` reports errors that have a deterministic safe fix, each one carries a `💡 Suggested fix:` line and the output ends with the exact `pipelex fix bundle` command to run. The fix command runs a validate → apply safe fixes → re-validate loop until the bundle is valid, out of fixes, or the iteration cap is reached — and names every change it makes.
@@ -62,6 +63,36 @@ pipelex fix bundle my_bundle.mthds --select match-sequence-output
Only fixes classified as SAFE are applied — deterministic corrections derived from the structured validation errors, never from guesswork. Errors without a safe deterministic fix are left in place and reported as remaining errors. An unsafe fix, such as `rename-model` for a model name the deck only nearly matches, still shows on its error, labelled `💡 Suggested fix (unsafe, confirm before applying):`, for you to check and apply by hand, and is not a code `--select` or `--ignore` accepts. One rule is safe on some errors and unsafe on others: `delete-redundant-dotted-input` deletes a dotted input key such as `"page.page_view" = "Image"` whose root `page` is declared beside it, and it is safe when the root keeps its presence marker and multiplicity. When the key, declared after its root, carries another presence marker or multiplicity than the root, as `"data.text" = "Text!"` after `data = "Text?"` does, deleting it would change what a run requires, so that fix is unsafe and names only what differs, the multiplicity, the presence marker or both, with the edit that carries it onto the root; `--select` accepts the code and applies only its safe fixes. The available fix rule codes (for `--select`/`--ignore`) are listed in the error message when you pass an unknown code.
+## Fix Function Names
+
+```bash
+pipelex fix function-names <PATH>
+```
+
+Rewrites every flat `PipeFunc` `function_name` of a method as the function's import path from the method's folder. A `PipeFunc` step names its function by import path, such as `funcs.totals.compute_totals` for the function `compute_totals` of `funcs/totals.py`, and a flat name, the key of the retired function registry, is refused as the bundle is read (see [PipeFunc](../../building-methods/pipes/pipe-operators/PipeFunc.md#migrating-flat-function-names)).
+
+For each step whose `function_name` holds no dot, the command finds the `@pipe_func` function of the method's tree that the flat name registered, its own name or the one its `@pipe_func(name=...)` marker gave it, and writes its import path; then it drops the argument of every `@pipe_func(name=...)` marker, since the marker takes none. It reads the method's files as text and syntax trees: nothing is imported, Pipelex does not boot, and every comment and alignment of the files is kept. A flat name that no module of the tree registers, or that two register, is left as it is and reported for you to decide, and so is the argument of each marker registering it, the only record left of which function the step meant. A marker whose name is not a string literal, such as `@pipe_func(name=NAME)`, is left and reported too, since its name cannot be read without running the module.
+
+**Arguments:**
+
+- `PATH` - The method's folder, the tree holding its bundles and its Python
+
+**Options:**
+
+- `--dry-run` - Show what would change without writing any file
+
+**Examples:**
+
+```bash
+# Migrate a method in place
+pipelex fix function-names invoicing/
+
+# Preview the migration
+pipelex fix function-names invoicing/ --dry-run
+```
+
+**Exit codes:** `0` when every flat name was rewritten (or there was none), `1` when a flat name or a marker's argument was left unresolved, `2` when the folder cannot be listed as a method tree or a file cannot be read or written.
+
## Related Documentation
- [Validate Commands](validate.md) - Where suggested fixes are surfaced
diff --git a/docs/under-the-hood/pipe-func-requirements.md b/docs/under-the-hood/pipe-func-requirements.md
index 8ac9844d9..4608331d8 100644
--- a/docs/under-the-hood/pipe-func-requirements.md
+++ b/docs/under-the-hood/pipe-func-requirements.md
@@ -11,7 +11,7 @@ A method declares the Python packages its own Python imports in a `requirements.
A method tree is checked when it holds a `requirements.txt` at its root and a `.py` file anywhere in it. Python is the proxy for "has code that could import a package" rather than "has a PipeFunc pipe", because a server receives a method's Python apart from its bundles: the runner loads a bundle's `.py` files from a directory it makes for the request, while the `.mthds` files arrive as content, so the directory holding the Python holds no pipe to judge it by.
-The check is `pipelex/libraries/method_requirements.py`, and a load runs it at three places, each before any Python of the tree is imported or captured and before any of its pipes is built:
+The check is `pipelex/libraries/method_requirements.py`, and a load runs it at three places, each before any Python of the tree is imported or captured and before any of its pipes is built. A load never imports a `PipeFunc`'s module anyway: the step imports its function's module by its import path when it first binds it, in a validation's dry run or its first run, which comes after the load and its check, and the only Python a direct-mode load imports, the modules defining a concept's structure class, is imported after the check too.
- `LibraryManager.load_libraries` checks every tree of its library directories, and the tree of each bundle file it is handed, once all of them are listed.
- `LibraryManager.load_from_blueprints` checks the tree of every bundle read from this host's disk, on every load: a blueprint an earlier load placed in its tree, a refused one included, is judged again rather than taken as judged.
@@ -28,7 +28,7 @@ A requirement whose marker is false in the environment is not needed there and i
| `direct` | `InterpreterPackageEnvironment` | The distributions installed in the interpreter Pipelex runs in, through `importlib.metadata`, and this interpreter's own marker environment. Extras are followed through each distribution's `Requires-Dist`. |
| any other mode | `OfferedPackageEnvironment` | The package set the plugin providing the mode states for its sandbox, and nothing else. A marker variable the offer does not state cannot be judged, and a requirement with extras cannot be followed, so both are refused rather than judged against this process's own values. |
-In the `direct` mode, a method's Python is imported into the process that loads it, which is the process the check judges. A distributed orchestrator, such as our Temporal plugin, never carries a `direct`-mode method's Python to its worker: the library crate carries Python sources only in a sandbox mode, where they are captured rather than imported. A worker in the `direct` mode therefore runs only the functions its own deployment installed, whose packages are that deployment's to install, and the API server refuses a caller's Python outright unless it is sandbox-hosted (`403 CustomCodeRequiresSandbox`). So no worker needs to repeat the check.
+In the `direct` mode, a method's Python is imported into the process that loads it, when its steps first bind their functions, and that is the process the check judges. A distributed orchestrator, such as our Temporal plugin, never carries a `direct`-mode method's Python to its worker: the library crate carries the files of a `PipeFunc` call (`pipe_func_files`) only in a sandbox mode, where they are captured rather than imported. A worker in the `direct` mode therefore runs only the functions its own deployment installed, whose packages are that deployment's to install, and the API server refuses a caller's Python outright unless it is sandbox-hosted (`403 CustomCodeRequiresSandbox`). So no worker needs to repeat the check.
## How a refusal names the method
diff --git a/docs/under-the-hood/pipelex-kernel.md b/docs/under-the-hood/pipelex-kernel.md
index 3453f2ce1..fe2bfc353 100644
--- a/docs/under-the-hood/pipelex-kernel.md
+++ b/docs/under-the-hood/pipelex-kernel.md
@@ -78,7 +78,7 @@ Both façade calls take the concept and the output class the caller wants, defau
| `pipelex.kernel.search_ops` | `resolve_search_setting`, `run_search` |
| `pipelex.kernel.judgment_ops` | `judgment_setting_of_choice`, `served_judgment_model`, `resolve_judgment_setting`, `make_verdict_content`, `run_judgment`, `AskedQuestion`, `run_multi_judgment` |
| `pipelex.kernel.compose_ops` | `build_compose_context`, `build_composed_content`, `run_compose_template` |
-| `pipelex.kernel.func_ops` | `call_registered_function`, `run_func` |
+| `pipelex.kernel.func_ops` | `call_function`, `run_func` |
| `pipelex.kernel.memory_ops` | `shape_inputs`, `store_result`, `extract_main_content` / `extract_named_content`, `extract_main_content_as_list` / `extract_named_content_as_list` |
| `pipelex.kernel.prompt_assembly` | `UserPromptContent`, `AssembledUserPrompt`, `assemble_user_prompt`, `PromptFiles` |
| `pipelex.kernel.llm_prompt_content` | `LlmPromptContent`, `assemble_llm_prompt` |
@@ -205,6 +205,6 @@ with kernel.log_context():
Two arms of the interpreter stayed interpreter-side, both because moving them would cost more than the caller gains:
- **`PipeCompose`'s construct mode.** Its semantics are `StructuredContentComposer` over a `ConstructBlueprint`, and a blueprint is an MTHDS language artifact with a language-side consumer. A programmatic caller holds real Python and builds its structured object directly rather than describing the construction declaratively. The template path *is* fully extracted, and the one thing both paths share — the three-layer context ordering — is single-sourced in `build_compose_context`.
-- **`PipeFunc`'s pluggable executor seam.** The protocol and its DTOs are typed on interpreter models (`PipeRunParams`, `LibraryCrate`), so the kernel cannot name them. What running a function *means* — registry lookup, async-vs-sync dispatch, content coercion — is single-sourced in `call_registered_function`, which both the in-process executor and the kernel's `run_func` ride. What stays outside is *where* the function runs, which is configured deployment machinery rather than operator semantics. So a kernel `run_func` always runs in this process.
+- **`PipeFunc`'s pluggable executor seam.** The protocol and its DTOs are typed on interpreter models (`PipeRunParams`, `LibraryCrate`), so the kernel cannot name them. What running a function *means* — async-vs-sync dispatch and content coercion — is single-sourced in `call_function`, which both the in-process executor and the kernel's `run_func` ride. Finding the function is not the kernel's: a `PipeFunc` step imports it by its import path from its method tree, which is a library's business, so `run_func` takes the function itself, as every other op takes resolved values. What stays outside is *where* the function runs, which is configured deployment machinery rather than operator semantics. So a kernel `run_func` always runs in this process.
Beyond those: controllers are out of scope entirely (`pipelex/pipe_controllers/` is the interpreter's), the kernel is not separately installable from PyPI, and "activity-shaped" is a design constraint on the call signatures rather than a distributed-execution deliverable.
diff --git a/pipelex/cli/_cli.py b/pipelex/cli/_cli.py
index 9eca8e06a..86fa74a5c 100644
--- a/pipelex/cli/_cli.py
+++ b/pipelex/cli/_cli.py
@@ -323,7 +323,7 @@ app.add_typer(
app.add_typer(
fix_app,
name="fix",
- help="Apply deterministic safe fixes to a bundle (.mthds) and re-validate until valid",
+ help="Apply deterministic safe fixes to a bundle (.mthds) and re-validate until valid, or migrate a method's function names",
)
app.command(name="resolve", help="Resolve a bundle closure into the normalized library crate (JSON or TOML) on stdout")(resolve_cmd)
app.add_typer(codegen_app, name="codegen", help="Project the resolved crate into typed, runnable artifacts (types, inputs) and check for drift")
diff --git a/pipelex/cli/agent_cli/CLAUDE.md b/pipelex/cli/agent_cli/CLAUDE.md
index 5a7ef16b7..6e5132322 100644
--- a/pipelex/cli/agent_cli/CLAUDE.md
+++ b/pipelex/cli/agent_cli/CLAUDE.md
@@ -19,7 +19,7 @@ Markdown structure per command:
- `run`: `# Pipeline run complete`, a `## Result` section (the rendered `main_stuff` markdown with `--with-memory`, otherwise the concept JSON in a fenced block), and output / graph file paths. With the graph on, the `graph_files` object of the `--with-memory` envelope names the viewer (`graph_html`), the graph on a live run (`graph_spec`, the graphspec renamed `live_run_graph.json`) and the graph's three companions written beside it under their canonical names (`pipe_io_contracts`, `input_form`, `output_form` — the validate artifacts describing the graph's data, keyed by `pipe_ref`); the agent CLI settles its graph inclusion (graphspec on, viewer on) before the run rather than at render time, since the companions are built inside the run's library window under that same flag; a viewer resolves them from the graph file's directory, so the rename never moves them. Cost is machine-first: on the local runner, when costs are on (`--costs`, the default) and the run did reportable work (any tokens or any cost), a best-effort `cost_report` object (`{total_cost, by_model: [...]}`, real USD) rides the JSON `--with-memory` envelope — there is **no** Rich cost table on stderr (unlike the human `pipelex run` CLI). The markdown renderer ignores the `cost_report` key, so it is JSON-only. A free/zero-price model still reports its token usage (with `total_cost` 0). The `cost_report` is **absent** when the run did no reportable work (dry runs), when `--no-costs` is passed, on a hosted run (which assembles no local usage), or if cost-summary aggregation fails (caught and skipped so it never fails an otherwise-successful run) — so consumers must treat it as optional. A local `run bundle` also carries `method_tree_digest`, the digest of the method tree it read, in its output file and its `--with-memory` envelope.
- `validate`: `# Validation passed`, the bundle path when relevant, and a list of validated pipes with their status. Every entry in `validated_pipes` (markdown and JSON) identifies its pipe by the namespaced `pipe_ref` (`domain.code`) — the same unambiguous identity across `validate all`, `validate bundle`, and `validate pipe`, so the same pipe is never reported under two identifiers. `validate bundle`, `validate method`, and `validate all` carry `pending_signatures` — the library-wide list of pipes still declared as `PipeSignature` (unimplemented forward declarations), namespaced by `pipe_ref` — rendered as a "Pending signatures" section in markdown. They also carry a derived `is_runnable` boolean (`true` ⇔ `pending_signatures` is empty), and the markdown adds an explicit plain-English runnability verdict on those surfaces: a complete bundle/library states it is runnable, while one with pending signatures states it is NOT yet runnable immediately above the verbatim "Pending signatures" heading. These whole-bundle/whole-library surfaces are **strict by default**: they exit non-zero on `not is_runnable` unless `--allow-signatures` is passed (which both tolerates the placeholders for the exit code and mock-runs the signature pipes in the sweep). Single-pipe surfaces make no library-wide runnability claim and never gate the exit code on it: bare `validate pipe <code>` omits the key and emits no verdict, while `validate bundle`/`validate method` with `--pipe` still surface the library-wide `pending_signatures` for information but do not gate on it (the requested slice can be fully implemented even when unrelated placeholders remain elsewhere). The whole-bundle and whole-library surfaces also carry a `warnings` array — the advisory lints, then what the sweep's dry runs found without failing (a `PipeDocGen` template check's `template_check_warning`s), rendered as a "Warnings" section in markdown — assembled by `pipelex/pipeline/advisory_warnings.py`, the ONE composition point every advisory-bearing channel calls (this CLI, the bare CLI and the protocol report). Bare `validate pipe <code>` carries the dry-run findings alone (`pipelex/pipeline/dry_run_warnings.py`), since one pipe has no flow to lint but what its dry run found is about it; `validate bundle` / `validate method` with `--pipe` carry every family, because they validate the whole bundle and only narrow the dry run, so the array stays bundle-wide there. Add a new advisory family there, never at a call site: the sites used to assemble their own and disagreed about which families they carried.
-- `fix`: `# Fix applied - bundle is valid` after one or more fixes, `# Bundle already valid` when the loop had nothing to apply, or `# Bundle valid but not runnable` when pending `PipeSignature` placeholders remain. The JSON success envelope carries `is_valid`, `bundle_path`, `iterations`, `fixes_applied`, `files_written`, `remaining_errors`, and, on successful validation, `pending_signatures` plus `is_runnable`. Whole-bundle fix runs are strict by default: they emit the success envelope but exit 1 on `not is_runnable` unless `--allow-signatures` is passed. A still-invalid verdict exits 1 via `FixBundleError` on stderr with the same structured result fields plus any `bail_reason`.
+- `fix`: `# Fix applied - bundle is valid` after one or more fixes, `# Bundle already valid` when the loop had nothing to apply, or `# Bundle valid but not runnable` when pending `PipeSignature` placeholders remain. The JSON success envelope carries `is_valid`, `bundle_path`, `iterations`, `fixes_applied`, `files_written`, `remaining_errors`, and, on successful validation, `pending_signatures` plus `is_runnable`. Whole-bundle fix runs are strict by default: they emit the success envelope but exit 1 on `not is_runnable` unless `--allow-signatures` is passed. A still-invalid verdict exits 1 via `FixBundleError` on stderr with the same structured result fields plus any `bail_reason`. `fix function-names <method folder>` is the migration of a method's flat PipeFunc `function_name`s into import paths from the method tree's root (`pipelex/migration/function_names.py`): `# Function names of the method at <folder>` then the rewrites, the `@pipe_func(...)` registration names dropped, the names left unresolved with their reason and candidates, and the markers whose argument it kept (one registering a name a step still holds unresolved, or one whose name is not a string literal); the JSON envelope carries `directory`, `is_dry_run`, `is_complete`, `changed_files`, `rewrites`, `marker_removals`, `unresolved` and `unresolved_markers`. It exits 1 after the envelope when a name or a marker is left unresolved, and 2 via `FunctionNamesMigrationError` when the folder is not a method tree.
- `migrate`: `# Configuration migration`, the mode (applied / dry run) and verdict, the directories walked, a files-walked/changed/written count and, when a former release left something, its file count, then one `## <path>` section per file the former-release cleanup touches, listing its changes and its backup, a `## Pipelex still cannot start` section when the cleanup's check, run on every run, found something, and one per file that is not clean listing the entries applied, the entries blocked with their reason and guidance, and any unexplained paths. Paths, operation kinds, ledger-supplied values and the names of retired backends and profiles only — no value read from the user's file is ever rendered, in either format.
- `init`: `# Pipelex initialized` with target directory and execution, then enabled backends and routing profile for a local setup, or whether a Pipelex API key is set for a hosted one.
- error path: `# Error: <error_type>`, the message, the hint as a `> 💡` callout, the `validation_errors` items (when the cause's report, or a hosted refusal, carries any) as the same grouped prose `validate` prints, and a `## Details` section. `error_source` (internal stack frames) is omitted from markdown — it remains in the JSON envelope (`--error-format json`) for programmatic consumers.
@@ -59,6 +59,7 @@ commands/
fix/ # fix — deterministic in-place bundle fixes
app.py # fix_app Typer, subcommand registration
bundle_cmd.py # fix bundle — run validate/fix/re-validate loop
+ function_names_cmd.py # fix function-names — migrate flat PipeFunc function names to import paths
inputs/ # inputs — generate example input JSON
app.py # inputs_app Typer, subcommand registration
pipe_cmd.py # inputs pipe — inputs for a pipe by code
@@ -85,7 +86,7 @@ commands/
| `init` | Initializes Pipelex configuration (non-interactive). Defaults to project `.pipelex/` at detected project root. Use `--global`/`-g` to target the home configuration directory (`~/.pipelex/`, or `PIPELEX_HOME`). Accepts `--config`/`-c` with inline JSON or file path for `execution` (`hosted` or `local`, written to `[run] execution` with `pipelex/cli/commands/init/setup_path.py`'s `write_run_execution`), backends and routing (via `primary_backend`); `"execution": "hosted"` skips backends and routing, refuses `backends`/`primary_backend`, and reports `execution` plus `api_key_set` (never the key) instead of `backends_enabled`/`routing_profile`; a `warnings` list is added when the project around the working directory sets another `[run] execution` in its own `pipelex.toml` (`find_project_execution_shadow`); telemetry is seeded from a template, not `--config`. `--format markdown\|json` (success, default: markdown) + `--error-format markdown\|json` (errors, defaults to `--format`'s value). |
| `run` | Executes a pipeline (pipe\|bundle\|method subcommands), returns main_stuff + working_memory. Graph visualizations on by default (`--no-graph` to disable). `--format markdown\|json` (success, default: markdown) + `--error-format markdown\|json` (errors, defaults to `--format`'s value). |
| `validate` | Dry-runs pipes/bundles/methods (pipe\|bundle\|method subcommands), returns validation status per pipe. Bundle subcommand supports `--graph` for graph visualization (with `--graph-format` for the graph renderer). `--format markdown\|json` (success, default: markdown) + `--error-format markdown\|json` (errors, defaults to `--format`'s value). |
-| `fix` | Applies deterministic safe fixes to a bundle file or directory (`fix bundle`) and re-validates until valid, out of fixes, or max iterations. Supports `--allow-signatures`, `--select`, `--ignore`, `--max-iterations`, and `-L/--library-dir`. `--format markdown\|json` (success, default: markdown) + `--error-format markdown\|json` (errors, defaults to `--format`'s value). |
+| `fix` | Applies deterministic safe fixes to a bundle file or directory (`fix bundle`) and re-validates until valid, out of fixes, or max iterations. `fix function-names` migrates a method folder's flat PipeFunc function names to import paths (`--dry-run` writes nothing). Supports `--allow-signatures`, `--select`, `--ignore`, `--max-iterations`, and `-L/--library-dir`. `--format markdown\|json` (success, default: markdown) + `--error-format markdown\|json` (errors, defaults to `--format`'s value). |
| `fmt` | Formats a .mthds/.toml/.plx file in-place (delegates to plxt) |
| `lint` | Lints a .mthds/.toml/.plx file for errors (delegates to plxt) |
| `inputs` | Generates an example inputs template for a pipe/bundle/method (pipe\|bundle\|method subcommands). `--format json\|toml` (template serialization, default: json — NOT the markdown\|json pair); `toml` prints raw TOML to stdout |
diff --git a/pipelex/cli/agent_cli/_agent_cli.py b/pipelex/cli/agent_cli/_agent_cli.py
index 5ab17e334..8146c939a 100644
--- a/pipelex/cli/agent_cli/_agent_cli.py
+++ b/pipelex/cli/agent_cli/_agent_cli.py
@@ -155,7 +155,7 @@ def app_callback(
app.command(name="init", help="Initialize Pipelex configuration (non-interactive)")(agent_init_cmd)
app.add_typer(run_app, name="run", help="Execute a pipeline and output JSON results")
app.add_typer(validate_app, name="validate", help="Validate a pipe, bundle, or all pipes and output JSON results")
-app.add_typer(fix_app, name="fix", help="Fix a bundle file or pipeline directory in place")
+app.add_typer(fix_app, name="fix", help="Fix a bundle file or pipeline directory in place, or migrate a method's function names")
app.command(name="fmt", help="Format a .mthds, .toml, or .plx file in-place")(fmt_cmd)
app.command(name="lint", help="Lint a .mthds, .toml, or .plx file")(lint_cmd)
app.add_typer(inputs_app, name="inputs", help="Generate example input JSON for a pipe")
diff --git a/pipelex/cli/agent_cli/commands/fix/app.py b/pipelex/cli/agent_cli/commands/fix/app.py
index e6190d9bd..b11db5417 100644
--- a/pipelex/cli/agent_cli/commands/fix/app.py
+++ b/pipelex/cli/agent_cli/commands/fix/app.py
@@ -3,6 +3,7 @@
import typer
from pipelex.cli.agent_cli.commands.fix.bundle_cmd import fix_bundle_cmd
+from pipelex.cli.agent_cli.commands.fix.function_names_cmd import fix_function_names_cmd
fix_app = typer.Typer(
add_completion=False,
@@ -10,3 +11,4 @@ fix_app = typer.Typer(
)
fix_app.command(name="bundle", help="Fix a bundle file (.mthds) or pipeline directory")(fix_bundle_cmd)
+fix_app.command(name="function-names", help="Rewrite a method's flat PipeFunc function names as import paths")(fix_function_names_cmd)
diff --git a/pipelex/cli/agent_cli/commands/fix/function_names_cmd.py b/pipelex/cli/agent_cli/commands/fix/function_names_cmd.py
new file mode 100644
index 000000000..4b06e5d11
--- /dev/null
+++ b/pipelex/cli/agent_cli/commands/fix/function_names_cmd.py
@@ -0,0 +1,56 @@
+"""Agent CLI fix function-names command."""
+
+from pathlib import Path
+from typing import Annotated, Any
+
+import typer
+
+from pipelex.cli.agent_cli.commands.agent_output import CliOutputFormat, agent_error, agent_success_formatted, set_agent_cli_error_format
+from pipelex.migration.exceptions import FunctionNamesMigrationError
+from pipelex.migration.function_names import FunctionNamesMigration, migrate_function_names_in_directory
+from pipelex.migration.function_names_render import format_function_names_markdown
+
+
+def _payload(*, migration: FunctionNamesMigration, directory: Path, is_dry_run: bool) -> dict[str, Any]:
+ return {
+ "success": True,
+ "directory": str(directory.resolve()),
+ "is_dry_run": is_dry_run,
+ "is_complete": migration.is_complete,
+ "changed_files": sorted(migration.changed_files),
+ **migration.model_dump(mode="json", include={"rewrites", "marker_removals", "unresolved", "unresolved_markers"}),
+ }
+
+
+def fix_function_names_cmd(
+ path: Annotated[
+ str,
+ typer.Argument(help="The folder of the method, whose tree holds its bundles and its Python"),
+ ],
+ dry_run: Annotated[
+ bool,
+ typer.Option("--dry-run", help="Report what would change without writing any file"),
+ ] = False,
+ output_format: Annotated[
+ CliOutputFormat,
+ typer.Option("--format", help="Success output format: markdown (default) or json (structured)"),
+ ] = CliOutputFormat.MARKDOWN,
+ error_format: Annotated[
+ CliOutputFormat | None,
+ typer.Option("--error-format", help="Error output format (defaults to --format value): markdown or json"),
+ ] = None,
+) -> None:
+ """Rewrite every flat PipeFunc function_name of a method as its import path; branch on `is_complete`."""
+ set_agent_cli_error_format(error_format or output_format)
+ directory = Path(path)
+ try:
+ migration = migrate_function_names_in_directory(directory=directory, is_dry_run=dry_run)
+ except FunctionNamesMigrationError as exc:
+ agent_error(exc.message, error_type=type(exc).__name__, cause=exc, exit_code=2)
+ agent_success_formatted(
+ _payload(migration=migration, directory=directory, is_dry_run=dry_run),
+ markdown_renderer=lambda _payload_dict: format_function_names_markdown(migration=migration, directory=path, is_dry_run=dry_run),
+ output_format=output_format,
+ )
+ if not migration.is_complete:
+ raise typer.Exit(1)
diff --git a/pipelex/cli/commands/fix/app.py b/pipelex/cli/commands/fix/app.py
index 6f019930e..23b28875f 100644
--- a/pipelex/cli/commands/fix/app.py
+++ b/pipelex/cli/commands/fix/app.py
@@ -1,10 +1,12 @@
import typer
from pipelex.cli.commands.fix.bundle_cmd import fix_bundle_cmd
+from pipelex.cli.commands.fix.function_names_cmd import fix_function_names_cmd
fix_app = typer.Typer(
- help="Apply deterministic safe fixes to a bundle and re-validate until valid",
+ help="Apply deterministic safe fixes to a bundle and re-validate until valid, or migrate a method's function names",
no_args_is_help=True,
)
fix_app.command("bundle", help="Fix a bundle file (.mthds) or pipeline directory in place")(fix_bundle_cmd)
+fix_app.command("function-names", help="Rewrite a method's flat PipeFunc function names as import paths, without booting")(fix_function_names_cmd)
diff --git a/pipelex/cli/commands/fix/function_names_cmd.py b/pipelex/cli/commands/fix/function_names_cmd.py
new file mode 100644
index 000000000..b645a3d76
--- /dev/null
+++ b/pipelex/cli/commands/fix/function_names_cmd.py
@@ -0,0 +1,40 @@
+from pathlib import Path
+from typing import Annotated
+
+import typer
+
+from pipelex.migration.exceptions import FunctionNamesMigrationError
+from pipelex.migration.function_names import migrate_function_names_in_directory
+from pipelex.migration.function_names_render import format_function_names_markdown
+
+
+def fix_function_names_cmd(
+ path: Annotated[
+ str,
+ typer.Argument(help="The folder of the method, whose tree holds its bundles and its Python"),
+ ],
+ dry_run: Annotated[
+ bool,
+ typer.Option("--dry-run", help="Show what would change without writing any file"),
+ ] = False,
+) -> None:
+ """Rewrite every flat PipeFunc function_name of a method as its import path from the method's folder.
+
+ A PipeFunc names its function by import path, such as 'funcs.totals.compute_totals' for the function
+ compute_totals of funcs/totals.py. This finds the @pipe_func function each flat name registered, writes its
+ import path, and drops the registration name of every @pipe_func(name=...) marker. Nothing is imported, and
+ Pipelex does not boot. A name it cannot resolve is left as it is and reported, with the markers registering it
+ and any marker whose name is not a string literal; the command then exits 1.
+
+ Examples:
+ pipelex fix function-names my_method/
+ pipelex fix function-names my_method/ --dry-run
+ """
+ try:
+ migration = migrate_function_names_in_directory(directory=Path(path), is_dry_run=dry_run)
+ except FunctionNamesMigrationError as exc:
+ typer.secho(f"Failed to migrate the function names: {exc.message}", fg=typer.colors.RED, err=True)
+ raise typer.Exit(2) from exc
+ typer.echo(format_function_names_markdown(migration=migration, directory=path, is_dry_run=dry_run))
+ if not migration.is_complete:
+ raise typer.Exit(1)
diff --git a/pipelex/cli/dev_cli/commands/generate_corpus_vocabulary_cmd.py b/pipelex/cli/dev_cli/commands/generate_corpus_vocabulary_cmd.py
index 6e6053773..cb5edfae0 100644
--- a/pipelex/cli/dev_cli/commands/generate_corpus_vocabulary_cmd.py
+++ b/pipelex/cli/dev_cli/commands/generate_corpus_vocabulary_cmd.py
@@ -134,11 +134,11 @@ _FEATURE_TAGS: dict[str, str] = {
# code leaves the registry breaks this module at import instead of rotting in the generated file.
_PIPE_TYPE_EXCLUSIONS: dict[PipeType, str] = {
PipeType.PIPE_FUNC: (
- "A PipeFunc names a Python function that is resolved against the runtime's function registry "
- "at validation time, so an entry declaring one fails with \"Function '<name>' not found in "
- 'registry" in every consumer that has not registered that exact function. A corpus entry has '
- "to validate everywhere, and the cross-language consumers cannot register a Python function at "
- "all, so no portable focused entry exists. Exercise PipeFunc in a consumer's own fixtures."
+ "A PipeFunc names a Python function by its import path in the method tree, and validation "
+ "locates that module among the tree's files and reads it, so an entry declaring one is refused "
+ "wherever its Python module does not travel with it. A corpus entry is a bundle that has to "
+ "validate everywhere, and the cross-language consumers do not read Python at all, so no portable "
+ "focused entry exists. Exercise PipeFunc in a consumer's own fixtures."
),
}
@@ -177,6 +177,12 @@ _ERROR_TYPE_EXCLUSIONS: dict[ValidationErrorType, str] = {
"address-based reference to a package that is not installed fails as `unresolved_package_dependency` instead. "
"Revisit if an entry can ever carry the packages it depends on."
),
+ PipeValidationErrorType.PIPE_FUNC_FUNCTION_REFUSED: (
+ "It refuses a PipeFunc step whose function its method tree does not hold or cannot be proven, so only a PipeFunc "
+ "step produces it, and the PipeFunc operator is excluded: a focused entry is one bundle that has to validate "
+ "everywhere, and its Python module would have to travel with it to consumers that do not read Python. "
+ "Exercise it in a consumer's own fixtures, beside the operator."
+ ),
PipeValidationErrorType.LLM_SETTING_REFUSED_BY_MODEL: (
"It refuses a model setting the model it resolves to refuses, such as reasoning on a model that does not "
"think, so the bundle that produces it must name a model the deck defines with that particular capability. "
diff --git a/pipelex/core/method_trees/exceptions.py b/pipelex/core/method_trees/exceptions.py
new file mode 100644
index 000000000..ed25d4ffc
--- /dev/null
+++ b/pipelex/core/method_trees/exceptions.py
@@ -0,0 +1,40 @@
+from pipelex.base_exceptions import ErrorDomain, PipelexError
+
+
+class TreePythonError(PipelexError):
+ """A method tree's Python cannot be imported in this process."""
+
+
+class TreePythonHeldError(TreePythonError):
+ """Another method tree's Python is held by a run in flight, so this tree's Python cannot load in this process.
+
+ A process imports one tree's Python at a time: the tree root goes first on `sys.path` and its modules keep their
+ real names, so two trees defining the same module could not share the process, and evicting the earlier tree's
+ modules would pull them from under its run. The message names both trees.
+ """
+
+
+class TreeModuleShadowingError(TreePythonError):
+ """A top-level module of a method tree shares its name with a standard-library module or a module importable outside the tree.
+
+ With the tree's root first on `sys.path`, the tree's module would replace the other one for the whole process,
+ and the failure would show up far from its cause. The message names the module, the tree file and what it
+ would shadow; it describes the caller's own method, so it is kept under STRICT disclosure.
+ """
+
+ error_domain = ErrorDomain.INPUT
+ retryable = False
+ _authors_caller_facing_message = True
+
+
+class TreeModuleExcludedError(ImportError):
+ """An import reached a file under a held method tree's root that the tree's membership rule keeps out of the tree.
+
+ Raised by the import guard from inside Python's import machinery, which expects an `ImportError`. The message
+ names the file by its tree path and the rule that excluded it, never the tree's root, which is a path on this host.
+ """
+
+ def __init__(self, message: str, *, module_name: str, tree_path: str):
+ super().__init__(message, name=module_name)
+ self.message = message
+ self.tree_path = tree_path
diff --git a/pipelex/core/method_trees/tree_files.py b/pipelex/core/method_trees/tree_files.py
new file mode 100644
index 000000000..a724bc7a3
--- /dev/null
+++ b/pipelex/core/method_trees/tree_files.py
@@ -0,0 +1,81 @@
+"""A method tree's files as they travel with a `PipeFunc` call: at their tree paths, in the standard's file entry forms.
+
+A call to a `PipeFunc` that runs out of this process receives every file of its tree except the `.mthds` files, so the
+function imports its sibling modules and reads a file beside its module as it does locally (`wip/extra-files/design.md`,
+"Inside a run"). Each file travels as a file entry of `mthds/docs/spec/method-tree.md`, "The File Entry": its text
+when its bytes are UTF-8, else `{"base64": "…"}`. A hash entry, which names bytes the receiver already holds, is
+resolved by whoever holds the blobs before the files reach the process that writes them.
+"""
+
+from collections.abc import Mapping
+from pathlib import Path
+from typing import Protocol, TypeAlias
+
+from mthds.method_tree.file_entry import Base64FileEntry
+from mthds.method_tree.models import MethodTree
+from mthds.method_tree.tree_path import tree_path_violation
+
+from pipelex.core.method_trees.exceptions import TreePythonError
+
+#: A file's bytes as they travel: its text when they are UTF-8, else in Base64.
+TreeFileEntry: TypeAlias = str | Base64FileEntry
+
+BUNDLE_SUFFIX = ".mthds"
+
+
+class CallFileFilter(Protocol):
+ """Decides, from a file's tree path and bytes, whether it travels with a call."""
+
+ def __call__(self, *, tree_path: str, data: bytes) -> bool: ...
+
+
+def encode_tree_file(*, data: bytes) -> TreeFileEntry:
+ """The entry a file's bytes travel as: the text when they are UTF-8, else Base64."""
+ try:
+ return data.decode("utf-8")
+ except UnicodeDecodeError:
+ return Base64FileEntry.make_from_bytes(data)
+
+
+def tree_file_bytes(*, entry: TreeFileEntry) -> bytes:
+ """The bytes a file entry carries."""
+ if isinstance(entry, Base64FileEntry):
+ return entry.data
+ return entry.encode("utf-8")
+
+
+def read_call_files(*, tree: MethodTree, keeps: CallFileFilter | None = None) -> dict[str, TreeFileEntry]:
+ """Every file of a tree listed from this host's disk except its `.mthds` files, read as the entries a call receives.
+
+ Args:
+ tree: A tree listed from a directory of this host.
+ keeps: Decides, from a file's tree path and bytes, whether it travels; every file travels when None.
+
+ Returns:
+ The files by tree path, in tree path order.
+ """
+ files: dict[str, TreeFileEntry] = {}
+ for tree_path in tree.tree_paths:
+ if tree_path.endswith(BUNDLE_SUFFIX):
+ continue
+ data = tree.file_location(tree_path).read_bytes()
+ if keeps is not None and not keeps(tree_path=tree_path, data=data):
+ continue
+ files[tree_path] = encode_tree_file(data=data)
+ return files
+
+
+def write_call_files(*, files: Mapping[str, TreeFileEntry], destination: Path) -> None:
+ """Write the files a call received into `destination`, each at its tree path.
+
+ Raises:
+ TreePythonError: A key is not a tree path, so it could name a place outside `destination`.
+ """
+ for tree_path, entry in files.items():
+ violation = tree_path_violation(tree_path)
+ if violation is not None:
+ msg = f"The PipeFunc call received a file at '{tree_path}', which is not a tree path: {violation}."
+ raise TreePythonError(msg)
+ target = destination.joinpath(*tree_path.split("/"))
+ target.parent.mkdir(parents=True, exist_ok=True)
+ target.write_bytes(tree_file_bytes(entry=entry))
diff --git a/pipelex/core/method_trees/tree_python.py b/pipelex/core/method_trees/tree_python.py
new file mode 100644
index 000000000..f1e4a60e8
--- /dev/null
+++ b/pipelex/core/method_trees/tree_python.py
@@ -0,0 +1,581 @@
+"""Importing a method tree's Python in this process: one tree at a time, guarded, and evicted before the next.
+
+A `PipeFunc` names its function by import path from the root of its method tree, so a run that calls one imports the
+tree's modules under their real names, with the tree's root first on `sys.path` (`wip/extra-files/design.md`, D3).
+Only the in-process `direct` mode does this; a sandbox-hosted deployment never imports a tree's Python in its own
+process. Four rules keep that safe:
+
+- **One tree's Python per process at a time** (D19). A run holds the tree whose Python it imports from its first
+ `PipeFunc` step until its library is torn down, and every run of the same tree shares the hold. Loading another
+ tree's Python while one is held is refused, naming both trees, since two trees' modules share one import namespace.
+- **Eviction before the next tree** (D8). Once no run holds a tree, its root leaves `sys.path`, and the next tree to
+ load first evicts from `sys.modules` every module imported from a file of the earlier tree, whatever imported it, such
+ as a helper a structure module imported while its library loaded, so a process running several methods one after the
+ other never runs one method's module for another's step.
+- **No shadowing** (D9). A top-level module of the tree named like a standard-library module, or like a module
+ importable outside the tree, is refused, naming both, since the tree's would replace it for the whole process; a
+ sandbox-hosted deployment, whose sandbox does not hold this process's packages, checks the standard library only. The
+ `structures` module keeps its documented exception: the runtime generates it from a method's concepts, and a
+ method's own copy of it is the intended one.
+- **The local import guard** (D22). While a tree is held, a finder first on `sys.meta_path` serves the modules under
+ its root: a file the tree's membership rule keeps out, such as a `.gitignore`d file or a virtual environment, is
+ refused with the rule that excluded it. A member is loaded without writing bytecode, so a `__pycache__` never
+ appears in the tree, which matters for a package cache digested as written. Python's own `open()` reads stay
+ unchecked locally: the guard covers imports only.
+
+A message names a tree by the bundle it holds and the start of its digest, never by its root, which is a path on this
+host that a caller's verdict must not show; the root is logged.
+"""
+
+import importlib
+import importlib.abc
+import importlib.machinery
+import sys
+import threading
+from collections.abc import Collection, Generator, Hashable, Iterable, Mapping, Sequence
+from contextlib import contextmanager
+from contextvars import ContextVar
+from dataclasses import dataclass, field
+from pathlib import Path
+from types import ModuleType
+from typing import NamedTuple, cast
+
+from mthds.method_tree.models import MethodTree
+from mthds.method_tree.tree_path import nfc
+from typing_extensions import Buffer, override
+
+from pipelex import log
+from pipelex.core.method_trees.exceptions import TreeModuleExcludedError, TreeModuleShadowingError, TreePythonError, TreePythonHeldError
+from pipelex.tools.typing.module_inspector import drop_module_dirs_inserted_under
+
+#: The module the runtime generates from a method's concepts, which a tree may ship itself (the shadowing exception).
+GENERATED_STRUCTURES_MODULE = "structures"
+
+_PYTHON_SUFFIX = ".py"
+_PACKAGE_INIT = "__init__"
+_MAIN_MODULE = "__main__"
+
+#: The module-name checks passed inside `sharing_module_name_checks`, or None outside it.
+_passed_module_name_checks: ContextVar[set[Hashable] | None] = ContextVar("passed_module_name_checks", default=None)
+
+
+def tree_label_holding(*, tree_path: str) -> str:
+ """How a message names a method tree: by the bundle it holds, at its tree path, never by its root on this host."""
+ return f"the method tree holding {tree_path}"
+
+
+@dataclass(frozen=True)
+class TreePythonIdentity:
+ """A tree whose Python a process imports: its root on this host, its digest, which changes with any file, and its label.
+
+ Two identities are the same tree when their roots and digests agree. The label is how a message names the tree,
+ such as by the bundle it holds, since its root is a path on this host that a caller's verdict must not show.
+ """
+
+ root: Path
+ digest: str
+ label: str = field(compare=False)
+
+ def describe(self) -> str:
+ """The tree, named for a message: its label and the start of its digest."""
+ return f"{self.label} ({self.digest[:19]})"
+
+
+def tree_python_identity(*, tree: MethodTree, label: str) -> TreePythonIdentity:
+ """The identity of a tree listed from a directory of this host, whose root is its import root.
+
+ Args:
+ tree: The tree.
+ label: How messages name it (`tree_label_holding`).
+
+ Raises:
+ TreePythonError: The tree was read from a map of file entries, or is the one-file tree of a lone bundle, so it
+ has no directory of Python to import from.
+ """
+ if tree.directory is None:
+ msg = "The method tree was read from a map of file entries and has no directory on this host, so its Python cannot be imported here."
+ raise TreePythonError(msg)
+ if tree.is_one_file:
+ msg = (
+ f"The bundle {tree.entry or ''} was read alone, as a one-file tree, which holds no Python: hand over the method's folder, "
+ "whose tree holds the bundle and its Python."
+ )
+ raise TreePythonError(msg)
+ return TreePythonIdentity(root=tree.directory.resolve(), digest=tree.digest, label=label)
+
+
+def top_level_module_names(*, tree_paths: Iterable[str]) -> dict[str, str]:
+ """The top-level modules a tree's Python defines, each with the first tree path that defines it.
+
+ A Python file at the root defines the module of its stem; a directory holding a Python file at any depth defines
+ the module, or namespace package, of its name. A name that is not an identifier cannot be imported and defines
+ nothing.
+ """
+ modules: dict[str, str] = {}
+ for tree_path in sorted(tree_paths):
+ if not tree_path.endswith(_PYTHON_SUFFIX):
+ continue
+ first_segment, separator, _ = tree_path.partition("/")
+ module_name = first_segment if separator else first_segment.removesuffix(_PYTHON_SUFFIX)
+ if module_name == _PACKAGE_INIT or not module_name.isidentifier():
+ continue
+ modules.setdefault(module_name, tree_path)
+ return modules
+
+
+def check_tree_module_names(*, top_level_modules: Mapping[str, str], tree_label: str, root: Path | None, checks_this_process: bool) -> None:
+ """Refuse a tree whose top-level module shares its name with a standard-library module or one importable outside the tree.
+
+ Args:
+ top_level_modules: The tree's top-level modules, each with the tree path that defines it.
+ tree_label: How the message names the tree.
+ root: The tree's root on this host, whose own modules are not outside it; None for files with no directory here.
+ checks_this_process: Whether the tree's Python is imported in this process, so that a module this process
+ imports or can import is one the tree's would shadow. When it is imported elsewhere, such as in the sandbox
+ of a sandbox-hosted deployment, whose packages are not this process's, only the standard library, which
+ every interpreter holds, is checked.
+
+ Raises:
+ TreeModuleShadowingError: A module would shadow another, named with the tree file and what it shadows.
+ """
+ earlier_roots = TREE_PYTHON_HOLD.loaded_roots()
+ passed_checks = _passed_module_name_checks.get()
+ # The paths defining the modules only name them in a message: the judgement reads their names.
+ check_key = (frozenset(top_level_modules), root, checks_this_process, tuple(earlier_roots))
+ if passed_checks is not None and check_key in passed_checks:
+ return
+ for module_name, tree_path in sorted(top_level_modules.items()):
+ if module_name == GENERATED_STRUCTURES_MODULE:
+ continue
+ shadowed = (
+ _module_outside_tree(module_name=module_name, root=root, earlier_roots=earlier_roots)
+ if checks_this_process
+ else _standard_library_module(module_name=module_name)
+ )
+ if shadowed is None:
+ continue
+ msg = (
+ f"The file {tree_path} of {tree_label} defines the top-level module '{module_name}', which shares its name with {shadowed}. "
+ "With the tree's root first on the import path, it would replace that module for the whole process: rename it, "
+ "for example by moving it into a package of the method's own."
+ )
+ raise TreeModuleShadowingError(msg)
+ if passed_checks is not None:
+ passed_checks.add(check_key)
+
+
+@contextmanager
+def sharing_module_name_checks() -> Generator[None, None, None]:
+ """Judge each tree's module names once for every step validated inside, such as the steps one library load builds.
+
+ The judgement reads this process's import path and loaded modules, which a load does not change, so the check one
+ step of a tree passed holds for the tree's other steps. Outside, every check is made again. A scope opened inside
+ another shares the outer one's checks.
+ """
+ if _passed_module_name_checks.get() is not None:
+ yield
+ return
+ token = _passed_module_name_checks.set(set())
+ try:
+ yield
+ finally:
+ _passed_module_name_checks.reset(token)
+
+
+def _standard_library_module(*, module_name: str) -> str | None:
+ """The standard-library module a top-level name names, described for a message, or None when it names none."""
+ if module_name in sys.stdlib_module_names or module_name in sys.builtin_module_names:
+ return f"the standard-library module '{module_name}'"
+ return None
+
+
+def _module_outside_tree(*, module_name: str, root: Path | None, earlier_roots: Collection[Path]) -> str | None:
+ """What a top-level name already names outside the tree, described for a message, or None when it names nothing."""
+ standard_library_module = _standard_library_module(module_name=module_name)
+ if standard_library_module is not None:
+ return standard_library_module
+ # The module is named without its location, which is a path on this host that a caller's verdict must not show.
+ loaded = sys.modules.get(module_name)
+ if loaded is not None:
+ location = _module_location(module=loaded)
+ if location is not None and _is_tree_location(location=location, root=root, earlier_roots=earlier_roots):
+ return None
+ return f"the module '{module_name}', already imported in this process"
+ for spec in _specs_outside_root(module_name=module_name, root=root):
+ if spec.origin is None or not spec.has_location:
+ # A namespace portion never shadows a regular module, and a regular module of the tree takes precedence over it.
+ continue
+ if _is_tree_location(location=Path(spec.origin), root=root, earlier_roots=earlier_roots):
+ continue
+ return f"the module '{module_name}', importable in this process outside the tree"
+ return None
+
+
+def _specs_outside_root(*, module_name: str, root: Path | None) -> list[importlib.machinery.ModuleSpec]:
+ """The specs this process's finders return for a top-level name when the tree's root is not searched."""
+ specs: list[importlib.machinery.ModuleSpec] = []
+ for finder in list(sys.meta_path):
+ if isinstance(finder, TreeModuleGuard):
+ continue
+ spec: importlib.machinery.ModuleSpec | None
+ if finder is importlib.machinery.PathFinder:
+ entries = [entry for entry in sys.path if root is None or not _is_same_directory(entry=entry, directory=root)]
+ spec = importlib.machinery.PathFinder.find_spec(module_name, entries)
+ else:
+ find_spec = getattr(finder, "find_spec", None)
+ if find_spec is None:
+ continue
+ try:
+ spec = find_spec(module_name, None)
+ except (ImportError, ValueError):
+ continue
+ if spec is not None:
+ specs.append(spec)
+ return specs
+
+
+def _module_location(*, module: ModuleType) -> Path | None:
+ """Where a loaded module's file, or its first package directory, lies on disk, else None."""
+ module_file = getattr(module, "__file__", None)
+ if isinstance(module_file, str):
+ return Path(module_file)
+ module_path = getattr(module, "__path__", None)
+ if module_path is not None:
+ for entry in module_path:
+ return Path(entry)
+ return None
+
+
+def _is_tree_location(*, location: Path, root: Path | None, earlier_roots: Collection[Path]) -> bool:
+ """Whether a location lies under the tree's root, or under an earlier tree's root that a load evicts first."""
+ candidates = [*earlier_roots, *([root] if root is not None else [])]
+ return any(_is_under(path=location, directory=directory) for directory in candidates)
+
+
+def _is_under(*, path: Path, directory: Path) -> bool:
+ try:
+ return path.resolve().is_relative_to(directory.resolve())
+ except OSError:
+ return False
+
+
+def _is_same_directory(*, entry: str, directory: Path) -> bool:
+ try:
+ return Path(entry or ".").resolve() == directory.resolve()
+ except OSError:
+ return False
+
+
+class _TreeSourceLoader(importlib.machinery.SourceFileLoader):
+ """Loads a module of a held tree from its source, and never writes bytecode beside it."""
+
+ @override
+ def set_data(self, path: str, data: Buffer, *, _mode: int = 0o666) -> None: # kw-only: ignore
+ # The only data a source loader writes is the bytecode cache: a tree on disk, a package cache digested as
+ # written included, is never written into.
+ return
+
+
+class TreeModuleGuard(importlib.abc.MetaPathFinder):
+ """The finder first on `sys.meta_path` while a tree is held: it serves the tree's modules, and refuses the files the tree excludes.
+
+ A top-level import is looked up at the tree's root first, as the root is first on `sys.path`; a submodule import
+ is looked up in its package's directories that lie under the root. A module found there whose file is a member of
+ the tree is loaded without writing bytecode; one whose file the tree's membership rule keeps out is refused,
+ naming the file and the rule. A namespace package is left to Python's own machinery, which merges its portions;
+ the modules under it are guarded as they are imported.
+ """
+
+ def __init__(self, *, tree: MethodTree, identity: TreePythonIdentity):
+ self._tree = tree
+ self._root = identity.root
+ self._identity = identity
+
+ @property
+ def root(self) -> Path:
+ return self._root
+
+ @override
+ def find_spec(
+ self,
+ fullname: str,
+ path: Sequence[str] | None,
+ target: ModuleType | None = None,
+ ) -> importlib.machinery.ModuleSpec | None:
+ search_entries: list[str]
+ if path is None:
+ search_entries = [str(self._root)]
+ else:
+ search_entries = [entry for entry in path if _is_under(path=Path(entry), directory=self._root)]
+ if not search_entries:
+ return None
+ spec = importlib.machinery.PathFinder.find_spec(fullname, search_entries)
+ if spec is None or spec.origin is None or not spec.has_location:
+ return None
+ location = Path(spec.origin).resolve()
+ if not location.is_relative_to(self._root):
+ return None
+ tree_path = nfc(location.relative_to(self._root).as_posix())
+ if tree_path not in self._tree.members:
+ log.debug(f"The import of '{fullname}' reached {location}, which the method tree at {self._root} keeps out")
+ msg = (
+ f"The import of '{fullname}' reached the file {tree_path} of {self._identity.describe()}, "
+ f"but the tree keeps that file out: {self._exclusion_reason(tree_path=tree_path)}. "
+ "A method's Python imports only the files of its own tree."
+ )
+ raise TreeModuleExcludedError(msg, module_name=fullname, tree_path=tree_path)
+ if isinstance(spec.loader, importlib.machinery.SourceFileLoader):
+ spec.loader = _TreeSourceLoader(fullname, str(location))
+ return spec
+
+ def _exclusion_reason(self, *, tree_path: str) -> str:
+ """The rule that keeps a tree path out of the tree, in words."""
+ exclusion = self._tree.excluded_files.get(tree_path)
+ if exclusion is None:
+ matching_directories = [directory for directory in self._tree.excluded_directories if tree_path.startswith(f"{directory}/")]
+ if matching_directories:
+ exclusion = self._tree.excluded_directories[max(matching_directories, key=len)]
+ if exclusion is None:
+ return "it is not a member of the tree as it was listed, which happens to a file created since"
+ return f"the membership rule excludes it, for {exclusion.rule.description}"
+
+
+class _LoadedTree(NamedTuple):
+ """The tree whose Python the process imports, held or not, as its load listed it."""
+
+ identity: TreePythonIdentity
+ tree: MethodTree
+
+
+class TreePythonHold:
+ """Which tree's Python this process imports, and which runs hold it; one per process (`TREE_PYTHON_HOLD`)."""
+
+ def __init__(self) -> None:
+ self._lock = threading.Lock()
+ self._loaded: _LoadedTree | None = None
+ self._holders: set[str] = set()
+ self._guard: TreeModuleGuard | None = None
+ # The entry the hold put first on `sys.path`, while it is there.
+ self._path_entry: str | None = None
+
+ def loaded_roots(self) -> list[Path]:
+ """The root of the tree whose Python is loaded, held or not, which the next other tree's load evicts."""
+ loaded = self._loaded
+ return [loaded.identity.root] if loaded is not None else []
+
+ def acquire(self, *, tree: MethodTree, holder: str, label: str) -> TreePythonIdentity:
+ """Hold the tree's Python for `holder`, loading it first when another tree's is loaded and no run holds it.
+
+ Loading checks the tree's top-level module names, evicts the earlier tree's modules, puts this tree's root first
+ on `sys.path` and installs the import guard first on `sys.meta_path`; taking the hold of a loaded tree no run
+ holds puts its root and its guard back.
+
+ Args:
+ tree: The tree, listed from a directory of this host.
+ holder: Who holds it: the library of the run, released when that library is torn down.
+ label: How messages name the tree (`tree_label_holding`).
+
+ Returns:
+ The identity of the held tree.
+
+ Raises:
+ TreePythonHeldError: Another tree's Python is held by a run in flight.
+ TreeModuleShadowingError: A top-level module of the tree would shadow another module.
+ TreePythonError: The tree has no directory of Python on this host.
+ """
+ identity = tree_python_identity(tree=tree, label=label)
+ with self._lock:
+ loaded = self._loaded
+ if loaded is not None and loaded.identity == identity:
+ if not self._holders:
+ self._put_root_on_path(root=identity.root)
+ self._install_guard(tree=tree, identity=identity)
+ importlib.invalidate_caches()
+ self._holders.add(holder)
+ return identity
+ if loaded is not None and self._holders:
+ log.debug(f"The method tree at {identity.root} cannot load while the one at {loaded.identity.root} is held")
+ msg = (
+ f"The Python of {identity.describe()} cannot load while a run holds the Python of {loaded.identity.describe()}: "
+ "a process imports one method tree's Python at a time, since the modules of two trees share one import "
+ "namespace. Run the two methods one after the other, or in separate processes."
+ )
+ raise TreePythonHeldError(msg)
+ check_tree_module_names(
+ top_level_modules=top_level_module_names(tree_paths=tree.members),
+ tree_label=identity.describe(),
+ root=identity.root,
+ checks_this_process=True,
+ )
+ if loaded is not None:
+ self._evict(loaded=loaded)
+ log.debug(f"The Python of the method tree at {identity.root} loads in this process")
+ self._loaded = _LoadedTree(identity=identity, tree=tree)
+ self._put_root_on_path(root=identity.root)
+ self._install_guard(tree=tree, identity=identity)
+ importlib.invalidate_caches()
+ self._holders.add(holder)
+ return identity
+
+ def release(self, *, holder: str) -> None:
+ """Release the hold `holder` took, if any.
+
+ Once no run holds the tree, its root and the entries introduced beneath it leave `sys.path` and its guard is
+ removed, so nothing imports a module of the tree by name between runs; the modules already imported stay loaded
+ until another tree's load evicts them.
+ """
+ with self._lock:
+ if holder not in self._holders:
+ return
+ self._holders.discard(holder)
+ if not self._holders:
+ self._unguard()
+
+ def release_all(self) -> None:
+ """Release every hold, as the process tears its libraries down."""
+ with self._lock:
+ self._holders.clear()
+ self._unguard()
+
+ def unload_if_unheld(self) -> None:
+ """Evict the loaded tree's modules and root now, if no run holds it, as before its directory is removed."""
+ with self._lock:
+ loaded = self._loaded
+ if loaded is None or self._holders:
+ return
+ self._evict(loaded=loaded)
+ self._loaded = None
+
+ def is_held(self, *, identity: TreePythonIdentity) -> bool:
+ """Whether a run holds this tree's Python."""
+ loaded = self._loaded
+ return loaded is not None and loaded.identity == identity and bool(self._holders)
+
+ def _install_guard(self, *, tree: MethodTree, identity: TreePythonIdentity) -> None:
+ self._remove_guard()
+ guard = TreeModuleGuard(tree=tree, identity=identity)
+ sys.meta_path.insert(0, guard)
+ self._guard = guard
+
+ def _remove_guard(self) -> None:
+ guard = self._guard
+ if guard is None:
+ return
+ sys.meta_path[:] = [finder for finder in sys.meta_path if finder is not guard]
+ self._guard = None
+
+ def _put_root_on_path(self, *, root: Path) -> None:
+ if self._path_entry is not None:
+ return
+ self._path_entry = str(root)
+ sys.path.insert(0, self._path_entry)
+
+ def _unguard(self) -> None:
+ """Remove the guard, and take the loaded tree's root and the entries introduced beneath it off `sys.path`."""
+ self._remove_guard()
+ loaded = self._loaded
+ if loaded is not None:
+ self._take_root_off_path(root=loaded.identity.root)
+
+ def _take_root_off_path(self, *, root: Path) -> None:
+ """Take off `sys.path` the root the hold put there, and every entry a module import introduced beneath it.
+
+ An entry the process held before is left, such as a working directory that is the tree's root.
+ """
+ path_entry = self._path_entry
+ self._path_entry = None
+ if path_entry is not None and path_entry in sys.path:
+ sys.path.remove(path_entry)
+ drop_module_dirs_inserted_under(directory=root)
+ importlib.invalidate_caches()
+
+ def _evict(self, *, loaded: _LoadedTree) -> None:
+ """Remove from `sys.modules` every module imported from a file of the tree, whatever its loader, and its root from `sys.path`.
+
+ A module is the tree's when its file is a member of the tree, or, for a namespace package, when each of its
+ portions is a directory holding members. A file under the root that the tree keeps out, such as one of a virtual
+ environment there, is not the tree's, and its module stays. The `__main__` module stays whatever its file.
+ """
+ root = loaded.identity.root
+ member_directories = _member_directories(tree=loaded.tree)
+ for module_name, module in list(sys.modules.items()):
+ if module_name == _MAIN_MODULE:
+ continue
+ if _is_tree_module(module=module, root=root, members=loaded.tree.members, member_directories=member_directories):
+ del sys.modules[module_name]
+ self._take_root_off_path(root=root)
+
+
+def _member_directories(*, tree: MethodTree) -> set[str]:
+ """The tree paths of the directories holding a member of the tree, at any depth."""
+ directories: set[str] = set()
+ for member in tree.members:
+ segments = member.split("/")[:-1]
+ directories.update("/".join(segments[:depth]) for depth in range(1, len(segments) + 1))
+ return directories
+
+
+def _tree_path_of(*, location: str, root: Path) -> str | None:
+ """The tree path of a location on this host under the tree's root, or None when it lies elsewhere."""
+ try:
+ resolved = Path(location).resolve()
+ except (OSError, RuntimeError):
+ return None
+ if not resolved.is_relative_to(root):
+ return None
+ return nfc(resolved.relative_to(root).as_posix())
+
+
+def _is_tree_module(*, module: ModuleType, root: Path, members: Collection[str], member_directories: Collection[str]) -> bool:
+ """Whether a loaded module was imported from a file of the tree, or is a namespace package of its directories only.
+
+ The module's own namespace is read, never its attributes, so a lazy module's hooks do not run.
+ """
+ # An object some library put in `sys.modules` in place of a module may have no namespace of its own.
+ namespace: object = getattr(module, "__dict__", None)
+ if not isinstance(namespace, dict):
+ return False
+ module_namespace = cast("dict[str, object]", namespace)
+ module_file = module_namespace.get("__file__")
+ if isinstance(module_file, str):
+ return _tree_path_of(location=module_file, root=root) in members
+ module_path = module_namespace.get("__path__")
+ if not isinstance(module_path, Iterable):
+ return False
+ portions = [portion for portion in cast("Iterable[object]", module_path) if isinstance(portion, str)]
+ return bool(portions) and all(_tree_path_of(location=portion, root=root) in member_directories for portion in portions)
+
+
+#: The process's one hold on a tree's Python.
+TREE_PYTHON_HOLD = TreePythonHold()
+
+
+def import_held_tree_module(*, identity: TreePythonIdentity, module_name: str) -> ModuleType:
+ """Import a module of the held tree under its real name, and check that the import reached the tree.
+
+ Args:
+ identity: The held tree.
+ module_name: The module's dotted import name.
+
+ Returns:
+ The imported module.
+
+ Raises:
+ TreePythonError: The tree is not held, or the import resolved to a module outside the tree, such as one
+ imported under the same name before the tree loaded.
+ ImportError: The module, or one it imports, cannot be imported; `TreeModuleExcludedError` for a file the tree
+ excludes.
+ """
+ if not TREE_PYTHON_HOLD.is_held(identity=identity):
+ msg = f"The Python of {identity.describe()} is not held, so its module '{module_name}' cannot be imported."
+ raise TreePythonError(msg)
+ module = importlib.import_module(module_name)
+ location = _module_location(module=module)
+ if location is None or not _is_under(path=location, directory=identity.root):
+ log.debug(f"The module '{module_name}' resolved to {location or 'a module with no file'}, outside the method tree at {identity.root}")
+ msg = (
+ f"The module '{module_name}' resolved outside {identity.describe()}: a module of that name was imported in this "
+ "process before the tree's Python loaded."
+ )
+ raise TreePythonError(msg)
+ return module
diff --git a/pipelex/kernel/func_ops.py b/pipelex/kernel/func_ops.py
index 97c68c977..4a1f8cf0d 100644
--- a/pipelex/kernel/func_ops.py
+++ b/pipelex/kernel/func_ops.py
@@ -1,11 +1,13 @@
-"""Function-call operator semantics: registry lookup, invocation, return coercion, memory write-back.
+"""Function-call operator semantics: invocation, return coercion, memory write-back.
These are the functions the interpreter's default `PipeFunc` executor calls, and the ones a
programmatic caller invokes on a `RuntimeBoot`-only process. Unlike every other operator's ops module
-this one reads no hub at all: running a registered function touches no inference service, only the
-process-global function registry.
+this one reads no hub at all: running a function touches no inference service, and the function is
+handed in already resolved. Where a `PipeFunc`'s function comes from, an import path from the root of
+its method tree, is the interpreter's business (`pipelex.pipe_operators.func.pipe_func_python`); the
+kernel is handed the callable, as every other op is handed resolved values.
-`call_registered_function` is where the semantics actually live — async-vs-sync dispatch and the
+`call_function` is where the semantics actually live — async-vs-sync dispatch and the
return coercion that lets a function return a bare `str` or a `list` and still produce one
`StuffContent` shape downstream. It is deliberately separate from `run_func`, because the
interpreter's out-of-process executors (a sandbox, a Temporal activity) replace *the call* and keep
@@ -18,7 +20,8 @@ configured deployment machinery, and its protocol is typed on interpreter models
import asyncio
import inspect
-from typing import cast
+from collections.abc import Callable
+from typing import Any, cast
from pipelex.core.concepts.concept import Concept
from pipelex.core.memory.working_memory import WorkingMemory
@@ -27,11 +30,10 @@ from pipelex.core.stuffs.stuff_content import StuffContent
from pipelex.core.stuffs.text_content import TextContent
from pipelex.kernel.func_results import FuncCallResult, FuncResult
from pipelex.kernel.memory_ops import store_result
-from pipelex.system.registries.func_registry import func_registry
-async def call_registered_function(*, function_name: str, memory: WorkingMemory) -> FuncCallResult:
- """Resolve a registered function by name, run it against the memory, and coerce what it returns.
+async def call_function(*, function: Callable[..., Any], memory: WorkingMemory, function_name: str | None = None) -> FuncCallResult:
+ """Run a function against the memory, and coerce what it returns.
The function is arbitrary user code and its failure surface is not enumerable, so nothing is
caught here: a caller that can add context to a failure (the interpreter's operator decorates it
@@ -40,8 +42,13 @@ async def call_registered_function(*, function_name: str, memory: WorkingMemory)
A sync function runs on a worker thread so it cannot block the event loop. The coercion accepts
the three shapes the language allows an author to return — a `StuffContent`, a list of them, or a
bare `str` — and normalizes them so every caller downstream sees one shape.
+
+ Args:
+ function: The function, called with the memory as its `working_memory` keyword.
+ memory: The working memory the function reads.
+ function_name: The name the step calls the function by, for messages; its qualified name when not given.
"""
- function = func_registry.get_required_function(function_name)
+ label = function_name or getattr(function, "__qualname__", repr(function))
if inspect.iscoroutinefunction(function):
func_output_object = await function(working_memory=memory)
@@ -57,30 +64,31 @@ async def call_registered_function(*, function_name: str, memory: WorkingMemory)
elif isinstance(func_output_object, str):
content = TextContent(text=func_output_object)
else:
- msg = f"Function '{function_name}' must return a StuffContent or a list, got {type(func_output_object)}"
+ msg = f"Function '{label}' must return a StuffContent or a list, got {type(func_output_object)}"
raise TypeError(msg)
return FuncCallResult(
content=content,
function_module=getattr(function, "__module__", None),
- function_qualname=getattr(function, "__qualname__", function_name),
+ function_qualname=getattr(function, "__qualname__", label),
)
async def run_func(
*,
memory: WorkingMemory,
- function_name: str,
+ function: Callable[..., Any],
concept: Concept,
result_name: str | None = None,
result_code: str | None = None,
) -> FuncResult:
- """A whole func step, run in this process: call the registered function and store what it returned.
+ """A whole func step, run in this process: call the function and store what it returned.
- Takes the output `concept` rather than deriving one, for the reason every other op does: turning
- a concept into anything is a library's business, and the kernel is handed the resolved value.
+ Takes the `function` and the output `concept` rather than deriving them, for the reason every other
+ op does: resolving a function or turning a concept into anything is a library's business, and the
+ kernel is handed the resolved values.
"""
- call_result = await call_registered_function(function_name=function_name, memory=memory)
+ call_result = await call_function(function=function, memory=memory)
return FuncResult(
memory=store_result(memory=memory, concept=concept, content=call_result.content, result_name=result_name, result_code=result_code),
content=call_result.content,
diff --git a/pipelex/libraries/library_crate.py b/pipelex/libraries/library_crate.py
index faf85a89a..34d60b496 100644
--- a/pipelex/libraries/library_crate.py
+++ b/pipelex/libraries/library_crate.py
@@ -8,9 +8,14 @@ from pipelex.core.bundle_resources.bundle_resource import BundleResources
from pipelex.core.concepts.concept_blueprint import ConceptBlueprint
from pipelex.core.domains.domain_blueprint import DomainBlueprint
from pipelex.core.method_trees.tree_bundle import TreeBundle, TreeBundleRef
+from pipelex.core.method_trees.tree_files import TreeFileEntry
from pipelex.libraries.exceptions import CrateIntegrityError
from pipelex.mthds_parsing.pipelex_bundle_blueprint import ElaborationMetadata, PipeBlueprintUnion
+#: The Python root of the steps whose bundles were parsed from content strings, which have no method tree: the files of
+#: the library directories loaded beside them, as a sandbox-hosted deployment captures them.
+SUBMITTED_PYTHON_ROOT_KEY = "submitted"
+
class LibraryCrate(BaseModel):
"""Complete library content as qualified blueprints, ready to load into a live Library.
@@ -44,14 +49,17 @@ class LibraryCrate(BaseModel):
``preliminary_text`` PipeLLM), so a refusal raised while building a helper is reported on the pipe
and the field the author wrote. Like ``source_map``, it is excluded from the fingerprint."""
- python_sources: dict[str, str] = Field(default_factory=dict)
- """relpath (within the library dir) -> Python source text, captured WITHOUT importing.
-
- Carries the customer's PipeFunc and structure-class .py source alongside the method so it can
- travel to a sandbox and be registered + executed there. Populated only in sandbox-hosted load
- mode; empty for local/direct loads. Deliberately EXCLUDED from the fingerprint (see
- compute_fingerprint_from_content): the fingerprint represents library structure for dedupe, and
- folding source into it would break the structural-idempotency contract in load_from_crate.
+ pipe_func_files: dict[str, dict[str, TreeFileEntry]] = Field(default_factory=dict)
+ """Python root -> the files a `PipeFunc` call of that root receives, by tree path, captured WITHOUT importing.
+
+ A root is the key of a method tree (`tree_key_of`) for a step whose bundle was read from a tree, holding every file
+ of that tree except its `.mthds` files, or `SUBMITTED_PYTHON_ROOT_KEY` for the steps of bundles parsed from content
+ strings, holding the files of the library directories loaded beside them, keyed by their path there. Each file is a
+ file entry of the standard: its text, or `{"base64": ...}`. A call carries its own root's files to the sandbox that
+ imports and runs the function (`PipeFuncExecutionRequest.files`). Populated only in sandbox-hosted load mode;
+ empty for local/direct loads, which import from the disk. Deliberately EXCLUDED from the fingerprint (see
+ compute_fingerprint_from_content): the fingerprint represents library structure for dedupe, and folding files into
+ it would break the structural-idempotency contract in load_from_crate.
"""
bundle_resources: BundleResources = Field(default_factory=dict)
@@ -61,7 +69,7 @@ class LibraryCrate(BaseModel):
crate is loaded again, a Temporal workflow included: a file read when the method loads travels inline, and one
read only when its step runs may travel by reference to the run's storage (see `relocate_run_resources`).
Empty for a library loaded from directories, whose steps read beside their files on disk. Like
- `python_sources`, it is transport-only: EXCLUDED from the fingerprint and from the normalized crate.
+ `pipe_func_files`, it is transport-only: EXCLUDED from the fingerprint and from the normalized crate.
"""
method_trees: dict[str, str] = Field(default_factory=dict)
@@ -72,7 +80,7 @@ class LibraryCrate(BaseModel):
A pipe reads a file its bundle names, such as a PipeDocGen `template_file`, only by resolving it in its bundle's
tree (`tree_bundles`), never through `source_map`: a source is a label, which a caller sends with content strings
and a bundle's own text can write, so a pipe loaded from content strings has no tree and reads only the files
- submitted with it (`bundle_resources`). Transport-only, like `python_sources`: EXCLUDED from the fingerprint and
+ submitted with it (`bundle_resources`). Transport-only, like `pipe_func_files`: EXCLUDED from the fingerprint and
from the normalized crate, so a host path never reaches a caller through `/resolve`.
"""
diff --git a/pipelex/libraries/library_crate_factory.py b/pipelex/libraries/library_crate_factory.py
index 6a70b1311..d895b9486 100644
--- a/pipelex/libraries/library_crate_factory.py
+++ b/pipelex/libraries/library_crate_factory.py
@@ -4,6 +4,7 @@ from pipelex.core.bundle_resources.bundle_resource import BundleResources
from pipelex.core.concepts.concept_factory import ConceptFactory
from pipelex.core.domains.domain_blueprint import DomainBlueprint
from pipelex.core.method_trees.tree_bundle import TreeBundleRef
+from pipelex.core.method_trees.tree_files import TreeFileEntry
from pipelex.libraries.collision_messages import duplicate_ref_msg
from pipelex.libraries.concept.exceptions import ConceptLibraryError
from pipelex.libraries.contract_match import contracts_match
@@ -38,7 +39,7 @@ class LibraryCrateFactory:
cls,
blueprints: list[PipelexBundleBlueprint],
*,
- python_sources: dict[str, str] | None = None,
+ pipe_func_files: dict[str, dict[str, TreeFileEntry]] | None = None,
bundle_resources: BundleResources | None = None,
) -> LibraryCrate:
"""Build a LibraryCrate from parsed bundle blueprints.
@@ -54,11 +55,11 @@ class LibraryCrateFactory:
Args:
blueprints: List of parsed MTHDS bundle blueprints
- python_sources: Optional relpath -> Python source text captured (without importing) in
- sandbox-hosted load mode. Attached verbatim to the crate; intentionally NOT folded
+ pipe_func_files: Optional Python root -> the files a PipeFunc call of that root receives, captured (without
+ importing) in sandbox-hosted load mode. Attached verbatim to the crate; intentionally NOT folded
into the fingerprint (the fingerprint represents library structure for dedupe).
bundle_resources: Optional canonical path -> file submitted with a bundle loaded from content strings.
- Attached verbatim to the crate and, like ``python_sources``, NOT folded into the fingerprint.
+ Attached verbatim to the crate and, like ``pipe_func_files``, NOT folded into the fingerprint.
Returns:
A LibraryCrate with all content merged and fingerprinted
@@ -191,7 +192,7 @@ class LibraryCrateFactory:
method_trees={tree_key: method_trees[tree_key].model_dump_json() for tree_key in sorted({ref.tree_key for ref in tree_bundles.values()})},
tree_bundles=tree_bundles,
elaboration_metadata=elaboration_metadata,
- python_sources=python_sources or {},
+ pipe_func_files={root_key: dict(files) for root_key, files in (pipe_func_files or {}).items()},
bundle_resources=dict(bundle_resources) if bundle_resources else {},
fingerprint=fingerprint,
)
diff --git a/pipelex/libraries/library_manager.py b/pipelex/libraries/library_manager.py
index 81b893d06..05c7d84b7 100644
--- a/pipelex/libraries/library_manager.py
+++ b/pipelex/libraries/library_manager.py
@@ -7,7 +7,7 @@ from typing import TYPE_CHECKING, Any, ForwardRef, Literal, get_args, get_origin
from kajson.class_registry import ClassRegistry
from kajson.kajson_manager import KajsonManager
-from mthds.method_tree.models import TreeLimits
+from mthds.method_tree.models import MethodTree, TreeLimits
from mthds.package.dependency_resolver import ResolvedDependency, determine_exported_pipes, resolve_all_dependencies
from mthds.package.discovery import find_package_manifest
from mthds.package.exceptions import DependencyResolveError, ManifestError
@@ -28,7 +28,10 @@ from pipelex.core.domains.domain_blueprint import DomainBlueprint
from pipelex.core.domains.domain_factory import DomainFactory
from pipelex.core.exceptions import PipelexBundleBlueprintValidationErrorData, PipesAndConceptValidationErrorData
from pipelex.core.method_trees.building_bundle import building_pipe_of
-from pipelex.core.pipes.exceptions import PipeLoadRefusalError, PipeOperatorModelChoiceError
+from pipelex.core.method_trees.tree_bundle import tree_key_of
+from pipelex.core.method_trees.tree_files import TreeFileEntry, encode_tree_file
+from pipelex.core.method_trees.tree_python import TREE_PYTHON_HOLD, sharing_module_name_checks
+from pipelex.core.pipes.exceptions import PipeLoadRefusalError, PipeOperatorModelChoiceError, PipeValidationError
from pipelex.core.qualified_ref import QualifiedRef
from pipelex.core.stuffs.structured_content import StructuredContent
from pipelex.core.validation import report_validation_error
@@ -42,25 +45,29 @@ from pipelex.libraries.exceptions import (
LibraryLoadingError,
)
from pipelex.libraries.library import Library
-from pipelex.libraries.library_crate import LibraryCrate
+from pipelex.libraries.library_crate import SUBMITTED_PYTHON_ROOT_KEY, LibraryCrate
from pipelex.libraries.library_crate_factory import LibraryCrateFactory
from pipelex.libraries.library_factory import LibraryFactory
from pipelex.libraries.library_manager_abstract import LibraryManagerAbstract
from pipelex.libraries.method_requirements import ensure_method_requirements_met, ensure_package_requirements_met
from pipelex.libraries.method_trees import (
LibraryBundleFile,
+ LibraryTree,
library_bundle_files_of,
list_library_dir_trees,
list_package_method_tree,
local_tree_limits,
+ submitted_python_files,
tree_bundle_of_lone_file,
tree_bundles_of,
+ tree_python_sources,
)
from pipelex.libraries.pipe.exceptions import PipeLibraryError
+from pipelex.libraries.pipe_func_call_files import holds_pipe_func_step, travels_with_call, tree_pipe_func_files
from pipelex.libraries.visibility_utils import check_visibility_for_blueprints, make_visibility_checker
from pipelex.methods.exceptions import MethodDependencyFetchError, MethodFetchDisabledError
from pipelex.methods.fetch_on_miss import find_vendored_method, resolve_address_based_method
-from pipelex.methods.structures_check import ensure_no_structured_content_in_library_sources, is_generated_structures_module
+from pipelex.methods.structures_check import ensure_no_structured_content_in_library_sources
from pipelex.mthds_parsing.exceptions import MthdsParserError
from pipelex.mthds_parsing.handle_pipe_errors import categorize_pipe_validation_error
from pipelex.mthds_parsing.parser import MthdsParser
@@ -75,7 +82,6 @@ from pipelex.pipe_machinery.pipe_abstract import PipeAbstract
from pipelex.pipe_machinery.pipe_factory import PipeFactory
from pipelex.runtime_hub import get_class_registry
from pipelex.system.registries.class_registry_utils import ClassRegistryUtils
-from pipelex.system.registries.func_registry_utils import FuncRegistryUtils
from pipelex.tools.log.error_fields import error_fields
from pipelex.tools.log.log_fields import USER_ACTION_FIELD
from pipelex.tools.misc.semver import SemVerError, parse_constraint, parse_version, version_satisfies
@@ -322,6 +328,25 @@ def _locating_pipe_build_refusals(
) from refusal
+def _python_sources_by_tree(*, library_trees: list[LibraryTree], bundle_files: list[LibraryBundleFile]) -> dict[Path, dict[str, str]]:
+ """The Python sources of every tree a load lists, each file keyed by its path from its library directory, grouped by tree root.
+
+ The library directories' trees come first, nested ones included, then the tree of each bundle file named alone; a
+ tree two of them list is read once.
+ """
+ trees: dict[str, tuple[MethodTree, Path | None]] = {}
+ for library_tree in library_trees:
+ trees.setdefault(tree_key_of(tree=library_tree.tree), (library_tree.tree, library_tree.library_dir))
+ for bundle_file in bundle_files:
+ trees.setdefault(bundle_file.tree_bundle.tree_key, (bundle_file.tree_bundle.tree, None))
+ sources_by_tree: dict[Path, dict[str, str]] = {}
+ for tree, library_dir in trees.values():
+ if tree.directory is None:
+ continue
+ sources_by_tree.setdefault(tree.directory, {}).update(tree_python_sources(tree=tree, library_dir=library_dir))
+ return sources_by_tree
+
+
class LibraryManager(LibraryManagerAbstract):
def __init__(self):
# UNTITLED library is the fallback library for all others
@@ -330,11 +355,11 @@ class LibraryManager(LibraryManagerAbstract):
self._blueprints: dict[str, list[PipelexBundleBlueprint]] = {} # library_id -> accumulated blueprints
self._crate_cache: dict[str, LibraryCrate] = {} # library_id -> cached crate from get_crate()
self._loaded_fingerprints: dict[str, set[str]] = {} # library_id -> set of loaded crate fingerprints
- # library_id -> {relpath: source}. Customer Python source captured (without importing) in
- # sandbox-hosted load mode, threaded into get_crate() so the crate can carry it to a sandbox.
- # Empty in local/direct mode. Per-library side-state because get_crate() rebuilds from
- # blueprints only and cannot see the original .py text otherwise.
- self._library_sources: dict[str, dict[str, str]] = {}
+ # library_id -> Python root -> {tree path: file entry}. The files a sandbox-hosted PipeFunc call receives,
+ # captured (without importing) from the library directories in sandbox-hosted load mode, or held from a
+ # transported crate, threaded into get_crate() so the crate carries them to a sandbox, and read by a
+ # PipeFunc's validation in place of a disk that is another host's. Empty in local/direct mode.
+ self._pipe_func_files: dict[str, dict[str, dict[str, TreeFileEntry]]] = {}
############################################################
# Manager lifecycle
@@ -370,7 +395,9 @@ class LibraryManager(LibraryManagerAbstract):
self._blueprints.pop(library_id, None)
self._crate_cache.pop(library_id, None)
self._loaded_fingerprints.pop(library_id, None)
- self._library_sources.pop(library_id, None)
+ self._pipe_func_files.pop(library_id, None)
+ # The library's runs are over: its hold on a method tree's Python, if a PipeFunc step took one, is released.
+ TREE_PYTHON_HOLD.release(holder=library_id)
return True
@override
@@ -392,7 +419,8 @@ class LibraryManager(LibraryManagerAbstract):
self._blueprints = {}
self._crate_cache = {}
self._loaded_fingerprints = {}
- self._library_sources = {}
+ self._pipe_func_files = {}
+ TREE_PYTHON_HOLD.release_all()
with ExitStack() as teardown_stack:
for library in libraries:
teardown_stack.callback(library.teardown)
@@ -539,14 +567,20 @@ class LibraryManager(LibraryManagerAbstract):
if not accumulated:
return None
library = self._libraries.get(library_id)
+ # The files a sandbox-hosted PipeFunc call receives were read as the library loaded: building the crate reads no
+ # file, so it builds after the directories the library was read from are gone.
crate = LibraryCrateFactory.make_from_blueprints(
blueprints=accumulated,
- python_sources=self._library_sources.get(library_id),
+ pipe_func_files=self._pipe_func_files.get(library_id),
bundle_resources=library.bundle_resources if library is not None else None,
)
self._crate_cache[library_id] = crate
return crate
+ @override
+ def get_pipe_func_files(self, *, library_id: str, root_key: str) -> dict[str, TreeFileEntry] | None:
+ return self._pipe_func_files.get(library_id, {}).get(root_key)
+
@override
def add_bundle_resources(self, *, library_id: str, bundle_resources: BundleResources) -> None:
if not bundle_resources:
@@ -622,43 +656,28 @@ class LibraryManager(LibraryManagerAbstract):
if is_pipe_func_sandbox_hosted():
# Sandbox-hosted mode imports no Python from a library directory, whatever its origin (an
# inline bundle, a stored method, a host directory): importing a file runs its module-level
- # code in this process. Every .py is captured as source text instead, to travel to the
- # sandbox, where it is imported and executed. A structure class cannot travel that way: it
- # would have to be imported here to back a concept, so the load is refused, naming each file
- # and class, before any source is kept. The refusal is a static scan, which a dynamically
- # built class escapes; such a file then travels as source and still never executes here.
- sources_by_dir = {library_dir: FuncRegistryUtils.read_py_sources(folder_path=library_dir) for library_dir in all_dirs}
- ensure_no_structured_content_in_library_sources(sources_by_dir=sources_by_dir)
- # Accumulate across dirs, but fail loud on a relpath collision: the sandbox writes
- # sources flat by relpath, so two dirs sharing a path would otherwise silently clobber one
- # customer's code and run the wrong PipeFunc body.
- captured_sources = self._library_sources.setdefault(library_id, {})
- for dir_sources in sources_by_dir.values():
- for relpath, source in dir_sources.items():
- if is_generated_structures_module(source=source):
- # The refusal accepted it as a copy of the method's own concepts, and the sandbox
- # generates its own from those concepts, qualified. Shipping this one too would give
- # the sandbox two `structures` modules, and a PipeFunc's import would reach either.
- continue
- if relpath in captured_sources and captured_sources[relpath] != source:
- msg = (
- f"Duplicate PipeFunc source path '{relpath}' across library dirs while loading library "
- f"'{library_id}'. Sandbox sources are keyed by relative path, so two dirs sharing a path "
- f"would clobber each other — give them distinct relative paths."
- )
- raise LibraryError(msg)
- captured_sources[relpath] = source
+ # code in this process. A structure class cannot travel to a sandbox: it would have to be
+ # imported here to back a concept, so the load is refused, naming each file and class, before
+ # any file is kept. Every tree the load lists is scanned, a package or repository nested in a
+ # library directory included. The refusal is a static scan, which a dynamically built class
+ # escapes; such a file then travels as source and still never executes here.
+ ensure_no_structured_content_in_library_sources(
+ sources_by_dir=_python_sources_by_tree(library_trees=library_trees, bundle_files=all_bundle_files),
+ )
+ # The files a step of a tree's bundle calls with are captured once its blueprints load, per tree;
+ # those of a library directory whose own tree holds no bundle are captured here, to travel with each
+ # call of a PipeFunc parsed from a content string, whose import paths start at that directory.
+ self._capture_submitted_python(library_id=library_id, library_trees=library_trees)
else:
for library_dir in all_dirs:
# Only import files that contain StructuredContent subclasses (uses AST pre-check), so
# concepts declared as `structure = "ClassName"` resolve that class from the registry.
+ # A PipeFunc's function is not imported here: its step imports it by its import path when
+ # it first runs.
ClassRegistryUtils.import_modules_in_folder(
folder_path=library_dir,
base_class_names=[StructuredContent.__name__],
)
- # Import files that contain @pipe_func decorated functions (uses AST pre-check) and
- # register them in the process-global func_registry.
- FuncRegistryUtils.register_funcs_in_folder(folder_path=library_dir)
# Auto-discover and register all StructuredContent classes from sys.modules
num_registered = ClassRegistryUtils.auto_register_all_subclasses(
@@ -738,10 +757,10 @@ class LibraryManager(LibraryManagerAbstract):
# Bind the target library as current for the whole load: PipeFactory and the concept
# factories resolve concepts and the class registry through the ambient current library,
# so a load into a non-current library must not read the caller's binding.
- with scoped_current_library(library_id=library_id):
+ with scoped_current_library(library_id=library_id), sharing_module_name_checks():
# Cache the crate for a DIRECT crate-load (the transported-crate path used by the Temporal
# workflow, which never populates _blueprints). This lets get_crate() hand the crate — with
- # its python_sources — to the sandbox executor. The blueprint-derived path (load_libraries ->
+ # its pipe_func_files — to the sandbox executor. The blueprint-derived path (load_libraries ->
# load_from_blueprints) leaves _blueprints populated and rebuilds from it instead, so it must
# NOT be shadowed by a cached crate here.
if library_id not in self._blueprints:
@@ -752,6 +771,13 @@ class LibraryManager(LibraryManagerAbstract):
if crate.bundle_resources:
self.get_library(library_id=library_id).add_bundle_resources(bundle_resources=crate.bundle_resources)
+ # The files a transported crate carries for its PipeFunc calls, held before the pipes are validated: a
+ # PipeFunc's validation reads its module from them, the trees' directories being the host's that listed them.
+ if crate.pipe_func_files:
+ held_files = self._pipe_func_files.setdefault(library_id, {})
+ for root_key, root_files in crate.pipe_func_files.items():
+ held_files.setdefault(root_key, dict(root_files))
+
# Fingerprint idempotency: skip if this crate was already loaded into this library
fingerprint = crate.fingerprint
loaded_set = self._loaded_fingerprints.setdefault(library_id, set())
@@ -877,7 +903,7 @@ class LibraryManager(LibraryManagerAbstract):
# Bind the target library as current for the whole load: the factories beneath
# (concept resolution, class-registry access) resolve through the ambient current
# library, so a load into a non-current library must not read the caller's binding.
- with scoped_current_library(library_id=library_id):
+ with scoped_current_library(library_id=library_id), sharing_module_name_checks():
# A bundle read from a file on this host that no loader placed in a method tree, such as a bundle file a CLI
# names alone, is read as a member of its own file's tree: the steps of every bundle read from this host's
# disk read only the files of their tree.
@@ -895,6 +921,10 @@ class LibraryManager(LibraryManagerAbstract):
trees=[blueprint.tree_bundle.tree for blueprint in blueprints if blueprint.tree_bundle is not None],
names_host_paths=True,
)
+ if is_pipe_func_sandbox_hosted():
+ # Each call of a PipeFunc read from a method tree receives the tree's files in the sandbox: they are read
+ # now, while the directory the tree was listed from is certain to exist, and the crate carries them.
+ self._capture_tree_pipe_func_files(library_id=library_id, blueprints=blueprints)
# Accumulate blueprints for later crate construction via get_crate()
self._blueprints.setdefault(library_id, []).extend(blueprints)
@@ -1085,6 +1115,55 @@ class LibraryManager(LibraryManagerAbstract):
# Private helper methods
############################################################
+ def _capture_submitted_python(self, *, library_id: str, library_trees: list[LibraryTree]) -> None:
+ """Capture the files the library directories submit beside bundles parsed from content strings, as one Python root.
+
+ Only a directory whose own tree holds no bundle submits files (`submitted_python_files`), and the sandbox writes
+ them all at their paths under one root, so two such directories holding different files at one path would
+ clobber each other and run the wrong function: the load fails loud instead.
+
+ Raises:
+ LibraryError: Two directories submit different files at one path.
+ """
+ submitted = dict(self._pipe_func_files.get(library_id, {}).get(SUBMITTED_PYTHON_ROOT_KEY, {}))
+ for library_tree in library_trees:
+ for relpath, data in submitted_python_files(library_tree=library_tree).items():
+ if not travels_with_call(tree_path=relpath, data=data):
+ # The refusal accepted it as a copy of the method's own concepts, and the sandbox generates its own
+ # from those concepts, qualified. Shipping this one too would give the sandbox two `structures`
+ # modules, and a PipeFunc's import would reach either.
+ continue
+ entry = encode_tree_file(data=data)
+ if relpath in submitted and submitted[relpath] != entry:
+ msg = (
+ f"Duplicate PipeFunc file path '{relpath}' across library dirs while loading library '{library_id}'. "
+ "A sandbox receives the files submitted beside a method's bundles at their paths, so two dirs sharing "
+ "a path would clobber each other — give them distinct relative paths."
+ )
+ raise LibraryError(msg)
+ submitted[relpath] = entry
+ if submitted:
+ self._pipe_func_files.setdefault(library_id, {})[SUBMITTED_PYTHON_ROOT_KEY] = submitted
+
+ def _capture_tree_pipe_func_files(self, *, library_id: str, blueprints: list[PipelexBundleBlueprint]) -> None:
+ """Capture, from this host's disk, the files of each method tree holding a `PipeFunc` step of these bundles.
+
+ A tree already captured for the library is not read again, and a tree holding no `PipeFunc` step is never read.
+ """
+ held_files = self._pipe_func_files.get(library_id, {})
+ captured: dict[str, dict[str, TreeFileEntry]] = {}
+ for blueprint in blueprints:
+ tree_bundle = blueprint.tree_bundle
+ if tree_bundle is None or tree_bundle.tree.directory is None or not holds_pipe_func_step(blueprint=blueprint):
+ continue
+ tree_key = tree_bundle.tree_key
+ if tree_key in held_files or tree_key in captured:
+ continue
+ captured[tree_key] = tree_pipe_func_files(tree=tree_bundle.tree)
+ if captured:
+ self._pipe_func_files.setdefault(library_id, {}).update(captured)
+ self._crate_cache.pop(library_id, None)
+
def _load_mthds_files_into_library(self, *, library_id: str, bundle_files: list[LibraryBundleFile]) -> list[PipeAbstract]:
"""Load MTHDS files into a specific library.
@@ -1510,6 +1589,9 @@ class LibraryManager(LibraryManagerAbstract):
blueprint=pipe_blueprint,
concept_codes_from_the_same_domain=domain_concept_codes.get(domain_code, []),
)
+ # What the pipe can tell is broken once built, such as a PipeFunc whose function the package does
+ # not hold, makes it a pipe the package could not build, refusing only a reference to it.
+ pipe.validate_once_built()
child_library.pipe_library.add_new_pipe(pipe=pipe)
except ValidationError as exc:
log.warning(
@@ -1517,6 +1599,12 @@ class LibraryManager(LibraryManagerAbstract):
fields={"dependency_alias": alias, "pipe_code": pipe_code, **error_fields(exc=exc)},
)
unbuilt_pipe_reasons[pipe_ref] = "; ".join(str(error["msg"]) for error in exc.errors()) or str(exc)
+ except PipeValidationError as exc:
+ log.warning(
+ "A pipe of a dependency could not be loaded",
+ fields={"dependency_alias": alias, "pipe_code": pipe_code, **error_fields(exc=exc)},
+ )
+ unbuilt_pipe_reasons[pipe_ref] = str(exc)
finally:
# Remove temporary concept entries from main library
library.concept_library.remove_concepts_by_concept_refs(concept_refs=temp_concept_refs)
diff --git a/pipelex/libraries/library_manager_abstract.py b/pipelex/libraries/library_manager_abstract.py
index 53050a259..432da7250 100644
--- a/pipelex/libraries/library_manager_abstract.py
+++ b/pipelex/libraries/library_manager_abstract.py
@@ -5,6 +5,7 @@ from typing import TYPE_CHECKING
from kajson.class_registry import ClassRegistry
from pipelex.core.bundle_resources.bundle_resource import BundleResources
+from pipelex.core.method_trees.tree_files import TreeFileEntry
from pipelex.libraries.library_crate import LibraryCrate
from pipelex.mthds_parsing.pipelex_bundle_blueprint import PipelexBundleBlueprint
from pipelex.pipe_machinery.pipe_abstract import PipeAbstract
@@ -98,6 +99,18 @@ class LibraryManagerAbstract(ABC):
A LibraryCrate built from all accumulated blueprints, or None
"""
+ @abstractmethod
+ def get_pipe_func_files(self, *, library_id: str, root_key: str) -> dict[str, TreeFileEntry] | None:
+ """The files a library captured for the `PipeFunc` calls of one Python root, or None when it captured none.
+
+ A sandbox-hosted load captures them from the library directories, and a transported crate carries them, so a
+ `PipeFunc`'s validation reads its module there rather than from a disk that may be another host's.
+
+ Args:
+ library_id: The library.
+ root_key: The Python root: a method tree's key, or `SUBMITTED_PYTHON_ROOT_KEY` for bundles parsed from strings.
+ """
+
@abstractmethod
def add_bundle_resources(self, *, library_id: str, bundle_resources: BundleResources) -> None:
"""Hold the files submitted with bundles loaded from content strings, for their pipes to read.
diff --git a/pipelex/libraries/method_trees.py b/pipelex/libraries/method_trees.py
index 43a6540cd..4fc67cb0e 100644
--- a/pipelex/libraries/method_trees.py
+++ b/pipelex/libraries/method_trees.py
@@ -476,3 +476,68 @@ def bundle_files_under(*, library_dirs: Iterable[Path]) -> list[Path]:
except OSError:
continue
return list(dict.fromkeys(bundle_files))
+
+
+def _spelling_prefix(*, tree_root: Path, library_dir: Path) -> tuple[str, str] | None:
+ """How a tree path is spelled from a library directory, as `(stripped, added)` prefixes, or None when they share nothing.
+
+ A tree rooted at the directory spells its paths as they are; one nested in it adds the tree's folder; one rooted above
+ it, at a `METHODS.toml`, strips the directory's folder from the paths lying in it.
+ """
+ if tree_root == library_dir:
+ return ("", "")
+ if tree_root.is_relative_to(library_dir):
+ return ("", f"{nfc(tree_root.relative_to(library_dir).as_posix())}/")
+ if library_dir.is_relative_to(tree_root):
+ return (f"{nfc(library_dir.relative_to(tree_root).as_posix())}/", "")
+ return None
+
+
+def tree_python_sources(*, tree: MethodTree, library_dir: Path | None) -> dict[str, str]:
+ """The text of every Python file of a tree listed from this host's disk, keyed by its path from the library directory.
+
+ A file outside the library directory, or of a tree that no library directory holds, such as the tree of a bundle file
+ named alone, is keyed by its tree path: the key names the file to a caller, never by a path of this host. These are
+ the sources a sandbox-hosted load scans for structure classes without importing them, so a file that is not UTF-8
+ text is read with replacement characters.
+ """
+ if tree.directory is None:
+ return {}
+ spelling = _spelling_prefix(tree_root=tree.directory.resolve(), library_dir=library_dir.resolve()) if library_dir is not None else None
+ sources: dict[str, str] = {}
+ for tree_path in tree.tree_paths:
+ if not tree_path.endswith(".py"):
+ continue
+ key = tree_path
+ if spelling is not None and tree_path.startswith(spelling[0]):
+ key = f"{spelling[1]}{tree_path.removeprefix(spelling[0])}"
+ sources[key] = tree.file_location(tree_path).read_bytes().decode("utf-8", errors="replace")
+ return sources
+
+
+def submitted_python_files(*, library_tree: LibraryTree) -> dict[str, bytes]:
+ """The files a library directory submits beside bundles parsed from content strings, keyed by their path from it.
+
+ A directory whose own tree holds no bundle, such as the directory a server writes a request's Python into beside the
+ request's bundles, is where those bundles' import paths start, so its files other than bundles travel with each call
+ of their `PipeFunc` steps. A tree holding a bundle submits nothing: its steps' calls carry their own tree's files, so a
+ directory's files never travel twice, and two such directories never share one import root. Neither does a tree
+ nested in a library directory. The membership rule decides what the tree holds, so a `.gitignore`d file or a virtual
+ environment never travels.
+
+ Args:
+ library_tree: A tree the library directory holds, as `list_library_dir_trees` listed it.
+ """
+ tree = library_tree.tree
+ if tree.directory is None or any(tree_path.endswith(BUNDLE_SUFFIX) for tree_path in tree.tree_paths):
+ return {}
+ spelling = _spelling_prefix(tree_root=tree.directory.resolve(), library_dir=library_tree.library_dir.resolve())
+ if spelling is None or spelling[1]:
+ # Not the directory's own tree: one nested in it.
+ return {}
+ stripped_prefix = spelling[0]
+ files: dict[str, bytes] = {}
+ for tree_path in tree.tree_paths:
+ if tree_path.startswith(stripped_prefix):
+ files[tree_path.removeprefix(stripped_prefix)] = tree.file_location(tree_path).read_bytes()
+ return files
diff --git a/pipelex/libraries/pipe_func_call_files.py b/pipelex/libraries/pipe_func_call_files.py
new file mode 100644
index 000000000..5785d3c10
--- /dev/null
+++ b/pipelex/libraries/pipe_func_call_files.py
@@ -0,0 +1,43 @@
+"""The files a sandbox-hosted `PipeFunc` call receives, captured onto the library crate without importing anything.
+
+A sandbox-hosted deployment never imports a method's Python in its own process: every call to a `PipeFunc` runs in a
+sandbox that receives the files of the step's Python root, every file except the `.mthds` files, at their paths
+(`wip/extra-files/design.md`, "Inside a run"). They are read once, as the library loads, for each method tree holding a
+`PipeFunc` step, and the crate carries them, per root, wherever the run travels, a Temporal workflow included, which may
+read no file itself (`LibraryCrate.pipe_func_files`); a directory the load read them from may be gone by then, such as
+the temporary one a server writes a request's files into.
+
+The generated `structures` module is the one file left behind: the sandbox generates its own from the crate's concepts,
+with the qualified class names the runtime checks a return type against, and a method's own copy beside it would give
+the sandbox two modules of that name.
+"""
+
+from mthds.method_tree.models import MethodTree
+
+from pipelex.core.method_trees.tree_files import TreeFileEntry, read_call_files
+from pipelex.methods.structures_check import is_generated_structures_module
+from pipelex.mthds_parsing.pipelex_bundle_blueprint import PipelexBundleBlueprint
+
+_PYTHON_SUFFIX = ".py"
+_PIPE_FUNC_TYPE = "PipeFunc"
+
+
+def travels_with_call(*, tree_path: str, data: bytes) -> bool:
+ """Whether a file of a `PipeFunc`'s Python root travels with its calls: every file but a generated `structures` module."""
+ if not tree_path.endswith(_PYTHON_SUFFIX):
+ return True
+ try:
+ source = data.decode("utf-8")
+ except UnicodeDecodeError:
+ return True
+ return not is_generated_structures_module(source=source)
+
+
+def holds_pipe_func_step(*, blueprint: PipelexBundleBlueprint) -> bool:
+ """Whether a bundle declares a `PipeFunc` step, whose calls carry its tree's files."""
+ return any(pipe_blueprint.type == _PIPE_FUNC_TYPE for pipe_blueprint in (blueprint.pipe or {}).values())
+
+
+def tree_pipe_func_files(*, tree: MethodTree) -> dict[str, TreeFileEntry]:
+ """The files every `PipeFunc` call of a method tree listed from this host's disk receives, read from the disk now."""
+ return read_call_files(tree=tree, keeps=travels_with_call)
diff --git a/pipelex/methods/structures_check.py b/pipelex/methods/structures_check.py
index 1488db8ee..7e02e7dd5 100644
--- a/pipelex/methods/structures_check.py
+++ b/pipelex/methods/structures_check.py
@@ -185,7 +185,7 @@ def scan_structured_content_sources(*, sources: Mapping[str, str]) -> list[Struc
`python-structures` projection, left as generated, is not a violation (`is_generated_structures_module`).
Args:
- sources: POSIX relative path to source text, as `FuncRegistryUtils.read_py_sources` returns it.
+ sources: POSIX relative path to source text, as the sandbox-hosted loader captures it.
Returns:
One violation per offending source, ordered by relative path.
diff --git a/pipelex/migration/exceptions.py b/pipelex/migration/exceptions.py
index cf17ed80d..175fb989a 100644
--- a/pipelex/migration/exceptions.py
+++ b/pipelex/migration/exceptions.py
@@ -37,6 +37,19 @@ class MigrationSnapshotRefusedError(MigrationError):
_declared_title = "Migration snapshot refused"
+class FunctionNamesMigrationError(PipelexError):
+ """The function names of a method could not be migrated to import paths.
+
+ Raised by `pipelex fix function-names` when the path it is handed is not a method's folder, its method tree cannot
+ be listed, or a file cannot be read or written. The flat names it cannot resolve are not an error: the migration
+ reports them for the author to decide.
+ """
+
+ error_domain = ErrorDomain.INPUT
+ retryable = False
+ _declared_title = "Function names migration error"
+
+
class FormerReleaseConfigError(PipelexSetupError):
"""The inference configuration still carries what a former release wrote for the Pipelex Gateway.
diff --git a/pipelex/migration/function_names.py b/pipelex/migration/function_names.py
new file mode 100644
index 000000000..5e1ffb2ba
--- /dev/null
+++ b/pipelex/migration/function_names.py
@@ -0,0 +1,441 @@
+"""The migration of a method's `PipeFunc` steps from flat function names to import paths.
+
+Until the function registry retired, a `PipeFunc` step named its function by a flat name, `function_name =
+"compute_totals"`, which the registry resolved among every `@pipe_func` function of every scanned directory: the
+function's own name, or the name its marker gave it, `@pipe_func(name="...")`. A step now names its function by its
+import path from the root of the method tree, `function_name = "funcs.totals.compute_totals"`, and a flat name is
+refused (`wip/extra-files/design.md`, D3).
+
+The migration is mechanical: for each `PipeFunc` step whose `function_name` holds no dot, find the module of the tree
+that defines the marked function the flat name registered, and write its import path; then drop the argument of every
+`@pipe_func(name=...)` marker, since the marker takes none. Nothing is imported: the modules are read as syntax trees,
+so a method whose Python cannot run here still migrates. A flat name no module of the tree registers, or that two
+register, is left as it is and reported, for the author to decide, and so is the name of each marker registering it,
+which is then the only record of which function the step meant; a marker whose name is not a string literal is left and
+reported too, since its name cannot be read without running the module.
+
+The engine is pure, over a map of tree paths to texts, so it serves a method folder on disk (`pipelex fix
+function-names`) and a method held anywhere else, such as a stored method rewritten in place, alike. Only the `.mthds`
+and `.py` files are read; the others are ignored.
+"""
+
+import ast
+import re
+from collections.abc import Mapping, MutableMapping
+from pathlib import Path
+from typing import Any, cast
+
+import tomlkit
+from mthds.method_tree.exceptions import MethodTreeRefusedError, TreeLimitError, TreeSourceError
+from mthds.method_tree.lister import list_method_tree
+from mthds.method_tree.python_modules import is_valid_function_segment
+from pydantic import BaseModel, ConfigDict, Field
+from tomlkit.exceptions import TOMLKitError
+from tomlkit.items import String
+
+from pipelex.migration.exceptions import FunctionNamesMigrationError
+from pipelex.pipe_operators.func.pipe_func_definition_check import marker_call, marker_references
+from pipelex.tools.typing.pydantic_utils import empty_list_factory_of
+
+_BUNDLE_SUFFIX = ".mthds"
+_PYTHON_SUFFIX = ".py"
+_PACKAGE_INIT = "__init__"
+_PIPE_FUNC_TYPE = "PipeFunc"
+_MARKER_NAME_ARGUMENT = "name"
+# A line of a module as the syntax tree counts lines, with its ending: Python ends a line only at `\r\n`, `\r` or `\n`,
+# where `str.splitlines` also breaks at a form feed, a vertical tab, the separators of the C1 and Unicode ranges.
+_SOURCE_LINE = re.compile(r".*?(?:\r\n|\r|\n)|.+\Z", flags=re.DOTALL)
+
+
+class FunctionNameRewrite(BaseModel):
+ """One `function_name` the migration rewrote: the step, the flat name it held and the import path it holds now."""
+
+ model_config = ConfigDict(frozen=True)
+
+ bundle: str = Field(description="The tree path of the .mthds file that defines the step")
+ pipe_code: str = Field(description="The step's pipe code")
+ flat_name: str = Field(description="The flat function name the step held")
+ import_path: str = Field(description="The import path it holds now, from the root of the method tree")
+
+
+class MarkerArgumentRemoval(BaseModel):
+ """One `@pipe_func(...)` marker whose registration name the migration dropped."""
+
+ model_config = ConfigDict(frozen=True)
+
+ module: str = Field(description="The tree path of the Python module that defines the function")
+ function: str = Field(description="The function's own name, which its import path ends with")
+ registered_name: str = Field(description="The name the marker registered the function under")
+
+
+class UnresolvedMarker(BaseModel):
+ """A `@pipe_func(...)` marker whose argument the migration left in place, and why."""
+
+ model_config = ConfigDict(frozen=True)
+
+ module: str = Field(description="The tree path of the Python module that defines the function")
+ function: str = Field(description="The function's own name")
+ registered_name: str | None = Field(default=None, description="The name the marker registered the function under, when it is a string literal")
+ reason: str = Field(description="Why the argument was left, and what to do")
+
+
+class UnresolvedFunctionName(BaseModel):
+ """A flat `function_name` the migration left as it is, and why."""
+
+ model_config = ConfigDict(frozen=True)
+
+ bundle: str = Field(description="The tree path of the .mthds file that defines the step")
+ pipe_code: str | None = Field(default=None, description="The step's pipe code, when the bundle could be read")
+ flat_name: str | None = Field(default=None, description="The flat function name the step holds, when the bundle could be read")
+ reason: str = Field(description="Why the name was not rewritten, and what to do")
+ candidates: list[str] = Field(default_factory=list, description="The import paths that register the name, when more than one does")
+
+
+class FunctionNamesMigration(BaseModel):
+ """What migrating one method's function names changed, and what it left for the author."""
+
+ model_config = ConfigDict(frozen=True)
+
+ rewrites: list[FunctionNameRewrite] = Field(default_factory=empty_list_factory_of(FunctionNameRewrite))
+ marker_removals: list[MarkerArgumentRemoval] = Field(default_factory=empty_list_factory_of(MarkerArgumentRemoval))
+ unresolved: list[UnresolvedFunctionName] = Field(default_factory=empty_list_factory_of(UnresolvedFunctionName))
+ unresolved_markers: list[UnresolvedMarker] = Field(default_factory=empty_list_factory_of(UnresolvedMarker))
+ changed_files: dict[str, str] = Field(default_factory=dict, description="Each file the migration changed, by tree path, with its new text")
+
+ @property
+ def is_complete(self) -> bool:
+ """Whether every `PipeFunc` step names its function by import path, and every marker takes no argument, once the changes are written."""
+ return not self.unresolved and not self.unresolved_markers
+
+ @property
+ def has_changes(self) -> bool:
+ return bool(self.changed_files)
+
+
+class _MarkedFunction(BaseModel):
+ """A top-level function of the tree carrying the marker, with the name it was registered under."""
+
+ model_config = ConfigDict(frozen=True)
+
+ module_path: str
+ module_name: str | None
+ function: str
+ registered_name: str | None
+ name_argument_span: tuple[int, int, int, int] | None
+
+ @property
+ def import_path(self) -> str | None:
+ return f"{self.module_name}.{self.function}" if self.module_name is not None else None
+
+
+def migrate_function_names(*, files: Mapping[str, str]) -> FunctionNamesMigration:
+ """Rewrite every flat `function_name` of a method's bundles into its import path, and drop the markers' names.
+
+ Args:
+ files: The method tree's files by tree path, each as its text; only the `.mthds` and `.py` files are read.
+
+ Returns:
+ The rewrites, the marker names dropped, the names left unresolved, and the new text of each changed file.
+ """
+ marked_functions = _marked_functions(files=files)
+ registrations: dict[str, list[_MarkedFunction]] = {}
+ for marked_function in marked_functions:
+ if marked_function.registered_name is not None:
+ registrations.setdefault(marked_function.registered_name, []).append(marked_function)
+
+ changed_files: dict[str, str] = {}
+ rewrites: list[FunctionNameRewrite] = []
+ unresolved: list[UnresolvedFunctionName] = []
+ for tree_path in sorted(files):
+ if not tree_path.endswith(_BUNDLE_SUFFIX):
+ continue
+ new_text = _migrate_bundle(
+ bundle=tree_path,
+ text=files[tree_path],
+ registrations=registrations,
+ rewrites=rewrites,
+ unresolved=unresolved,
+ )
+ if new_text is not None:
+ changed_files[tree_path] = new_text
+
+ # A marker's name is the only record of which function a step left unresolved meant, so it stays until every step
+ # naming it is rewritten.
+ unresolved_names = {name.flat_name: name for name in unresolved if name.flat_name is not None}
+ marker_removals: list[MarkerArgumentRemoval] = []
+ unresolved_markers: list[UnresolvedMarker] = []
+ spans_by_module: dict[str, list[tuple[int, int, int, int]]] = {}
+ for marked_function in marked_functions:
+ if marked_function.name_argument_span is None:
+ continue
+ if marked_function.registered_name is None:
+ unresolved_markers.append(
+ UnresolvedMarker(
+ module=marked_function.module_path,
+ function=marked_function.function,
+ reason=(
+ "The marker's name is not a string literal, so the name the retired registry gave the function cannot be read "
+ "without running the module: write the function's import path in each step that names it, then drop the "
+ "marker's argument."
+ ),
+ )
+ )
+ continue
+ unresolved_step = unresolved_names.get(marked_function.registered_name)
+ if unresolved_step is not None:
+ unresolved_markers.append(
+ UnresolvedMarker(
+ module=marked_function.module_path,
+ function=marked_function.function,
+ registered_name=marked_function.registered_name,
+ reason=(
+ f"The step '{unresolved_step.pipe_code}' of {unresolved_step.bundle} still names '{marked_function.registered_name}', "
+ "which the migration left unresolved: the marker keeps its name until that step names its function by import "
+ "path, then the migration drops it."
+ ),
+ )
+ )
+ continue
+ spans_by_module.setdefault(marked_function.module_path, []).append(marked_function.name_argument_span)
+ marker_removals.append(
+ MarkerArgumentRemoval(
+ module=marked_function.module_path,
+ function=marked_function.function,
+ registered_name=marked_function.registered_name,
+ )
+ )
+ for module_path, spans in sorted(spans_by_module.items()):
+ changed_files[module_path] = _without_spans(text=files[module_path], spans=spans)
+
+ return FunctionNamesMigration(
+ rewrites=rewrites,
+ marker_removals=marker_removals,
+ unresolved=unresolved,
+ unresolved_markers=unresolved_markers,
+ changed_files=dict(sorted(changed_files.items())),
+ )
+
+
+def migrate_function_names_in_directory(*, directory: Path, is_dry_run: bool) -> FunctionNamesMigration:
+ """Migrate the function names of the method tree rooted at a directory, writing the changed files unless it is a dry run.
+
+ The tree is listed by the standard's membership rule, so a `.gitignore`d file or a virtual environment is never
+ read nor written. A file that is not UTF-8 text is left out of the migration.
+
+ Raises:
+ FunctionNamesMigrationError: The path is not a directory, or its tree cannot be listed or a file written.
+ """
+ if not directory.is_dir():
+ msg = f"'{directory}' is not a directory: hand over the folder of the method, whose tree holds its bundles and its Python."
+ raise FunctionNamesMigrationError(msg)
+ try:
+ tree = list_method_tree(directory)
+ except (MethodTreeRefusedError, TreeLimitError, TreeSourceError) as exc:
+ msg = f"The method tree at '{directory}' cannot be listed: {exc}"
+ raise FunctionNamesMigrationError(msg) from exc
+
+ files: dict[str, str] = {}
+ for tree_path in tree.tree_paths:
+ if not tree_path.endswith((_BUNDLE_SUFFIX, _PYTHON_SUFFIX)):
+ continue
+ try:
+ files[tree_path] = tree.file_location(tree_path).read_bytes().decode("utf-8")
+ except UnicodeDecodeError:
+ continue
+ except OSError as exc:
+ msg = f"The file '{tree_path}' of the method tree at '{directory}' cannot be read: {exc}"
+ raise FunctionNamesMigrationError(msg) from exc
+
+ migration = migrate_function_names(files=files)
+ if is_dry_run:
+ return migration
+ for tree_path, new_text in migration.changed_files.items():
+ try:
+ tree.file_location(tree_path).write_bytes(new_text.encode("utf-8"))
+ except OSError as exc:
+ msg = f"The file '{tree_path}' of the method tree at '{directory}' cannot be written: {exc}"
+ raise FunctionNamesMigrationError(msg) from exc
+ return migration
+
+
+def _module_name_of(*, tree_path: str) -> str | None:
+ """The import name of a Python file of the tree, from its root, or None when it cannot be imported by name."""
+ segments = tree_path.removesuffix(_PYTHON_SUFFIX).split("/")
+ if segments[-1] == _PACKAGE_INIT:
+ segments = segments[:-1]
+ if not segments or not all(is_valid_function_segment(segment) for segment in segments):
+ return None
+ return ".".join(segments)
+
+
+def _marked_functions(*, files: Mapping[str, str]) -> list[_MarkedFunction]:
+ """Every top-level function of the tree's Python decorated with the marker, with the name the registry gave it."""
+ marked_functions: list[_MarkedFunction] = []
+ for tree_path in sorted(files):
+ if not tree_path.endswith(_PYTHON_SUFFIX):
+ continue
+ try:
+ module = ast.parse(files[tree_path], filename=tree_path)
+ except SyntaxError:
+ continue
+ marker_refs = marker_references(module=module)
+ if not marker_refs:
+ continue
+ for statement in module.body:
+ if not isinstance(statement, (ast.FunctionDef, ast.AsyncFunctionDef)):
+ continue
+ decorator = marker_call(definition=statement, marker_refs=marker_refs)
+ if decorator is None:
+ continue
+ registered_name, has_name_argument = _registered_name(decorator=decorator, function=statement.name)
+ marked_functions.append(
+ _MarkedFunction(
+ module_path=tree_path,
+ module_name=_module_name_of(tree_path=tree_path),
+ function=statement.name,
+ registered_name=registered_name,
+ name_argument_span=_arguments_span(decorator=decorator) if has_name_argument else None,
+ )
+ )
+ return marked_functions
+
+
+def _registered_name(*, decorator: ast.Call, function: str) -> tuple[str | None, bool]:
+ """The name the retired registry registered a marked function under, and whether the marker passed one.
+
+ The name is the marker's `name` argument, positional or by keyword, when it is a string literal, and the function's
+ own name when the marker passes none. A name the syntax tree cannot read, such as a variable, is None.
+ """
+ has_arguments = bool(decorator.args or decorator.keywords)
+ name_node: ast.expr | None = None
+ if decorator.args:
+ name_node = decorator.args[0]
+ for keyword in decorator.keywords:
+ if keyword.arg == _MARKER_NAME_ARGUMENT:
+ name_node = keyword.value
+ if name_node is None or (isinstance(name_node, ast.Constant) and name_node.value is None):
+ return function, has_arguments
+ if isinstance(name_node, ast.Constant) and isinstance(name_node.value, str):
+ return name_node.value, has_arguments
+ return None, has_arguments
+
+
+def _arguments_span(*, decorator: ast.Call) -> tuple[int, int, int, int] | None:
+ """The span of a marker call's arguments between its parentheses, as (start line, start column, end line, end column).
+
+ Lines count from 1 and columns are UTF-8 byte offsets, as the syntax tree gives them.
+ """
+ func_end_line = decorator.func.end_lineno
+ func_end_col = decorator.func.end_col_offset
+ call_end_line = decorator.end_lineno
+ call_end_col = decorator.end_col_offset
+ if func_end_line is None or func_end_col is None or call_end_line is None or call_end_col is None:
+ return None
+ return (func_end_line, func_end_col, call_end_line, call_end_col)
+
+
+def _without_spans(*, text: str, spans: list[tuple[int, int, int, int]]) -> str:
+ """The module's text with each marker call's arguments replaced by `()`, the last span first so offsets hold."""
+ lines = _SOURCE_LINE.findall(text)
+ encoded_lines = [line.encode("utf-8") for line in lines]
+ for start_line, start_col, end_line, end_col in sorted(spans, reverse=True):
+ head = encoded_lines[start_line - 1][:start_col]
+ tail = encoded_lines[end_line - 1][end_col:]
+ encoded_lines[start_line - 1 : end_line] = [head + b"()" + tail]
+ return b"".join(encoded_lines).decode("utf-8")
+
+
+def _migrate_bundle(
+ *,
+ bundle: str,
+ text: str,
+ registrations: Mapping[str, list[_MarkedFunction]],
+ rewrites: list[FunctionNameRewrite],
+ unresolved: list[UnresolvedFunctionName],
+) -> str | None:
+ """Rewrite the flat names of one bundle's `PipeFunc` steps; its new text, or None when nothing changed."""
+ try:
+ document = tomlkit.parse(text)
+ except TOMLKitError as exc:
+ if _holds_flat_function_name_hint(text=text):
+ unresolved.append(UnresolvedFunctionName(bundle=bundle, reason=f"The bundle does not parse as TOML, so its steps cannot be read: {exc}"))
+ return None
+ # tomlkit's containers are untyped mappings: each level is read as a mapping of strings once checked to be one.
+ pipes = cast("Mapping[str, object]", document).get("pipe")
+ if not isinstance(pipes, Mapping):
+ return None
+ has_changed = False
+ for pipe_code, pipe_item in cast("Mapping[str, object]", pipes).items():
+ if not isinstance(pipe_item, MutableMapping):
+ continue
+ pipe_table = cast("MutableMapping[str, Any]", pipe_item)
+ if pipe_table.get("type") != _PIPE_FUNC_TYPE:
+ continue
+ function_name = pipe_table.get("function_name")
+ if not isinstance(function_name, str) or "." in function_name:
+ continue
+ candidates = registrations.get(function_name, [])
+ import_paths = sorted({candidate.import_path for candidate in candidates if candidate.import_path is not None})
+ if not candidates:
+ unresolved.append(
+ UnresolvedFunctionName(
+ bundle=bundle,
+ pipe_code=pipe_code,
+ flat_name=function_name,
+ reason=(
+ f"No module of the method tree defines a @pipe_func function registered as '{function_name}'. If the function lives "
+ "outside the method's folder, move its module into the folder, then run the migration again, or write its import "
+ "path from the folder's root by hand."
+ ),
+ )
+ )
+ continue
+ if len(candidates) > 1:
+ unresolved.append(
+ UnresolvedFunctionName(
+ bundle=bundle,
+ pipe_code=pipe_code,
+ flat_name=function_name,
+ reason=(
+ f"More than one @pipe_func function of the method tree is registered as '{function_name}', which the retired registry "
+ "refused as a collision: write the import path of the one the step calls."
+ ),
+ candidates=[candidate.import_path or candidate.module_path for candidate in candidates],
+ )
+ )
+ continue
+ if not import_paths:
+ unresolved.append(
+ UnresolvedFunctionName(
+ bundle=bundle,
+ pipe_code=pipe_code,
+ flat_name=function_name,
+ reason=(
+ f"The function registered as '{function_name}' lives in {candidates[0].module_path}, whose path is not an import path: "
+ "rename its folders and file to Python identifiers, then run the migration again."
+ ),
+ )
+ )
+ continue
+ import_path = import_paths[0]
+ _replace_string_value(table=pipe_table, key="function_name", value=import_path)
+ rewrites.append(FunctionNameRewrite(bundle=bundle, pipe_code=pipe_code, flat_name=function_name, import_path=import_path))
+ has_changed = True
+ return tomlkit.dumps(document) if has_changed else None # pyright: ignore[reportUnknownMemberType]
+
+
+def _replace_string_value(*, table: MutableMapping[str, Any], key: str, value: str) -> None:
+ """Replace a string value in a tomlkit table, keeping the comment and the spacing around it."""
+ new_item = tomlkit.string(value)
+ old_item = table[key]
+ if isinstance(old_item, String):
+ new_item.trivia.indent = old_item.trivia.indent
+ new_item.trivia.comment_ws = old_item.trivia.comment_ws
+ new_item.trivia.comment = old_item.trivia.comment
+ new_item.trivia.trail = old_item.trivia.trail
+ table[key] = new_item
+
+
+def _holds_flat_function_name_hint(*, text: str) -> bool:
+ """Whether a bundle that does not parse mentions a PipeFunc, so its failure to parse is worth reporting."""
+ return _PIPE_FUNC_TYPE in text and "function_name" in text
diff --git a/pipelex/migration/function_names_render.py b/pipelex/migration/function_names_render.py
new file mode 100644
index 000000000..5726ac599
--- /dev/null
+++ b/pipelex/migration/function_names_render.py
@@ -0,0 +1,43 @@
+"""The report of a function names migration, as Markdown for a person or an agent to read."""
+
+from pipelex.migration.function_names import FunctionNamesMigration
+from pipelex.pipe_operators.func.function_name import FUNCTION_NAMES_MIGRATION_COMMAND
+
+
+def format_function_names_markdown(*, migration: FunctionNamesMigration, directory: str, is_dry_run: bool) -> str:
+ """The migration's report: what it rewrote, or would rewrite in a dry run, and what it left for the author."""
+ verb = "would rewrite" if is_dry_run else "rewrote"
+ lines: list[str] = [f"# Function names of the method at {directory}", ""]
+ if not migration.rewrites and not migration.marker_removals and not migration.unresolved and not migration.unresolved_markers:
+ lines.append("Every PipeFunc step already names its function by import path: nothing to migrate.")
+ return "\n".join(lines)
+
+ if migration.rewrites:
+ lines.extend([f"The migration {verb} these function names as import paths:", ""])
+ for rewrite in migration.rewrites:
+ lines.append(f"- `{rewrite.bundle}`, pipe `{rewrite.pipe_code}`: `{rewrite.flat_name}` → `{rewrite.import_path}`")
+ lines.append("")
+ if migration.marker_removals:
+ drop = "would drop" if is_dry_run else "dropped"
+ lines.extend([f"The migration {drop} the registration name of these `@pipe_func(...)` markers, which take no argument now:", ""])
+ for removal in migration.marker_removals:
+ lines.append(f"- `{removal.module}`, function `{removal.function}`, registered as `{removal.registered_name}`")
+ lines.append("")
+ if migration.unresolved:
+ lines.extend(["These function names are left as they are, for you to decide:", ""])
+ for unresolved in migration.unresolved:
+ where = f"`{unresolved.bundle}`, pipe `{unresolved.pipe_code}`" if unresolved.pipe_code is not None else f"`{unresolved.bundle}`"
+ name = f" `{unresolved.flat_name}`" if unresolved.flat_name is not None else ""
+ lines.append(f"- {where}{name}: {unresolved.reason}")
+ for candidate in unresolved.candidates:
+ lines.append(f" - candidate `{candidate}`")
+ lines.append("")
+ if migration.unresolved_markers:
+ lines.extend(["These `@pipe_func(...)` markers keep their argument, for you to decide:", ""])
+ for marker in migration.unresolved_markers:
+ registered = f", registered as `{marker.registered_name}`" if marker.registered_name is not None else ""
+ lines.append(f"- `{marker.module}`, function `{marker.function}`{registered}: {marker.reason}")
+ lines.append("")
+ if is_dry_run and migration.has_changes:
+ lines.append(f"Nothing was written. Run `{FUNCTION_NAMES_MIGRATION_COMMAND} {directory}` to write these changes.")
+ return "\n".join(lines).rstrip() + "\n"
diff --git a/pipelex/pipe_machinery/pipe_abstract.py b/pipelex/pipe_machinery/pipe_abstract.py
index d7c16d708..25d0186e5 100644
--- a/pipelex/pipe_machinery/pipe_abstract.py
+++ b/pipelex/pipe_machinery/pipe_abstract.py
@@ -520,6 +520,16 @@ class PipeAbstract(ABC, BaseModel):
def validate_output_static(self):
pass
+ def validate_once_built(self) -> None:
+ """Refuse what makes this pipe unusable that it can tell as soon as it is built, before the library around it is complete.
+
+ A dependency package's pipe refused here counts as one its package could not build, so it refuses only a
+ reference to it, as a pipe whose construction failed does. Nothing by default: a `PipeFunc` locates its function.
+
+ Raises:
+ PipeValidationError: The pipe is unusable.
+ """
+
def _scan_input_presence(self, *, working_memory: WorkingMemory) -> InputPresenceScan:
"""Apply the runtime trichotomy (D3) to this pipe's needed inputs.
diff --git a/pipelex/pipe_operators/func/direct_pipe_func_executor.py b/pipelex/pipe_operators/func/direct_pipe_func_executor.py
index 8285c44d0..8d0a4e8a6 100644
--- a/pipelex/pipe_operators/func/direct_pipe_func_executor.py
+++ b/pipelex/pipe_operators/func/direct_pipe_func_executor.py
@@ -1,62 +1,63 @@
import asyncio
import shutil
-import sys
import tempfile
from pathlib import Path
+from mthds.method_tree.exceptions import MethodTreeRefusedError, TreeSourceError
+from mthds.method_tree.written_tree import read_written_tree
from typing_extensions import override
from pipelex import log
from pipelex.codegen.emitters.python_structures import emit_python_structures
from pipelex.codegen.resolved_concepts import resolve_concepts_from_crate
from pipelex.core.memory.working_memory import WorkingMemory
-from pipelex.core.stuffs.structured_content import StructuredContent
+from pipelex.core.method_trees.exceptions import TreePythonError
+from pipelex.core.method_trees.tree_files import write_call_files
+from pipelex.core.method_trees.tree_python import TREE_PYTHON_HOLD
from pipelex.core.stuffs.stuff_factory import StuffFactory
-from pipelex.interpreter_hub import get_library_manager
-from pipelex.kernel.func_ops import call_registered_function
+from pipelex.interpreter_hub import get_library_manager, get_required_pipe
+from pipelex.kernel.func_ops import call_function
+from pipelex.libraries.library_crate import LibraryCrate
from pipelex.pipe_operators.func.exceptions import PipeFuncExecutionError
+from pipelex.pipe_operators.func.pipe_func import PipeFunc
+from pipelex.pipe_operators.func.pipe_func_call_context import function_of_call_in_flight
from pipelex.pipe_operators.func.pipe_func_execution_dtos import PipeFuncExecutionRequest, PipeFuncExecutionResponse
from pipelex.pipe_operators.func.pipe_func_executor_protocol import PipeFuncExecutionResult, PipeFuncExecutorProtocol
from pipelex.pipe_run.pipe_run_params import PipeRunParams
from pipelex.runtime_bridge.primitives.rehydration import rehydrate_library_and_memory
from pipelex.system.job_metadata import JobMetadata
-from pipelex.system.registries.class_registry_utils import ClassRegistryUtils
-from pipelex.system.registries.func_registry_utils import FuncRegistryUtils
# The library id the transported run rebuilds under. Internal to one process (a fresh sandbox box or a
# one-shot subprocess), so a fixed id is safe — there is no other library to collide with.
_TRANSPORTED_LIBRARY_ID = "pipe_func_transported_lib"
-def bundle_import_dirs(*, workdir: Path) -> list[Path]:
- """The directories a materialized bundle's modules import from, in `sys.path` search order.
+def resolve_pipe_func(*, pipe_code: str) -> PipeFunc:
+ """The `PipeFunc` step of the current library a call names.
- The bundle root comes first and the subdirectories holding a `.py` file follow in sorted order, so a
- module name that two of them share resolves to the same file in every process. In particular, the
- `structures` module generated at the root wins over any `structures.py` a subdirectory ships.
-
- Args:
- workdir: The directory the bundle's sources were materialized into.
-
- Returns:
- The root followed by its source subdirectories, each listed once.
+ Raises:
+ PipeFuncExecutionError: The step the call names is not a `PipeFunc`.
"""
- subdirs = {py_file.parent for py_file in workdir.rglob("*.py")} - {workdir}
- return [workdir, *sorted(subdirs)]
+ pipe = get_required_pipe(pipe_code=pipe_code)
+ if not isinstance(pipe, PipeFunc):
+ msg = f"The pipe '{pipe_code}' a PipeFunc call names is a {pipe.type}, not a PipeFunc."
+ raise PipeFuncExecutionError(msg)
+ return pipe
class DirectPipeFuncExecutor(PipeFuncExecutorProtocol):
- """The ``direct`` execution mode: resolve the function from the process-global registry and run it here.
+ """The ``direct`` execution mode: import the step's function in this process and run it here.
The free, in-process, trusted mode core owns (mirrors ``DirectOrchestrator`` on the orchestration
- axis). It is used for local runs and inside a sandbox box (where the real function IS registered);
- a sandbox-dispatching executor from the out-of-tree plugin runs it out-of-process instead when
+ axis). It is used for local runs and inside a sandbox box, where the call's files were written to
+ disk; a sandbox-dispatching executor from the out-of-tree plugin runs it out-of-process instead when
``execution_mode`` names a sandbox backend.
- What running a function *means* — the registry lookup, async-vs-sync dispatch, and the
- str/list -> StuffContent coercion — is the kernel's ``call_registered_function``, so this executor
- is what is left once that is factored out: the in-process wiring of the live arm, and the whole
- transported arm below.
+ The function is the step's own: the `PipeFunc` imports it by its import path, under the process's
+ hold on its method tree's Python, and checks it (`PipeFunc.bind_function`). What running it *means*
+ — async-vs-sync dispatch and the str/list -> StuffContent coercion — is the kernel's
+ ``call_function``, so this executor is what is left once that is factored out: the in-process
+ wiring of the live arm, and the whole transported arm below.
"""
@override
@@ -70,12 +71,14 @@ class DirectPipeFuncExecutor(PipeFuncExecutorProtocol):
pipe_run_params: PipeRunParams,
) -> PipeFuncExecutionResult:
log.verbose(f"Running PipeFunc in-process with function '{function_name}'")
- # The call itself — registry lookup, async/sync dispatch, return coercion — is the kernel's,
- # so an out-of-process executor and this one share one definition of what running a function
- # means. Nothing is caught here: the function is arbitrary user code whose failure surface is
- # not enumerable, and the operator wraps this call to decorate the failure with the pipe's
- # inputs/output for a diagnostic PipeRunError.
- call_result = await call_registered_function(function_name=function_name, memory=working_memory)
+ # The step imported and checked its function before the run (its `_validate_before_run`), and names it for its
+ # call in flight; a call that names none, the transported run's, finds the step in the current library, which
+ # binds it once. The call itself — async/sync dispatch, return coercion — is the kernel's, so an out-of-process
+ # executor and this one share one definition of what running a function means. Nothing is caught here: the
+ # function is arbitrary user code whose failure surface is not enumerable, and the operator wraps this call to
+ # decorate the failure with the pipe's inputs/output for a diagnostic PipeRunError.
+ function = function_of_call_in_flight(pipe_ref=pipe_code) or resolve_pipe_func(pipe_code=pipe_code).bind_function()
+ call_result = await call_function(function=function, function_name=function_name, memory=working_memory)
return PipeFuncExecutionResult(
content=call_result.content,
function_module=call_result.function_module,
@@ -84,7 +87,7 @@ class DirectPipeFuncExecutor(PipeFuncExecutorProtocol):
@override
async def run_pipe_func_transported(self, *, request: PipeFuncExecutionRequest) -> PipeFuncExecutionResponse:
- """Register the customer's real code from the crate, rebuild the library + memory, run the one PipeFunc, transport the output.
+ """Write the call's files, rebuild the library + memory, run the one PipeFunc, transport the output.
The far-from-caller half of out-of-process execution, and the single source of truth for it: a
sandbox box runs exactly this (its entrypoint is a thin shell over this call), and it pulls in
@@ -92,79 +95,108 @@ class DirectPipeFuncExecutor(PipeFuncExecutorProtocol):
runner/worker, it deliberately imports and executes customer code, because it runs INSIDE the
isolation boundary. The output rides back wrapped in a working memory so it round-trips with its
concept (and thus its dynamic-class identity), rebound against the receiver's registry.
+
+ The call's files are written into a fresh temporary directory, removed on every path.
"""
- # Resolved, because importing a file puts its resolved directory on sys.path: under a symlinked
- # temp dir (`/var` is `/private/var` on macOS) those entries would jump ahead of the bundle's
- # import order and escape the prune below.
- workdir = Path(tempfile.mkdtemp(prefix="pipelex_pipe_func_")).resolve()
+ # Resolved, so the root the hold puts on sys.path is the one the import system reports for its modules
+ # (`/var` is `/private/var` on macOS).
+ tree_root = Path(tempfile.mkdtemp(prefix="pipelex_pipe_func_")).resolve()
try:
- return await self._run_transported_from_workdir(request=request, workdir=workdir)
+ return await self.run_pipe_func_transported_at(request=request, tree_root=tree_root)
finally:
- # The imported modules live on in memory; the materialized sources are only needed during
- # import, so the workdir is removed on every path (success, timeout, failure). The sys.path
- # entries inserted for that import are pruned too — the mkdtemp path is unique, so removing
- # exactly the entries under this workdir is safe even with concurrent transported runs
- # (unlike a snapshot/restore of the whole list).
- sys.path[:] = [entry for entry in sys.path if not Path(entry).is_relative_to(workdir)]
- shutil.rmtree(workdir, ignore_errors=True)
-
- async def _run_transported_from_workdir(self, *, request: PipeFuncExecutionRequest, workdir: Path) -> PipeFuncExecutionResponse:
- for relpath, source in request.crate.python_sources.items():
- target = workdir / relpath
- target.parent.mkdir(parents=True, exist_ok=True)
- target.write_text(source, encoding="utf-8")
-
- # Generate the method's concept structure classes from the crate — the .mthds is the single
- # source of truth for them (`resolve_concepts_from_crate` reads the crate's ConceptBlueprints,
- # `emit_python_structures` projects a `structures.py`). So a PipeFunc can
- # `from structures import <domain>__<Concept>` without the method having to STORE or transport a
- # hand-written copy that could drift from the .mthds. A file the method actually shipped at the
- # same path wins (already materialized above) — never clobber customer source.
- #
- # Force domain-qualified class names (`atlas_devis__PricedQuote`): the runtime ALWAYS qualifies a
- # concept's structure class (`concept.structure_class_name`), and that is the exact name a
- # PipeFunc's return-type validation checks against — so the generated classes must match the
- # runtime spelling, not the emitter's default collision-only qualification (which would emit a
- # bare `PricedQuote` the validator rejects).
- resolved_library = resolve_concepts_from_crate(request.crate)
+ shutil.rmtree(tree_root, ignore_errors=True)
+
+ async def run_pipe_func_transported_at(self, *, request: PipeFuncExecutionRequest, tree_root: Path) -> PipeFuncExecutionResponse:
+ """Run a transported call with its Python root at `tree_root`, a directory the caller may have filled already.
+
+ A sandbox client may unpack the call's tree into the sandbox itself, such as from one archive per call, and hand
+ the directory over; the files the request carries are written there too, at their tree paths. The directory is
+ the call's import root: it goes first on `sys.path` under the process's hold on its Python, and the function is
+ imported from it under its real import path.
+
+ Raises:
+ PipeFuncExecutionError: The call's files cannot be read back as a tree, or the function runs past its timeout.
+ """
+ write_call_files(files=request.files, destination=tree_root)
+ self._write_generated_structures(crate=request.crate, tree_root=tree_root)
+ try:
+ tree = read_written_tree(tree_root)
+ except (MethodTreeRefusedError, TreeSourceError) as exc:
+ msg = f"The files of the PipeFunc call for pipe '{request.pipe_code}' cannot be read back as a method tree: {exc}"
+ raise PipeFuncExecutionError(msg) from exc
+
+ # The call's Python is held for the transported library until the run ends, and unloaded before its directory
+ # is removed.
+ TREE_PYTHON_HOLD.acquire(tree=tree, holder=_TRANSPORTED_LIBRARY_ID, label=f"the method tree of the call to '{request.pipe_code}'")
+ try:
+ return await self._run_transported(request=request, crate=self._crate_rooted_at(request=request, tree_json=tree.model_dump_json()))
+ finally:
+ TREE_PYTHON_HOLD.release(holder=_TRANSPORTED_LIBRARY_ID)
+ TREE_PYTHON_HOLD.unload_if_unheld()
+
+ @staticmethod
+ def _write_generated_structures(*, crate: LibraryCrate, tree_root: Path) -> None:
+ """Generate the method's concept structure classes from the crate as the `structures` module, unless the call shipped one.
+
+ The .mthds is the single source of truth for them (`resolve_concepts_from_crate` reads the crate's
+ ConceptBlueprints, `emit_python_structures` projects a `structures.py`), so a PipeFunc can
+ `from structures import <domain>__<Concept>` without the method having to store or transport a
+ hand-written copy that could drift from the .mthds. A file the method actually shipped at the
+ same path wins — never clobber customer source.
+
+ Domain-qualified class names are forced (`atlas_devis__PricedQuote`): the runtime ALWAYS qualifies a
+ concept's structure class (`concept.structure_class_name`), and that is the exact name a
+ PipeFunc's return-type validation checks against.
+ """
+ resolved_library = resolve_concepts_from_crate(crate)
qualified_library = resolved_library.model_copy(
update={"concepts": [concept.model_copy(update={"needs_qualification": True}) for concept in resolved_library.concepts]}
)
for emitted in emit_python_structures(qualified_library):
- generated_target = workdir / emitted.filename
+ generated_target = tree_root / emitted.filename
if generated_target.exists():
log.verbose(f"The structures file '{emitted.filename}' the method shipped was kept over the generated one")
continue
generated_target.parent.mkdir(parents=True, exist_ok=True)
generated_target.write_text(emitted.content, encoding="utf-8")
- # Put the bundle's directories on sys.path so a PipeFunc (or structure class) can import a sibling
- # file from the same bundle. This is only reached in-box (transported path): the crate is
- # re-materialized into a throwaway workdir that is NOT on sys.path, and each module is imported by
- # file path (spec_from_file_location), which does NOT add its directory to sys.path either — so a
- # top-level `from helpers import ...` between sibling files raises ModuleNotFoundError, the module
- # never finishes importing, and its @pipe_func never registers. A local direct run does not hit
- # this: its funcs were registered at boot from the real on-disk project already on sys.path. Add
- # workdir (resolves `from funcs.helpers import ...`) and every subdirectory holding a .py file
- # (resolves the sibling-relative `from helpers import ...`); the caller's finally prunes these
- # entries once the import phase is over, so they never outlive the run.
- sys.path[0:0] = [str(source_dir) for source_dir in bundle_import_dirs(workdir=workdir)]
-
- # Register the customer's structure classes and @pipe_func functions from their real source,
- # before rehydration seeds a per-run registry from the global one and before the function lookup.
- ClassRegistryUtils.import_modules_in_folder(folder_path=workdir, base_class_names=[StructuredContent.__name__])
- ClassRegistryUtils.auto_register_all_subclasses(base_class=StructuredContent)
- FuncRegistryUtils.register_funcs_in_folder(folder_path=workdir)
+ @staticmethod
+ def _crate_rooted_at(*, request: PipeFuncExecutionRequest, tree_json: str) -> LibraryCrate:
+ """The request's crate with the called step's method tree replaced by the tree written here.
+ A step read from a tree resolves its function, and any file its tree's other steps name, in that tree, which the
+ crate carries as the host that assembled it listed it; here the tree is the call's files as written in this
+ process. A step parsed from a content string has no tree, and imports from this process's import path, which
+ the hold starts with the call's root.
+ """
+ crate = request.crate
+ tree_bundle_ref = crate.tree_bundles.get(request.pipe_code)
+ if tree_bundle_ref is None:
+ return crate
+ method_trees = dict(crate.method_trees)
+ method_trees[tree_bundle_ref.tree_key] = tree_json
+ return crate.model_copy(update={"method_trees": method_trees})
+
+ async def _run_transported(self, *, request: PipeFuncExecutionRequest, crate: LibraryCrate) -> PipeFuncExecutionResponse:
+ # The crate was validated where it was assembled; its other steps may name files of trees this process never
+ # receives, so it is loaded as it is, and the one step called checks its function as it binds it.
working_memory = rehydrate_library_and_memory(
library_id=_TRANSPORTED_LIBRARY_ID,
- crate=request.crate,
+ crate=crate,
working_memory_raw=request.working_memory_raw,
+ is_crate_prevalidated=True,
)
if working_memory is None:
msg = f"Transported rehydration produced no working memory for pipe '{request.pipe_code}'"
raise PipeFuncExecutionError(msg)
+ pipe_func = resolve_pipe_func(pipe_code=request.pipe_code)
+ try:
+ pipe_func.bind_function()
+ except TreePythonError as exc:
+ msg = f"PipeFunc '{request.pipe_code}' cannot call its function '{request.function_name}': {exc.message}"
+ raise PipeFuncExecutionError(msg) from exc
+
# Runaway-code guard: time out the PipeFunc if it runs past the request's (plan-dependent)
# deadline. Best-effort in-process — a blocking sync function keeps its worker thread; the
# hard kill is the execution boundary's job (the sandbox destroys the box, the subprocess is
diff --git a/pipelex/pipe_operators/func/exceptions.py b/pipelex/pipe_operators/func/exceptions.py
index 3b6c5fed9..08bf5b450 100644
--- a/pipelex/pipe_operators/func/exceptions.py
+++ b/pipelex/pipe_operators/func/exceptions.py
@@ -1,4 +1,4 @@
-from pipelex.base_exceptions import PipelexError
+from pipelex.base_exceptions import ErrorDomain, PipelexError
class PipeFuncTransportError(PipelexError):
@@ -16,3 +16,17 @@ class PipeFuncExecutionError(PipelexError):
rehydration yields no working memory or the function exceeds its (plan-dependent) timeout — as
opposed to the customer function itself raising, which propagates untouched.
"""
+
+
+class PipeFuncFunctionError(PipelexError):
+ """A `PipeFunc`'s `function_name` names no function the step may call.
+
+ Raised when the module the import path names is not found, when the function is not a top-level `def` or
+ `async def` of that module carrying the `@pipe_func()` marker, or, once the module is imported, when the function
+ does not follow the calling convention. The message names the step and the function, and describes the
+ caller's own method, so it is kept under STRICT disclosure.
+ """
+
+ error_domain = ErrorDomain.INPUT
+ retryable = False
+ _authors_caller_facing_message = True
diff --git a/pipelex/pipe_operators/func/function_name.py b/pipelex/pipe_operators/func/function_name.py
new file mode 100644
index 000000000..d71a91004
--- /dev/null
+++ b/pipelex/pipe_operators/func/function_name.py
@@ -0,0 +1,73 @@
+"""The form of a `PipeFunc`'s `function_name`: a Python import path from the root of the method tree.
+
+`function_name = "funcs.totals.compute_totals"` names the function `compute_totals` of the module `funcs.totals`, the
+file `funcs/totals.py` or `funcs/totals/__init__.py` of the tree, imported with the tree's root as the import root
+(`mthds/docs/spec/mthds-format.md`, "Locating the function"). The name is normalized to NFC, then split at its last
+dot: a name with no dot names no module and is refused, and each segment must be a Python identifier that is not a
+keyword and that NFKC leaves unchanged, so a ligature or a fullwidth letter is refused rather than mapped to another
+name. This is the form alone, checked as the bundle is parsed; whether the tree holds the module, and the module the
+function, is checked when the library validates the step.
+
+A flat name, such as `compute_totals`, is the key of the retired function registry: it is refused with the command that
+rewrites it, `pipelex fix function-names`.
+"""
+
+from typing import NamedTuple
+
+from mthds.method_tree.python_modules import is_valid_function_segment
+from mthds.method_tree.tree_path import nfc
+
+#: The command that rewrites every flat `function_name` of a method into its import path.
+FUNCTION_NAMES_MIGRATION_COMMAND = "pipelex fix function-names"
+
+
+class FunctionImportPath(NamedTuple):
+ """A `function_name` split at its last dot: the module's dotted import name and the function's name in it."""
+
+ module_name: str
+ function: str
+
+
+def function_name_form_violation(*, function_name: str) -> str | None:
+ """Why `function_name` is not an import path a `PipeFunc` can name, or None when its form is valid.
+
+ Args:
+ function_name: The `function_name` as the bundle writes it.
+
+ Returns:
+ None when the form is valid, else the refusal message, which names the `function_name`.
+ """
+ normalized = nfc(function_name)
+ if "." not in normalized:
+ return (
+ f"The function_name '{function_name}' holds no dot, so it names no module: a PipeFunc names its function by its import path "
+ "from the root of the method tree, such as 'funcs.totals.compute_totals' for the function compute_totals of the file "
+ f"funcs/totals.py. A flat name was the key of the retired function registry; run `{FUNCTION_NAMES_MIGRATION_COMMAND} "
+ "<method folder>` to rewrite it."
+ )
+ for segment in normalized.split("."):
+ if not is_valid_function_segment(segment):
+ return (
+ f"The function_name '{function_name}' holds the segment '{segment}', which is not a Python identifier, is a Python keyword, "
+ "or changes under NFKC normalization."
+ )
+ return None
+
+
+def split_function_name(*, function_name: str) -> FunctionImportPath:
+ """Split a `function_name` whose form is valid into its module's import name and the function's name.
+
+ Args:
+ function_name: A `function_name` that `function_name_form_violation` accepts.
+
+ Returns:
+ The module's dotted import name and the function's name, both in NFC.
+
+ Raises:
+ ValueError: When the form is not valid.
+ """
+ violation = function_name_form_violation(function_name=function_name)
+ if violation is not None:
+ raise ValueError(violation)
+ module_name, _, function = nfc(function_name).rpartition(".")
+ return FunctionImportPath(module_name=module_name, function=function)
diff --git a/pipelex/pipe_operators/func/pipe_func.py b/pipelex/pipe_operators/func/pipe_func.py
index d2eca4448..ef0695f43 100644
--- a/pipelex/pipe_operators/func/pipe_func.py
+++ b/pipelex/pipe_operators/func/pipe_func.py
@@ -1,6 +1,9 @@
+from collections.abc import Callable
+from contextlib import nullcontext
from typing import Any, Literal, cast, get_args, get_origin, get_type_hints
-from pydantic import field_validator
+from mthds.method_tree.tree_path import nfc
+from pydantic import PrivateAttr, field_validator
from typing_extensions import override
from pipelex import log
@@ -9,19 +12,38 @@ from pipelex.core.concepts.concept_factory import ConceptFactory
from pipelex.core.memory.exceptions import WorkingMemoryStuffNotFoundError
from pipelex.core.memory.working_memory import WorkingMemory
from pipelex.core.memory.working_memory_factory import WorkingMemoryFactory
-from pipelex.core.pipes.exceptions import PipeRunError
+from pipelex.core.method_trees.exceptions import TreePythonError
+from pipelex.core.method_trees.tree_bundle import TreeBundle
+from pipelex.core.method_trees.tree_python import check_tree_module_names, top_level_module_names, tree_label_holding
+from pipelex.core.pipes.exceptions import PipeRunError, PipeValidationError
from pipelex.core.pipes.inputs.input_stuff_specs import InputStuffSpecs, TypedNamedStuffSpec
from pipelex.core.pipes.pipe_output import PipeOutput
from pipelex.core.stuffs.list_content import ListContent
from pipelex.core.stuffs.stuff_content import StuffContent
from pipelex.core.stuffs.stuff_factory import StuffFactory
-from pipelex.interpreter_hub import get_pipe_func_executor
+from pipelex.interpreter_hub import get_current_library_id_or_none, get_library_manager, get_pipe_func_executor
from pipelex.kernel.memory_ops import store_result
+from pipelex.libraries.library_crate import SUBMITTED_PYTHON_ROOT_KEY
+from pipelex.pipe_operators.func.exceptions import PipeFuncFunctionError
+from pipelex.pipe_operators.func.function_name import function_name_form_violation
+from pipelex.pipe_operators.func.pipe_func_call_context import pipe_func_call_in_flight
+from pipelex.pipe_operators.func.pipe_func_python import (
+ PipeFuncModuleSource,
+ bind_import_path_function,
+ bind_tree_function,
+ check_pipe_func_definition,
+ locate_in_files,
+ locate_in_tree,
+ locate_on_import_path,
+)
from pipelex.pipe_operators.pipe_operator import PipeOperator
from pipelex.pipe_run.pipe_run_params import PipeRunParams
from pipelex.runtime_hub import get_class_registry
from pipelex.system.job_metadata import JobMetadata
-from pipelex.system.registries.func_registry import func_registry
+from pipelex.validation_error_types import PipeValidationErrorType
+
+#: Who holds a tree's Python for a run with no current library, such as a pipe run directly by a test.
+_PROCESS_HOLDER = "process"
class PipeFuncOutput(PipeOutput):
@@ -29,9 +51,27 @@ class PipeFuncOutput(PipeOutput):
class PipeFunc(PipeOperator[PipeFuncOutput]):
+ """Calls a Python function the step names by import path from the root of its method tree.
+
+ The function is located and its form proven at validation, without running Python; it is imported, and its calling
+ convention and return type checked, when the step first runs, dry or live. In `direct` mode it is imported in
+ this process, under the process's hold on the tree's Python; in a sandbox-hosted mode it never is, and the call
+ travels to the sandbox with the tree's files. See `pipe_func_python` for where a `function_name` resolves.
+ """
+
type: Literal["PipeFunc"] = "PipeFunc"
function_name: str
+ # The member of its method tree the step's bundle was read as, which the function's import path resolves in; None
+ # for a bundle parsed from a content string. Set by the factory, never serialized: a pipe is rebuilt from its
+ # blueprint wherever it travels, and the crate carries the tree.
+ _tree_bundle: TreeBundle | None = PrivateAttr(default=None)
+ # The function once imported and checked, with the holder its import was made for.
+ _bound_function: tuple[str, Callable[..., Any]] | None = PrivateAttr(default=None)
+ # Whether validation located the function and proved its form, which a dependency package's step, validated as it is
+ # built and again with its consumer's library, then need not do twice.
+ _is_function_proven: bool = PrivateAttr(default=False)
+
@override
def required_variables(self) -> set[str]:
return set()
@@ -42,32 +82,22 @@ class PipeFunc(PipeOperator[PipeFuncOutput]):
@field_validator("function_name", mode="before")
@classmethod
- def validate_function_name(cls, function_name: str) -> str:
- if is_pipe_func_sandbox_hosted():
- # Sandbox-hosted mode: the customer's function is not registered in this process (its
- # source only travels on the crate), so the registry lookup + return-type inspection
- # cannot run here. The sandbox registers the real function and validates it for real
- # when it loads. Accept the declared name verbatim.
+ def validate_function_name(cls, function_name: object) -> object:
+ """Refuse a `function_name` that is not an import path, such as a flat name of the retired registry."""
+ if not isinstance(function_name, str):
return function_name
- function = func_registry.get_function(function_name)
- if not function:
- # Check if this function was found but is ineligible (e.g., missing return type)
- ineligible_info = func_registry.get_ineligible_function_info(function_name)
- if ineligible_info:
- msg = f"Function '{function_name}' has @pipe_func() decorator but is not eligible for registration: {ineligible_info.reason}"
- else:
- msg = f"Function '{function_name}' not found in registry"
- raise ValueError(msg)
+ violation = function_name_form_violation(function_name=function_name)
+ if violation is not None:
+ raise ValueError(violation)
+ return nfc(function_name)
- return_type = get_type_hints(function).get("return")
+ def place_in_tree(self, *, tree_bundle: TreeBundle | None) -> None:
+ """Record the member of its method tree the step's bundle was read as, or None for a bundle parsed from a string."""
+ self._tree_bundle = tree_bundle
- if return_type is None:
- msg = f"Function '{function_name}' has no return type annotation"
- raise ValueError(msg)
- if not issubclass(return_type, StuffContent):
- msg = f"Function '{function_name}' return type {return_type} is not a subclass of StuffContent"
- raise TypeError(msg)
- return function_name
+ @property
+ def tree_bundle(self) -> TreeBundle | None:
+ return self._tree_bundle
@override
def validate_inputs_static(self):
@@ -83,26 +113,125 @@ class PipeFunc(PipeOperator[PipeFuncOutput]):
@override
def validate_output_with_library(self):
- if is_pipe_func_sandbox_hosted():
- # Sandbox-hosted mode: the function is not in this process, so its return type cannot be
- # inspected to cross-check against the output concept's structure class. The sandbox does
- # this check for real at registration time. Skip the whole library-output validation here.
+ """Locate the function's module and prove the function's form from its source, without running Python, once."""
+ if self._is_function_proven:
return
- function = func_registry.get_required_function(self.function_name)
+ try:
+ self._check_function_statically()
+ except (PipeFuncFunctionError, TreePythonError) as exc:
+ msg = f"PipeFunc '{self.code}' cannot call its function '{self.function_name}': {exc.message}"
+ raise PipeValidationError(
+ message=msg,
+ error_type=PipeValidationErrorType.PIPE_FUNC_FUNCTION_REFUSED,
+ domain_code=self.domain_code,
+ pipe_code=self.code,
+ field_name="function_name",
+ ) from exc
+ self._is_function_proven = True
+
+ @override
+ def validate_once_built(self) -> None:
+ """Locate the function and prove its form: a step whose function cannot be found is one that cannot be built."""
+ self.validate_output_with_library()
+
+ def _check_function_statically(self) -> None:
+ """Locate the module where the step's import root puts it, prove the function's form, and check the tree's module names.
+
+ A sandbox-hosted deployment imports the function in a sandbox, whose packages are not this process's, so there
+ a module name is checked against the standard library only.
+
+ Raises:
+ PipeFuncFunctionError: The module is not found, or the form is not proven.
+ TreeModuleShadowingError: A top-level module of the tree, or of the submitted Python, would shadow another.
+ """
+ library_id = get_current_library_id_or_none()
+ imports_in_this_process = not is_pipe_func_sandbox_hosted()
+ tree_bundle = self._tree_bundle
+ module_source: PipeFuncModuleSource
+ if tree_bundle is not None:
+ tree = tree_bundle.tree
+ # A crate loaded on another host carries the tree's files when its library captured them; the tree's
+ # directory is then the host that listed it, so the files are read in place of the disk.
+ captured = (
+ get_library_manager().get_pipe_func_files(library_id=library_id, root_key=tree_bundle.tree_key) if library_id is not None else None
+ )
+ try:
+ module_source = locate_in_tree(function_name=self.function_name, tree=tree, files=captured)
+ except OSError as exc:
+ msg = f"The module of the function_name '{self.function_name}' cannot be read from the method tree on this host."
+ raise PipeFuncFunctionError(msg) from exc
+ check_pipe_func_definition(module_source=module_source)
+ on_disk = captured is None and tree.directory is not None
+ check_tree_module_names(
+ top_level_modules=top_level_module_names(tree_paths=captured.keys() if captured is not None else tree.members),
+ # Named by the bundle it holds, never by its directory, which is a path on this host.
+ tree_label=tree_label_holding(tree_path=tree_bundle.tree_path),
+ root=tree.directory.resolve() if on_disk and tree.directory is not None else None,
+ checks_this_process=imports_in_this_process,
+ )
+ elif not imports_in_this_process:
+ submitted = (
+ get_library_manager().get_pipe_func_files(library_id=library_id, root_key=SUBMITTED_PYTHON_ROOT_KEY)
+ if library_id is not None
+ else None
+ )
+ module_source = locate_in_files(function_name=self.function_name, files=submitted or {})
+ check_pipe_func_definition(module_source=module_source)
+ check_tree_module_names(
+ top_level_modules=top_level_module_names(tree_paths=(submitted or {}).keys()),
+ tree_label="the Python submitted with the method",
+ root=None,
+ checks_this_process=False,
+ )
+ else:
+ module_source = locate_on_import_path(function_name=self.function_name)
+ check_pipe_func_definition(module_source=module_source)
+
+ def bind_function(self) -> Callable[..., Any]:
+ """Import the function in this process and check it, once per holder: the `direct` mode's way to reach it.
+
+ The import is made for the current library, which holds the tree's Python until it is torn down.
+
+ Raises:
+ PipeFuncFunctionError: The function cannot be located, its form is not proven, it does not follow the
+ calling convention, or its return type disagrees with the step's output.
+ TreePythonError: The tree's Python cannot load in this process, such as while another tree's is held.
+ """
+ holder = get_current_library_id_or_none() or _PROCESS_HOLDER
+ bound = self._bound_function
+ if bound is not None and bound[0] == holder:
+ return bound[1]
+ function: Callable[..., Any]
+ if self._tree_bundle is not None:
+ function = bind_tree_function(
+ function_name=self.function_name,
+ tree=self._tree_bundle.tree,
+ holder=holder,
+ tree_label=tree_label_holding(tree_path=self._tree_bundle.tree_path),
+ )
+ else:
+ function = bind_import_path_function(function_name=self.function_name)
+ self._check_return_type(function=function)
+ self._bound_function = (holder, function)
+ return function
+
+ def _check_return_type(self, *, function: Callable[..., Any]) -> None:
+ """Check the function's return type against the step's output concept and multiplicity.
+
+ Raises:
+ PipeFuncFunctionError: They disagree.
+ """
return_type: type[StuffContent] | None = get_type_hints(function).get("return")
if return_type is None:
- msg = (
- f"PipeFunc '{self.code}' failed to validate output with library: The return type of the function is None. "
- "It should be a subclass of StuffContent."
- )
- raise TypeError(msg)
+ msg = f"The return type of the function '{self.function_name}' is None. It should be a subclass of StuffContent."
+ raise PipeFuncFunctionError(msg)
if self.output.multiplicity and not issubclass(return_type, ListContent):
msg = (
f"PipeFunc '{self.code}' output multiplicity is '{self.output.multiplicity}', but the function '{self.function_name}' "
f"return type {return_type} is not a subclass of ListContent. The output of your PipeFunc is "
f"'{self.output.to_bundle_representation()}'. The return type of your function should be a subclass of ListContent."
)
- raise TypeError(msg)
+ raise PipeFuncFunctionError(msg)
if not self.output.multiplicity and issubclass(return_type, ListContent):
msg = (
f"PipeFunc '{self.code}' output multiplicity is '{self.output.multiplicity}', but the function '{self.function_name}' "
@@ -110,7 +239,7 @@ class PipeFunc(PipeOperator[PipeFuncOutput]):
f"'{self.output.concept.concept_ref}{self.output.to_bundle_representation()}' "
f"when it should be '{self.output.concept.concept_ref}' (no multiplicity)."
)
- raise TypeError(msg)
+ raise PipeFuncFunctionError(msg)
# Validate that the function's return type matches the concept's structure class
concept_structure_class = get_class_registry().get_class(name=self.output.concept.structure_class_name)
@@ -121,7 +250,7 @@ class PipeFunc(PipeOperator[PipeFuncOutput]):
f"The class may live in a Python module that was not part of the request — include that module, "
f"or express the type as MTHDS concepts."
)
- raise TypeError(msg)
+ raise PipeFuncFunctionError(msg)
# When multiplicity is set (e.g., output = "Expense[]"), the return type should be ListContent[T]
# where T matches the concept's structure class
@@ -159,7 +288,7 @@ class PipeFunc(PipeOperator[PipeFuncOutput]):
f"(from {item_type.__module__}.{item_type.__name__}). "
f"The item type of your ListContent should be '{self.output.concept.structure_class_name}' or a subclass of it."
)
- raise TypeError(msg)
+ raise PipeFuncFunctionError(msg)
# If no type_args found, return_type is raw ListContent without generic parameter - we already validated it's a ListContent subclass
else:
# No multiplicity - return type must match the concept's structure class
@@ -174,7 +303,7 @@ class PipeFunc(PipeOperator[PipeFuncOutput]):
f"but the function '{self.function_name}' return type is '{return_type.__name__}'. "
f"The return type of your function should be '{self.output.concept.structure_class_name}' or a subclass of it."
)
- raise TypeError(msg)
+ raise PipeFuncFunctionError(msg)
@override
async def _live_run_operator_pipe(
@@ -185,16 +314,21 @@ class PipeFunc(PipeOperator[PipeFuncOutput]):
pipe_run_params: PipeRunParams,
output_name: str | None = None,
) -> PipeFuncOutput:
+ # In `direct` mode the step runs the function it bound before the run (`_validate_before_run`, a cache hit here),
+ # and names it for the in-process executor, since the step may be one no library holds; out of process, the
+ # function is imported where the call lands.
+ bound_function = None if is_pipe_func_sandbox_hosted() else self._bind_for_run(pipe_run_params=pipe_run_params)
try:
- execution_result = await get_pipe_func_executor().run_pipe_func(
- job_metadata=job_metadata,
- # Qualified: the executor resolves this against the transported library, which is
- # keyed by pipe_ref. A bare code no longer resolves there.
- pipe_code=self.pipe_ref,
- function_name=self.function_name,
- working_memory=working_memory,
- pipe_run_params=pipe_run_params,
- )
+ with pipe_func_call_in_flight(pipe_ref=self.pipe_ref, function=bound_function) if bound_function is not None else nullcontext():
+ execution_result = await get_pipe_func_executor().run_pipe_func(
+ job_metadata=job_metadata,
+ # Qualified: the executor resolves this against the transported library, which is
+ # keyed by pipe_ref. A bare code no longer resolves there.
+ pipe_code=self.pipe_ref,
+ function_name=self.function_name,
+ working_memory=working_memory,
+ pipe_run_params=pipe_run_params,
+ )
except Exception as exc:
# PipeFunc runs arbitrary user code — in-process, or (in hosted mode) inside a sandbox
# reached through a Temporal activity — whose failure surface is not enumerable; any
@@ -258,11 +392,12 @@ class PipeFunc(PipeOperator[PipeFuncOutput]):
output_name: str | None = None,
) -> PipeFuncOutput:
log.debug("The dry run of a PipeFunc validates its inputs and return type without running the function", fields={"pipe_code": self.code})
- # Sandbox-hosted mode: the customer function is not registered in THIS process (its source only
- # travels on the crate to the sandbox), so its return type cannot be inspected here. Build the
- # mock output from the DECLARED output concept's structure class instead — mirroring PipeLLM's
- # dry run — and let the sandbox validate the real function's return type when it registers it.
- function = None if is_pipe_func_sandbox_hosted() else func_registry.get_required_function(self.function_name)
+ # Sandbox-hosted mode: the function is never imported in THIS process (the tree's files travel to the
+ # sandbox with the call), so its return type cannot be inspected here. Build the mock output from the
+ # DECLARED output concept's structure class instead — mirroring PipeLLM's dry run — and let the sandbox
+ # check the real function when it imports it. In direct mode the function was imported and checked
+ # before the run (`_validate_before_run`).
+ function = None if is_pipe_func_sandbox_hosted() else self._bind_for_run(pipe_run_params=pipe_run_params)
return_type: type[StuffContent]
if function is None:
return_type = get_class_registry().get_required_subclass(
@@ -327,22 +462,23 @@ class PipeFunc(PipeOperator[PipeFuncOutput]):
self, *, job_metadata: JobMetadata, working_memory: WorkingMemory, pipe_run_params: PipeRunParams, output_name: str | None = None
):
if is_pipe_func_sandbox_hosted():
- # Sandbox-hosted mode: the function is not registered in this process, so its return type
- # cannot be inspected here. The sandbox validates the real function when it registers it.
+ # Sandbox-hosted mode: the function is never imported in this process; the sandbox imports and checks it.
return
- function = func_registry.get_required_function(self.function_name)
- return_type = get_type_hints(function).get("return")
- # TODO: this should not happend ever. The correct way to do this would be to have a unit test making sure
- # that the FuncRegistry DOES CALL the 'is_eligible_function' function, and this function should be unit tested.
- if return_type is None:
- msg = f"Dry run failed for {self.type} '{self.code}': function '{self.function_name}' has no return type annotation"
- raise PipeRunError(message=msg, run_mode=pipe_run_params.run_mode, pipe_code=self.code)
- if not issubclass(return_type, StuffContent):
- msg = (
- f"Dry run failed for pipe {self.type} '{self.code}': "
- f"function '{self.function_name}' return type {return_type} is not a subclass of StuffContent"
- )
- raise PipeRunError(message=msg, run_mode=pipe_run_params.run_mode, pipe_code=self.code)
+ self._bind_for_run(pipe_run_params=pipe_run_params)
+
+ def _bind_for_run(self, *, pipe_run_params: PipeRunParams) -> Callable[..., Any]:
+ """The function, imported and checked for this run, or a `PipeRunError` on the step saying why it cannot be called."""
+ try:
+ return self.bind_function()
+ except (PipeFuncFunctionError, TreePythonError) as exc:
+ msg = f"PipeFunc '{self.code}' cannot call its function '{self.function_name}': {exc.message}"
+ raise PipeRunError(message=msg, run_mode=pipe_run_params.run_mode, pipe_code=self.code) from exc
+ except Exception as exc:
+ # Importing the function's module runs the method's own module-level code, whose failure surface is not
+ # enumerable: an ImportError, the guard's refusal of a file the tree excludes included, or anything the
+ # code raises. Wrapped on the step, as a failure of the function itself is, and never swallowed.
+ msg = f"PipeFunc '{self.code}' cannot import the module of its function '{self.function_name}': {type(exc).__name__}: {exc}"
+ raise PipeRunError(message=msg, run_mode=pipe_run_params.run_mode, pipe_code=self.code) from exc
@override
async def _validate_after_run(
diff --git a/pipelex/pipe_operators/func/pipe_func_blueprint.py b/pipelex/pipe_operators/func/pipe_func_blueprint.py
index b5201cb15..1050d838c 100644
--- a/pipelex/pipe_operators/func/pipe_func_blueprint.py
+++ b/pipelex/pipe_operators/func/pipe_func_blueprint.py
@@ -1,15 +1,35 @@
from typing import Literal
-from pydantic import Field
+from mthds.method_tree.tree_path import nfc
+from pydantic import Field, field_validator
from typing_extensions import override
from pipelex.pipe_machinery.pipe_blueprint import PipeBlueprint
+from pipelex.pipe_operators.func.function_name import function_name_form_violation
class PipeFuncBlueprint(PipeBlueprint):
type: Literal["PipeFunc"] = "PipeFunc"
pipe_category: Literal["PipeOperator"] = "PipeOperator"
- function_name: str = Field(description="The name of the function to call.")
+ function_name: str = Field(
+ description=(
+ "The import path of the function to call, from the root of the method tree: the module's dotted path, then the "
+ "function's name, such as 'funcs.totals.compute_totals' for the function compute_totals of funcs/totals.py. "
+ "The function must be a top-level def or async def of that module, decorated with @pipe_func()."
+ ),
+ json_schema_extra={"pattern": r"^[^.]+(\.[^.]+)+$"},
+ )
+
+ @field_validator("function_name", mode="before")
+ @classmethod
+ def validate_function_name(cls, function_name: object) -> object:
+ """Refuse a `function_name` that is not an import path, a flat registry key included, and keep it in NFC."""
+ if not isinstance(function_name, str):
+ return function_name
+ violation = function_name_form_violation(function_name=function_name)
+ if violation is not None:
+ raise ValueError(violation)
+ return nfc(function_name)
@override
def validate_inputs(self):
diff --git a/pipelex/pipe_operators/func/pipe_func_call_context.py b/pipelex/pipe_operators/func/pipe_func_call_context.py
new file mode 100644
index 000000000..e9f4d0aec
--- /dev/null
+++ b/pipelex/pipe_operators/func/pipe_func_call_context.py
@@ -0,0 +1,39 @@
+"""The function of the `PipeFunc` call in flight in this context, for the in-process executor to run.
+
+A `PipeFunc` step imports and checks its own function before it runs (`PipeFunc.bind_function`), and the executor it
+hands the call to receives only the step's ref and `function_name`, the shape every executor shares, sandboxes
+included. In `direct` mode the step holds the function the executor must run, and it may be a step no library holds,
+such as one built from its blueprint by a programmatic caller: the step names the function it bound here, for the
+duration of the call, and the in-process executor takes it from here before it looks the step up in the library.
+"""
+
+from collections.abc import Callable, Generator
+from contextlib import contextmanager
+from contextvars import ContextVar
+from typing import Any, NamedTuple
+
+
+class _CallInFlight(NamedTuple):
+ pipe_ref: str
+ function: Callable[..., Any]
+
+
+_CALL_IN_FLIGHT: ContextVar[_CallInFlight | None] = ContextVar("pipe_func_call_in_flight", default=None)
+
+
+@contextmanager
+def pipe_func_call_in_flight(*, pipe_ref: str, function: Callable[..., Any]) -> Generator[None]:
+ """Name the function the step `pipe_ref` bound as the one its call runs, for the scope."""
+ token = _CALL_IN_FLIGHT.set(_CallInFlight(pipe_ref=pipe_ref, function=function))
+ try:
+ yield
+ finally:
+ _CALL_IN_FLIGHT.reset(token)
+
+
+def function_of_call_in_flight(*, pipe_ref: str) -> Callable[..., Any] | None:
+ """The function the step `pipe_ref` bound for its call in flight in this context, or None when no such call is."""
+ call_in_flight = _CALL_IN_FLIGHT.get()
+ if call_in_flight is None or call_in_flight.pipe_ref != pipe_ref:
+ return None
+ return call_in_flight.function
diff --git a/pipelex/pipe_operators/func/pipe_func_definition_check.py b/pipelex/pipe_operators/func/pipe_func_definition_check.py
new file mode 100644
index 000000000..0d887ffb6
--- /dev/null
+++ b/pipelex/pipe_operators/func/pipe_func_definition_check.py
@@ -0,0 +1,168 @@
+"""The static check of a `PipeFunc`'s function: its form, proven from the module's syntax tree without running Python.
+
+The function a `function_name` names must be a top-level `def` or `async def` of its module, bound there once, and
+decorated with the `@pipe_func()` marker, directly or through an import alias of it or of its module. Anything the
+syntax tree cannot decide is refused rather than guessed at: a re-export (`from .impl import compute_totals`), an
+assignment (`compute_totals = build()`), a class, a definition under an `if`, a `try` or a loop, and a name bound twice.
+This is the strict form of `wip/extra-files/design.md`'s D10: validation proves it from the source alone, and the
+calling convention, which needs the annotations resolved, is checked once the module is imported
+(`pipe_func_signature`).
+"""
+
+import ast
+from functools import lru_cache
+
+from pipelex.pipe_operators.func.function_name import FUNCTION_NAMES_MIGRATION_COMMAND
+from pipelex.system.registries.func_registry import PIPE_FUNC_MARKER_MODULE, PIPE_FUNC_MARKER_NAME
+
+_MARKER_PARENT_MODULE, _, _MARKER_MODULE_LEAF = PIPE_FUNC_MARKER_MODULE.rpartition(".")
+_MARKER_SPELLING = f"@{PIPE_FUNC_MARKER_NAME}()"
+_COMPOUND_STATEMENTS = (ast.If, ast.Try, ast.TryStar, ast.With, ast.AsyncWith, ast.For, ast.AsyncFor, ast.While, ast.Match)
+# The modules parsed lately: the steps calling functions of one module, and every validation of a method loaded again,
+# read the same source, which is parsed once. A syntax tree here is only read, never changed.
+_PARSED_MODULES_KEPT = 64
+
+
+@lru_cache(maxsize=_PARSED_MODULES_KEPT)
+def _parsed_module(*, source: str, module_label: str) -> ast.Module | str:
+ """The module's syntax tree, or the message refusing a source that does not parse."""
+ try:
+ return ast.parse(source, filename=module_label)
+ except SyntaxError as exc:
+ return f"The module {module_label} does not parse as Python: {exc.msg} (line {exc.lineno})."
+
+
+def pipe_func_definition_violation(*, source: str, function: str, module_label: str) -> str | None:
+ """Why the module's source does not define `function` as a `PipeFunc` may call it, or None when it does.
+
+ Args:
+ source: The module's source text.
+ function: The name of the function in the module, the last segment of the `function_name`.
+ module_label: How the messages name the module, such as its tree path.
+
+ Returns:
+ None when the form is proven, else a refusal message naming the module and the function.
+ """
+ module = _parsed_module(source=source, module_label=module_label)
+ if isinstance(module, str):
+ return module
+
+ definitions: list[ast.FunctionDef | ast.AsyncFunctionDef] = []
+ for statement in module.body:
+ if isinstance(statement, (ast.FunctionDef, ast.AsyncFunctionDef)):
+ if statement.name == function:
+ definitions.append(statement)
+ continue
+ other_binding = _other_binding_kind(statement=statement, name=function)
+ if other_binding is not None:
+ return (
+ f"The module {module_label} binds '{function}' through {other_binding}, which validation cannot prove to be a function "
+ f"carrying {_MARKER_SPELLING}: define it as a top-level def or async def of {module_label} itself."
+ )
+
+ if not definitions:
+ return (
+ f"The module {module_label} defines no top-level function '{function}': the function must be a def or async def at the top of the module."
+ )
+ if len(definitions) > 1:
+ return f"The module {module_label} defines '{function}' more than once at top level, so which one runs depends on the order: keep one."
+ return _marker_violation(definition=definitions[0], marker_refs=marker_references(module=module), module_label=module_label)
+
+
+def _other_binding_kind(*, statement: ast.stmt, name: str) -> str | None:
+ """How a top-level statement other than a function definition binds `name`, described for a message, or None."""
+ if isinstance(statement, ast.ClassDef):
+ return "a class definition" if statement.name == name else None
+ if isinstance(statement, (ast.Import, ast.ImportFrom)):
+ for alias in statement.names:
+ bound_name = alias.asname or alias.name.partition(".")[0]
+ if bound_name == name:
+ return "an import, which re-exports a function defined elsewhere"
+ return None
+ if isinstance(statement, _COMPOUND_STATEMENTS):
+ return "a definition under a condition, a loop or a context manager" if _binds_name_anywhere(node=statement, name=name) else None
+ if _binds_name_anywhere(node=statement, name=name):
+ return "an assignment"
+ return None
+
+
+def _binds_name_anywhere(*, node: ast.AST, name: str) -> bool:
+ """Whether `node` binds `name` in the module's scope, at any depth short of a nested function or class body."""
+ pending: list[ast.AST] = [node]
+ while pending:
+ current = pending.pop()
+ if isinstance(current, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)):
+ if current.name == name:
+ return True
+ continue
+ if isinstance(current, ast.Lambda):
+ continue
+ if isinstance(current, ast.Name) and current.id == name and isinstance(current.ctx, (ast.Store, ast.Del)):
+ return True
+ if isinstance(current, (ast.Import, ast.ImportFrom)):
+ for alias in current.names:
+ if (alias.asname or alias.name.partition(".")[0]) == name:
+ return True
+ if isinstance(current, ast.ExceptHandler) and current.name == name:
+ return True
+ if isinstance(current, (ast.MatchAs, ast.MatchStar)) and current.name == name:
+ return True
+ pending.extend(ast.iter_child_nodes(current))
+ return False
+
+
+def marker_references(*, module: ast.Module) -> frozenset[str]:
+ """The dotted spellings under which the module's top-level imports make the marker reachable."""
+ references: set[str] = set()
+ for statement in module.body:
+ if isinstance(statement, ast.ImportFrom) and statement.level == 0:
+ for alias in statement.names:
+ if statement.module == PIPE_FUNC_MARKER_MODULE and alias.name == PIPE_FUNC_MARKER_NAME:
+ references.add(alias.asname or alias.name)
+ elif statement.module == _MARKER_PARENT_MODULE and alias.name == _MARKER_MODULE_LEAF:
+ references.add(f"{alias.asname or alias.name}.{PIPE_FUNC_MARKER_NAME}")
+ elif isinstance(statement, ast.Import):
+ for alias in statement.names:
+ if alias.name != PIPE_FUNC_MARKER_MODULE:
+ continue
+ spelled_module = alias.asname or PIPE_FUNC_MARKER_MODULE
+ references.add(f"{spelled_module}.{PIPE_FUNC_MARKER_NAME}")
+ return frozenset(references)
+
+
+def _dotted_name(*, expression: ast.expr) -> str | None:
+ """The dotted spelling of a name or attribute chain, such as `registries.func_registry.pipe_func`, else None."""
+ if isinstance(expression, ast.Name):
+ return expression.id
+ if isinstance(expression, ast.Attribute):
+ owner = _dotted_name(expression=expression.value)
+ return f"{owner}.{expression.attr}" if owner is not None else None
+ return None
+
+
+def marker_call(*, definition: ast.FunctionDef | ast.AsyncFunctionDef, marker_refs: frozenset[str]) -> ast.Call | None:
+ """The decorator of a function that calls the marker, such as `@pipe_func()` or `@pipe_func(name="x")`, else None."""
+ for decorator in definition.decorator_list:
+ if isinstance(decorator, ast.Call) and _dotted_name(expression=decorator.func) in marker_refs:
+ return decorator
+ return None
+
+
+def _marker_violation(*, definition: ast.FunctionDef | ast.AsyncFunctionDef, marker_refs: frozenset[str], module_label: str) -> str | None:
+ """Why the function's decorators do not include the marker as `@pipe_func()`, or None when they do."""
+ function = definition.name
+ for decorator in definition.decorator_list:
+ if isinstance(decorator, ast.Call) and _dotted_name(expression=decorator.func) in marker_refs:
+ if decorator.args or decorator.keywords:
+ return (
+ f"The function '{function}' of {module_label} passes arguments to {_MARKER_SPELLING}, which takes none: the import path "
+ "in function_name is the function's only name, so the registration name the marker once took is gone. "
+ f"Run `{FUNCTION_NAMES_MIGRATION_COMMAND} <method folder>` to rewrite the method."
+ )
+ return None
+ if _dotted_name(expression=decorator) in marker_refs:
+ return f"The function '{function}' of {module_label} is decorated with the marker without parentheses: write {_MARKER_SPELLING}."
+ return (
+ f"The function '{function}' of {module_label} is not decorated with {_MARKER_SPELLING}, imported from {PIPE_FUNC_MARKER_MODULE}: "
+ "only a function carrying the marker can be called from a PipeFunc step."
+ )
diff --git a/pipelex/pipe_operators/func/pipe_func_execution_dtos.py b/pipelex/pipe_operators/func/pipe_func_execution_dtos.py
index 768bce4f9..505297d85 100644
--- a/pipelex/pipe_operators/func/pipe_func_execution_dtos.py
+++ b/pipelex/pipe_operators/func/pipe_func_execution_dtos.py
@@ -2,6 +2,7 @@ from typing import Any
from pydantic import BaseModel, ConfigDict, Field
+from pipelex.core.method_trees.tree_files import TreeFileEntry
from pipelex.libraries.library_crate import LibraryCrate
from pipelex.pipe_run.pipe_run_params import PipeRunParams
from pipelex.system.job_metadata import JobMetadata
@@ -25,14 +26,27 @@ class PipeFuncExecutionRequest(BaseModel):
The generic transport primitive for out-of-process PipeFunc execution — sibling to the
runtime-bridge serialization, carrying only open pipelex types. A backend (a sandbox box, a local
subprocess) runs it and returns a ``PipeFuncExecutionResponse``; a host runtime (a Temporal
- activity) forwards it across its boundary. Carries the crate (which includes ``python_sources`` —
- the customer's .py to register wherever it runs), the transported working memory (the function's
- inputs), and the identity of the pipe/function to run. Deliberately carries NO secrets.
+ activity) forwards it across its boundary. Carries the crate (the method, without the files of its
+ PipeFunc calls), the call's files (``files``, the Python root of the step called), the transported
+ working memory (the function's inputs), and the identity of the pipe/function to run. Deliberately
+ carries NO secrets.
"""
model_config = ConfigDict(extra="forbid")
crate: LibraryCrate
+ files: dict[str, TreeFileEntry] = Field(
+ default_factory=dict,
+ description=(
+ "The files of the called step's Python root, by tree path, written where the call runs before the function is "
+ "imported: every file of the step's method tree except its .mthds files, or for a step parsed from a content "
+ "string, the files submitted beside it. Each is a file entry of the MTHDS standard, its text or {'base64': ...}. "
+ "The root goes first on sys.path and the function is imported by its import path from there, so it imports "
+ "its sibling modules and reads a file beside its module as it does locally. A sandbox client may move them "
+ "out of the request and unpack them itself, such as from one archive per call, and hand the directory to the "
+ "entrypoint."
+ ),
+ )
working_memory_raw: dict[str, Any]
pipe_code: str
function_name: str
diff --git a/pipelex/pipe_operators/func/pipe_func_execution_transport.py b/pipelex/pipe_operators/func/pipe_func_execution_transport.py
index 8f2ca3036..84028034e 100644
--- a/pipelex/pipe_operators/func/pipe_func_execution_transport.py
+++ b/pipelex/pipe_operators/func/pipe_func_execution_transport.py
@@ -1,5 +1,6 @@
from pipelex.core.memory.working_memory import WorkingMemory
from pipelex.interpreter_hub import get_current_library, get_library_manager
+from pipelex.libraries.library_crate import SUBMITTED_PYTHON_ROOT_KEY
from pipelex.pipe_operators.func.exceptions import PipeFuncTransportError
from pipelex.pipe_operators.func.pipe_func_execution_dtos import (
DEFAULT_PIPE_FUNC_TIMEOUT_SECONDS,
@@ -26,7 +27,7 @@ def build_pipe_func_execution_request(
pipe_run_params: PipeRunParams,
timeout_seconds: float = DEFAULT_PIPE_FUNC_TIMEOUT_SECONDS,
) -> PipeFuncExecutionRequest:
- """Package one PipeFunc call for out-of-process execution: the current library's crate (with sources) + the transported inputs.
+ """Package one PipeFunc call for out-of-process execution: the current library's crate, the call's files, and the transported inputs.
Shared by every out-of-process executor — the in-process sandbox one and the Temporal in-workflow
one — so the request shape is defined once. Reads the crate from the current library via the
@@ -45,12 +46,18 @@ def build_pipe_func_execution_request(
if crate is None:
msg = (
f"Cannot run PipeFunc '{function_name}' out-of-process: no crate is available for the current library "
- f"'{library_id}'. The library must be loaded in sandbox-hosted mode so the customer sources travel on the crate."
+ f"'{library_id}'. The library must be loaded in sandbox-hosted mode so the customer files travel on the crate."
)
raise PipeFuncTransportError(msg)
+ # The call carries its own Python root's files and no other: the files of the method tree its step was read from,
+ # or the files submitted beside the bundles parsed from content strings. The crate travels without them.
+ tree_bundle_ref = crate.tree_bundles.get(pipe_code)
+ root_key = tree_bundle_ref.tree_key if tree_bundle_ref is not None else SUBMITTED_PYTHON_ROOT_KEY
+ call_files = crate.pipe_func_files.get(root_key, {})
return PipeFuncExecutionRequest(
- crate=crate,
+ crate=crate.model_copy(update={"pipe_func_files": {}}),
+ files=dict(call_files),
working_memory_raw=working_memory.dump_for_transport(),
pipe_code=pipe_code,
function_name=function_name,
diff --git a/pipelex/pipe_operators/func/pipe_func_factory.py b/pipelex/pipe_operators/func/pipe_func_factory.py
index 5ab027a54..4cad9bf83 100644
--- a/pipelex/pipe_operators/func/pipe_func_factory.py
+++ b/pipelex/pipe_operators/func/pipe_func_factory.py
@@ -2,6 +2,7 @@ from typing import Any
from typing_extensions import override
+from pipelex.core.method_trees.building_bundle import get_building_tree_bundle
from pipelex.core.pipes.inputs.input_stuff_specs import InputStuffSpecs
from pipelex.core.pipes.stuff_spec.stuff_spec import StuffSpec
from pipelex.pipe_machinery.pipe_factory import PipeFactoryProtocol
@@ -24,8 +25,7 @@ class PipeFuncFactory(PipeFactoryProtocol[PipeFuncBlueprint, PipeFunc]):
output: StuffSpec,
blueprint: PipeFuncBlueprint,
) -> PipeFunc:
- # TODO: make function_name into a callable in PipeFunc
- return PipeFunc(
+ pipe_func = PipeFunc(
domain_code=domain_code,
code=pipe_code,
description=description,
@@ -33,3 +33,7 @@ class PipeFuncFactory(PipeFactoryProtocol[PipeFuncBlueprint, PipeFunc]):
output=output,
function_name=blueprint.function_name,
)
+ # The function's import path resolves from the root of the method tree the bundle was read in, recorded here
+ # without reading anything: the module is located and checked when the library validates the step.
+ pipe_func.place_in_tree(tree_bundle=get_building_tree_bundle())
+ return pipe_func
diff --git a/pipelex/pipe_operators/func/pipe_func_python.py b/pipelex/pipe_operators/func/pipe_func_python.py
new file mode 100644
index 000000000..59231ecf3
--- /dev/null
+++ b/pipelex/pipe_operators/func/pipe_func_python.py
@@ -0,0 +1,220 @@
+"""Where a `PipeFunc`'s function lives, how its form is proven without running Python, and how it is imported to be called.
+
+A `function_name` is an import path from an import root, which is one of three places:
+
+- **The method tree** the step's bundle was read as a member of, on this host's disk (`mthds/docs/spec/method-tree.md`,
+ "Python Modules"). The module is located among the tree's files with `mthds`' `locate_function`, and imported under
+ the process's hold on the tree's Python (`pipelex.core.method_trees.tree_python`).
+- **The Python submitted with a bundle parsed from a content string**, which a sandbox-hosted deployment captures from
+ its library directories and sends to the sandbox with the call. Nothing imports it in this process: the step is
+ validated from the captured text.
+- **This process's own import path**, for a bundle parsed from a content string in `direct` mode, such as a bundle a
+ program builds in memory beside its own modules. The module is located through the import system's finders without
+ importing it, then imported as any module of the program is.
+
+Validation never runs Python: it locates the module, reads its source, and proves from the syntax tree that the
+function is a top-level `def` or `async def` carrying the `@pipe_func()` marker (`pipe_func_definition_check`). The
+function's calling convention is checked once it is imported, when its step first runs.
+"""
+
+import importlib
+import importlib.machinery
+import importlib.util
+import inspect
+import sys
+from collections.abc import Callable, Mapping
+from pathlib import Path
+from types import ModuleType
+from typing import Any, NamedTuple
+
+from mthds.method_tree.digest import digest_of_contents, tree_files_from_contents
+from mthds.method_tree.exceptions import MethodTreeRefusedError
+from mthds.method_tree.models import MethodTree
+from mthds.method_tree.python_modules import locate_function
+
+from pipelex.core.method_trees.tree_files import TreeFileEntry, tree_file_bytes
+from pipelex.core.method_trees.tree_python import TREE_PYTHON_HOLD, import_held_tree_module
+from pipelex.pipe_operators.func.exceptions import PipeFuncFunctionError
+from pipelex.pipe_operators.func.function_name import split_function_name
+from pipelex.pipe_operators.func.pipe_func_definition_check import pipe_func_definition_violation
+from pipelex.pipe_operators.func.pipe_func_signature import pipe_func_signature_violation
+from pipelex.system.registries.func_registry import is_marked_pipe_func
+
+
+class PipeFuncModuleSource(NamedTuple):
+ """The module a `function_name` names, located and read: its import name, the function's name, how messages name it, its text."""
+
+ module_name: str
+ function: str
+ module_label: str
+ source: str
+
+
+def locate_in_tree(*, function_name: str, tree: MethodTree, files: Mapping[str, TreeFileEntry] | None = None) -> PipeFuncModuleSource:
+ """Locate the module of a `function_name` among a method tree's files, and read it.
+
+ Args:
+ function_name: The step's `function_name`.
+ tree: The method tree the step's bundle is a member of.
+ files: The tree's files as captured for a call, read in place of the disk when given.
+
+ Raises:
+ PipeFuncFunctionError: The tree holds no such module, or holds it twice.
+ """
+ import_path = split_function_name(function_name=function_name)
+ try:
+ location = locate_function(function_name, tree=tree)
+ except MethodTreeRefusedError as exc:
+ raise PipeFuncFunctionError(str(exc)) from exc
+ source_bytes: bytes
+ if files is not None and location.module in files:
+ source_bytes = tree_file_bytes(entry=files[location.module])
+ else:
+ source_bytes = tree.file_location(location.module).read_bytes()
+ return PipeFuncModuleSource(
+ module_name=import_path.module_name,
+ function=import_path.function,
+ module_label=location.module,
+ source=_decoded_source(source_bytes=source_bytes, module_label=location.module),
+ )
+
+
+def locate_in_files(*, function_name: str, files: Mapping[str, TreeFileEntry]) -> PipeFuncModuleSource:
+ """Locate the module of a `function_name` among submitted files keyed by their path from the import root, and read it.
+
+ Raises:
+ PipeFuncFunctionError: The files hold no such module, or hold it twice.
+ """
+ contents = {tree_path: tree_file_bytes(entry=entry) for tree_path, entry in files.items()}
+ tree = MethodTree(files=tree_files_from_contents(contents), digest=digest_of_contents(contents))
+ return locate_in_tree(function_name=function_name, tree=tree, files=files)
+
+
+def locate_on_import_path(*, function_name: str) -> PipeFuncModuleSource:
+ """Locate the module of a `function_name` on this process's import path without importing it, and read it.
+
+ Each package of the path is found through the finders, not imported, so no package's code runs before the
+ function's form is proven; a package already imported is read from `sys.modules`.
+
+ Raises:
+ PipeFuncFunctionError: No such module is found, or it is not a Python source file.
+ """
+ import_path = split_function_name(function_name=function_name)
+ module_file = _find_source_file(module_name=import_path.module_name, function_name=function_name)
+ # Named by its import name, never by its file, which is a path on this host.
+ module_label = f"'{import_path.module_name}'"
+ return PipeFuncModuleSource(
+ module_name=import_path.module_name,
+ function=import_path.function,
+ module_label=module_label,
+ source=_decoded_source(source_bytes=module_file.read_bytes(), module_label=module_label),
+ )
+
+
+def check_pipe_func_definition(*, module_source: PipeFuncModuleSource) -> None:
+ """Prove from the module's syntax tree that the function is a top-level `def` or `async def` carrying the marker.
+
+ Raises:
+ PipeFuncFunctionError: The form is not proven.
+ """
+ violation = pipe_func_definition_violation(
+ source=module_source.source,
+ function=module_source.function,
+ module_label=module_source.module_label,
+ )
+ if violation is not None:
+ raise PipeFuncFunctionError(violation)
+
+
+def bind_tree_function(*, function_name: str, tree: MethodTree, holder: str, tree_label: str) -> Callable[..., Any]:
+ """Import a method tree's function under the process's hold on the tree's Python, and check it.
+
+ The hold is taken for `holder`, the library of the run, and released when that library is torn down. Messages name
+ the tree by `tree_label`.
+
+ Raises:
+ PipeFuncFunctionError: The static form is not proven, or the imported function does not follow the
+ calling convention.
+ TreePythonHeldError: Another tree's Python is held by a run in flight.
+ TreeModuleShadowingError: A top-level module of the tree would shadow another module.
+ """
+ module_source = locate_in_tree(function_name=function_name, tree=tree)
+ check_pipe_func_definition(module_source=module_source)
+ identity = TREE_PYTHON_HOLD.acquire(tree=tree, holder=holder, label=tree_label)
+ module = import_held_tree_module(identity=identity, module_name=module_source.module_name)
+ return checked_pipe_func(module=module, module_source=module_source)
+
+
+def bind_import_path_function(*, function_name: str) -> Callable[..., Any]:
+ """Import a function from this process's import path, and check it.
+
+ Raises:
+ PipeFuncFunctionError: The static form is not proven, or the imported function does not follow the
+ calling convention.
+ """
+ module_source = locate_on_import_path(function_name=function_name)
+ check_pipe_func_definition(module_source=module_source)
+ module = importlib.import_module(module_source.module_name)
+ return checked_pipe_func(module=module, module_source=module_source)
+
+
+def checked_pipe_func(*, module: ModuleType, module_source: PipeFuncModuleSource) -> Callable[..., Any]:
+ """The function of an imported module, once it is proven to be the module's own function, marked, and conventional.
+
+ Raises:
+ PipeFuncFunctionError: It is missing, not a function defined in the module, unmarked, or does not follow the
+ calling convention.
+ """
+ function_label = f"'{module_source.function}' of {module_source.module_label}"
+ func = getattr(module, module_source.function, None)
+ if func is None:
+ msg = f"The function {function_label} is missing once the module is imported."
+ raise PipeFuncFunctionError(msg)
+ if not inspect.isroutine(func) or getattr(func, "__module__", None) != module.__name__:
+ msg = f"The attribute {function_label} is not a function the module defines itself once it is imported."
+ raise PipeFuncFunctionError(msg)
+ if not is_marked_pipe_func(func=func):
+ msg = f"The function {function_label} does not carry the @pipe_func() marker once the module is imported."
+ raise PipeFuncFunctionError(msg)
+ signature_violation = pipe_func_signature_violation(func=func)
+ if signature_violation is not None:
+ msg = f"The function {function_label} {signature_violation}."
+ raise PipeFuncFunctionError(msg)
+ return func
+
+
+def _decoded_source(*, source_bytes: bytes, module_label: str) -> str:
+ try:
+ return source_bytes.decode("utf-8")
+ except UnicodeDecodeError as exc:
+ msg = f"The module {module_label} is not UTF-8 text, so its source cannot be read."
+ raise PipeFuncFunctionError(msg) from exc
+
+
+def _find_source_file(*, module_name: str, function_name: str) -> Path:
+ """The source file of a module on the import path, found without importing it or any package above it."""
+ segments = module_name.split(".")
+ spec: importlib.machinery.ModuleSpec | None = None
+ for index in range(len(segments)):
+ dotted = ".".join(segments[: index + 1])
+ loaded = sys.modules.get(dotted)
+ if loaded is not None and getattr(loaded, "__spec__", None) is not None:
+ spec = loaded.__spec__
+ elif index == 0:
+ try:
+ spec = importlib.util.find_spec(dotted)
+ except (ImportError, ValueError):
+ spec = None
+ else:
+ parent_locations = spec.submodule_search_locations if spec is not None else None
+ if parent_locations is None:
+ msg = f"The function_name '{function_name}' names the module '{module_name}', but '{'.'.join(segments[:index])}' is not a package."
+ raise PipeFuncFunctionError(msg)
+ spec = importlib.machinery.PathFinder.find_spec(dotted, list(parent_locations))
+ if spec is None:
+ msg = f"The function_name '{function_name}' names the module '{module_name}', which is not found on this process's import path."
+ raise PipeFuncFunctionError(msg)
+ if spec is None or spec.origin is None or not spec.has_location or not spec.origin.endswith(".py"):
+ msg = f"The function_name '{function_name}' names the module '{module_name}', which is not a Python source file."
+ raise PipeFuncFunctionError(msg)
+ return Path(spec.origin)
diff --git a/pipelex/pipe_operators/func/pipe_func_signature.py b/pipelex/pipe_operators/func/pipe_func_signature.py
new file mode 100644
index 000000000..8c57d8a5e
--- /dev/null
+++ b/pipelex/pipe_operators/func/pipe_func_signature.py
@@ -0,0 +1,65 @@
+"""The calling convention a `PipeFunc`'s function must follow, checked on the imported function.
+
+The function takes exactly one parameter, named `working_memory` and typed `WorkingMemory`, and declares a return type
+that is a `StuffContent` subclass, such as `TextContent` or `ListContent[...]`. Reading the annotations takes
+`get_type_hints`, which needs the imported module, so this check runs when the function's module is imported, as the
+step first runs, never at validation: validation proves the form from the syntax tree without running Python
+(`pipe_func_definition_check`).
+"""
+
+import inspect
+from collections.abc import Callable
+from typing import Any, get_origin, get_type_hints
+
+from pipelex.core.memory.working_memory import WorkingMemory
+from pipelex.core.stuffs.stuff_content import StuffContent
+
+
+def pipe_func_signature_violation(*, func: Callable[..., Any]) -> str | None:
+ """Why `func` does not follow the `PipeFunc` calling convention, or None when it does.
+
+ Args:
+ func: The imported function.
+
+ Returns:
+ None when the function is eligible, else a reason that completes the sentence "The function ...".
+ """
+ try:
+ signature = inspect.signature(func)
+ except (ValueError, TypeError) as exc:
+ return f"has a signature that cannot be inspected: {exc}"
+
+ params = list(signature.parameters.values())
+ if not params:
+ return "must have exactly one parameter named 'working_memory', but has no parameters"
+ if len(params) > 1:
+ return f"must have exactly one parameter named 'working_memory', but has {len(params)} parameters"
+ param = params[0]
+ if param.name != "working_memory":
+ return f"must name its parameter 'working_memory', but names it '{param.name}'"
+
+ try:
+ type_hints = get_type_hints(func)
+ except (NameError, TypeError) as exc:
+ return f"has type annotations that cannot be resolved: {exc}"
+
+ if "working_memory" not in type_hints:
+ return "must annotate its parameter 'working_memory' with the type 'WorkingMemory'"
+ param_type = type_hints["working_memory"]
+ if param_type is not WorkingMemory:
+ return f"must annotate its parameter 'working_memory' with the type 'WorkingMemory', but annotates it '{param_type}'"
+
+ if "return" not in type_hints:
+ return "must have a return type annotation that is a subclass of StuffContent"
+ return_type = type_hints["return"]
+ if _is_stuff_content_type(annotation=return_type):
+ return None
+ return f"must return a subclass of StuffContent, but its return type is '{return_type}'"
+
+
+def _is_stuff_content_type(*, annotation: Any) -> bool:
+ """Whether a return annotation is a `StuffContent` subclass, or a parametrized generic of one such as `ListContent[X]`."""
+ if inspect.isclass(annotation) and issubclass(annotation, StuffContent):
+ return True
+ origin = get_origin(annotation)
+ return inspect.isclass(origin) and issubclass(origin, StuffContent)
diff --git a/pipelex/pipe_operators/func/pipe_func_transported_entrypoint.py b/pipelex/pipe_operators/func/pipe_func_transported_entrypoint.py
index 0354541c9..2ab8cf037 100644
--- a/pipelex/pipe_operators/func/pipe_func_transported_entrypoint.py
+++ b/pipelex/pipe_operators/func/pipe_func_transported_entrypoint.py
@@ -2,13 +2,17 @@
Invoked as a subprocess INSIDE an isolated sandbox (a Daytona box or a local subprocess): it reads a
serialized ``PipeFuncExecutionRequest``, runs it via the ``direct`` executor's transported path (which
-registers the customer's real code from the crate and runs the one function), and writes the
-serialized ``PipeFuncExecutionResponse``.
+writes the call's files, imports the step's function by its import path from their root, and runs the
+one function), and writes the serialized ``PipeFuncExecutionResponse``.
+
+An optional third argument names a directory the sandbox client already unpacked the call's files into,
+such as from one archive per call: it becomes the call's import root, and is left in place. Without it,
+the files the request carries are written into a temporary directory, removed after the call.
It lives in open ``pipelex`` and imports no sandbox code, so a box needs only ``pipelex`` installed —
the sandbox *client* (box provisioning, upload/download, teardown) is a worker-side concern that stays
in the out-of-tree plugin. This is the module a sandbox client runs as
-``python -m pipelex.pipe_operators.func.pipe_func_transported_entrypoint <request_json> <result_json>``.
+``python -m pipelex.pipe_operators.func.pipe_func_transported_entrypoint <request_json> <result_json> [<tree_root>]``.
On any failure the process exits non-zero with a diagnostic on stderr, which the calling sandbox client
turns into a sandbox error.
@@ -23,17 +27,22 @@ from pipelex.pipe_operators.func.pipe_func_execution_dtos import PipeFuncExecuti
def main() -> None:
- """argv: <request_json_path> <result_json_path>."""
+ """argv: <request_json_path> <result_json_path> [<tree_root>]."""
request_path = Path(sys.argv[1])
result_path = Path(sys.argv[2])
+ tree_root = Path(sys.argv[3]).resolve() if len(sys.argv) > 3 else None
request = PipeFuncExecutionRequest.model_validate_json(request_path.read_text(encoding="utf-8"))
# Import here so module import (e.g. for tests) stays cheap and free of boot side effects.
from pipelex.pipelex import Pipelex # ruff: ignore[import-outside-top-level]
Pipelex.make(needs_inference=False)
+ executor = DirectPipeFuncExecutor()
try:
- result = asyncio.run(DirectPipeFuncExecutor().run_pipe_func_transported(request=request))
+ if tree_root is None:
+ result = asyncio.run(executor.run_pipe_func_transported(request=request))
+ else:
+ result = asyncio.run(executor.run_pipe_func_transported_at(request=request, tree_root=tree_root))
finally:
Pipelex.teardown_if_needed()
result_path.write_text(result.model_dump_json(), encoding="utf-8")
diff --git a/pipelex/pipeline/validate_bundle_translation.py b/pipelex/pipeline/validate_bundle_translation.py
index 0c9cf52ac..4fe5ce840 100644
--- a/pipelex/pipeline/validate_bundle_translation.py
+++ b/pipelex/pipeline/validate_bundle_translation.py
@@ -44,7 +44,6 @@ from pipelex.mthds_parsing.handle_pipe_errors import (
)
from pipelex.pipe_run.exceptions import DryRunError
from pipelex.pipeline.exceptions import ValidateBundleError
-from pipelex.system.registries.exceptions import FuncRegistryError
from pipelex.tools.misc.file_utils import find_files_in_dir
from pipelex.tools.typing.module_inspector import convert_file_path_to_module_path
@@ -162,13 +161,6 @@ def translate_to_validate_bundle_error() -> Generator[None, None, None]:
pipelex_bundle_blueprint_validation_errors=blueprint_validation_errors,
pipe_validation_errors=pipe_concept_validation_errors,
) from library_error
- except FuncRegistryError as func_registry_error:
- # A duplicate @pipe_func name across the scanned library dirs. Raised while loading the
- # library, so it lands here rather than on the PipeFunc field validator's ValueError path.
- # It is caller-fixable input (rename one with @pipe_func(name=...)), so it must produce a
- # verdict — `is_valid: false` — not the "no verdict could be produced" exit 2 / 5xx an
- # untranslated error would give.
- raise ValidateBundleError(message=func_registry_error.message) from func_registry_error
except PipeRunError as pipe_run_error:
# A pipe run failing outside the sweep's per-pipe classification: one dry-run item on its pipe,
# whose text follows the disclosure rule, since the verdict carries it past STRICT disclosure.
diff --git a/pipelex/runtime_boot.py b/pipelex/runtime_boot.py
index 6c296fd47..25932dabd 100644
--- a/pipelex/runtime_boot.py
+++ b/pipelex/runtime_boot.py
@@ -97,7 +97,6 @@ from pipelex.system.configuration.config_root import ConfigRoot
from pipelex.system.configuration.config_surface import INFERENCE_BACKEND_CONFIG_SURFACE_ID, PIPELEX_CONFIG_SURFACE_ID
from pipelex.system.configuration.configs import PipelexConfig
from pipelex.system.registries.class_registry_access import class_registry_scoping
-from pipelex.system.registries.func_registry import FuncRegistry, func_registry
from pipelex.system.registries.singleton import MetaSingleton
from pipelex.system.runtime import IntegrationMode, runtime_manager
from pipelex.system.telemetry.observer_telemetry import ObserverTelemetry
@@ -263,7 +262,6 @@ class RuntimeBoot(metaclass=MetaSingleton):
# tools
self.class_registry: ClassRegistryAbstract | None = None
- self.func_registry: FuncRegistry | None = None
# plugins — the registrar is built in setup() (build_registrar); held for the slot-claim /
# CLI-command / teardown apply-points. Declared here so teardown() can guard on it.
self._plugin_registrar: PluginRegistrar | None = None
@@ -464,8 +462,6 @@ If you need help, drop by our Discord: we're happy to assist: {URLs.discord}.
KajsonManager.teardown()
self.kajson_manager = KajsonManager(class_registry=self.class_registry)
- self.func_registry = func_registry or FuncRegistry()
- self.runtime_hub.set_func_registry(func_registry=self.func_registry)
self.runtime_hub.set_secrets_provider(secrets_provider=secrets_provider)
# Storage is selected from the config-driven StorageProviderRegistry, built from the plugin
# registrar (constructed above, just before the secrets provider). Its resolution and hub-set
@@ -727,7 +723,7 @@ If you need help, drop by our Discord: we're happy to assist: {URLs.discord}.
What a raising step *does* still skip is the releases that follow it in the ``try`` — the ones
before it have already run. That set is now only the *dangling* half: an SDK client left open, a
- reporting buffer unflushed, the previous boot's ``func_registry`` entries carried forward.
+ reporting buffer unflushed.
Everything that would instead **poison** the next boot moved into the ``finally``, so the two
release paths guarantee the same set. Closing the dangling half means collapsing this path and
``_release_after_failed_boot`` into one list.
@@ -745,7 +741,6 @@ If you need help, drop by our Discord: we're happy to assist: {URLs.discord}.
# tools
if self.class_registry:
self.class_registry.teardown()
- func_registry.teardown()
log.verbose(f"The {PACKAGE_NAME} {PACKAGE_VERSION} runtime was torn down, except its config and logs")
finally:
@@ -808,7 +803,7 @@ If you need help, drop by our Discord: we're happy to assist: {URLs.discord}.
would otherwise *poison the next boot* is here — the hub config and its boot-scoped flags, the class-registry scoping, the
``KajsonManager``, the template registries, the telemetry singleton and the ``MetaSingleton``
registration. What is deliberately absent is ``sdk_client_manager``, ``reporting_delegate``,
- ``func_registry``, ``inference_manager`` and ``class_registry``. The first three leave resources
+ ``inference_manager`` and ``class_registry``. The first two leave resources
dangling rather than corrupting the next boot; the last two are the very attributes a partial
``setup()`` may not have assigned, which is the reason this path exists at all. Adding any of
them here would widen a second hand-maintained copy of the teardown list that is bound to
diff --git a/pipelex/runtime_bridge/primitives/rehydration.py b/pipelex/runtime_bridge/primitives/rehydration.py
index 8a81c8bbd..ac2cec3b3 100644
--- a/pipelex/runtime_bridge/primitives/rehydration.py
+++ b/pipelex/runtime_bridge/primitives/rehydration.py
@@ -11,6 +11,7 @@ def rehydrate_library_and_memory(
library_id: str,
crate: LibraryCrate,
working_memory_raw: dict[str, Any] | None,
+ is_crate_prevalidated: bool = False,
) -> WorkingMemory | None:
"""Open a fresh library from a transported crate and hydrate a transported working memory.
@@ -29,13 +30,17 @@ def rehydrate_library_and_memory(
2. ``load_from_crate`` — rebuilds domains/concepts/pipes and registers the dynamic classes.
3. Hydrate the working memory (needs step 2 to have registered the classes it references).
+ ``is_crate_prevalidated`` skips the pipes' validation, for a crate the sender validated where it was assembled,
+ such as the one a `PipeFunc` call carries to a sandbox, whose other steps may name files of trees the sandbox
+ never receives.
+
Returns the hydrated WorkingMemory, or None when there was nothing to hydrate.
"""
library_manager = get_library_manager()
library_manager.open_fresh_library(library_id=library_id)
set_current_library(library_id=library_id)
- library_manager.load_from_crate(library_id=library_id, crate=crate)
+ library_manager.load_from_crate(library_id=library_id, crate=crate, is_crate_prevalidated=is_crate_prevalidated)
if working_memory_raw is None:
return None
diff --git a/pipelex/runtime_hub.py b/pipelex/runtime_hub.py
index 2af9f62d9..87f051f8a 100644
--- a/pipelex/runtime_hub.py
+++ b/pipelex/runtime_hub.py
@@ -46,7 +46,6 @@ from pipelex.system.configuration.config_root import ConfigRoot
from pipelex.system.console_target import ConsoleTarget
from pipelex.system.pipe_run_mode import PipeRunMode
from pipelex.system.registries.class_registry_access import get_class_registry as _get_active_class_registry
-from pipelex.system.registries.func_registry import FuncRegistry
from pipelex.system.telemetry.telemetry_manager_abstract import TelemetryManagerAbstract
from pipelex.tools.log.log_fields import USER_ACTION_FIELD
from pipelex.tools.misc.pretty import PrettyPrinter, PrettyPrintMode
@@ -112,7 +111,6 @@ class RuntimeHub:
self._secrets_provider: SecretsProviderAbstract | None = None
self._storage_provider: StorageProviderAbstract | None = None
self._telemetry_manager: TelemetryManagerAbstract | None = None
- self._func_registry: FuncRegistry | None = None
# cogt
self._models_manager: ModelManagerAbstract | None = None
self._sdk_client_manager: SdkClientManager | None = None
@@ -324,9 +322,6 @@ class RuntimeHub:
def set_isolated_execution_probe(self, probe: Callable[[], bool]) -> None:
self._isolated_execution_probe = probe
- def set_func_registry(self, func_registry: FuncRegistry):
- self._func_registry = func_registry
-
############################################################
# Getters
############################################################
@@ -390,12 +385,6 @@ class RuntimeHub:
raise RuntimeError(msg)
return self._telemetry_manager
- def get_func_registry(self) -> FuncRegistry:
- if self._func_registry is None:
- msg = "FuncRegistry is not initialized"
- raise RuntimeError(msg)
- return self._func_registry
-
# cogt
def get_required_models_manager(self) -> ModelManagerAbstract:
@@ -518,10 +507,6 @@ def get_class_registry() -> ClassRegistryAbstract:
return _get_active_class_registry()
-def get_func_registry() -> FuncRegistry:
- return get_runtime_hub().get_func_registry()
-
-
def get_telemetry_manager() -> TelemetryManagerAbstract:
return get_runtime_hub().get_telemetry_manager()
diff --git a/pipelex/system/configuration/pipe_func_config.py b/pipelex/system/configuration/pipe_func_config.py
index dc8890e91..8e82c67de 100644
--- a/pipelex/system/configuration/pipe_func_config.py
+++ b/pipelex/system/configuration/pipe_func_config.py
@@ -8,10 +8,9 @@ class PipeFuncConfig(ConfigModel):
# plugins populate (the config-selected-singleton seam, sibling of runtime.storage.method). An open
# string token: "direct" (core, in-process — imports and runs the customer function here) is the
# default; "daytona" (our Daytona sandbox plugin) runs it out-of-process. Any non-"direct" mode is a remote/sandbox
- # backend: library loading captures the customer .py source as text (onto the crate) instead of
- # registering it in the func_registry, and the PipeFunc validators skip the func_registry lookup +
- # return-type checks — the real function is registered and validated inside the sandbox, not here.
- # "direct" is byte-identical to the pre-existing behavior. This is a hosted-deploy concern, not a
+ # backend: library loading captures the method's files as they are (onto the crate) instead of
+ # importing any of them, a PipeFunc's validation proves its function's form from the captured source,
+ # and the function is imported, checked and run inside the sandbox, never here. This is a hosted-deploy concern, not a
# client preference, so it is intentionally absent from the .pipelex/ override file.
#
# Lives in its own module (not configs.py) so the PipeFuncExecutorRegistry can type its factory
diff --git a/pipelex/system/registries/exceptions.py b/pipelex/system/registries/exceptions.py
deleted file mode 100644
index a58c455d5..000000000
--- a/pipelex/system/registries/exceptions.py
+++ /dev/null
@@ -1,5 +0,0 @@
-from pipelex.system.exceptions import ToolError
-
-
-class FuncRegistryError(ToolError):
- pass
diff --git a/pipelex/system/registries/func_registry.py b/pipelex/system/registries/func_registry.py
index 4fea9ae24..aec9af8fd 100644
--- a/pipelex/system/registries/func_registry.py
+++ b/pipelex/system/registries/func_registry.py
@@ -1,393 +1,57 @@
-import inspect
-from collections.abc import Callable
-from typing import Any, TypeVar, cast, get_origin, get_type_hints
-
-from pydantic import Field, PrivateAttr, RootModel
-from typing_extensions import override
+"""The `@pipe_func` marker: the opt-in that lets a `PipeFunc` step call a Python function.
-from pipelex.system.registries.exceptions import FuncRegistryError
-from pipelex.urls import URLs
+A `PipeFunc` names its function by import path from the root of its method tree (`function_name =
+"funcs.totals.compute_totals"`), and the function it names must carry this marker, so a `.mthds` file can never
+call an arbitrary function such as `os.system`. The marker is Pipelex's, as is the calling convention it stands for:
+one parameter named `working_memory` typed `WorkingMemory`, and a return type that is a `StuffContent`.
-# Type variable for generic function types
-T = TypeVar("T")
-FuncRegistryDict = dict[str, Callable[..., Any]]
+The process-wide function registry this module once held is retired: nothing registers a function anymore, and the
+import path is the only name a function has. The module keeps its name because every method's Python imports the
+marker from here, as `from pipelex.system.registries.func_registry import pipe_func`, and that line must keep working.
-# Attribute name used by the decorator to mark functions for registration
-PIPE_FUNC_MARKER = "_is_pipe_func"
+The marker takes no argument. Its former `name=` argument, which registered a function under another name, is gone
+with the registry: `pipelex fix function-names` rewrites a method that used it.
+"""
+from collections.abc import Callable
+from typing import Any, TypeVar
-def _describe_function_origin(*, func: Callable[..., Any]) -> str:
- """Renders where a registered function came from, for collision diagnostics.
+T = TypeVar("T")
- Modules imported by file path get a name mangled from their absolute path, so the module name
- alone is enough to point the author at the offending file.
- """
- module_name = getattr(func, "__module__", None) or "<unknown module>"
- qualified_name = getattr(func, "__qualname__", None) or getattr(func, "__name__", "<unknown function>")
- return f"'{qualified_name}' from module '{module_name}'"
+#: The attribute the marker sets on the function it decorates.
+PIPE_FUNC_MARKER = "_is_pipe_func"
+#: The module the marker is imported from, which the static check of a `PipeFunc`'s module recognizes.
+PIPE_FUNC_MARKER_MODULE = "pipelex.system.registries.func_registry"
-def pipe_func(name: str | None = None) -> Callable[[T], T]:
- """Decorator to mark a function for automatic registration in the func_registry.
+#: The name of the marker in its module.
+PIPE_FUNC_MARKER_NAME = "pipe_func"
- This decorator marks functions to be discovered and registered for use in PipeFunc operators.
- Functions marked with this decorator must follow the PipeFunc signature:
- - Accept exactly one parameter named "working_memory" of type WorkingMemory
- - Return a StuffContent or subclass
- Args:
- name: Optional custom name for registration. If not provided, uses function's __name__
+def pipe_func() -> Callable[[T], T]:
+ """Mark a function as callable from a `PipeFunc` step, which names it by its import path.
- Returns:
- The decorated function unchanged, but marked for registration
+ The function must be a top-level `def` or `async def` of its module, decorated with this marker directly or
+ through an import alias, and follow the calling convention: exactly one parameter named `working_memory` of type
+ `WorkingMemory`, and a return type that is a `StuffContent` subclass, such as `TextContent` or `ListContent[...]`.
+ An `async def` is awaited; a plain `def` runs on a worker thread.
Example:
@pipe_func()
- async def my_custom_function(working_memory: WorkingMemory) -> TextContent:
- result = working_memory.get_stuff("input")
- return TextContent(text=f"Processed: {result}")
-
- @pipe_func(name="custom_name")
- async def another_function(working_memory: WorkingMemory) -> MyContent:
- return MyContent(data="example")
+ async def compute_totals(working_memory: WorkingMemory) -> TextContent:
+ ...
+ Returns:
+ The decorator, which returns the function unchanged, marked.
"""
def decorator(func: T) -> T:
- # Mark the function with the attribute
setattr(func, PIPE_FUNC_MARKER, True)
- # Store custom name if provided
- if name is not None:
- func._pipe_func_name = name # type: ignore[attr-defined] # ruff: ignore[private-member-access]
return func
return decorator
-class IneligibleFunctionInfo:
- """Information about a function that has @pipe_func decorator but failed eligibility checks."""
-
- def __init__(self, func_name: str, reason: str, source_file: str | None = None):
- self.func_name = func_name
- self.reason = reason
- self.source_file = source_file
-
- @override
- def __str__(self) -> str:
- if self.source_file:
- return f"Function '{self.func_name}' in '{self.source_file}': {self.reason}"
- return f"Function '{self.func_name}': {self.reason}"
-
-
-# Type for tracking ineligible decorated functions
-IneligibleFuncsDict = dict[str, IneligibleFunctionInfo]
-
-
-def _make_ineligible_funcs_dict() -> IneligibleFuncsDict:
- """Factory function for creating an empty ineligible functions dict with proper typing."""
- return {}
-
-
-class FuncRegistry(RootModel[FuncRegistryDict]):
- root: FuncRegistryDict = Field(default_factory=dict)
- _ineligible_decorated_funcs: IneligibleFuncsDict = PrivateAttr(default_factory=_make_ineligible_funcs_dict)
-
- def teardown(self) -> None:
- """Resets the registry to an empty state."""
- self.root.clear()
- self._ineligible_decorated_funcs.clear()
-
- def register_function(
- self,
- func: Callable[..., Any],
- *,
- name: str | None = None,
- ) -> None:
- """Registers a function in the registry with a name if it meets eligibility criteria.
-
- Raises:
- FuncRegistryError: If a *different* function is already registered under the same name.
- Registering the same function object again is a no-op, so a folder scanned twice
- (the same module object is reused from ``sys.modules``) stays idempotent.
-
- """
- if not self.is_eligible_function(func):
- return
-
- key = name or func.__name__
- already_registered = self.root.get(key)
- if already_registered is not None:
- if already_registered is func:
- return
- msg = (
- f"Function name '{key}' is already registered by a different function: "
- f"{_describe_function_origin(func=already_registered)} would be replaced by {_describe_function_origin(func=func)}. "
- f"PipeFunc names share a single flat, process-wide name space, so which one wins would depend on "
- f'scan order. Give one of them a distinct name with @pipe_func(name="..."). '
- f"See: {URLs.pipe_func_docs}"
- )
- raise FuncRegistryError(msg)
-
- self.root[key] = func
-
- def unregister_function(self, func: Callable[..., Any]) -> None:
- """Unregisters a function from the registry."""
- key = func.__name__
- if key not in self.root:
- msg = f"Function '{key}' not found in registry"
- raise FuncRegistryError(msg)
- del self.root[key]
-
- def unregister_function_by_name(self, name: str) -> None:
- """Unregisters a function from the registry by its name."""
- if name not in self.root:
- msg = f"Function '{name}' not found in registry"
- raise FuncRegistryError(msg)
- del self.root[name]
-
- def register_functions_dict(self, functions: dict[str, Callable[..., Any]]) -> None:
- """Registers multiple functions in the registry with names if they meet eligibility criteria."""
- for name, func in functions.items():
- self.register_function(func=func, name=name)
-
- def register_functions(self, functions: list[Callable[..., Any]]) -> None:
- """Registers multiple functions in the registry with names if they meet eligibility criteria."""
- for func in functions:
- self.register_function(func=func)
-
- def get_function(self, name: str) -> Callable[..., Any] | None:
- """Retrieves a function from the registry by its name. Returns None if not found."""
- return self.root.get(name)
-
- def get_required_function(self, name: str) -> Callable[..., Any]:
- """Retrieves a function from the registry by its name. Raises an error if not found."""
- if name not in self.root:
- # Check if this function was found but is ineligible
- ineligible_info = self._ineligible_decorated_funcs.get(name)
- if ineligible_info:
- msg = (
- f"Function '{name}' has @pipe_func() decorator but is not eligible for registration: "
- f"{ineligible_info.reason}. "
- f"See: {URLs.pipe_func_docs}"
- )
- else:
- msg = (
- f"Function '{name}' not found in registry. "
- f"Since v0.12.0, custom functions require the @pipe_func() decorator for auto-discovery. "
- f"Add @pipe_func() above your function definition. "
- f"See: {URLs.pipe_func_docs}"
- )
- raise FuncRegistryError(msg)
- return self.root[name]
-
- def get_required_function_with_signature(self, name: str) -> Callable[..., object]:
- """Retrieves a function from the registry by its name and verifies it matches the expected signature.
- Raises an error if not found or if signature doesn't match.
- """
- if name not in self.root:
- msg = f"Function '{name}' not found in registry"
- raise FuncRegistryError(msg)
-
- func = self.root[name]
- # Note: This is a basic signature check. For more thorough type checking,
- # you might want to use typing.get_type_hints() or a more sophisticated type checker
- if not callable(func):
- msg = f"'{name}' is not a callable function"
- raise FuncRegistryError(msg)
- return func
-
- def has_function(self, name: str) -> bool:
- """Checks if a function is in the registry by its name."""
- return name in self.root
-
- def is_marked_pipe_func(self, func: Any) -> bool:
- """Checks if a function is marked with the @pipe_func decorator.
-
- Args:
- func: The function to check
-
- Returns:
- True if the function has the pipe_func marker attribute
-
- """
- return hasattr(func, PIPE_FUNC_MARKER) and getattr(func, PIPE_FUNC_MARKER) is True
-
- # TODO: refactor this into a subclass of FuncRegistry dedicated to pipe funcs, avoid the circular import issue, avoid the code-smell
- def is_eligible_function(self, func: Any, *, require_decorator: bool = False) -> bool:
- """Checks if a function matches the criteria for PipeFunc registration:
- - Must be callable
- - Exactly 1 parameter named "working_memory" with type WorkingMemory
- - Return type that is a subclass of StuffContent
- - Optionally must be marked with @pipe_func decorator if require_decorator=True
-
- Args:
- func: The function to check
- require_decorator: If True, only functions marked with @pipe_func are eligible
-
- Returns:
- True if the function meets all eligibility criteria
-
- """
- if not callable(func):
- return False
-
- # If decorator is required, check for it first (fast check)
- if require_decorator and not self.is_marked_pipe_func(func):
- return False
-
- the_function = cast("Callable[..., Any]", func)
-
- # Import here to avoid circular imports
- # TODO: code-smell
- from pipelex.core.memory.working_memory import WorkingMemory # ruff: ignore[import-outside-top-level]
- from pipelex.core.stuffs.stuff_content import StuffContent # ruff: ignore[import-outside-top-level]
-
- # Get function signature
- sig = inspect.signature(the_function)
- params = list(sig.parameters.values())
-
- # Check parameter count and name
- if len(params) != 1:
- return False
-
- param = params[0]
- if param.name != "working_memory":
- return False
-
- # Get type hints
- type_hints = get_type_hints(the_function)
-
- # Check parameter type
- if "working_memory" not in type_hints:
- return False
-
- param_type = type_hints["working_memory"]
- if param_type != WorkingMemory:
- return False
-
- # Check return type
- if "return" not in type_hints:
- return False
-
- return_type = type_hints["return"]
-
- # Check if return type is a subclass of StuffContent
- try:
- if inspect.isclass(return_type) and issubclass(return_type, StuffContent):
- return True
- # Handle generic types like ListContent[SomeType]
- origin = get_origin(return_type)
- if inspect.isclass(origin) and issubclass(origin, StuffContent):
- return True
- except TypeError:
- # Handle cases where issubclass fails on generic types
- pass
-
- return False
-
- def check_function_eligibility(self, func: Any) -> str | None:
- """Check if a function is eligible for PipeFunc registration and return error message if not.
-
- This method provides detailed error messages explaining why a function is not eligible,
- which is useful for debugging when a @pipe_func decorated function is not being registered.
-
- Args:
- func: The function to check
-
- Returns:
- None if the function is eligible, or an error message string explaining why not
- """
- if not callable(func):
- return "not callable"
-
- the_function = cast("Callable[..., Any]", func)
-
- # Import here to avoid circular imports
- from pipelex.core.memory.working_memory import WorkingMemory # ruff: ignore[import-outside-top-level]
- from pipelex.core.stuffs.stuff_content import StuffContent # ruff: ignore[import-outside-top-level]
-
- # Get function signature
- try:
- sig = inspect.signature(the_function)
- except (ValueError, TypeError) as exc:
- return f"could not inspect signature: {exc}"
-
- params = list(sig.parameters.values())
-
- # Check parameter count
- if len(params) == 0:
- return "must have exactly one parameter named 'working_memory', but has no parameters"
- if len(params) > 1:
- return f"must have exactly one parameter named 'working_memory', but has {len(params)} parameters"
-
- # Check parameter name
- param = params[0]
- if param.name != "working_memory":
- return f"parameter must be named 'working_memory', but is named '{param.name}'"
-
- # Get type hints
- try:
- type_hints = get_type_hints(the_function)
- except (NameError, TypeError) as exc:
- return f"could not get type hints: {exc}"
-
- # Check parameter type annotation
- if "working_memory" not in type_hints:
- return "parameter 'working_memory' must have type annotation 'WorkingMemory'"
-
- param_type = type_hints["working_memory"]
- if param_type != WorkingMemory:
- return f"parameter 'working_memory' must have type 'WorkingMemory', but has type '{param_type}'"
-
- # Check return type annotation
- if "return" not in type_hints:
- return "must have a return type annotation that is a subclass of StuffContent"
-
- return_type = type_hints["return"]
-
- # Check if return type is a subclass of StuffContent
- try:
- if inspect.isclass(return_type) and issubclass(return_type, StuffContent):
- return None # Eligible
- # Handle generic types like ListContent[SomeType]
- origin = get_origin(return_type)
- if inspect.isclass(origin) and issubclass(origin, StuffContent):
- return None # Eligible
- except TypeError:
- pass
-
- return f"return type must be a subclass of StuffContent, but is '{return_type}'"
-
- def register_ineligible_function(
- self,
- func: Callable[..., Any],
- *,
- reason: str,
- source_file: str | None = None,
- ) -> None:
- """Register a function that has @pipe_func decorator but failed eligibility checks.
-
- This allows us to provide better error messages when the function is later looked up.
-
- Args:
- func: The function that failed eligibility
- reason: The reason the function is not eligible
- source_file: Optional source file path where the function was found
- """
- func_name = getattr(func, "_pipe_func_name", None) or func.__name__
- info = IneligibleFunctionInfo(func_name=func_name, reason=reason, source_file=source_file)
- self._ineligible_decorated_funcs[func_name] = info
-
- def get_ineligible_function_info(self, name: str) -> IneligibleFunctionInfo | None:
- """Get information about an ineligible @pipe_func decorated function.
-
- Args:
- name: The function name to look up
-
- Returns:
- IneligibleFunctionInfo if the function was found but ineligible, None otherwise
- """
- return self._ineligible_decorated_funcs.get(name)
-
-
-func_registry = FuncRegistry()
+def is_marked_pipe_func(*, func: Any) -> bool:
+ """Whether `func` carries the `@pipe_func` marker."""
+ return getattr(func, PIPE_FUNC_MARKER, None) is True
diff --git a/pipelex/system/registries/func_registry_utils.py b/pipelex/system/registries/func_registry_utils.py
deleted file mode 100644
index 08c875c36..000000000
--- a/pipelex/system/registries/func_registry_utils.py
+++ /dev/null
@@ -1,277 +0,0 @@
-import importlib
-import inspect
-import pkgutil
-from collections.abc import Callable
-from pathlib import Path
-from typing import Any
-
-from pipelex import log
-from pipelex.config import get_config
-from pipelex.system.registries.func_registry import func_registry, pipe_func
-from pipelex.system.telemetry.otel_constants import OTelLogAttr
-from pipelex.tools.log.error_fields import error_fields
-from pipelex.tools.misc.file_utils import find_files_in_dir
-from pipelex.tools.typing.exceptions import ModuleFileError
-from pipelex.tools.typing.module_inspector import import_module_from_file_if_has_decorated_functions
-
-
-class FuncRegistryUtils:
- @classmethod
- def register_pipe_funcs_from_package(cls, package_name: str, *, package: Any) -> int:
- """Register all @pipe_func decorated functions from a package.
-
- Args:
- package_name: Full name of the package (e.g. "my_project.pipe_funcs")
- package: The imported package object
-
- Returns:
- Number of functions registered
-
- """
- functions_registered = 0
-
- if not hasattr(package, "__path__"):
- log.warning("Nothing was registered from a PipeFunc package that is a plain module", fields={"module_name": package_name})
- return 0
-
- log.verbose(f"Walking package {package_name} at {package.__path__}")
-
- for _importer, modname, _ispkg in pkgutil.walk_packages(
- path=package.__path__,
- prefix=f"{package_name}.",
- onerror=lambda _: None,
- ):
- # Import the module
- module = importlib.import_module(modname)
-
- # Find @pipe_func decorated functions in this module
- decorated_functions = cls._find_functions_in_module(module)
-
- for func in decorated_functions:
- func_name = cls._get_function_registration_name(func)
-
- # Check if the function is eligible for registration
- eligibility_error = func_registry.check_function_eligibility(func)
-
- if eligibility_error is None:
- # Function is eligible - register it
- func_registry.register_function(
- func=func,
- name=func_name,
- )
- functions_registered += 1
- else:
- # Function has @pipe_func but is not eligible - track it for better error messages
- func_registry.register_ineligible_function(
- func=func,
- reason=eligibility_error,
- source_file=modname,
- )
- log.warning(
- "An ineligible PipeFunc function was not registered",
- fields={"function_name": func_name, "module_name": modname, "eligibility_error": eligibility_error},
- )
-
- return functions_registered
-
- @classmethod
- def register_funcs_in_folder(
- cls,
- folder_path: Path,
- *,
- is_recursive: bool = True,
- ) -> None:
- """Discovers and attempts to register all functions in Python files within a folder.
- Only functions that meet the eligibility criteria will be registered:
- - Must be an async function
- - Exactly 1 parameter named "working_memory" with type WorkingMemory
- - Return type that is a subclass of StuffContent
- - Must be marked with the @pipe_func decorator
-
- Uses AST parsing to first check if files contain @pipe_func decorated functions
- before importing them. This avoids executing module-level code in files that
- don't contain the functions you're looking for.
-
- The function name is used as the registry key (or custom name if provided to decorator).
-
- Args:
- folder_path: Path to folder containing Python files
- is_recursive: Whether to search recursively in subdirectories
-
- """
- python_files = find_files_in_dir(
- dir_path=folder_path,
- pattern="*.py",
- is_recursive=is_recursive,
- excluded_dirs=list(get_config().interpreter.scan.excluded_dirs),
- )
-
- for python_file in python_files:
- cls._register_funcs_in_file(file_path=python_file)
-
- @classmethod
- def read_py_sources(
- cls,
- folder_path: Path,
- *,
- is_recursive: bool = True,
- ) -> dict[str, str]:
- """Capture the text of every Python file in a folder WITHOUT importing or executing any of it.
-
- This is the sandbox-hosted counterpart of ``register_funcs_in_folder``: instead of importing
- the customer's modules and registering ``@pipe_func`` functions in this process, it reads the
- raw source so the code can travel (on the crate) to an isolated sandbox where it is registered
- and run. It performs NO import — ``sys.modules`` is left untouched — which is what keeps the
- runner/worker from ever executing customer code.
-
- Discovery mirrors ``register_funcs_in_folder`` (same ``find_files_in_dir`` + ``excluded_dirs``)
- so the captured set matches what the local path would have imported. Every file is captured,
- PipeFunc bodies and their helpers alike. A structure class never travels: the sandbox-hosted load
- scans this very mapping and refuses a library that declares one before keeping any of it
- (``ensure_no_structured_content_in_library_sources``).
-
- Args:
- folder_path: Path to the folder containing Python files.
- is_recursive: Whether to search recursively in subdirectories.
-
- Returns:
- Mapping of POSIX relpath (relative to ``folder_path``) -> source text.
- """
- python_files = find_files_in_dir(
- dir_path=folder_path,
- pattern="*.py",
- is_recursive=is_recursive,
- excluded_dirs=list(get_config().interpreter.scan.excluded_dirs),
- )
-
- sources: dict[str, str] = {}
- for python_file in python_files:
- relative_path = python_file.relative_to(folder_path).as_posix()
- sources[relative_path] = python_file.read_text(encoding="utf-8")
- return sources
-
- @classmethod
- def _register_funcs_in_file(
- cls,
- file_path: Path,
- ) -> None:
- """Processes a Python file to find and register eligible @pipe_func decorated functions.
-
- Uses AST parsing to check if the file contains @pipe_func decorated functions before
- importing. Only functions marked with @pipe_func decorator are registered.
-
- If a function has @pipe_func decorator but is not eligible (e.g., missing return type),
- it is tracked as an ineligible function so that helpful error messages can be provided
- when the function is later looked up.
-
- Args:
- file_path: Path to the Python file
-
- """
- try:
- # Import the module only if it has @pipe_func decorated functions
- module = import_module_from_file_if_has_decorated_functions(
- file_path,
- decorator_names=[pipe_func.__name__],
- )
- # If no decorated functions found, module will be None
- if module is None:
- return
-
- # Find functions that match criteria (have @pipe_func decorator)
- decorated_functions = cls._find_functions_in_module(module)
-
- for func in decorated_functions:
- func_name = cls._get_function_registration_name(func)
-
- # Check if the function is eligible for registration
- eligibility_error = func_registry.check_function_eligibility(func)
-
- if eligibility_error is None:
- # Function is eligible - register it
- func_registry.register_function(
- func=func,
- name=func_name,
- )
- else:
- # Function has @pipe_func but is not eligible - track it for better error messages
- func_registry.register_ineligible_function(
- func=func,
- reason=eligibility_error,
- source_file=str(file_path),
- )
- log.warning(
- "An ineligible PipeFunc function was not registered",
- fields={"function_name": func_name, OTelLogAttr.FILE_PATH: str(file_path), "eligibility_error": eligibility_error},
- )
- except ModuleFileError:
- # Expected: file validation issues (directories with .py extension, etc.)
- pass
- except ImportError as exc:
- # A module that fails to import (missing sibling module, circular import, bad relative
- # import) cannot contribute its @pipe_func functions — they never register. Surfacing this
- # is essential: the only downstream symptom is an opaque "Function '<name>' not found in
- # registry" raised much later by the PipeFunc validator, with the real ImportError (the
- # actual cause) otherwise swallowed here and invisible.
- log.warning(
- "A Python file could not be imported while registering PipeFuncs",
- fields={OTelLogAttr.FILE_PATH: str(file_path), **error_fields(exc=exc)},
- )
- except SyntaxError as exc:
- # Potentially problematic: invalid Python syntax may indicate broken code
- log.warning(
- "A Python file with a syntax error was skipped while registering PipeFuncs",
- fields={OTelLogAttr.FILE_PATH: str(file_path), **error_fields(exc=exc)},
- )
-
- @classmethod
- def _find_functions_in_module(
- cls,
- module: Any,
- ) -> list[Callable[..., Any]]:
- """Finds all @pipe_func decorated functions in a module.
-
- Only functions marked with @pipe_func decorator are included.
- Full eligibility (signature, return type) will be checked during registration.
-
- Args:
- module: The module to search for functions
-
- Returns:
- List of @pipe_func decorated functions found in the module
-
- """
- functions: list[Callable[..., Any]] = []
- module_name = module.__name__
-
- # Find all functions in the module (not imported ones)
- for _, obj in inspect.getmembers(module, inspect.isfunction):
- # Skip functions imported from other modules
- if obj.__module__ != module_name:
- continue
-
- # Only include functions marked with @pipe_func
- if not func_registry.is_marked_pipe_func(obj):
- continue
-
- # Add function - full eligibility will be checked by func_registry.register_function
- functions.append(obj)
-
- return functions
-
- @classmethod
- def _get_function_registration_name(cls, func: Callable[..., Any]) -> str:
- """Extract the registration name for a function.
-
- If the function has a custom name from the @pipe_func decorator, use that.
- Otherwise, use the function's __name__.
-
- Args:
- func: The function to get the registration name for
-
- Returns:
- The name to use when registering the function
-
- """
- custom_name = getattr(func, "_pipe_func_name", None)
- return custom_name if custom_name is not None else func.__name__
diff --git a/pipelex/test_extras/mthds_corpus/vocabulary.toml b/pipelex/test_extras/mthds_corpus/vocabulary.toml
index 135f0ace6..b2ca0fc48 100644
--- a/pipelex/test_extras/mthds_corpus/vocabulary.toml
+++ b/pipelex/test_extras/mthds_corpus/vocabulary.toml
@@ -70,7 +70,7 @@ code = "Composite"
[operator.func]
code = "PipeFunc"
-excluded = "A PipeFunc names a Python function that is resolved against the runtime's function registry at validation time, so an entry declaring one fails with \"Function '<name>' not found in registry\" in every consumer that has not registered that exact function. A corpus entry has to validate everywhere, and the cross-language consumers cannot register a Python function at all, so no portable focused entry exists. Exercise PipeFunc in a consumer's own fixtures."
+excluded = "A PipeFunc names a Python function by its import path in the method tree, and validation locates that module among the tree's files and reads it, so an entry declaring one is refused wherever its Python module does not travel with it. A corpus entry is a bundle that has to validate everywhere, and the cross-language consumers do not read Python at all, so no portable focused entry exists. Exercise PipeFunc in a consumer's own fixtures."
[operator.img_gen]
code = "PipeImgGen"
@@ -220,6 +220,10 @@ excluded = "It refuses a reference to a pipe an installed dependency package loa
code = "unknown_model"
fails_at = "runtime"
+[error.pipe_func_function_refused]
+code = "pipe_func_function_refused"
+excluded = "It refuses a PipeFunc step whose function its method tree does not hold or cannot be proven, so only a PipeFunc step produces it, and the PipeFunc operator is excluded: a focused entry is one bundle that has to validate everywhere, and its Python module would have to travel with it to consumers that do not read Python. Exercise it in a consumer's own fixtures, beside the operator."
+
[error.unknown_validation_error]
code = "unknown_validation_error"
excluded = "The generic fallback for a validation failure that matched no specific code. Reaching it on purpose would mean authoring a bundle that breaks in a way the categorizer does not recognize — which is a state to fix in the runtime, not to pin in a corpus entry that would then have to be rewritten the moment the fault gets a code of its own."
diff --git a/pipelex/tools/typing/module_inspector.py b/pipelex/tools/typing/module_inspector.py
index bfce74a02..befb3d8a3 100644
--- a/pipelex/tools/typing/module_inspector.py
+++ b/pipelex/tools/typing/module_inspector.py
@@ -7,6 +7,10 @@ from typing import Any
from pipelex.tools.typing.exceptions import ModuleFileError
+#: The directories `import_module_from_file` put on `sys.path`, each once, so that whoever unloads the modules of a
+#: directory, such as the process's hold on a method tree's Python, takes the entries introduced beneath it off again.
+_inserted_module_dirs: list[str] = []
+
def import_module_from_file(file_path: Path) -> Any:
"""Imports a module from a file path.
@@ -58,6 +62,8 @@ def import_module_from_file(file_path: Path) -> Any:
module_dir = str(file_path.resolve().parent)
if module_dir not in sys.path:
sys.path.insert(0, module_dir)
+ if module_dir not in _inserted_module_dirs:
+ _inserted_module_dirs.append(module_dir)
# Execute the module
spec.loader.exec_module(module)
@@ -65,6 +71,20 @@ def import_module_from_file(file_path: Path) -> Any:
return module
+def drop_module_dirs_inserted_under(*, directory: Path) -> None:
+ """Take off `sys.path` every entry `import_module_from_file` put there for a file at or beneath `directory`.
+
+ Args:
+ directory: A resolved directory, such as the root of a method tree whose modules are being unloaded.
+ """
+ for module_dir in list(_inserted_module_dirs):
+ if not Path(module_dir).is_relative_to(directory):
+ continue
+ _inserted_module_dirs.remove(module_dir)
+ if module_dir in sys.path:
+ sys.path.remove(module_dir)
+
+
def convert_file_path_to_module_path(file_path: Path) -> str:
"""Convert a file path to a valid module identifier.
@@ -168,104 +188,6 @@ def find_class_names_in_file(file_path: Path, *, base_class_names: list[str] | N
return class_names
-def find_decorated_function_names_in_file(
- file_path: Path,
- *,
- decorator_names: list[str],
-) -> list[str]:
- """Find function names decorated with specific decorators without executing the file.
-
- This uses AST parsing to find functions with specific decorators.
-
- Args:
- file_path: Path to the Python file to analyze
- decorator_names: List of decorator names to look for (e.g. ["pipe_func", "register_func"])
-
- Returns:
- List of function names that have the specified decorators
-
- Raises:
- ModuleFileError: If the file cannot be read or parsed
-
- """
- # Validate that the file is a Python file
- if file_path.suffix != ".py":
- msg = f"File {file_path} is not a Python file (must end with .py)"
- raise ModuleFileError(msg)
-
- # Validate that the path exists and is a file
- if not file_path.exists() or not file_path.is_file():
- msg = f"Path {file_path} does not exist or is not a file"
- raise ModuleFileError(msg)
-
- try:
- # Read and parse the file
- source = file_path.read_text(encoding="utf-8")
- tree = ast.parse(source, filename=str(file_path))
- except (OSError, SyntaxError, ValueError) as exc:
- msg = f"Failed to parse {file_path}: {exc}"
- raise ModuleFileError(msg) from exc
-
- function_names: list[str] = []
-
- # Walk through the AST to find function definitions with decorators
- for node in ast.walk(tree):
- if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)):
- # Check if function has any of the specified decorators
- for decorator in node.decorator_list:
- decorator_name = None
-
- # Handle simple decorator names like @pipe_func
- if isinstance(decorator, ast.Name):
- decorator_name = decorator.id
- # Handle decorator calls like @pipe_func() or @pipe_func(name="foo")
- elif isinstance(decorator, ast.Call):
- if isinstance(decorator.func, ast.Name):
- decorator_name = decorator.func.id
- # Handle qualified names like @registry.pipe_func()
- elif isinstance(decorator, ast.Attribute):
- decorator_name = decorator.attr
-
- if decorator_name in decorator_names:
- function_names.append(node.name)
- break # Found a matching decorator, no need to check others
-
- return function_names
-
-
-def import_module_from_file_if_has_decorated_functions(
- file_path: Path,
- *,
- decorator_names: list[str],
-) -> Any | None:
- """Import a module only if it contains functions with specific decorators.
-
- This function uses AST parsing to check if the file contains functions decorated
- with specific decorators before importing, avoiding execution of modules that don't
- have the functions you're looking for.
-
- Args:
- file_path: Path to the Python file to potentially import
- decorator_names: List of decorator names to look for (e.g. ["pipe_func"])
-
- Returns:
- The imported module if it contains decorated functions, None otherwise
-
- Raises:
- ModuleFileError: If the file is not a Python file or cannot be loaded
-
- """
- # First, use AST to check if file has decorated functions
- function_names = find_decorated_function_names_in_file(file_path, decorator_names=decorator_names)
-
- # If no decorated functions found, skip import
- if not function_names:
- return None
-
- # File has decorated functions, import it
- return import_module_from_file(file_path)
-
-
def import_module_from_file_if_has_classes(
file_path: Path,
*,
diff --git a/pipelex/validation_error_types.py b/pipelex/validation_error_types.py
index 2c227375a..788c1c463 100644
--- a/pipelex/validation_error_types.py
+++ b/pipelex/validation_error_types.py
@@ -132,6 +132,12 @@ class PipeValidationErrorType(StrEnum):
# model type and the deck's close matches, so the author can pick one.
UNKNOWN_MODEL = "unknown_model"
+ # A PipeFunc whose function_name names no function the step may call: the module is not in the method tree, the
+ # function is not a top-level def or async def of it decorated with @pipe_func(), or a top-level module of the
+ # tree shares its name with a standard-library module or one importable outside the tree. Detected when the pipe
+ # is validated against the library, from the module's syntax tree, without running Python.
+ PIPE_FUNC_FUNCTION_REFUSED = "pipe_func_function_refused"
+
# Generic fallback for unexpected validation errors
UNKNOWN_VALIDATION_ERROR = "unknown_validation_error"
@@ -171,6 +177,7 @@ class PipeValidationErrorType(StrEnum):
| PipeValidationErrorType.UNRESOLVED_PACKAGE_DEPENDENCY
| PipeValidationErrorType.UNEXPORTED_PIPE_DEPENDENCY
| PipeValidationErrorType.UNKNOWN_MODEL
+ | PipeValidationErrorType.PIPE_FUNC_FUNCTION_REFUSED
| PipeValidationErrorType.UNKNOWN_VALIDATION_ERROR
):
return False
@@ -208,6 +215,7 @@ class PipeValidationErrorType(StrEnum):
| PipeValidationErrorType.UNRESOLVED_PACKAGE_DEPENDENCY
| PipeValidationErrorType.UNEXPORTED_PIPE_DEPENDENCY
| PipeValidationErrorType.UNKNOWN_MODEL
+ | PipeValidationErrorType.PIPE_FUNC_FUNCTION_REFUSED
| PipeValidationErrorType.UNKNOWN_VALIDATION_ERROR
):
return False
@@ -252,6 +260,7 @@ class PipeValidationErrorType(StrEnum):
| PipeValidationErrorType.UNRESOLVED_PACKAGE_DEPENDENCY
| PipeValidationErrorType.UNEXPORTED_PIPE_DEPENDENCY
| PipeValidationErrorType.UNKNOWN_MODEL
+ | PipeValidationErrorType.PIPE_FUNC_FUNCTION_REFUSED
| PipeValidationErrorType.UNKNOWN_VALIDATION_ERROR
):
return False
@@ -290,6 +299,7 @@ class PipeValidationErrorType(StrEnum):
| PipeValidationErrorType.UNRESOLVED_PACKAGE_DEPENDENCY
| PipeValidationErrorType.UNEXPORTED_PIPE_DEPENDENCY
| PipeValidationErrorType.UNKNOWN_MODEL
+ | PipeValidationErrorType.PIPE_FUNC_FUNCTION_REFUSED
| PipeValidationErrorType.UNKNOWN_VALIDATION_ERROR
):
return False
@@ -333,6 +343,7 @@ class PipeValidationErrorType(StrEnum):
| PipeValidationErrorType.UNRESOLVED_PACKAGE_DEPENDENCY
| PipeValidationErrorType.UNEXPORTED_PIPE_DEPENDENCY
| PipeValidationErrorType.UNKNOWN_MODEL
+ | PipeValidationErrorType.PIPE_FUNC_FUNCTION_REFUSED
| PipeValidationErrorType.UNKNOWN_VALIDATION_ERROR
):
return False
@@ -376,6 +387,7 @@ class PipeValidationErrorType(StrEnum):
| PipeValidationErrorType.UNRESOLVED_PACKAGE_DEPENDENCY
| PipeValidationErrorType.UNEXPORTED_PIPE_DEPENDENCY
| PipeValidationErrorType.UNKNOWN_MODEL
+ | PipeValidationErrorType.PIPE_FUNC_FUNCTION_REFUSED
| PipeValidationErrorType.UNKNOWN_VALIDATION_ERROR
):
return False
@@ -414,6 +426,7 @@ class PipeValidationErrorType(StrEnum):
| PipeValidationErrorType.UNRESOLVED_PIPE_DEPENDENCY
| PipeValidationErrorType.UNRESOLVED_PACKAGE_DEPENDENCY
| PipeValidationErrorType.UNEXPORTED_PIPE_DEPENDENCY
+ | PipeValidationErrorType.PIPE_FUNC_FUNCTION_REFUSED
| PipeValidationErrorType.UNKNOWN_VALIDATION_ERROR
):
return False
diff --git a/subject_grants.toml b/subject_grants.toml
index 3abe53544..ac1eacb4e 100644
--- a/subject_grants.toml
+++ b/subject_grants.toml
@@ -5232,10 +5232,6 @@ rationale = "Hub setter: installs the print target; single self-labelling operan
param = "content_generator"
rationale = "Hub setter: installs the content generator; single self-labelling operand."
-["pipelex/runtime_hub.py::RuntimeHub.set_func_registry"]
-param = "func_registry"
-rationale = "Hub setter: installs the func registry; single self-labelling operand."
-
["pipelex/runtime_hub.py::RuntimeHub.set_inference_backend_registry"]
param = "inference_backend_registry"
rationale = "Hub setter: installs the inference backend registry; single self-labelling operand."
@@ -5380,94 +5376,10 @@ rationale = "Verb-object: registers classes in the file; options stay keyword."
param = "folder_path"
rationale = "Verb-object: registers classes under the folder path; options stay keyword."
-["pipelex/system/registries/func_registry.py::FuncRegistry.check_function_eligibility"]
-param = "func"
-rationale = "Verb-object: checks the function; single operand."
-
-["pipelex/system/registries/func_registry.py::FuncRegistry.get_function"]
-param = "name"
-rationale = "Registry getter keyed by the function name; single operand."
-
-["pipelex/system/registries/func_registry.py::FuncRegistry.get_ineligible_function_info"]
-param = "name"
-rationale = "Registry getter keyed by the function name; single operand."
-
-["pipelex/system/registries/func_registry.py::FuncRegistry.get_required_function"]
-param = "name"
-rationale = "Registry getter keyed by the function name; single operand."
-
-["pipelex/system/registries/func_registry.py::FuncRegistry.get_required_function_with_signature"]
-param = "name"
-rationale = "Registry getter keyed by the function name; single operand."
-
-["pipelex/system/registries/func_registry.py::FuncRegistry.has_function"]
-param = "name"
-rationale = "Predicate on the function named; single operand."
-
-["pipelex/system/registries/func_registry.py::FuncRegistry.is_eligible_function"]
-param = "func"
-rationale = "Predicate on the function; single operand."
-
-["pipelex/system/registries/func_registry.py::FuncRegistry.is_marked_pipe_func"]
-param = "func"
-rationale = "Predicate on the function; single operand."
-
-["pipelex/system/registries/func_registry.py::FuncRegistry.register_function"]
-param = "func"
-rationale = "Verb-object: registers the function; options stay keyword."
-
-["pipelex/system/registries/func_registry.py::FuncRegistry.register_functions"]
-param = "functions"
-rationale = "Verb-object: registers the functions; options stay keyword."
-
-["pipelex/system/registries/func_registry.py::FuncRegistry.register_functions_dict"]
-param = "functions"
-rationale = "Verb-object: registers the functions mapping; single operand."
-
-["pipelex/system/registries/func_registry.py::FuncRegistry.register_ineligible_function"]
-param = "func"
-rationale = "Verb-object: records the function; reason stays keyword."
-
-["pipelex/system/registries/func_registry.py::FuncRegistry.unregister_function"]
-param = "func"
-rationale = "Verb-object: unregisters the function; single operand."
-
-["pipelex/system/registries/func_registry.py::FuncRegistry.unregister_function_by_name"]
-param = "name"
-rationale = "Verb-object: the name of the function being unregistered; single operand."
-
-["pipelex/system/registries/func_registry.py::pipe_func"]
-param = "name"
-rationale = "Decorator factory: the registration name, idiomatically positional at decoration sites."
-
["pipelex/system/registries/func_registry.py::pipe_func.decorator"]
param = "func"
rationale = "Positional Callable protocol: decorator applied to the function by @ syntax."
-["pipelex/system/registries/func_registry_utils.py::FuncRegistryUtils._find_functions_in_module"]
-param = "module"
-rationale = "Verb-object: scans the module; single operand."
-
-["pipelex/system/registries/func_registry_utils.py::FuncRegistryUtils._get_function_registration_name"]
-param = "func"
-rationale = "Derivation from the function; single operand."
-
-["pipelex/system/registries/func_registry_utils.py::FuncRegistryUtils._register_funcs_in_file"]
-param = "file_path"
-rationale = "Verb-object: registers funcs in the file; options stay keyword."
-
-["pipelex/system/registries/func_registry_utils.py::FuncRegistryUtils.read_py_sources"]
-param = "folder_path"
-rationale = "Folder reader: captures py sources under the folder; single self-labelling operand, consistent with register_funcs_in_folder."
-
-["pipelex/system/registries/func_registry_utils.py::FuncRegistryUtils.register_funcs_in_folder"]
-param = "folder_path"
-rationale = "Verb-object: registers funcs under the folder path; options stay keyword."
-
-["pipelex/system/registries/func_registry_utils.py::FuncRegistryUtils.register_pipe_funcs_from_package"]
-param = "package_name"
-rationale = "Verb-object: registers funcs from the named package; single operand."
-
["pipelex/system/registries/singleton.py::MetaSingleton.clear_subclass_instances"]
param = "base_cls"
rationale = "Verb-object: clears instances of the base class; single operand."
@@ -6752,10 +6664,6 @@ rationale = "Extracts/derives from the file_path; single operand."
param = "module"
rationale = "Extracts/derives from the module; single operand."
-["pipelex/tools/typing/module_inspector.py::find_decorated_function_names_in_file"]
-param = "file_path"
-rationale = "Extracts/derives from the file_path; single operand."
-
["pipelex/tools/typing/module_inspector.py::import_module_from_file"]
param = "file_path"
rationale = "Inspects the file_path; single operand."
@@ -6764,10 +6672,6 @@ rationale = "Inspects the file_path; single operand."
param = "file_path"
rationale = "Inspects the file_path; single operand."
-["pipelex/tools/typing/module_inspector.py::import_module_from_file_if_has_decorated_functions"]
-param = "file_path"
-rationale = "Inspects the file_path; single operand."
-
["pipelex/tools/typing/pydantic_utils.py::_truncated_rich_repr_from_items"]
param = "rich_repr_items"
rationale = "Transforms the rich_repr_items; single operand."
diff --git a/tests/cases/source_code.py b/tests/cases/source_code.py
index f25c318aa..c4a3cdd97 100644
--- a/tests/cases/source_code.py
+++ b/tests/cases/source_code.py
@@ -2,8 +2,10 @@
from pipelex.core.memory.working_memory import WorkingMemory
from pipelex.core.stuffs.text_content import TextContent
+from pipelex.system.registries.func_registry import pipe_func
+@pipe_func()
def wrap_lines(working_memory: WorkingMemory) -> TextContent:
"""Wraps each line of source code text with HTML span tags.
diff --git a/tests/data/errors/error_identity.txt b/tests/data/errors/error_identity.txt
index 54b3561a6..79afcf4b7 100644
--- a/tests/data/errors/error_identity.txt
+++ b/tests/data/errors/error_identity.txt
@@ -109,7 +109,7 @@ FileTypeError | File type | https://docs.pipelex.com/latest/errors/file-type-err
FixTransactionError | Fix transaction | https://docs.pipelex.com/latest/errors/fix-transaction-error/
FixWriteConflictError | Fix write conflict | https://docs.pipelex.com/latest/errors/fix-write-conflict-error/
FormerReleaseConfigError | Configuration left by a former release | https://docs.pipelex.com/latest/errors/former-release-config-error/
-FuncRegistryError | Func registry | https://docs.pipelex.com/latest/errors/func-registry-error/
+FunctionNamesMigrationError | Function names migration error | https://docs.pipelex.com/latest/errors/function-names-migration-error/
GcpLogSinkCredentialsError | Log sink credentials missing or refused | https://docs.pipelex.com/latest/errors/gcp-log-sink-credentials-error/
GeneratedImageError | Generated image | https://docs.pipelex.com/latest/errors/generated-image-error/
GoogleImgGenWorkerError | Google img gen worker | https://docs.pipelex.com/latest/errors/google-img-gen-worker-error/
@@ -248,6 +248,7 @@ PipeExecutionError | Pipe execution | https://docs.pipelex.com/latest/errors/pip
PipeExtractFactoryError | Pipe extract factory | https://docs.pipelex.com/latest/errors/pipe-extract-factory-error/
PipeFactoryError | Pipe factory | https://docs.pipelex.com/latest/errors/pipe-factory-error/
PipeFuncExecutionError | Pipe func execution | https://docs.pipelex.com/latest/errors/pipe-func-execution-error/
+PipeFuncFunctionError | Pipe func function | https://docs.pipelex.com/latest/errors/pipe-func-function-error/
PipeFuncPackageSetUnstatedError | Pipe func package set unstated | https://docs.pipelex.com/latest/errors/pipe-func-package-set-unstated-error/
PipeFuncTransportError | Pipe func transport | https://docs.pipelex.com/latest/errors/pipe-func-transport-error/
PipeIOContractError | Pipe IO contract | https://docs.pipelex.com/latest/errors/pipe-io-contract-error/
@@ -351,6 +352,9 @@ TomlError | TOML parse error | https://docs.pipelex.com/latest/errors/toml-error
ToolError | Tool error | https://docs.pipelex.com/latest/errors/tool-error/
TraceEventModeError | Trace event mode | https://docs.pipelex.com/latest/errors/trace-event-mode-error/
TracebackMessageError | Traceback message | https://docs.pipelex.com/latest/errors/traceback-message-error/
+TreeModuleShadowingError | Tree module shadowing | https://docs.pipelex.com/latest/errors/tree-module-shadowing-error/
+TreePythonError | Tree python | https://docs.pipelex.com/latest/errors/tree-python-error/
+TreePythonHeldError | Tree python held | https://docs.pipelex.com/latest/errors/tree-python-held-error/
TypesafeError | Typesafe | https://docs.pipelex.com/latest/errors/typesafe-error/
TypesafeJudgmentResponseError | Typesafe judgment response | https://docs.pipelex.com/latest/errors/typesafe-judgment-response-error/
TypesafeQuestionUnsupportedError | Typesafe question unsupported | https://docs.pipelex.com/latest/errors/typesafe-question-unsupported-error/
diff --git a/tests/e2e/pipelex/pipes/optionals/guarded_condition/guarded_condition.mthds b/tests/e2e/pipelex/pipes/optionals/guarded_condition/guarded_condition.mthds
index b39d90a00..dfbf8d9b2 100644
--- a/tests/e2e/pipelex/pipes/optionals/guarded_condition/guarded_condition.mthds
+++ b/tests/e2e/pipelex/pipes/optionals/guarded_condition/guarded_condition.mthds
@@ -6,13 +6,13 @@ main_pipe = "ogc_route_on_flag"
type = "PipeFunc"
description = "Branch taken when the flag is present"
output = "Text"
-function_name = "ogc_make_flagged_note"
+function_name = "guarded_condition_funcs.ogc_make_flagged_note"
[pipe.ogc_make_plain_note]
type = "PipeFunc"
description = "Branch taken when the flag is absent"
output = "Text"
-function_name = "ogc_make_plain_note"
+function_name = "guarded_condition_funcs.ogc_make_plain_note"
[pipe.ogc_route_on_flag]
type = "PipeCondition"
diff --git a/tests/e2e/pipelex/pipes/optionals/guarded_condition/guarded_condition_funcs.py b/tests/e2e/pipelex/pipes/optionals/guarded_condition/guarded_condition_funcs.py
index 26e96d5d9..6af30e0d9 100644
--- a/tests/e2e/pipelex/pipes/optionals/guarded_condition/guarded_condition_funcs.py
+++ b/tests/e2e/pipelex/pipes/optionals/guarded_condition/guarded_condition_funcs.py
@@ -1,5 +1,5 @@
-"""PipeFunc implementations for the guarded-condition bundle, auto-registered at library load via
-the @pipe_func decorator (FuncRegistryUtils.register_funcs_in_folder).
+"""PipeFunc implementations for the guarded-condition bundle, which names each by its import path from this folder,
+such as `guarded_condition_funcs.<function>`.
"""
from pipelex.core.memory.working_memory import WorkingMemory
diff --git a/tests/e2e/pipelex/pipes/optionals/lift_parity/lift_parity.mthds b/tests/e2e/pipelex/pipes/optionals/lift_parity/lift_parity.mthds
index f0431cbb2..01d17507b 100644
--- a/tests/e2e/pipelex/pipes/optionals/lift_parity/lift_parity.mthds
+++ b/tests/e2e/pipelex/pipes/optionals/lift_parity/lift_parity.mthds
@@ -7,14 +7,14 @@ type = "PipeFunc"
description = "Analyzes the source (plain input: lifts when source is absent)"
inputs = { source = "Text" }
output = "Text"
-function_name = "opar_make_analysis"
+function_name = "lift_parity_funcs.opar_make_analysis"
[pipe.opar_summarize]
type = "PipeFunc"
description = "Summarizes the analysis (plain input: chained lift)"
inputs = { analysis = "Text" }
output = "Text"
-function_name = "opar_summarize"
+function_name = "lift_parity_funcs.opar_summarize"
[pipe.opar_flow]
type = "PipeSequence"
diff --git a/tests/e2e/pipelex/pipes/optionals/lift_parity/lift_parity_funcs.py b/tests/e2e/pipelex/pipes/optionals/lift_parity/lift_parity_funcs.py
index 0be9db1de..99f1126fc 100644
--- a/tests/e2e/pipelex/pipes/optionals/lift_parity/lift_parity_funcs.py
+++ b/tests/e2e/pipelex/pipes/optionals/lift_parity/lift_parity_funcs.py
@@ -1,5 +1,5 @@
-"""PipeFunc implementations for the lift-parity bundle, auto-registered at library load via the
-@pipe_func decorator (FuncRegistryUtils.register_funcs_in_folder).
+"""PipeFunc implementations for the lift-parity bundle, which names each by its import path from this folder,
+such as `lift_parity_funcs.<function>`.
"""
from pipelex.core.memory.working_memory import WorkingMemory
diff --git a/tests/e2e/pipelex/pipes/optionals/optionals_showcase/optionals_showcase.mthds b/tests/e2e/pipelex/pipes/optionals/optionals_showcase/optionals_showcase.mthds
index 864213ef9..5699ac94e 100644
--- a/tests/e2e/pipelex/pipes/optionals/optionals_showcase/optionals_showcase.mthds
+++ b/tests/e2e/pipelex/pipes/optionals/optionals_showcase/optionals_showcase.mthds
@@ -7,21 +7,21 @@ type = "PipeFunc"
description = "Analyzes the clause (plain input: lifts when clause is absent)"
inputs = { clause = "Text" }
output = "Text"
-function_name = "oshow_analyze_clause"
+function_name = "optionals_showcase_funcs.oshow_analyze_clause"
[pipe.oshow_summarize_analysis]
type = "PipeFunc"
description = "Summarizes the analysis (plain input: chained lift)"
inputs = { analysis = "Text" }
output = "Text"
-function_name = "oshow_summarize_analysis"
+function_name = "optionals_showcase_funcs.oshow_summarize_analysis"
[pipe.oshow_flag_risk]
type = "PipeFunc"
description = "Produces the risk note for a flagged contract"
inputs = { contract = "Text" }
output = "Text"
-function_name = "oshow_flag_risk"
+function_name = "optionals_showcase_funcs.oshow_flag_risk"
[pipe.oshow_assess_risk]
type = "PipeCondition"
@@ -64,7 +64,7 @@ type = "PipeFunc"
description = "Asserts presence of the clause with a force marker"
inputs = { clause = "Text!" }
output = "Text"
-function_name = "oshow_force_extract"
+function_name = "optionals_showcase_funcs.oshow_force_extract"
[pipe.oshow_force_flow]
type = "PipeSequence"
diff --git a/tests/e2e/pipelex/pipes/optionals/optionals_showcase/optionals_showcase_funcs.py b/tests/e2e/pipelex/pipes/optionals/optionals_showcase/optionals_showcase_funcs.py
index b9165676e..066ced7a3 100644
--- a/tests/e2e/pipelex/pipes/optionals/optionals_showcase/optionals_showcase_funcs.py
+++ b/tests/e2e/pipelex/pipes/optionals/optionals_showcase/optionals_showcase_funcs.py
@@ -1,5 +1,5 @@
-"""PipeFunc implementations for the optionals showcase bundle, auto-registered at library load
-via the @pipe_func decorator (FuncRegistryUtils.register_funcs_in_folder).
+"""PipeFunc implementations for the optionals showcase bundle, which names each by its import path from this folder,
+such as `optionals_showcase_funcs.<function>`.
"""
from pipelex.core.memory.working_memory import WorkingMemory
diff --git a/tests/e2e/pipelex/pipes/optionals/test_guarded_condition_mthds.py b/tests/e2e/pipelex/pipes/optionals/test_guarded_condition_mthds.py
index 590ef6702..b899cb8bf 100644
--- a/tests/e2e/pipelex/pipes/optionals/test_guarded_condition_mthds.py
+++ b/tests/e2e/pipelex/pipes/optionals/test_guarded_condition_mthds.py
@@ -12,7 +12,6 @@ from pipelex.core.memory.absence import AbsenceKind
from pipelex.pipeline.exceptions import ValidateBundleError
from pipelex.pipeline.pipeline_response import RunState
from pipelex.pipeline.runner import PipelexMTHDSProtocol
-from pipelex.system.registries.func_registry_utils import FuncRegistryUtils
from pipelex.validation_error_types import PipeValidationErrorType
_FIXTURE_DIR = Path(__file__).parent / "guarded_condition"
@@ -58,12 +57,6 @@ class TestGuardedConditionExpression:
)
assert unguarded_text != guarded_text
- # Register the bundle's @pipe_func impls so PipeFunc construction succeeds and the load-time
- # guard lint — not a missing-function error — is what rejects the unguarded variant. Without
- # this, the raw-content load path never walks a folder to register funcs, so construction
- # fails first and the failure "passes" for the wrong reason (and only on some xdist workers).
- FuncRegistryUtils.register_funcs_in_folder(folder_path=_FIXTURE_DIR)
-
runner = PipelexMTHDSProtocol()
with pytest.raises(ValidateBundleError) as exc_info:
await runner.execute(mthds_contents=[unguarded_text], inputs={})
diff --git a/tests/integration/pipelex/libraries/test_data.py b/tests/integration/pipelex/libraries/test_data.py
index 3ca73fde9..41fcf9304 100644
--- a/tests/integration/pipelex/libraries/test_data.py
+++ b/tests/integration/pipelex/libraries/test_data.py
@@ -162,12 +162,12 @@ prompt = "Orphan: $data"
f'steps = [{{ pipe = "{DEP_ALIAS}->helper", result = "helped" }}]',
)
- #: The same package with `helper` a PipeFunc naming a function no registry holds, so it fails to build.
+ #: The same package with `helper` a PipeFunc naming a function of a module its package does not hold, so it fails to build.
DEP_BUNDLE_WITH_AN_UNBUILDABLE_HELPER: ClassVar[str] = DEP_BUNDLE.replace(
'type = "PipeLLM"\ndescription = "DEPENDENCY helper"\ninputs = { data = "Text" }\n'
'output = "Text"\nprompt = "DEPENDENCY helper: $data"',
'type = "PipeFunc"\ndescription = "DEPENDENCY helper"\ninputs = { data = "Text" }\n'
- 'output = "Text"\nfunction_name = "no_such_function_anywhere"',
+ 'output = "Text"\nfunction_name = "probe_funcs.no_such_function_anywhere"',
)
#: A consumer whose sequence calls the package's entry.
@@ -334,7 +334,7 @@ type = "PipeFunc"
description = "The exported x, which fails to build"
inputs = { data = "Text" }
output = "Text"
-function_name = "no_such_function_anywhere"
+function_name = "probe_funcs.no_such_function_anywhere"
[pipe.entry]
type = "PipeSequence"
diff --git a/tests/integration/pipelex/libraries/test_hosted_pipe_func_load.py b/tests/integration/pipelex/libraries/test_hosted_pipe_func_load.py
index 040e41011..92a29f98a 100644
--- a/tests/integration/pipelex/libraries/test_hosted_pipe_func_load.py
+++ b/tests/integration/pipelex/libraries/test_hosted_pipe_func_load.py
@@ -1,11 +1,14 @@
+import shutil
+import sys
import tempfile
from pathlib import Path
import pytest
+from pipelex.core.pipes.exceptions import PipeValidationError
from pipelex.interpreter_hub import get_library_manager, scoped_current_library
from pipelex.libraries.exceptions import LibraryError
-from pipelex.system.registries.func_registry import func_registry
+from pipelex.libraries.library_crate import SUBMITTED_PYTHON_ROOT_KEY
HOSTED_DEMO_MTHDS = """\
domain = "hosted_demo"
@@ -16,10 +19,10 @@ Note = "A note produced by customer code"
[pipe.make_note]
type = "PipeFunc"
-description = "Build a note via customer-supplied code that lives only in the sandbox"
+description = "Build a note via customer-supplied code that runs only in the sandbox"
inputs = {}
output = "Note"
-function_name = "make_note_in_sandbox"
+function_name = "hosted_demo_funcs.make_note_in_sandbox"
"""
CUSTOMER_FUNC_PY = """\
@@ -28,45 +31,96 @@ from pipelex.core.stuffs.text_content import TextContent
from pipelex.system.registries.func_registry import pipe_func
-@pipe_func(name="make_note_in_sandbox")
+@pipe_func()
async def make_note_in_sandbox(working_memory: WorkingMemory) -> TextContent:
return TextContent(text="hello from the sandbox")
"""
+SIDE_BY_SIDE_MTHDS = """\
+domain = "side_by_side_{name}"
+description = "One of two methods loaded side by side"
+
+[pipe.make_note]
+type = "PipeFunc"
+description = "Build a note"
+inputs = {{}}
+output = "Text"
+function_name = "{name}_funcs.make_note_in_sandbox"
+"""
+
+SIDE_BY_SIDE_MANIFEST = '[package]\naddress = "github.com/pipelex-tests/{name}-side-by-side"\nversion = "1.0.0"\ndescription = "The {name} method"\n'
+
+
+def _write_method(*, method_dir: Path, name: str) -> None:
+ """A method folder whose README.md, requirements.txt and METHODS.toml say which method it is."""
+ method_dir.mkdir(parents=True)
+ (method_dir / "METHODS.toml").write_text(SIDE_BY_SIDE_MANIFEST.format(name=name), encoding="utf-8")
+ (method_dir / "README.md").write_text(f"# The {name} method\n", encoding="utf-8")
+ (method_dir / "requirements.txt").write_text(f"# The {name} method needs nothing beyond Pipelex\n", encoding="utf-8")
+ (method_dir / f"{name}.mthds").write_text(SIDE_BY_SIDE_MTHDS.format(name=name), encoding="utf-8")
+ (method_dir / f"{name}_funcs.py").write_text(CUSTOMER_FUNC_PY, encoding="utf-8")
+
+
+SHADOWING_MTHDS = """\
+domain = "shadowing_probe"
+description = "A method shipping a module named like a package of the runner"
+
+[pipe.make_note]
+type = "PipeFunc"
+description = "Build a note"
+inputs = {}
+output = "Text"
+function_name = "shadowing_probe_funcs.make_note_in_sandbox"
+"""
+
+
+def _write_shadowing_method(*, library_dir: Path) -> None:
+ """A method shipping a module named like a package this test process imports, which a sandbox need not hold."""
+ library_dir.mkdir()
+ (library_dir / "shadowing_probe.mthds").write_text(SHADOWING_MTHDS, encoding="utf-8")
+ (library_dir / "shadowing_probe_funcs.py").write_text(CUSTOMER_FUNC_PY, encoding="utf-8")
+ (library_dir / "pytest.py").write_text("SHADOW = True\n", encoding="utf-8")
+
+
class TestHostedPipeFuncLoad:
- """Sandbox-hosted load: the customer's PipeFunc .py is captured as crate source and is NEVER
- registered/executed in this process, yet the method still loads (validators skip the callable).
+ """Sandbox-hosted load: the method's files are captured on the crate and NEVER imported in this process,
+ yet the method still loads, its PipeFunc validated from its module's source.
"""
@pytest.mark.usefixtures("sandbox_hosted_mode")
- def test_hosted_load_captures_source_without_registering(self):
+ def test_hosted_load_captures_the_tree_files_without_importing(self):
library_manager = get_library_manager()
- function_name = "make_note_in_sandbox"
- # Guard the precondition: the customer function must not already be in the process registry.
- assert func_registry.get_function(function_name) is None
+ # Guard the precondition: the customer module must not already be imported in this process.
+ assert "hosted_demo_funcs" not in sys.modules
with tempfile.TemporaryDirectory() as tmp_dir:
library_dir = Path(tmp_dir)
(library_dir / "make_note.mthds").write_text(HOSTED_DEMO_MTHDS, encoding="utf-8")
- (library_dir / "customer_func.py").write_text(CUSTOMER_FUNC_PY, encoding="utf-8")
+ (library_dir / "hosted_demo_funcs.py").write_text(CUSTOMER_FUNC_PY, encoding="utf-8")
+ (library_dir / "notes").mkdir()
+ (library_dir / "notes" / "greeting.txt").write_text("hello", encoding="utf-8")
+ (library_dir / ".gitignore").write_text("scratch.py\n", encoding="utf-8")
+ (library_dir / "scratch.py").write_text("NOT_SHIPPED = True\n", encoding="utf-8")
library_id, _ = library_manager.open_library()
try:
with scoped_current_library(library_id=library_id):
- # Loads cleanly even though the function is absent here: hosted-mode validators skip
- # the func_registry lookup and the return-type check.
+ # Loads cleanly: the PipeFunc's module is located and its form proven from its source.
pipes = library_manager.load_libraries(library_id=library_id, library_dirs=[library_dir])
assert any(pipe.code == "make_note" for pipe in pipes)
- # The customer code was NOT imported/registered in this process.
- assert func_registry.get_function(function_name) is None
+ # The customer code was NOT imported in this process.
+ assert "hosted_demo_funcs" not in sys.modules
- # The .py source rode along on the crate, ready to travel to a sandbox.
+ # Every file of the step's tree but its bundles rode along on the crate, ready to travel to a
+ # sandbox; a file the tree excludes did not.
crate = library_manager.get_crate(library_id=library_id)
assert crate is not None
- assert crate.python_sources.get("customer_func.py") == CUSTOMER_FUNC_PY
- # Carrying source leaves the structural fingerprint untouched.
+ (tree_key,) = {tree_bundle_ref.tree_key for tree_bundle_ref in crate.tree_bundles.values()}
+ assert sorted(crate.pipe_func_files[tree_key]) == ["hosted_demo_funcs.py", "notes/greeting.txt"]
+ assert crate.pipe_func_files[tree_key]["hosted_demo_funcs.py"] == CUSTOMER_FUNC_PY
+ # Carrying files leaves the structural fingerprint untouched.
assert crate.fingerprint == crate.compute_fingerprint()
finally:
library_manager.teardown(library_id=library_id)
@@ -76,8 +130,8 @@ class TestHostedPipeFuncLoad:
@pytest.mark.usefixtures("sandbox_hosted_mode")
def test_hosted_load_rejects_colliding_source_paths(self):
- """Two library dirs sharing a source relpath (with different content) must fail loud, not
- silently overwrite one customer's PipeFunc body — the sandbox writes sources flat by relpath.
+ """Two library dirs sharing a file path (with different content) must fail loud, not
+ silently overwrite one customer's PipeFunc body — the sandbox writes the files at their paths.
"""
library_manager = get_library_manager()
@@ -87,12 +141,101 @@ class TestHostedPipeFuncLoad:
dir_a.mkdir()
dir_b.mkdir()
# Same relative path in both dirs, different content -> a genuine conflict.
- (dir_a / "customer_func.py").write_text(CUSTOMER_FUNC_PY, encoding="utf-8")
- (dir_b / "customer_func.py").write_text(CUSTOMER_FUNC_PY.replace("hello from the sandbox", "a different body"), encoding="utf-8")
+ (dir_a / "hosted_demo_funcs.py").write_text(CUSTOMER_FUNC_PY, encoding="utf-8")
+ (dir_b / "hosted_demo_funcs.py").write_text(CUSTOMER_FUNC_PY.replace("hello from the sandbox", "a different body"), encoding="utf-8")
library_id, _ = library_manager.open_library()
try:
- with scoped_current_library(library_id=library_id), pytest.raises(LibraryError, match="Duplicate PipeFunc source path"):
+ with scoped_current_library(library_id=library_id), pytest.raises(LibraryError, match="Duplicate PipeFunc file path"):
library_manager.load_libraries(library_id=library_id, library_dirs=[dir_a, dir_b])
finally:
library_manager.teardown(library_id=library_id)
+
+ @pytest.mark.usefixtures("sandbox_hosted_mode")
+ def test_two_methods_shipping_different_files_at_one_path_load_side_by_side(self, tmp_path: Path):
+ """Each method's calls carry its own tree's files, so two methods' README.md, requirements.txt and METHODS.toml never clash."""
+ library_manager = get_library_manager()
+ _write_method(method_dir=tmp_path / "first", name="first")
+ _write_method(method_dir=tmp_path / "second", name="second")
+
+ library_id, _ = library_manager.open_library()
+ try:
+ with scoped_current_library(library_id=library_id):
+ library_manager.load_libraries(library_id=library_id, library_dirs=[tmp_path / "first", tmp_path / "second"])
+ crate = library_manager.get_crate(library_id=library_id)
+ assert crate is not None
+ assert SUBMITTED_PYTHON_ROOT_KEY not in crate.pipe_func_files
+ assert len(crate.pipe_func_files) == 2
+ readmes = sorted(str(call_files["README.md"]) for call_files in crate.pipe_func_files.values())
+ assert readmes == ["# The first method\n", "# The second method\n"]
+ finally:
+ library_manager.teardown(library_id=library_id)
+
+ @pytest.mark.usefixtures("sandbox_hosted_mode")
+ def test_the_crate_builds_after_the_directory_its_files_were_read_from_is_removed(self, tmp_path: Path):
+ library_manager = get_library_manager()
+ library_dir = tmp_path / "request"
+ library_dir.mkdir()
+ (library_dir / "make_note.mthds").write_text(HOSTED_DEMO_MTHDS, encoding="utf-8")
+ (library_dir / "hosted_demo_funcs.py").write_text(CUSTOMER_FUNC_PY, encoding="utf-8")
+ (library_dir / "notes").mkdir()
+ (library_dir / "notes" / "greeting.txt").write_text("hello", encoding="utf-8")
+
+ library_id, _ = library_manager.open_library()
+ try:
+ with scoped_current_library(library_id=library_id):
+ library_manager.load_libraries(library_id=library_id, library_dirs=[library_dir])
+ shutil.rmtree(library_dir)
+
+ crate = library_manager.get_crate(library_id=library_id)
+
+ assert crate is not None
+ (call_files,) = crate.pipe_func_files.values()
+ assert call_files["hosted_demo_funcs.py"] == CUSTOMER_FUNC_PY
+ assert call_files["notes/greeting.txt"] == "hello"
+ finally:
+ library_manager.teardown(library_id=library_id)
+
+ @pytest.mark.usefixtures("sandbox_hosted_mode")
+ def test_a_tree_holding_no_pipe_func_step_is_not_carried(self, tmp_path: Path):
+ library_manager = get_library_manager()
+ library_dir = tmp_path / "no_steps"
+ library_dir.mkdir()
+ (library_dir / "concepts.mthds").write_text('domain = "no_steps"\n\n[concept]\nMemo = "A memo"\n', encoding="utf-8")
+ (library_dir / "helpers.py").write_text("VALUE = 1\n", encoding="utf-8")
+
+ library_id, _ = library_manager.open_library()
+ try:
+ with scoped_current_library(library_id=library_id):
+ library_manager.load_libraries(library_id=library_id, library_dirs=[library_dir])
+ crate = library_manager.get_crate(library_id=library_id)
+ assert crate is not None
+ assert crate.pipe_func_files == {}
+ finally:
+ library_manager.teardown(library_id=library_id)
+
+ @pytest.mark.usefixtures("sandbox_hosted_mode")
+ def test_a_module_named_like_a_runner_package_passes_on_a_sandbox_hosted_deployment(self, tmp_path: Path):
+ library_manager = get_library_manager()
+ library_dir = tmp_path / "shadowing"
+ _write_shadowing_method(library_dir=library_dir)
+
+ library_id, _ = library_manager.open_library()
+ try:
+ with scoped_current_library(library_id=library_id):
+ pipes = library_manager.load_libraries(library_id=library_id, library_dirs=[library_dir])
+ assert any(pipe.code == "make_note" for pipe in pipes)
+ finally:
+ library_manager.teardown(library_id=library_id)
+
+ def test_a_module_named_like_a_runner_package_is_refused_in_direct_mode(self, tmp_path: Path):
+ library_manager = get_library_manager()
+ library_dir = tmp_path / "shadowing"
+ _write_shadowing_method(library_dir=library_dir)
+
+ library_id, _ = library_manager.open_library()
+ try:
+ with scoped_current_library(library_id=library_id), pytest.raises(PipeValidationError, match="the module 'pytest'"):
+ library_manager.load_libraries(library_id=library_id, library_dirs=[library_dir])
+ finally:
+ library_manager.teardown(library_id=library_id)
diff --git a/tests/integration/pipelex/libraries/test_hosted_structures_refusal.py b/tests/integration/pipelex/libraries/test_hosted_structures_refusal.py
index 14373693d..bd3df8036 100644
--- a/tests/integration/pipelex/libraries/test_hosted_structures_refusal.py
+++ b/tests/integration/pipelex/libraries/test_hosted_structures_refusal.py
@@ -66,7 +66,7 @@ class {class_name}(StructuredContent):
total: float
-@pipe_func(name="make_hosted_refusal_total")
+@pipe_func()
async def make_hosted_refusal_total(working_memory: WorkingMemory) -> {class_name}:
return {class_name}(total=1.0)
@@ -92,6 +92,28 @@ description = "A receipt"
total = { type = "number", description = "The total", required = true }
"""
+GREETING_MTHDS = """\
+domain = "hosted_greeting"
+description = "A step calling a helper of the method's tree"
+
+[pipe.greet]
+type = "PipeFunc"
+description = "Greet"
+output = "Text"
+function_name = "helpers.greet"
+"""
+
+GREETING_HELPERS_PY = """\
+from pipelex.core.memory.working_memory import WorkingMemory
+from pipelex.core.stuffs.text_content import TextContent
+from pipelex.system.registries.func_registry import pipe_func
+
+
+@pipe_func()
+async def greet(working_memory: WorkingMemory) -> TextContent:
+ return TextContent(text="hello")
+"""
+
CLASS_BACKED_MTHDS = """\
domain = "hosted_refusal"
description = "A concept backed by a Python class"
@@ -200,6 +222,28 @@ class TestHostedStructuresRefusal:
assert not marker.exists()
assert not _is_imported(path=py_file)
+ @pytest.mark.usefixtures("sandbox_hosted_mode")
+ def test_a_structure_file_of_a_nested_repository_is_refused(self, tmp_path: Path, load_empty_library: Callable[[], str]):
+ """A repository nested in a library directory is a tree of its own, whose bundles load too, so its Python is scanned."""
+ library_dir = tmp_path / "library"
+ nested_dir = library_dir / "checkouts" / "invoicing"
+ (nested_dir / ".git").mkdir(parents=True)
+ (nested_dir / "invoice.mthds").write_text('domain = "nested_refusal"\n', encoding="utf-8")
+ (library_dir / "top.mthds").write_text('domain = "nested_refusal_top"\n', encoding="utf-8")
+ marker = tmp_path / "sentinel.ran"
+ structure_file = nested_dir / "models" / "invoice.py"
+ _write_structure_file(path=structure_file, template=STRUCTURE_FILE_PY, class_name="HostedRefusalNestedInvoice", marker=marker)
+
+ library_id = load_empty_library()
+ with pytest.raises(MethodStructuresRefusedError) as exc_info:
+ get_library_manager().load_libraries(library_id=library_id, library_dirs=[library_dir])
+
+ message = str(exc_info.value)
+ assert "checkouts/invoicing/models/invoice.py defines HostedRefusalNestedInvoice" in message
+ assert str(tmp_path.resolve()) not in message
+ assert not marker.exists()
+ assert not _is_imported(path=structure_file)
+
@pytest.mark.usefixtures("sandbox_hosted_mode")
def test_one_refusal_lists_every_directory(self, tmp_path: Path, load_empty_library: Callable[[], str]):
dir_a = tmp_path / "a"
@@ -265,16 +309,17 @@ class TestHostedStructuresRefusal:
generated_file = library_dir / "structures" / "structures.py"
generated_source = generated_file.read_text(encoding="utf-8")
assert "(StructuredContent)" in generated_source
- helper_source = "GREETING = 'hello'\n"
- (library_dir / "helpers.py").write_text(helper_source, encoding="utf-8")
+ (library_dir / "helpers.py").write_text(GREETING_HELPERS_PY, encoding="utf-8")
+ (library_dir / "greet.mthds").write_text(GREETING_MTHDS, encoding="utf-8")
library_id = load_empty_library()
library_manager.load_libraries(library_id=library_id, library_dirs=[library_dir])
hosted_crate = library_manager.get_crate(library_id=library_id)
assert hosted_crate is not None
- assert "structures/structures.py" not in hosted_crate.python_sources
- assert hosted_crate.python_sources.get("helpers.py") == helper_source
+ (call_files,) = hosted_crate.pipe_func_files.values()
+ assert "structures/structures.py" not in call_files
+ assert call_files.get("helpers.py") == GREETING_HELPERS_PY
assert not _is_imported(path=generated_file)
@pytest.mark.usefixtures("sandbox_hosted_mode")
diff --git a/tests/integration/pipelex/libraries/test_method_requirements_load.py b/tests/integration/pipelex/libraries/test_method_requirements_load.py
index d699366b7..cb05322af 100644
--- a/tests/integration/pipelex/libraries/test_method_requirements_load.py
+++ b/tests/integration/pipelex/libraries/test_method_requirements_load.py
@@ -1,19 +1,23 @@
+import sys
from pathlib import Path
import pytest
from pytest_mock import MockerFixture
-from pipelex.interpreter_hub import get_library_manager, scoped_current_library
+from pipelex.core.method_trees.tree_python import TREE_PYTHON_HOLD
+from pipelex.interpreter_hub import get_library_manager, get_required_pipe, scoped_current_library
from pipelex.libraries.exceptions import MethodRequirementsNotMetError, PipeFuncPackageSetUnstatedError
from pipelex.mthds_parsing.parser import MthdsParser
+from pipelex.pipe_operators.func.pipe_func import PipeFunc
from pipelex.plugins.pipe_func_package_offer_registry import PipeFuncPackageOffer, PipeFuncPackageOfferFactoryFn, PipeFuncPackageOfferRegistry
from pipelex.system.configuration.pipe_func_config import PipeFuncConfig
-from pipelex.system.registries.func_registry import func_registry
from tests.integration.pipelex.libraries.installed_packages import isolate_installed_methods, vendor_probe
from tests.integration.pipelex.libraries.test_data import ProbePackageTestData
_ABSENT_PACKAGE = "pipelex-requirements-test-absent"
_FUNCTION_NAME = "requirements_test_crunch"
+# The module of the method's function: distinctive, since a tree's top-level module is imported under its own name.
+_MODULE_NAME = "requirements_load_funcs"
_OFFER = PipeFuncPackageOffer(
environment="the Daytona sandbox snapshot 'pipelex-base-test'",
distributions={"pandas": "2.2.3", "openpyxl": "3.1.5"},
@@ -25,15 +29,12 @@ _MTHDS = f"""\
domain = "requirements_load"
main_pipe = "crunch"
-[concept]
-Note = "A note produced by the method's own code"
-
[pipe.crunch]
type = "PipeFunc"
description = "Crunch numbers in the method's own code"
inputs = {{}}
-output = "Note"
-function_name = "{_FUNCTION_NAME}"
+output = "Text"
+function_name = "{_MODULE_NAME}.{_FUNCTION_NAME}"
"""
@@ -49,7 +50,7 @@ from pipelex.system.registries.func_registry import pipe_func
Path({str(sentinel)!r}).write_text("imported", encoding="utf-8")
-@pipe_func(name="{_FUNCTION_NAME}")
+@pipe_func()
async def {_FUNCTION_NAME}(working_memory: WorkingMemory) -> TextContent:
return TextContent(text="crunched")
"""
@@ -58,7 +59,7 @@ async def {_FUNCTION_NAME}(working_memory: WorkingMemory) -> TextContent:
def _method(*, directory: Path, requirements: str, sentinel: Path) -> Path:
directory.mkdir(parents=True, exist_ok=True)
(directory / "main.mthds").write_text(_MTHDS, encoding="utf-8")
- (directory / "funcs.py").write_text(_funcs_py(sentinel=sentinel), encoding="utf-8")
+ (directory / f"{_MODULE_NAME}.py").write_text(_funcs_py(sentinel=sentinel), encoding="utf-8")
(directory / "requirements.txt").write_text(requirements, encoding="utf-8")
return directory
@@ -89,7 +90,31 @@ class TestMethodRequirementsLoad:
assert f"line 2 '{_ABSENT_PACKAGE}>=1'" in exc_info.value.message
assert "uv pip install --python" in exc_info.value.message
assert not sentinel.exists(), "the method's Python was imported before its requirements were checked"
- assert func_registry.get_function(_FUNCTION_NAME) is None
+ assert _MODULE_NAME not in sys.modules
+
+ def test_locally_a_met_requirement_loads_the_method_and_its_python_is_imported_only_when_its_step_binds(self, tmp_path: Path) -> None:
+ """The check comes first, the load imports none of the method's Python, and the step's first bind imports its module."""
+ sentinel = tmp_path / "funcs-module-ran"
+ library_dir = _method(directory=tmp_path / "method", requirements="pydantic>=2\n", sentinel=sentinel)
+ library_manager = get_library_manager()
+ library_id, _ = library_manager.open_library()
+ try:
+ with scoped_current_library(library_id=library_id):
+ pipes = library_manager.load_libraries(library_id=library_id, library_dirs=[library_dir])
+ assert [pipe.code for pipe in pipes] == ["crunch"]
+ assert not sentinel.exists(), "the load imported the method's Python"
+ assert _MODULE_NAME not in sys.modules
+
+ pipe_func = get_required_pipe(pipe_code="requirements_load.crunch")
+ assert isinstance(pipe_func, PipeFunc)
+ function = pipe_func.bind_function()
+ assert function.__module__ == _MODULE_NAME
+ assert sentinel.read_text(encoding="utf-8") == "imported"
+ finally:
+ library_manager.teardown(library_id=library_id)
+ TREE_PYTHON_HOLD.unload_if_unheld()
+
+ assert _MODULE_NAME not in sys.modules
def test_a_bundle_file_named_alone_loads_under_its_trees_requirements_on_every_load(self, tmp_path: Path) -> None:
"""A retry of the same blueprint, placed in its tree by the refused load, is refused again rather than taken as judged."""
@@ -114,7 +139,7 @@ class TestMethodRequirementsLoad:
methods_dir = vendor_probe(methods_dir=tmp_path / "request-methods")
package_dir = methods_dir / ProbePackageTestData.METHOD_NAME
sentinel = tmp_path / "funcs-module-ran"
- (package_dir / "funcs.py").write_text(_funcs_py(sentinel=sentinel), encoding="utf-8")
+ (package_dir / f"{_MODULE_NAME}.py").write_text(_funcs_py(sentinel=sentinel), encoding="utf-8")
(package_dir / "requirements.txt").write_text(f"pydantic>=2\n{_ABSENT_PACKAGE}>=1\n", encoding="utf-8")
blueprint = MthdsParser.make_pipelex_bundle_blueprint(mthds_content=ProbePackageTestData.CONSUMER_BUNDLE)
library_manager = get_library_manager()
@@ -151,7 +176,7 @@ class TestMethodRequirementsLoad:
assert [pipe.code for pipe in pipes] == ["crunch"]
crate = library_manager.get_crate(library_id=library_id)
assert crate is not None
- assert "funcs.py" in crate.python_sources
+ assert any(f"{_MODULE_NAME}.py" in call_files for call_files in crate.pipe_func_files.values())
finally:
library_manager.teardown(library_id=library_id)
diff --git a/tests/integration/pipelex/pipeline/invocation_paths/conftest.py b/tests/integration/pipelex/pipeline/invocation_paths/conftest.py
deleted file mode 100644
index d3d441330..000000000
--- a/tests/integration/pipelex/pipeline/invocation_paths/conftest.py
+++ /dev/null
@@ -1,15 +0,0 @@
-from collections.abc import Iterator
-
-import pytest
-
-from pipelex.system.registries.func_registry import func_registry
-from tests.integration.pipelex.pipeline.invocation_paths.invocation_paths_helpers import invocation_paths_condense_filing
-
-
[email protected](autouse=True)
-def register_condense_filing() -> Iterator[None]:
- """Register the function the worked example's `condense_filing` runs, for the test's duration."""
- func_registry.register_function(invocation_paths_condense_filing)
- yield
- if func_registry.has_function(invocation_paths_condense_filing.__name__):
- func_registry.unregister_function_by_name(invocation_paths_condense_filing.__name__)
diff --git a/tests/integration/pipelex/pipeline/invocation_paths/invocation_paths_helpers.py b/tests/integration/pipelex/pipeline/invocation_paths/invocation_paths_helpers.py
index bc25b8be2..64f16ab29 100644
--- a/tests/integration/pipelex/pipeline/invocation_paths/invocation_paths_helpers.py
+++ b/tests/integration/pipelex/pipeline/invocation_paths/invocation_paths_helpers.py
@@ -15,6 +15,7 @@ from pipelex.pipe_run.pipe_job import PipeJob
from pipelex.pipe_run.pipe_router import PipeRouter
from pipelex.runtime_hub import get_event_log_override
from pipelex.system.job_metadata import JobMetadata
+from pipelex.system.registries.func_registry import pipe_func
if TYPE_CHECKING:
from pipelex.system.trace_context import TraceContext
@@ -23,6 +24,7 @@ if TYPE_CHECKING:
_IS_LOST_ATTEMPT: ContextVar[bool] = ContextVar("invocation_paths_is_lost_attempt", default=False)
+@pipe_func()
def invocation_paths_condense_filing(working_memory: WorkingMemory) -> TextContent:
reading = working_memory.get_stuff_as_str(name="reading")
if _IS_LOST_ATTEMPT.get():
diff --git a/tests/integration/pipelex/pipeline/invocation_paths/test_data.py b/tests/integration/pipelex/pipeline/invocation_paths/test_data.py
index 5c5ccc1ac..eb8a29b48 100644
--- a/tests/integration/pipelex/pipeline/invocation_paths/test_data.py
+++ b/tests/integration/pipelex/pipeline/invocation_paths/test_data.py
@@ -117,7 +117,7 @@ type = "PipeFunc"
description = "Condense the reading of a filing"
inputs = { reading = "Text" }
output = "Text"
-function_name = "invocation_paths_condense_filing"
+function_name = "tests.integration.pipelex.pipeline.invocation_paths.invocation_paths_helpers.invocation_paths_condense_filing"
[pipe.assemble_report]
type = "PipeCompose"
diff --git a/tests/integration/pipelex/pipeline/test_file_input_consumers.py b/tests/integration/pipelex/pipeline/test_file_input_consumers.py
index ffac65ee0..7a7c2be50 100644
--- a/tests/integration/pipelex/pipeline/test_file_input_consumers.py
+++ b/tests/integration/pipelex/pipeline/test_file_input_consumers.py
@@ -22,7 +22,7 @@ from pipelex.mthds_parsing.parser import MthdsParser
from pipelex.pipeline import file_input_consumers as file_input_consumers_module
from pipelex.pipeline.file_input_consumers import FileConsumerKind, FileInputConsumer, collect_file_input_consumers
from pipelex.runtime_hub import get_model_deck
-from pipelex.system.registries.func_registry import func_registry
+from pipelex.system.registries.func_registry import pipe_func
PDF_ONLY_MODEL = "pypdfium2-extract-pdf"
DOCLING_MODEL = "docling-extract-text"
@@ -46,6 +46,7 @@ model = "{DOCLING_MODEL}"
"""
+@pipe_func()
def file_consumers_replace_transcript(working_memory: WorkingMemory) -> DocumentContent:
return DocumentContent(url=working_memory.get_stuff_as_document(name="transcript").url)
@@ -67,13 +68,6 @@ def _summary(consumers: list[FileInputConsumer]) -> list[tuple[str, str, bool]]:
class TestFileInputConsumers:
- @pytest.fixture(autouse=True)
- def register_funcs(self):
- func_registry.register_function(file_consumers_replace_transcript)
- yield
- if func_registry.has_function(file_consumers_replace_transcript.__name__):
- func_registry.unregister_function_by_name(file_consumers_replace_transcript.__name__)
-
def test_a_sequence_step_consumes_the_input(self, load_empty_library: Callable[[], str]):
pipes = """
[pipe.main]
@@ -110,7 +104,7 @@ type = "PipeFunc"
description = "Replace the transcript with another document"
inputs = { transcript = "Document" }
output = "Document"
-function_name = "file_consumers_replace_transcript"
+function_name = "tests.integration.pipelex.pipeline.test_file_input_consumers.file_consumers_replace_transcript"
"""
consumers = _consumers(load_empty_library=load_empty_library, domain="fic_overwrite", pipes=pipes)
@@ -176,7 +170,7 @@ type = "PipeFunc"
description = "Replace the transcript with another document"
inputs = {{ transcript = "Document" }}
output = "Document"
-function_name = "file_consumers_replace_transcript"
+function_name = "tests.integration.pipelex.pipeline.test_file_input_consumers.file_consumers_replace_transcript"
"""
consumers = _consumers(load_empty_library=load_empty_library, domain=domain, pipes=pipes)
@@ -386,7 +380,7 @@ type = "PipeFunc"
description = "Replace the transcript with another document"
inputs = { transcript = "Document" }
output = "Document"
-function_name = "file_consumers_replace_transcript"
+function_name = "tests.integration.pipelex.pipeline.test_file_input_consumers.file_consumers_replace_transcript"
[pipe.compose_note]
type = "PipeCompose"
@@ -525,7 +519,7 @@ type = "PipeFunc"
description = "Make a new document from the transcript"
inputs = { transcript = "Document" }
output = "Document"
-function_name = "file_consumers_replace_transcript"
+function_name = "tests.integration.pipelex.pipeline.test_file_input_consumers.file_consumers_replace_transcript"
[pipe.summarize_fresh_document]
type = "PipeLLM"
diff --git a/tests/integration/pipelex/pipeline/test_optional_method_inputs.py b/tests/integration/pipelex/pipeline/test_optional_method_inputs.py
index 58b521ad3..c0971f177 100644
--- a/tests/integration/pipelex/pipeline/test_optional_method_inputs.py
+++ b/tests/integration/pipelex/pipeline/test_optional_method_inputs.py
@@ -21,10 +21,11 @@ from pipelex.pipe_operators.func.pipe_func_blueprint import PipeFuncBlueprint
from pipelex.pipeline.execution_seams import prepare_pipe_job
from pipelex.system.job_metadata import JobMetadata
from pipelex.system.pipe_run_mode import PipeRunMode
-from pipelex.system.registries.func_registry import func_registry
+from pipelex.system.registries.func_registry import pipe_func
from tests.integration.pipelex.fixtures.pipe_job_helpers import make_mode_guarded_run_params
+@pipe_func()
def optionals_method_report(working_memory: WorkingMemory) -> TextContent:
topic = working_memory.get_stuff_as_str(name="topic")
source_stuff = working_memory.get_optional_stuff(name="source")
@@ -41,22 +42,13 @@ def _make_method_pipe() -> PipeFunc:
description="Method with a required topic and an optional source",
inputs={"topic": "Text", "source": "Text?"},
output="Text",
- function_name="optionals_method_report",
+ function_name=f"{__name__}.optionals_method_report",
),
)
@pytest.mark.asyncio(loop_scope="class")
class TestOptionalMethodInputs:
- @classmethod
- def setup_class(cls):
- func_registry.register_function(optionals_method_report)
-
- @classmethod
- def teardown_class(cls):
- if func_registry.has_function("optionals_method_report"):
- func_registry.unregister_function_by_name("optionals_method_report")
-
async def test_omitted_optional_input_records_not_provided(self, load_empty_library: Callable[[], str]):
"""prepare_pipe_job seeds a not-provided absence record for each omitted `?` input."""
library_id = load_empty_library()
diff --git a/tests/integration/pipelex/pipeline/test_run_graph_io_multiplicity.py b/tests/integration/pipelex/pipeline/test_run_graph_io_multiplicity.py
index 10e9bf97f..253284c51 100644
--- a/tests/integration/pipelex/pipeline/test_run_graph_io_multiplicity.py
+++ b/tests/integration/pipelex/pipeline/test_run_graph_io_multiplicity.py
@@ -24,7 +24,7 @@ from pipelex.graph.graphspec import GraphSpec, IOSpec, NodeSpec, NodeStatus
from pipelex.pipeline.dry_run_pipeline import dry_run_pipeline
from pipelex.pipeline.runner import PipelexMTHDSProtocol
from pipelex.runtime_hub import scoped_event_log
-from pipelex.system.registries.func_registry import func_registry
+from pipelex.system.registries.func_registry import pipe_func
from pipelex.tracing.in_memory_event_log import InMemoryEventLog
_DRY_BUNDLE = '''
@@ -146,7 +146,7 @@ type = "PipeFunc"
description = "A plural output: lifted to an empty list"
inputs = { source = "Text" }
output = "Text[]"
-function_name = "io_multiplicity_find_many"
+function_name = "tests.integration.pipelex.pipeline.test_run_graph_io_multiplicity.io_multiplicity_find_many"
[pipe.branch_out]
type = "PipeParallel"
@@ -164,32 +164,35 @@ type = "PipeFunc"
description = "A singular branch"
inputs = { source = "Text" }
output = "Text"
-function_name = "io_multiplicity_find_one"
+function_name = "tests.integration.pipelex.pipeline.test_run_graph_io_multiplicity.io_multiplicity_find_one"
[pipe.find_many_again]
type = "PipeFunc"
description = "A plural branch"
inputs = { source = "Text" }
output = "Text[]"
-function_name = "io_multiplicity_find_many"
+function_name = "tests.integration.pipelex.pipeline.test_run_graph_io_multiplicity.io_multiplicity_find_many"
[pipe.count_findings]
type = "PipeFunc"
description = "Consumes the lifted lists"
inputs = { findings = "Text[]", many_out = "Text[]", one_out = "Text?" }
output = "Text"
-function_name = "io_multiplicity_count_findings"
+function_name = "tests.integration.pipelex.pipeline.test_run_graph_io_multiplicity.io_multiplicity_count_findings"
"""
+@pipe_func()
def io_multiplicity_find_many(working_memory: WorkingMemory) -> ListContent[TextContent]:
return ListContent[TextContent](items=[TextContent(text=f"found:{working_memory.get_stuff_as_str(name='source')}")])
+@pipe_func()
def io_multiplicity_find_one(working_memory: WorkingMemory) -> TextContent:
return TextContent(text=f"found:{working_memory.get_stuff_as_str(name='source')}")
+@pipe_func()
def io_multiplicity_count_findings(working_memory: WorkingMemory) -> TextContent:
findings = working_memory.get_stuff(name="findings").content
many_out = working_memory.get_stuff(name="many_out").content
@@ -198,9 +201,6 @@ def io_multiplicity_count_findings(working_memory: WorkingMemory) -> TextContent
return TextContent(text=f"{findings.nb_items} findings, {many_out.nb_items} more")
-_TEST_FUNCS = [io_multiplicity_find_many, io_multiplicity_find_one, io_multiplicity_count_findings]
-
-
def _io_items(graph: GraphSpec) -> list[tuple[NodeSpec, IOSpec]]:
return [(node, io_spec) for node in graph.nodes for io_spec in [*node.node_io.inputs, *node.node_io.outputs]]
@@ -237,17 +237,6 @@ def _in_process_graph(teardown_spy: Any) -> GraphSpec:
@pytest.mark.asyncio(loop_scope="class")
class TestRunGraphIoMultiplicity:
- @classmethod
- def setup_class(cls):
- for func in _TEST_FUNCS:
- func_registry.register_function(func)
-
- @classmethod
- def teardown_class(cls):
- for func in _TEST_FUNCS:
- if func_registry.has_function(func.__name__):
- func_registry.unregister_function_by_name(func.__name__)
-
async def test_dry_run_marks_every_list(self, mocker: MockerFixture) -> None:
teardown_spy = mocker.spy(GraphTracer, "teardown")
diff --git a/tests/integration/pipelex/pipeline/test_run_path_bundle_verdict.py b/tests/integration/pipelex/pipeline/test_run_path_bundle_verdict.py
index b9f82fc24..42342b5cf 100644
--- a/tests/integration/pipelex/pipeline/test_run_path_bundle_verdict.py
+++ b/tests/integration/pipelex/pipeline/test_run_path_bundle_verdict.py
@@ -90,8 +90,8 @@ prompt = "A painted wooden harbour board announcing the tide times"
aspect_ratio = "7:3"
"""
-# A check that fires inside a pipe's pydantic validator while the pipe is built: the function a
-# PipeFunc names is not registered.
+# A check that fires inside a pipe's pydantic validator while the bundle is read: the function a
+# PipeFunc names is a flat name, the key of the retired function registry, not an import path.
_MODEL_VALIDATOR_FAULT_BUNDLE = """
domain = "harbour_notices"
description = "Format notices for the harbour board"
diff --git a/tests/integration/pipelex/pipeline/test_verdict_names_no_host_path.py b/tests/integration/pipelex/pipeline/test_verdict_names_no_host_path.py
index 0e410a069..769e305be 100644
--- a/tests/integration/pipelex/pipeline/test_verdict_names_no_host_path.py
+++ b/tests/integration/pipelex/pipeline/test_verdict_names_no_host_path.py
@@ -33,7 +33,6 @@ from pipelex.pipeline.runner import PipelexMTHDSProtocol
from pipelex.pipeline.validate_bundle import validate_bundle
from pipelex.pipeline.validate_bundle_translation import HOST_LIBRARY_FILE_PLACEHOLDER
from pipelex.system.pipe_run_mode import PipeRunMode
-from pipelex.system.registries.func_registry import func_registry
from pipelex.validation_error_types import PipeValidationErrorType
_DEPENDENCY_ADDRESS = "github.com/acme-tests/harbour-methods/tides"
@@ -148,19 +147,29 @@ prompt = "Write a short notice for the harbour board from these tide times:
"""
-# Two host library files registering a PipeFunc under the same name: the registry refuses the second, naming
-# both by the module names it imported them under, which it spells from their absolute paths.
-_HOST_PIPE_FUNC_NAME = "format_harbour_tide_board_for_host_path_test"
+# A host library bundle whose PipeFunc's tree holds a module named like a standard-library module: the
+# refusal names the module, the tree file and the bundle, never the tree's directory on the host.
+_HOST_PIPE_FUNC_BUNDLE = """
+domain = "harbour_board"
+description = "Format notices for the harbour board"
+
+[pipe.format_board_notice]
+type = "PipeFunc"
+description = "Format the notice for the harbour board"
+inputs = { notice = "Text" }
+output = "Text"
+function_name = "board_formatting.format_board_notice"
+"""
-_HOST_PIPE_FUNC_MODULE = f"""
+_HOST_PIPE_FUNC_MODULE = """
from pipelex.core.memory.working_memory import WorkingMemory
from pipelex.core.stuffs.text_content import TextContent
from pipelex.system.registries.func_registry import pipe_func
@pipe_func()
-async def {_HOST_PIPE_FUNC_NAME}(working_memory: WorkingMemory) -> TextContent:
- return TextContent(text="{{flavor}}")
+async def format_board_notice(working_memory: WorkingMemory) -> TextContent:
+ return TextContent(text="board")
"""
@@ -384,6 +393,21 @@ class TestVerdictNamesNoHostPath:
assert item.message.count(HOST_LIBRARY_FILE_PLACEHOLDER) >= 2
_assert_names_no_host_path(payload=_strict_payload(raised.value), tmp_path=tmp_path)
+ async def test_a_host_library_pipe_func_refusal_names_no_host_path(self, tmp_path: Path) -> None:
+ host_library_dir = tmp_path / "host" / "library"
+ _write(path=host_library_dir / "harbour_board.mthds", content=_HOST_PIPE_FUNC_BUNDLE)
+ _write(path=host_library_dir / "board_formatting.py", content=_HOST_PIPE_FUNC_MODULE)
+ _write(path=host_library_dir / "json.py", content="SHADOW = True\n")
+
+ with pytest.raises(ValidateBundleError) as raised:
+ await validate_bundle(mthds_contents=[_VALID_CALLER_BUNDLE], library_dirs=[host_library_dir])
+
+ items = raised.value.to_error_report().validation_errors or []
+ refused_items = [item for item in items if item.error_type == PipeValidationErrorType.PIPE_FUNC_FUNCTION_REFUSED]
+ assert refused_items, f"expected a pipe_func_function_refused item, got {items!r}"
+ assert "standard-library module 'json'" in refused_items[0].message
+ _assert_names_no_host_path(payload=_strict_payload(raised.value), tmp_path=tmp_path)
+
async def test_a_run_answers_a_host_library_refusal_without_a_host_path(self, tmp_path: Path) -> None:
host_library_dir = tmp_path / "host" / "library"
_write(path=host_library_dir / "harbour_board.mthds", content=_MISSPELLED_CONCEPT_LIBRARY_BUNDLE)
@@ -409,26 +433,6 @@ class TestVerdictNamesNoHostPath:
assert concept_items, f"expected an unresolved_concept item, got {items!r}"
assert all(item.source == str(library_file) for item in concept_items)
- async def test_a_host_library_pipe_func_collision_names_no_host_module(self, tmp_path: Path) -> None:
- host_library_dir = tmp_path / "host" / "library"
- _write(path=host_library_dir / "board_formatting.py", content=_HOST_PIPE_FUNC_MODULE.format(flavor="board"))
- _write(path=host_library_dir / "notice_formatting.py", content=_HOST_PIPE_FUNC_MODULE.format(flavor="notice"))
-
- try:
- with pytest.raises(ValidateBundleError) as raised:
- await validate_bundle(mthds_contents=[_VALID_CALLER_BUNDLE], library_dirs=[host_library_dir])
- finally:
- if func_registry.has_function(_HOST_PIPE_FUNC_NAME):
- func_registry.unregister_function_by_name(_HOST_PIPE_FUNC_NAME)
-
- (item,) = raised.value.to_error_report().validation_errors or []
- assert _HOST_PIPE_FUNC_NAME in item.message
- assert item.message.count(HOST_LIBRARY_FILE_PLACEHOLDER) == 2
- strict_payload = _strict_payload(raised.value)
- _assert_names_no_host_path(payload=strict_payload, tmp_path=tmp_path)
- # The module names are the host paths spelled with underscores.
- assert "host_library_board_formatting" not in json.dumps(strict_payload)
-
@pytest.mark.parametrize("library_dirs_are_callers", [False, True], ids=["hosts_library", "callers_library"])
async def test_a_runtime_validates_content_against_the_library_it_is_told_is_whose(self, library_dirs_are_callers: bool, tmp_path: Path) -> None:
library_dir = tmp_path / "library"
diff --git a/tests/integration/pipelex/pipes/controller/pipe_batch/test_pipe_batch_branch_dispatch.py b/tests/integration/pipelex/pipes/controller/pipe_batch/test_pipe_batch_branch_dispatch.py
index 9a381a86d..ecf2a2070 100644
--- a/tests/integration/pipelex/pipes/controller/pipe_batch/test_pipe_batch_branch_dispatch.py
+++ b/tests/integration/pipelex/pipes/controller/pipe_batch/test_pipe_batch_branch_dispatch.py
@@ -31,11 +31,12 @@ from pipelex.pipe_run.pipe_router import PipeRouter
from pipelex.pipe_run.pipe_run_params_factory import PipeRunParamsFactory
from pipelex.system.job_metadata import JobMetadata
from pipelex.system.pipe_run_mode import PipeRunMode
-from pipelex.system.registries.func_registry import func_registry
+from pipelex.system.registries.func_registry import pipe_func
_DOMAIN_CODE = "test_batch_branch_dispatch"
+@pipe_func()
def batch_branch_dispatch_shout_item(working_memory: WorkingMemory) -> TextContent:
return TextContent(text=working_memory.get_stuff_as_str(name="item").upper())
@@ -68,7 +69,7 @@ def _build_batch() -> PipeBatch:
description="Uppercase the batch item",
inputs={"item": "Text"},
output="Text",
- function_name="batch_branch_dispatch_shout_item",
+ function_name=f"{__name__}.batch_branch_dispatch_shout_item",
),
)
pipe_library = get_pipe_library()
@@ -103,15 +104,6 @@ def _make_items_memory(batch: PipeBatch, texts: list[str]):
@pytest.mark.asyncio(loop_scope="class")
class TestPipeBatchBranchDispatch:
- @classmethod
- def setup_class(cls):
- func_registry.register_function(batch_branch_dispatch_shout_item)
-
- @classmethod
- def teardown_class(cls):
- if func_registry.has_function(batch_branch_dispatch_shout_item.__name__):
- func_registry.unregister_function_by_name(batch_branch_dispatch_shout_item.__name__)
-
async def test_branches_dispatch_through_the_hook(self, job_metadata: JobMetadata, load_empty_library: Callable[[], str]):
"""One `run_batch_branch` call per item, and the batch itself never enters through the hook."""
load_empty_library()
diff --git a/tests/integration/pipelex/pipes/controller/pipe_batch/test_pipe_batch_compaction.py b/tests/integration/pipelex/pipes/controller/pipe_batch/test_pipe_batch_compaction.py
index c3801eaa6..983a9cc80 100644
--- a/tests/integration/pipelex/pipes/controller/pipe_batch/test_pipe_batch_compaction.py
+++ b/tests/integration/pipelex/pipes/controller/pipe_batch/test_pipe_batch_compaction.py
@@ -29,19 +29,17 @@ from pipelex.pipe_operators.func.pipe_func_blueprint import PipeFuncBlueprint
from pipelex.pipe_run.pipe_run_params import PipeRunParams
from pipelex.system.job_metadata import JobMetadata
from pipelex.system.pipe_run_mode import PipeRunMode
-from pipelex.system.registries.func_registry import func_registry
+from pipelex.system.registries.func_registry import pipe_func
from tests.integration.pipelex.fixtures.pipe_job_helpers import make_mode_guarded_run_params
_DOMAIN_CODE = "test_optionals_batch"
+@pipe_func()
def optionals_batch_shout_item(working_memory: WorkingMemory) -> TextContent:
return TextContent(text=working_memory.get_stuff_as_str(name="item").upper())
-_TEST_FUNCS = [optionals_batch_shout_item]
-
-
def _make_live_run_params() -> PipeRunParams:
"""Build through the factory so the batch fans out under the *configured* bound.
@@ -61,7 +59,7 @@ def _build_compacting_batch() -> PipeBatch:
description="Uppercase the batch item",
inputs={"item": "Text"},
output="Text",
- function_name="optionals_batch_shout_item",
+ function_name=f"{__name__}.optionals_batch_shout_item",
),
)
gate_pipe = PipeFactory[PipeCondition].make_from_blueprint(
@@ -98,17 +96,6 @@ def _build_compacting_batch() -> PipeBatch:
@pytest.mark.asyncio(loop_scope="class")
class TestPipeBatchCompaction:
- @classmethod
- def setup_class(cls):
- for func in _TEST_FUNCS:
- func_registry.register_function(func)
-
- @classmethod
- def teardown_class(cls):
- for func in _TEST_FUNCS:
- if func_registry.has_function(func.__name__):
- func_registry.unregister_function_by_name(func.__name__)
-
async def test_absent_branch_results_are_dropped(self, job_metadata: JobMetadata, load_empty_library: Callable[[], str]):
"""Mixed items: rejected items resolve absent (continue) and are dropped from the list."""
load_empty_library()
diff --git a/tests/integration/pipelex/pipes/controller/pipe_batch/test_pipe_batch_item_isolation.py b/tests/integration/pipelex/pipes/controller/pipe_batch/test_pipe_batch_item_isolation.py
index 8d33ef9d1..3ad7522bd 100644
--- a/tests/integration/pipelex/pipes/controller/pipe_batch/test_pipe_batch_item_isolation.py
+++ b/tests/integration/pipelex/pipes/controller/pipe_batch/test_pipe_batch_item_isolation.py
@@ -4,7 +4,7 @@ from pipelex.core.memory.working_memory import WorkingMemory
from pipelex.core.stuffs.text_content import TextContent
from pipelex.pipeline.runner import PipelexMTHDSProtocol
from pipelex.system.pipe_run_mode import PipeRunMode
-from pipelex.system.registries.func_registry import func_registry
+from pipelex.system.registries.func_registry import pipe_func
_BUNDLE = """
domain = "batch_item_isolation"
@@ -25,10 +25,11 @@ type = "PipeFunc"
description = "Shouts one note, rewriting the item it was handed in place"
inputs = { note = "Text" }
output = "Text"
-function_name = "batch_item_isolation_shout_note"
+function_name = "tests.integration.pipelex.pipes.controller.pipe_batch.test_pipe_batch_item_isolation.batch_item_isolation_shout_note"
"""
+@pipe_func()
def batch_item_isolation_shout_note(working_memory: WorkingMemory) -> TextContent:
note = working_memory.get_stuff_as_text(name="note")
note.text = note.text.upper()
@@ -37,15 +38,6 @@ def batch_item_isolation_shout_note(working_memory: WorkingMemory) -> TextConten
@pytest.mark.asyncio(loop_scope="class")
class TestPipeBatchItemIsolation:
- @classmethod
- def setup_class(cls):
- func_registry.register_function(batch_item_isolation_shout_note)
-
- @classmethod
- def teardown_class(cls):
- if func_registry.has_function(batch_item_isolation_shout_note.__name__):
- func_registry.unregister_function_by_name(batch_item_isolation_shout_note.__name__)
-
async def test_a_branch_rewriting_its_item_leaves_the_batched_list_unchanged(self) -> None:
"""Each branch gets its own copy of its item, as it gets its own copy of the rest of working memory."""
result = await PipelexMTHDSProtocol(pipe_run_mode=PipeRunMode.LIVE).execute(
diff --git a/tests/integration/pipelex/pipes/controller/pipe_parallel/test_pipe_parallel_absence.py b/tests/integration/pipelex/pipes/controller/pipe_parallel/test_pipe_parallel_absence.py
index 6c2473a3b..d07c1bbd7 100644
--- a/tests/integration/pipelex/pipes/controller/pipe_parallel/test_pipe_parallel_absence.py
+++ b/tests/integration/pipelex/pipes/controller/pipe_parallel/test_pipe_parallel_absence.py
@@ -37,7 +37,7 @@ from pipelex.pipe_run.pipe_run_params import PipeRunParams
from pipelex.runtime_hub import get_class_registry
from pipelex.system.job_metadata import JobMetadata
from pipelex.system.pipe_run_mode import PipeRunMode
-from pipelex.system.registries.func_registry import func_registry
+from pipelex.system.registries.func_registry import pipe_func
from tests.integration.pipelex.fixtures.pipe_job_helpers import make_mode_guarded_run_params
_DOMAIN_CODE = "test_optionals_par"
@@ -57,17 +57,16 @@ class ParAbsStrictReport(StructuredContent):
found_result: TextContent = Field(description="Result of the maybe-lifted branch")
+@pipe_func()
def optionals_par_echo_source(working_memory: WorkingMemory) -> TextContent:
return TextContent(text=f"found:{working_memory.get_stuff_as_str(name='source')}")
+@pipe_func()
def optionals_par_echo_topic(working_memory: WorkingMemory) -> TextContent:
return TextContent(text=f"base:{working_memory.get_stuff_as_str(name='topic')}")
-_TEST_FUNCS = [optionals_par_echo_source, optionals_par_echo_topic]
-
-
def _make_live_run_params() -> PipeRunParams:
return make_mode_guarded_run_params(pipe_run_mode=PipeRunMode.LIVE)
@@ -86,7 +85,7 @@ def _build_parallel(*, output_ref: str, structure_class_names: list[str], add_ea
description="Consumes the maybe-absent source (plain input): lifted when source is absent",
inputs={"source": "Text"},
output="Text",
- function_name="optionals_par_echo_source",
+ function_name=f"{__name__}.optionals_par_echo_source",
),
)
branch_base = PipeFactory[PipeFunc].make_from_blueprint(
@@ -96,7 +95,7 @@ def _build_parallel(*, output_ref: str, structure_class_names: list[str], add_ea
description="Consumes the always-present topic",
inputs={"topic": "Text"},
output="Text",
- function_name="optionals_par_echo_topic",
+ function_name=f"{__name__}.optionals_par_echo_topic",
),
)
pipe_library = get_pipe_library()
@@ -148,17 +147,9 @@ def _make_absent_source_memory() -> WorkingMemory:
class TestPipeParallelAbsence:
@classmethod
def setup_class(cls):
- for func in _TEST_FUNCS:
- func_registry.register_function(func)
get_class_registry().register_class(ParAbsReport)
get_class_registry().register_class(ParAbsStrictReport)
- @classmethod
- def teardown_class(cls):
- for func in _TEST_FUNCS:
- if func_registry.has_function(func.__name__):
- func_registry.unregister_function_by_name(func.__name__)
-
async def test_composite_output_omits_absent_component_with_ledger_note(self, job_metadata: JobMetadata, load_empty_library: Callable[[], str]):
"""Composite output: the absent component is omitted; a ledger note keeps observability."""
load_empty_library()
diff --git a/tests/integration/pipelex/pipes/operator/pipe_func/test_pipe_func.py b/tests/integration/pipelex/pipes/operator/pipe_func/test_pipe_func.py
index 14334c246..283f0ef64 100644
--- a/tests/integration/pipelex/pipes/operator/pipe_func/test_pipe_func.py
+++ b/tests/integration/pipelex/pipes/operator/pipe_func/test_pipe_func.py
@@ -16,24 +16,11 @@ from pipelex.pipe_run.pipe_job_factory import PipeJobFactory
from pipelex.pipe_run.pipe_run_params_factory import PipeRunParamsFactory
from pipelex.system.job_metadata import JobMetadata
from pipelex.system.pipe_run_mode import PipeRunMode
-from pipelex.system.registries.func_registry import func_registry
-from tests.cases.source_code import wrap_lines
@pytest.mark.dry_runnable
@pytest.mark.asyncio(loop_scope="class")
class TestPipeFunc:
- @classmethod
- def setup_class(cls):
- """Register test functions before running tests."""
- func_registry.register_function(wrap_lines)
-
- @classmethod
- def teardown_class(cls):
- """Clean up registered functions after tests."""
- if func_registry.has_function("wrap_lines"):
- func_registry.unregister_function_by_name("wrap_lines")
-
async def test_wrap_lines_pipe_func(
self,
job_metadata: JobMetadata,
@@ -62,7 +49,7 @@ if __name__ == "__main__":
# Create the PipeFunc job
pipe_func_blueprint = PipeFuncBlueprint(
description="Function pipe for wrapping lines",
- function_name="wrap_lines",
+ function_name="tests.cases.source_code.wrap_lines",
output=NativeConceptCode.TEXT,
)
diff --git a/tests/integration/pipelex/pipes/operator/pipe_func/test_pipe_func_validation_errors.py b/tests/integration/pipelex/pipes/operator/pipe_func/test_pipe_func_validation_errors.py
index 4e4cf9af9..eec2fe04a 100644
--- a/tests/integration/pipelex/pipes/operator/pipe_func/test_pipe_func_validation_errors.py
+++ b/tests/integration/pipelex/pipes/operator/pipe_func/test_pipe_func_validation_errors.py
@@ -1,8 +1,8 @@
"""Tests for PipeFunc validation error reporting.
This module tests that when a @pipe_func decorated function has issues
-(like missing return type), the validation provides clear error messages
-instead of just "function not found".
+(like a missing return type), validating its method provides a clear error message
+naming the function and the issue, the module being imported by the step's import path.
"""
import tempfile
@@ -11,10 +11,10 @@ from typing import ClassVar
import pytest
+from pipelex.interpreter_hub import clear_current_library, get_current_library_id_or_none, get_library_manager
from pipelex.libraries.exceptions import LibraryError
from pipelex.pipeline.exceptions import ValidateBundleError
-from pipelex.pipeline.validate_bundle import validate_bundle
-from pipelex.system.registries.func_registry import func_registry
+from pipelex.pipeline.validate_bundle import ValidateBundleResult, validate_bundle
class TestData:
@@ -30,7 +30,7 @@ description = "Test bundle for pipe_func validation error reporting"
[pipe.test_pipe_func]
type = "PipeFunc"
description = "Test pipe that uses a function"
-function_name = "{function_name}"
+function_name = "my_funcs.{function_name}"
output = "Text"
"""
@@ -41,7 +41,7 @@ description = "Test bundle for pipe_func validation error reporting"
[pipe.test_pipe_func]
type = "PipeFunc"
description = "Test pipe that uses a function without return type"
-function_name = "my_func_no_return_type"
+function_name = "my_funcs.my_func_no_return_type"
output = "Text"
"""
@@ -165,23 +165,25 @@ from pipelex.system.registries.func_registry import pipe_func
async def func_missing_param_type(working_memory) -> TextContent:
return TextContent(text="test")
""",
- "type annotation",
+ "must annotate its parameter 'working_memory'",
),
]
+async def _validate_and_release(*, mthds_file_path: Path, library_dirs: list[Path]) -> ValidateBundleResult:
+ """Validate, then tear down the library a successful validation leaves open and current, which holds its tree's Python."""
+ result = await validate_bundle(mthds_file_path=mthds_file_path, library_dirs=library_dirs)
+ library_id = get_current_library_id_or_none()
+ if library_id is not None:
+ get_library_manager().teardown(library_id=library_id)
+ clear_current_library()
+ return result
+
+
@pytest.mark.asyncio(loop_scope="class")
class TestPipeFuncValidationErrors:
"""Tests for PipeFunc validation error reporting when @pipe_func decorated functions have issues."""
- def setup_method(self):
- """Clear the func_registry before each test."""
- func_registry.teardown()
-
- def teardown_method(self):
- """Clean up the func_registry after each test."""
- func_registry.teardown()
-
async def test_pipe_func_missing_return_type_reports_clear_error(self):
"""Test that a @pipe_func decorated function without return type gives a clear error.
@@ -252,7 +254,7 @@ class TestPipeFuncValidationErrors:
py_file.write_text(TestData.FUNC_WITH_DECORATOR_VALID)
# Validate the bundle - should succeed
- result = await validate_bundle(
+ result = await _validate_and_release(
mthds_file_path=mthds_file,
library_dirs=[temp_path],
)
@@ -350,9 +352,6 @@ class TestPipeFuncValidationErrors:
# The error should mention the function name
assert function_name in error_message, f"Error message should mention the function name '{function_name}'. Got: {error_message}"
- # The error should mention that the function is not eligible (has decorator but failed)
- assert "not eligible" in error_message.lower(), f"Error should mention that the function is 'not eligible'. Got: {error_message}"
-
# The error should contain the expected error substring explaining the specific issue
assert expected_error_substring.lower() in error_message.lower(), (
f"Error should mention '{expected_error_substring}' to explain the specific issue. Got: {error_message}"
@@ -389,7 +388,7 @@ description = "Test bundle for pipe_func return type validation"
[pipe.test_pipe_func]
type = "PipeFunc"
description = "Test pipe expecting Text output but function returns StructuredContent"
-function_name = "func_wrong_structure_class"
+function_name = "my_funcs.func_wrong_structure_class"
output = "Text"
"""
with tempfile.TemporaryDirectory() as temp_dir:
@@ -447,7 +446,7 @@ description = "Test bundle for ListContent validation"
[pipe.test_list_content_pipe]
type = "PipeFunc"
description = "Test pipe with array output"
-function_name = "func_returns_list_content"
+function_name = "my_funcs.func_returns_list_content"
output = "Text[]"
"""
with tempfile.TemporaryDirectory() as temp_dir:
@@ -462,7 +461,7 @@ output = "Text[]"
py_file.write_text(func_code)
# Validate the bundle - should succeed
- result = await validate_bundle(
+ result = await _validate_and_release(
mthds_file_path=mthds_file,
library_dirs=[temp_path],
)
@@ -499,7 +498,7 @@ description = "Test bundle for ListContent validation error"
[pipe.test_wrong_list_content_pipe]
type = "PipeFunc"
description = "Test pipe with mismatched list item type"
-function_name = "func_returns_wrong_list_content"
+function_name = "my_funcs.func_returns_wrong_list_content"
output = "Text[]"
"""
with tempfile.TemporaryDirectory() as temp_dir:
@@ -557,7 +556,7 @@ description = "Test bundle for ListContent requirement"
[pipe.test_array_requires_list_content]
type = "PipeFunc"
description = "Test pipe with array output expecting ListContent"
-function_name = "func_returns_single_instead_of_list"
+function_name = "my_funcs.func_returns_single_instead_of_list"
output = "Text[]"
"""
with tempfile.TemporaryDirectory() as temp_dir:
diff --git a/tests/integration/pipelex/pipes/optionals/test_absent_main_output_surfaces.py b/tests/integration/pipelex/pipes/optionals/test_absent_main_output_surfaces.py
index eeec35af5..9a26c730b 100644
--- a/tests/integration/pipelex/pipes/optionals/test_absent_main_output_surfaces.py
+++ b/tests/integration/pipelex/pipes/optionals/test_absent_main_output_surfaces.py
@@ -15,7 +15,7 @@ from pipelex.core.stuffs.text_content import TextContent
from pipelex.graph.graphspec import NodeStatus
from pipelex.pipeline.pipeline_response import RunState
from pipelex.pipeline.runner import PipelexMTHDSProtocol
-from pipelex.system.registries.func_registry import func_registry
+from pipelex.system.registries.func_registry import pipe_func
_BUNDLE = """
domain = "opt_surfaces"
@@ -27,14 +27,14 @@ type = "PipeFunc"
description = "Analyzes the source (plain input: lifts when source is absent)"
inputs = { source = "Text" }
output = "Text"
-function_name = "opt_surfaces_make_analysis"
+function_name = "tests.integration.pipelex.pipes.optionals.test_absent_main_output_surfaces.opt_surfaces_make_analysis"
[pipe.opt_surfaces_summarize]
type = "PipeFunc"
description = "Summarizes the analysis (plain input: lifts in chain)"
inputs = { analysis = "Text" }
output = "Text"
-function_name = "opt_surfaces_summarize"
+function_name = "tests.integration.pipelex.pipes.optionals.test_absent_main_output_surfaces.opt_surfaces_summarize"
[pipe.opt_flow]
type = "PipeSequence"
@@ -48,30 +48,18 @@ steps = [
"""
+@pipe_func()
def opt_surfaces_make_analysis(working_memory: WorkingMemory) -> TextContent:
return TextContent(text=f"analysis of {working_memory.get_stuff_as_str(name='source')}")
+@pipe_func()
def opt_surfaces_summarize(working_memory: WorkingMemory) -> TextContent:
return TextContent(text=f"summary: {working_memory.get_stuff_as_str(name='analysis')}")
-_TEST_FUNCS = [opt_surfaces_make_analysis, opt_surfaces_summarize]
-
-
@pytest.mark.asyncio(loop_scope="class")
class TestAbsentMainOutputSurfaces:
- @classmethod
- def setup_class(cls):
- for func in _TEST_FUNCS:
- func_registry.register_function(func)
-
- @classmethod
- def teardown_class(cls):
- for func in _TEST_FUNCS:
- if func_registry.has_function(func.__name__):
- func_registry.unregister_function_by_name(func.__name__)
-
async def test_execute_with_absent_main_output_succeeds_across_surfaces(self):
"""Omitting the optional input lifts the whole chain: the execute response is a success
naming the declared slot, the ledger chains provenance, and the graph marks skips.
diff --git a/tests/integration/pipelex/pipes/optionals/test_condition_optional_output_validation.py b/tests/integration/pipelex/pipes/optionals/test_condition_optional_output_validation.py
index 30a89699a..472dd7503 100644
--- a/tests/integration/pipelex/pipes/optionals/test_condition_optional_output_validation.py
+++ b/tests/integration/pipelex/pipes/optionals/test_condition_optional_output_validation.py
@@ -22,19 +22,17 @@ from pipelex.pipe_controllers.condition.pipe_condition_blueprint import PipeCond
from pipelex.pipe_machinery.pipe_factory import PipeFactory
from pipelex.pipe_operators.func.pipe_func import PipeFunc
from pipelex.pipe_operators.func.pipe_func_blueprint import PipeFuncBlueprint
-from pipelex.system.registries.func_registry import func_registry
+from pipelex.system.registries.func_registry import pipe_func
from pipelex.validation_error_types import PipeValidationErrorType
_DOMAIN_CODE = "test_optionals_cond"
+@pipe_func()
def optionals_cond_echo_topic(working_memory: WorkingMemory) -> TextContent:
return TextContent(text=f"echo:{working_memory.get_stuff_as_str(name='topic')}")
-_TEST_FUNCS = [optionals_cond_echo_topic]
-
-
def _make_condition(*, output_ref: str, outcomes: dict[str, str], default_outcome: str) -> PipeCondition:
return PipeFactory[PipeCondition].make_from_blueprint(
domain_code=_DOMAIN_CODE,
@@ -58,24 +56,13 @@ def _register_target_pipe(*, pipe_code: str, output_ref: str) -> None:
description="Mapped target pipe",
inputs={"topic": "Text"},
output=output_ref,
- function_name="optionals_cond_echo_topic",
+ function_name=f"{__name__}.optionals_cond_echo_topic",
),
)
get_pipe_library().add_new_pipe(pipe=target)
class TestConditionOptionalOutputValidation:
- @classmethod
- def setup_class(cls):
- for func in _TEST_FUNCS:
- func_registry.register_function(func)
-
- @classmethod
- def teardown_class(cls):
- for func in _TEST_FUNCS:
- if func_registry.has_function(func.__name__):
- func_registry.unregister_function_by_name(func.__name__)
-
# ---- OPTIONAL_OUTPUT_REQUIRED (static, at pipe construction) ----
def test_continue_outcome_with_plain_output_is_rejected(self, load_empty_library: Callable[[], str]):
diff --git a/tests/integration/pipelex/pipes/optionals/test_lift_sequence.py b/tests/integration/pipelex/pipes/optionals/test_lift_sequence.py
index d7bb44e5c..179faf49a 100644
--- a/tests/integration/pipelex/pipes/optionals/test_lift_sequence.py
+++ b/tests/integration/pipelex/pipes/optionals/test_lift_sequence.py
@@ -26,20 +26,23 @@ from pipelex.pipe_run.pipe_run_params import PipeRunParams
from pipelex.pipeline.execution_seams import prepare_pipe_job
from pipelex.system.job_metadata import JobMetadata
from pipelex.system.pipe_run_mode import PipeRunMode
-from pipelex.system.registries.func_registry import func_registry
+from pipelex.system.registries.func_registry import pipe_func
from tests.integration.pipelex.fixtures.pipe_job_helpers import make_mode_guarded_run_params
_DOMAIN_CODE = "test_optionals_seq"
+@pipe_func()
def optionals_seq_echo_source(working_memory: WorkingMemory) -> TextContent:
return TextContent(text=f"a:{working_memory.get_stuff_as_str(name='source')}")
+@pipe_func()
def optionals_seq_echo_a_out(working_memory: WorkingMemory) -> TextContent:
return TextContent(text=f"b:{working_memory.get_stuff_as_str(name='a_out')}")
+@pipe_func()
def optionals_seq_sink(working_memory: WorkingMemory) -> TextContent:
topic = working_memory.get_stuff_as_str(name="topic")
b_out_stuff = working_memory.get_optional_stuff(name="b_out")
@@ -48,9 +51,6 @@ def optionals_seq_sink(working_memory: WorkingMemory) -> TextContent:
return TextContent(text=f"report about {topic}: {b_out_stuff.as_text.text}")
-_TEST_FUNCS = [optionals_seq_echo_source, optionals_seq_echo_a_out, optionals_seq_sink]
-
-
def _make_live_run_params() -> PipeRunParams:
return make_mode_guarded_run_params(pipe_run_mode=PipeRunMode.LIVE)
@@ -68,7 +68,7 @@ def _build_lift_sequence() -> PipeSequence:
description="Consumes the maybe-absent source (plain input): lifted when source is absent",
inputs={"source": "Text"},
output="Text",
- function_name="optionals_seq_echo_source",
+ function_name=f"{__name__}.optionals_seq_echo_source",
),
)
step_b = PipeFactory[PipeFunc].make_from_blueprint(
@@ -78,7 +78,7 @@ def _build_lift_sequence() -> PipeSequence:
description="Consumes step A's output (plain input): lifted in chain",
inputs={"a_out": "Text"},
output="Text",
- function_name="optionals_seq_echo_a_out",
+ function_name=f"{__name__}.optionals_seq_echo_a_out",
),
)
sink_c = PipeFactory[PipeFunc].make_from_blueprint(
@@ -88,7 +88,7 @@ def _build_lift_sequence() -> PipeSequence:
description="Absorbing sink: declares b_out optional and handles both arms",
inputs={"topic": "Text", "b_out": "Text?"},
output="Text",
- function_name="optionals_seq_sink",
+ function_name=f"{__name__}.optionals_seq_sink",
),
)
pipe_library = get_pipe_library()
@@ -117,17 +117,6 @@ def _build_lift_sequence() -> PipeSequence:
@pytest.mark.asyncio(loop_scope="class")
class TestLiftSequence:
- @classmethod
- def setup_class(cls):
- for func in _TEST_FUNCS:
- func_registry.register_function(func)
-
- @classmethod
- def teardown_class(cls):
- for func in _TEST_FUNCS:
- if func_registry.has_function(func.__name__):
- func_registry.unregister_function_by_name(func.__name__)
-
async def test_sequence_lifts_steps_and_sinks_absence(self, job_metadata: JobMetadata, load_empty_library: Callable[[], str]):
"""With source absent (recorded), steps A and B skip with chained provenance and the
absorbing sink C still delivers the sequence's main output.
@@ -184,7 +173,7 @@ class TestLiftSequence:
description="Consumes the maybe-absent source (plain input): lifted when source is absent",
inputs={"source": "Text"},
output="Text",
- function_name="optionals_seq_echo_source",
+ function_name=f"{__name__}.optionals_seq_echo_source",
),
)
working_memory = WorkingMemoryFactory.make_from_single_stuff(
diff --git a/tests/integration/pipelex/pipes/optionals/test_lift_skip_trichotomy.py b/tests/integration/pipelex/pipes/optionals/test_lift_skip_trichotomy.py
index 825151b35..1a72fe77c 100644
--- a/tests/integration/pipelex/pipes/optionals/test_lift_skip_trichotomy.py
+++ b/tests/integration/pipelex/pipes/optionals/test_lift_skip_trichotomy.py
@@ -26,16 +26,18 @@ from pipelex.pipe_operators.func.pipe_func_blueprint import PipeFuncBlueprint
from pipelex.pipe_run.pipe_run_params import PipeRunParams
from pipelex.system.job_metadata import JobMetadata, RunMetadata
from pipelex.system.pipe_run_mode import PipeRunMode
-from pipelex.system.registries.func_registry import func_registry
+from pipelex.system.registries.func_registry import pipe_func
from tests.integration.pipelex.fixtures.pipe_job_helpers import make_mode_guarded_run_params
from tests.unit.pipelex.graph.conftest import make_trace_context
+@pipe_func()
def optionals_echo_source(working_memory: WorkingMemory) -> TextContent:
source_text = working_memory.get_stuff_as_str(name="source")
return TextContent(text=f"echo:{source_text}")
+@pipe_func()
def optionals_absorbing_sink(working_memory: WorkingMemory) -> TextContent:
source_stuff = working_memory.get_optional_stuff(name="source")
if source_stuff is None:
@@ -43,15 +45,13 @@ def optionals_absorbing_sink(working_memory: WorkingMemory) -> TextContent:
return TextContent(text=f"report:{source_stuff.as_text.text}")
+@pipe_func()
def optionals_list_from_source(working_memory: WorkingMemory) -> ListContent[TextContent]:
source_text = working_memory.get_stuff_as_str(name="source")
return ListContent[TextContent](items=[TextContent(text=source_text)])
-_TEST_FUNCS = [optionals_echo_source, optionals_absorbing_sink, optionals_list_from_source]
-
-
-def _make_pipe_func(pipe_code: str, *, source_ref: str, function_name: str = "optionals_echo_source", output: str = "Text") -> PipeFunc:
+def _make_pipe_func(pipe_code: str, *, source_ref: str, function_name: str = f"{__name__}.optionals_echo_source", output: str = "Text") -> PipeFunc:
blueprint = PipeFuncBlueprint(
description=f"Optionals trichotomy test pipe {pipe_code}",
inputs={"source": source_ref},
@@ -88,17 +88,6 @@ def _make_live_run_params() -> PipeRunParams:
@pytest.mark.asyncio(loop_scope="class")
class TestLiftSkipTrichotomy:
- @classmethod
- def setup_class(cls):
- for func in _TEST_FUNCS:
- func_registry.register_function(func)
-
- @classmethod
- def teardown_class(cls):
- for func in _TEST_FUNCS:
- if func_registry.has_function(func.__name__):
- func_registry.unregister_function_by_name(func.__name__)
-
async def test_plain_input_absent_with_record_lifts_pipe(self, job_metadata: JobMetadata, load_empty_library: Callable[[], str]):
"""A plain input fed a recorded absence skips the pipe and records a chained skipped-absence."""
load_empty_library()
@@ -137,7 +126,7 @@ class TestLiftSkipTrichotomy:
description="Second chained consumer",
inputs={"echoed": "Text"},
output="Text",
- function_name="optionals_absorbing_sink",
+ function_name=f"{__name__}.optionals_absorbing_sink",
)
pipe_second = PipeFactory[PipeFunc].make_from_blueprint(
domain_code="test_optionals",
@@ -171,7 +160,7 @@ class TestLiftSkipTrichotomy:
async def test_optional_input_absent_runs_pipe(self, job_metadata: JobMetadata, load_empty_library: Callable[[], str]):
"""A `?` input absorbs absence: the pipe runs and the taint terminates."""
load_empty_library()
- pipe = _make_pipe_func("opt_absorber", source_ref="Text?", function_name="optionals_absorbing_sink")
+ pipe = _make_pipe_func("opt_absorber", source_ref="Text?", function_name=f"{__name__}.optionals_absorbing_sink")
working_memory = WorkingMemoryFactory.make_empty()
working_memory.record_absence(_make_seeded_absence())
@@ -250,7 +239,7 @@ class TestLiftSkipTrichotomy:
pipe = _make_pipe_func(
"opt_plural_consumer",
source_ref="Text",
- function_name="optionals_list_from_source",
+ function_name=f"{__name__}.optionals_list_from_source",
output="Text[]",
)
working_memory = WorkingMemoryFactory.make_empty()
diff --git a/tests/integration/pipelex/pipes/optionals/test_liftable_pipes_inventory.py b/tests/integration/pipelex/pipes/optionals/test_liftable_pipes_inventory.py
index 46212605e..9732b91d1 100644
--- a/tests/integration/pipelex/pipes/optionals/test_liftable_pipes_inventory.py
+++ b/tests/integration/pipelex/pipes/optionals/test_liftable_pipes_inventory.py
@@ -19,31 +19,32 @@ from pipelex.pipe_operators.func.pipe_func_blueprint import PipeFuncBlueprint
from pipelex.pipeline.controller_taint import collect_controller_taint_analyses
from pipelex.pipeline.liftable_pipes import build_liftable_pipes
from pipelex.pipeline.validation_report import PipelexValidationReport
-from pipelex.system.registries.func_registry import func_registry
+from pipelex.system.registries.func_registry import pipe_func
_DOMAIN_CODE = "test_optionals_inventory"
+@pipe_func()
def optionals_inv_echo_source(working_memory: WorkingMemory) -> TextContent:
return TextContent(text=f"a:{working_memory.get_stuff_as_str(name='source')}")
+@pipe_func()
def optionals_inv_echo_a_out(working_memory: WorkingMemory) -> TextContent:
return TextContent(text=f"b:{working_memory.get_stuff_as_str(name='a_out')}")
+@pipe_func()
def optionals_inv_sink(working_memory: WorkingMemory) -> TextContent:
b_out_stuff = working_memory.get_optional_stuff(name="b_out")
return TextContent(text="ok" if b_out_stuff is None else b_out_stuff.as_text.text)
+@pipe_func()
def optionals_inv_echo_topic(working_memory: WorkingMemory) -> TextContent:
return TextContent(text=f"base:{working_memory.get_stuff_as_str(name='topic')}")
-_TEST_FUNCS = [optionals_inv_echo_source, optionals_inv_echo_a_out, optionals_inv_sink, optionals_inv_echo_topic]
-
-
def _build_lift_chain_pipes() -> list[str]:
"""Register the lift-chain sequence (A and B liftable, C absorbs) and return the pipe codes."""
step_a = PipeFactory[PipeFunc].make_from_blueprint(
@@ -53,7 +54,7 @@ def _build_lift_chain_pipes() -> list[str]:
description="Liftable when source is absent",
inputs={"source": "Text"},
output="Text",
- function_name="optionals_inv_echo_source",
+ function_name=f"{__name__}.optionals_inv_echo_source",
),
)
step_b = PipeFactory[PipeFunc].make_from_blueprint(
@@ -63,7 +64,7 @@ def _build_lift_chain_pipes() -> list[str]:
description="Liftable in chain",
inputs={"a_out": "Text"},
output="Text",
- function_name="optionals_inv_echo_a_out",
+ function_name=f"{__name__}.optionals_inv_echo_a_out",
),
)
sink_c = PipeFactory[PipeFunc].make_from_blueprint(
@@ -73,7 +74,7 @@ def _build_lift_chain_pipes() -> list[str]:
description="Absorbing sink",
inputs={"b_out": "Text?"},
output="Text",
- function_name="optionals_inv_sink",
+ function_name=f"{__name__}.optionals_inv_sink",
),
)
pipe_library = get_pipe_library()
@@ -98,17 +99,6 @@ def _build_lift_chain_pipes() -> list[str]:
class TestLiftablePipesInventory:
- @classmethod
- def setup_class(cls):
- for func in _TEST_FUNCS:
- func_registry.register_function(func)
-
- @classmethod
- def teardown_class(cls):
- for func in _TEST_FUNCS:
- if func_registry.has_function(func.__name__):
- func_registry.unregister_function_by_name(func.__name__)
-
def test_sequence_lift_chain_is_inventoried(self, load_empty_library: Callable[[], str]):
load_empty_library()
pipe_codes = _build_lift_chain_pipes()
@@ -136,7 +126,7 @@ class TestLiftablePipesInventory:
description="Liftable branch",
inputs={"source": "Text"},
output="Text",
- function_name="optionals_inv_echo_source",
+ function_name=f"{__name__}.optionals_inv_echo_source",
),
)
branch_base = PipeFactory[PipeFunc].make_from_blueprint(
@@ -146,7 +136,7 @@ class TestLiftablePipesInventory:
description="Guaranteed branch",
inputs={"topic": "Text"},
output="Text",
- function_name="optionals_inv_echo_topic",
+ function_name=f"{__name__}.optionals_inv_echo_topic",
),
)
pipe_library = get_pipe_library()
diff --git a/tests/integration/pipelex/pipes/optionals/test_lifted_parallel_companion_slots.py b/tests/integration/pipelex/pipes/optionals/test_lifted_parallel_companion_slots.py
index f6b831855..567d3f6cd 100644
--- a/tests/integration/pipelex/pipes/optionals/test_lifted_parallel_companion_slots.py
+++ b/tests/integration/pipelex/pipes/optionals/test_lifted_parallel_companion_slots.py
@@ -29,20 +29,23 @@ from pipelex.pipe_operators.func.pipe_func_blueprint import PipeFuncBlueprint
from pipelex.pipe_run.pipe_run_params import PipeRunParams
from pipelex.system.job_metadata import JobMetadata
from pipelex.system.pipe_run_mode import PipeRunMode
-from pipelex.system.registries.func_registry import func_registry
+from pipelex.system.registries.func_registry import pipe_func
from tests.integration.pipelex.fixtures.pipe_job_helpers import make_mode_guarded_run_params
_DOMAIN_CODE = "test_optionals_companion"
+@pipe_func()
def optionals_comp_find(working_memory: WorkingMemory) -> TextContent:
return TextContent(text=f"found:{working_memory.get_stuff_as_str(name='source')}")
+@pipe_func()
def optionals_comp_find_many(working_memory: WorkingMemory) -> ListContent[TextContent]:
return ListContent[TextContent](items=[TextContent(text=f"item:{working_memory.get_stuff_as_str(name='source')}")])
+@pipe_func()
def optionals_comp_sink(working_memory: WorkingMemory) -> TextContent:
a_out_stuff = working_memory.get_optional_stuff(name="a_out")
if a_out_stuff is None:
@@ -50,9 +53,6 @@ def optionals_comp_sink(working_memory: WorkingMemory) -> TextContent:
return TextContent(text=f"findings: {a_out_stuff.as_text.text}")
-_TEST_FUNCS = [optionals_comp_find, optionals_comp_find_many, optionals_comp_sink]
-
-
def _make_live_run_params() -> PipeRunParams:
return make_mode_guarded_run_params(pipe_run_mode=PipeRunMode.LIVE)
@@ -69,7 +69,7 @@ def _build_sequence_with_liftable_parallel() -> PipeSequence:
description="Singular branch over source",
inputs={"source": "Text"},
output="Text",
- function_name="optionals_comp_find",
+ function_name=f"{__name__}.optionals_comp_find",
),
)
branch_plural = PipeFactory[PipeFunc].make_from_blueprint(
@@ -79,7 +79,7 @@ def _build_sequence_with_liftable_parallel() -> PipeSequence:
description="Plural branch over source",
inputs={"source": "Text"},
output="Text[]",
- function_name="optionals_comp_find_many",
+ function_name=f"{__name__}.optionals_comp_find_many",
),
)
sink = PipeFactory[PipeFunc].make_from_blueprint(
@@ -89,7 +89,7 @@ def _build_sequence_with_liftable_parallel() -> PipeSequence:
description="Absorbs the singular branch slot",
inputs={"a_out": "Text?"},
output="Text",
- function_name="optionals_comp_sink",
+ function_name=f"{__name__}.optionals_comp_sink",
),
)
parallel = PipeFactory[PipeParallel].make_from_blueprint(
@@ -127,17 +127,6 @@ def _build_sequence_with_liftable_parallel() -> PipeSequence:
class TestLiftedParallelCompanionSlots:
- @classmethod
- def setup_class(cls):
- for func in _TEST_FUNCS:
- func_registry.register_function(func)
-
- @classmethod
- def teardown_class(cls):
- for func in _TEST_FUNCS:
- if func_registry.has_function(func.__name__):
- func_registry.unregister_function_by_name(func.__name__)
-
@pytest.mark.asyncio(loop_scope="class")
async def test_lifted_parallel_resolves_branch_slots(self, job_metadata: JobMetadata, load_empty_library: Callable[[], str]):
"""Static validation blesses the flow AND the runtime resolves every branch slot on lift:
diff --git a/tests/integration/pipelex/pipes/optionals/test_parallel_optional_combine_validation.py b/tests/integration/pipelex/pipes/optionals/test_parallel_optional_combine_validation.py
index 2d0bb864b..e0cf994e5 100644
--- a/tests/integration/pipelex/pipes/optionals/test_parallel_optional_combine_validation.py
+++ b/tests/integration/pipelex/pipes/optionals/test_parallel_optional_combine_validation.py
@@ -22,7 +22,7 @@ from pipelex.pipe_machinery.pipe_factory import PipeFactory
from pipelex.pipe_operators.func.pipe_func import PipeFunc
from pipelex.pipe_operators.func.pipe_func_blueprint import PipeFuncBlueprint
from pipelex.runtime_hub import get_class_registry
-from pipelex.system.registries.func_registry import func_registry
+from pipelex.system.registries.func_registry import pipe_func
from pipelex.validation_error_types import PipeValidationErrorType
_DOMAIN_CODE = "test_optionals_par_static"
@@ -42,17 +42,16 @@ class ParTaintStrictReport(StructuredContent):
found_result: TextContent = Field(description="Result of the maybe-absent branch")
+@pipe_func()
def optionals_par_static_find(working_memory: WorkingMemory) -> TextContent:
return TextContent(text=f"found:{working_memory.get_stuff_as_str(name='source')}")
+@pipe_func()
def optionals_par_static_base(working_memory: WorkingMemory) -> TextContent:
return TextContent(text=f"base:{working_memory.get_stuff_as_str(name='topic')}")
-_TEST_FUNCS = [optionals_par_static_find, optionals_par_static_base]
-
-
def _register_branch_pipes(*, find_output_ref: str = "Text") -> None:
branch_found = PipeFactory[PipeFunc].make_from_blueprint(
domain_code=_DOMAIN_CODE,
@@ -61,7 +60,7 @@ def _register_branch_pipes(*, find_output_ref: str = "Text") -> None:
description="Consumes source plain: liftable when the parallel's source is optional",
inputs={"source": "Text"},
output=find_output_ref,
- function_name="optionals_par_static_find",
+ function_name=f"{__name__}.optionals_par_static_find",
),
)
branch_base = PipeFactory[PipeFunc].make_from_blueprint(
@@ -71,7 +70,7 @@ def _register_branch_pipes(*, find_output_ref: str = "Text") -> None:
description="Consumes the always-present topic",
inputs={"topic": "Text"},
output="Text",
- function_name="optionals_par_static_base",
+ function_name=f"{__name__}.optionals_par_static_base",
),
)
pipe_library = get_pipe_library()
@@ -111,17 +110,9 @@ def _build_parallel(*, output_ref: str, structure_class_names: list[str], inputs
class TestParallelOptionalCombineValidation:
@classmethod
def setup_class(cls):
- for func in _TEST_FUNCS:
- func_registry.register_function(func)
get_class_registry().register_class(ParTaintReport)
get_class_registry().register_class(ParTaintStrictReport)
- @classmethod
- def teardown_class(cls):
- for func in _TEST_FUNCS:
- if func_registry.has_function(func.__name__):
- func_registry.unregister_function_by_name(func.__name__)
-
def test_required_field_fed_by_liftable_branch_is_rejected(self, load_empty_library: Callable[[], str]):
"""The parallel's `?` input makes the find branch liftable, and 'found_result' is required."""
load_empty_library()
diff --git a/tests/integration/pipelex/pipes/optionals/test_redundant_force_warning.py b/tests/integration/pipelex/pipes/optionals/test_redundant_force_warning.py
index 2ee8824b0..51150b6fd 100644
--- a/tests/integration/pipelex/pipes/optionals/test_redundant_force_warning.py
+++ b/tests/integration/pipelex/pipes/optionals/test_redundant_force_warning.py
@@ -18,23 +18,22 @@ from pipelex.pipe_operators.func.pipe_func import PipeFunc
from pipelex.pipe_operators.func.pipe_func_blueprint import PipeFuncBlueprint
from pipelex.pipeline.controller_taint import collect_controller_taint_analyses
from pipelex.pipeline.optionality_warnings import build_optionality_warnings
-from pipelex.system.registries.func_registry import func_registry
+from pipelex.system.registries.func_registry import pipe_func
from pipelex.validation_error_types import PipeValidationErrorType
_DOMAIN_CODE = "test_redundant_force"
+@pipe_func()
def redundant_force_make_a(working_memory: WorkingMemory) -> TextContent:
return TextContent(text=f"a:{working_memory.get_stuff_as_str(name='topic')}")
+@pipe_func()
def redundant_force_consume_a(working_memory: WorkingMemory) -> TextContent:
return TextContent(text=f"b:{working_memory.get_stuff_as_str(name='a_out')}")
-_TEST_FUNCS = [redundant_force_make_a, redundant_force_consume_a]
-
-
def _build_force_pipes() -> tuple[PipeSequence, PipeSequence]:
"""Register the step pipes and build the two flows over the same `!`-consuming pipe.
@@ -50,7 +49,7 @@ def _build_force_pipes() -> tuple[PipeSequence, PipeSequence]:
description="Always produces a_out",
inputs={"topic": "Text"},
output="Text",
- function_name="redundant_force_make_a",
+ function_name=f"{__name__}.redundant_force_make_a",
),
)
step_force = PipeFactory[PipeFunc].make_from_blueprint(
@@ -60,7 +59,7 @@ def _build_force_pipes() -> tuple[PipeSequence, PipeSequence]:
description="Consumes a_out with a force marker",
inputs={"a_out": "Text!"},
output="Text",
- function_name="redundant_force_consume_a",
+ function_name=f"{__name__}.redundant_force_consume_a",
),
)
pipe_library = get_pipe_library()
@@ -98,17 +97,6 @@ def _build_force_pipes() -> tuple[PipeSequence, PipeSequence]:
class TestRedundantForceWarning:
- @classmethod
- def setup_class(cls):
- for func in _TEST_FUNCS:
- func_registry.register_function(func)
-
- @classmethod
- def teardown_class(cls):
- for func in _TEST_FUNCS:
- if func_registry.has_function(func.__name__):
- func_registry.unregister_function_by_name(func.__name__)
-
def test_force_on_guaranteed_slot_warns(self, load_empty_library: Callable[[], str]):
"""A `!` consumption whose slot is guaranteed in its only analyzed flow yields one warning."""
load_empty_library()
diff --git a/tests/integration/pipelex/pipes/optionals/test_taint_sequence_validation.py b/tests/integration/pipelex/pipes/optionals/test_taint_sequence_validation.py
index aba300a69..74da96719 100644
--- a/tests/integration/pipelex/pipes/optionals/test_taint_sequence_validation.py
+++ b/tests/integration/pipelex/pipes/optionals/test_taint_sequence_validation.py
@@ -22,20 +22,23 @@ from pipelex.pipe_controllers.sub_pipe_blueprint import SubPipeBlueprint
from pipelex.pipe_machinery.pipe_factory import PipeFactory
from pipelex.pipe_operators.func.pipe_func import PipeFunc
from pipelex.pipe_operators.func.pipe_func_blueprint import PipeFuncBlueprint
-from pipelex.system.registries.func_registry import func_registry
+from pipelex.system.registries.func_registry import pipe_func
from pipelex.validation_error_types import PipeValidationErrorType
_DOMAIN_CODE = "test_optionals_taint_seq"
+@pipe_func()
def optionals_taint_echo_source(working_memory: WorkingMemory) -> TextContent:
return TextContent(text=f"a:{working_memory.get_stuff_as_str(name='source')}")
+@pipe_func()
def optionals_taint_echo_a_out(working_memory: WorkingMemory) -> TextContent:
return TextContent(text=f"b:{working_memory.get_stuff_as_str(name='a_out')}")
+@pipe_func()
def optionals_taint_sink(working_memory: WorkingMemory) -> TextContent:
b_out_stuff = working_memory.get_optional_stuff(name="b_out")
if b_out_stuff is None:
@@ -43,9 +46,6 @@ def optionals_taint_sink(working_memory: WorkingMemory) -> TextContent:
return TextContent(text=f"analysis: {b_out_stuff.as_text.text}")
-_TEST_FUNCS = [optionals_taint_echo_source, optionals_taint_echo_a_out, optionals_taint_sink]
-
-
def _register_step_pipes(*, source_input_ref: str = "Text") -> None:
"""Register step A (consumes `source`), step B (consumes `a_out`), and the absorbing sink."""
step_a = PipeFactory[PipeFunc].make_from_blueprint(
@@ -55,7 +55,7 @@ def _register_step_pipes(*, source_input_ref: str = "Text") -> None:
description="Consumes source",
inputs={"source": source_input_ref},
output="Text",
- function_name="optionals_taint_echo_source",
+ function_name=f"{__name__}.optionals_taint_echo_source",
),
)
step_b = PipeFactory[PipeFunc].make_from_blueprint(
@@ -65,7 +65,7 @@ def _register_step_pipes(*, source_input_ref: str = "Text") -> None:
description="Consumes step A's output",
inputs={"a_out": "Text"},
output="Text",
- function_name="optionals_taint_echo_a_out",
+ function_name=f"{__name__}.optionals_taint_echo_a_out",
),
)
sink_c = PipeFactory[PipeFunc].make_from_blueprint(
@@ -75,7 +75,7 @@ def _register_step_pipes(*, source_input_ref: str = "Text") -> None:
description="Absorbing sink: declares b_out optional",
inputs={"b_out": "Text?"},
output="Text",
- function_name="optionals_taint_sink",
+ function_name=f"{__name__}.optionals_taint_sink",
),
)
pipe_library = get_pipe_library()
@@ -99,17 +99,6 @@ def _build_sequence(*, output_ref: str, steps: list[SubPipeBlueprint], inputs: d
class TestSequenceTaintValidation:
- @classmethod
- def setup_class(cls):
- for func in _TEST_FUNCS:
- func_registry.register_function(func)
-
- @classmethod
- def teardown_class(cls):
- for func in _TEST_FUNCS:
- if func_registry.has_function(func.__name__):
- func_registry.unregister_function_by_name(func.__name__)
-
def test_taint_reaching_plain_sequence_output_is_rejected(self, load_empty_library: Callable[[], str]):
"""Optional sequence input feeds a lift chain that ends the sequence: plain output → error
naming the source, the tainted variable, and the three fixes.
@@ -270,7 +259,7 @@ class TestSequenceTaintValidation:
description="Per-item finder that may find nothing",
inputs={"item": "Text"},
output="Text?",
- function_name="optionals_taint_echo_source",
+ function_name=f"{__name__}.optionals_taint_echo_source",
),
)
list_sink = PipeFactory[PipeFunc].make_from_blueprint(
@@ -280,7 +269,7 @@ class TestSequenceTaintValidation:
description="Consumes the compacted list — a guaranteed slot",
inputs={"found_items": "Text[]"},
output="Text",
- function_name="optionals_taint_sink",
+ function_name=f"{__name__}.optionals_taint_sink",
),
)
pipe_library = get_pipe_library()
@@ -309,7 +298,7 @@ class TestSequenceTaintValidation:
description="Guaranteed branch",
inputs={"topic": "Text"},
output="Text",
- function_name="optionals_taint_echo_source",
+ function_name=f"{__name__}.optionals_taint_echo_source",
),
)
get_pipe_library().add_new_pipe(pipe=branch_base)
diff --git a/tests/integration/pipelex/pipes/test_bracket_notation_operators/test_bracket_notation_operators.py b/tests/integration/pipelex/pipes/test_bracket_notation_operators/test_bracket_notation_operators.py
index 56fe78430..7499463ce 100644
--- a/tests/integration/pipelex/pipes/test_bracket_notation_operators/test_bracket_notation_operators.py
+++ b/tests/integration/pipelex/pipes/test_bracket_notation_operators/test_bracket_notation_operators.py
@@ -1,4 +1,3 @@
-from pathlib import Path
from typing import Callable
from pipelex.core.concepts.native.concept_native import NativeConceptCode
@@ -20,7 +19,7 @@ from pipelex.system.registries.func_registry import pipe_func
# Test function for PipeFunc bracket notation test
-@pipe_func(name="process_function")
+@pipe_func()
def process_function(working_memory: WorkingMemory) -> ListContent[TextContent]:
"""Test function that processes items and returns a list."""
items = working_memory.get_stuff_as_list(name="two_texts", item_type=TextContent).items
@@ -111,14 +110,14 @@ class TestBracketNotationInOperators:
assert pipe.output.concept.code == "Image"
assert pipe.output_multiplicity == 3
- def test_pipe_func_with_bracket_input_and_output(self, load_test_library: Callable[[list[Path]], None]):
- load_test_library([Path(Path(__file__).parent)])
+ def test_pipe_func_with_bracket_input_and_output(self, load_empty_library: Callable[[], None]):
+ load_empty_library()
blueprint = PipeFuncBlueprint(
description="Process items",
inputs={"two_texts": f"{NativeConceptCode.TEXT}[2]"},
output=f"{NativeConceptCode.TEXT}[]",
- function_name="process_function",
+ function_name=f"{__name__}.process_function",
)
pipe = PipeFactory[PipeFunc].make_from_blueprint(
diff --git a/tests/unit/pipelex/cli/test_fix_function_names_cmds.py b/tests/unit/pipelex/cli/test_fix_function_names_cmds.py
new file mode 100644
index 000000000..575d9f36d
--- /dev/null
+++ b/tests/unit/pipelex/cli/test_fix_function_names_cmds.py
@@ -0,0 +1,124 @@
+import json
+from pathlib import Path
+
+import pytest
+import typer
+
+from pipelex.cli.agent_cli.commands.agent_output import CliOutputFormat
+from pipelex.cli.agent_cli.commands.fix.function_names_cmd import fix_function_names_cmd as agent_fix_function_names_cmd
+from pipelex.cli.commands.fix.function_names_cmd import fix_function_names_cmd
+
+_MODULE = """\
+from pipelex.system.registries.func_registry import pipe_func
+
+
+@pipe_func(name="invoice_totals")
+def compute_totals(working_memory):
+ return working_memory
+"""
+
+_BUNDLE = """\
+domain = "invoicing"
+
+[pipe.totals]
+type = "PipeFunc"
+description = "Compute the totals"
+output = "Text"
+function_name = "{flat_name}"
+"""
+
+
+def _method_folder(*, root: Path, flat_name: str) -> Path:
+ method_dir = root / "invoicing"
+ (method_dir / "funcs").mkdir(parents=True)
+ (method_dir / "invoicing.mthds").write_text(_BUNDLE.format(flat_name=flat_name), encoding="utf-8")
+ (method_dir / "funcs" / "totals.py").write_text(_MODULE, encoding="utf-8")
+ return method_dir
+
+
+class TestFixFunctionNamesCmds:
+ def test_the_command_rewrites_the_method_and_reports_it(self, tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None:
+ method_dir = _method_folder(root=tmp_path, flat_name="invoice_totals")
+
+ fix_function_names_cmd(path=str(method_dir))
+
+ out = capsys.readouterr().out
+ assert "`invoice_totals` → `funcs.totals.compute_totals`" in out
+ assert 'function_name = "funcs.totals.compute_totals"' in (method_dir / "invoicing.mthds").read_text(encoding="utf-8")
+ assert "@pipe_func()\n" in (method_dir / "funcs" / "totals.py").read_text(encoding="utf-8")
+
+ def test_the_command_writes_nothing_on_a_dry_run(self, tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None:
+ method_dir = _method_folder(root=tmp_path, flat_name="invoice_totals")
+
+ fix_function_names_cmd(path=str(method_dir), dry_run=True)
+
+ assert "Nothing was written" in capsys.readouterr().out
+ assert (method_dir / "invoicing.mthds").read_text(encoding="utf-8") == _BUNDLE.format(flat_name="invoice_totals")
+
+ @pytest.mark.parametrize(
+ ("flat_name", "expected_exit_code"),
+ [pytest.param("no_such_totals", 1, id="unresolved"), pytest.param("invoice_totals", None, id="complete")],
+ )
+ def test_the_command_exits_one_when_a_name_is_left(self, tmp_path: Path, flat_name: str, expected_exit_code: int | None) -> None:
+ method_dir = _method_folder(root=tmp_path, flat_name=flat_name)
+
+ if expected_exit_code is None:
+ fix_function_names_cmd(path=str(method_dir))
+ return
+ with pytest.raises(typer.Exit) as exc_info:
+ fix_function_names_cmd(path=str(method_dir))
+ assert exc_info.value.exit_code == expected_exit_code
+
+ def test_the_command_exits_two_on_a_path_that_is_no_folder(self, tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None:
+ with pytest.raises(typer.Exit) as exc_info:
+ fix_function_names_cmd(path=str(tmp_path / "missing"))
+
+ assert exc_info.value.exit_code == 2
+ assert "is not a directory" in capsys.readouterr().err
+
+ def test_the_agent_command_reports_its_verdict_in_json(self, tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None:
+ method_dir = _method_folder(root=tmp_path, flat_name="no_such_totals")
+
+ with pytest.raises(typer.Exit) as exc_info:
+ agent_fix_function_names_cmd(path=str(method_dir), dry_run=True, output_format=CliOutputFormat.JSON)
+
+ assert exc_info.value.exit_code == 1
+ payload = json.loads(capsys.readouterr().out)
+ assert payload["success"] is True
+ assert payload["is_complete"] is False
+ assert payload["is_dry_run"] is True
+ assert payload["changed_files"] == ["funcs/totals.py"]
+ assert payload["rewrites"] == []
+ assert payload["marker_removals"] == [{"module": "funcs/totals.py", "function": "compute_totals", "registered_name": "invoice_totals"}]
+ (unresolved,) = payload["unresolved"]
+ assert unresolved["pipe_code"] == "totals"
+ assert unresolved["flat_name"] == "no_such_totals"
+ assert payload["unresolved_markers"] == []
+
+ def test_a_marker_whose_name_is_not_a_literal_exits_one(self, tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None:
+ method_dir = _method_folder(root=tmp_path, flat_name="funcs.totals.compute_totals")
+ module_file = method_dir / "funcs" / "totals.py"
+ module_file.write_text(_MODULE.replace('(name="invoice_totals")', "(name=TOTALS_NAME)"), encoding="utf-8")
+
+ with pytest.raises(typer.Exit) as exc_info:
+ agent_fix_function_names_cmd(path=str(method_dir), output_format=CliOutputFormat.JSON)
+
+ assert exc_info.value.exit_code == 1
+ payload = json.loads(capsys.readouterr().out)
+ assert payload["is_complete"] is False
+ assert payload["changed_files"] == []
+ (marker,) = payload["unresolved_markers"]
+ assert marker["module"] == "funcs/totals.py"
+ assert marker["function"] == "compute_totals"
+ assert marker["registered_name"] is None
+ assert "(name=TOTALS_NAME)" in module_file.read_text(encoding="utf-8")
+
+ def test_the_agent_command_reports_in_markdown_by_default(self, tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None:
+ method_dir = _method_folder(root=tmp_path, flat_name="invoice_totals")
+
+ agent_fix_function_names_cmd(path=str(method_dir))
+
+ out = capsys.readouterr().out
+ assert out.startswith("# Function names of the method at")
+ with pytest.raises(json.JSONDecodeError):
+ json.loads(out)
diff --git a/tests/unit/pipelex/core/method_trees/conftest.py b/tests/unit/pipelex/core/method_trees/conftest.py
new file mode 100644
index 000000000..df42b3515
--- /dev/null
+++ b/tests/unit/pipelex/core/method_trees/conftest.py
@@ -0,0 +1,13 @@
+from collections.abc import Iterator
+
+import pytest
+
+from pipelex.core.method_trees.tree_python import TREE_PYTHON_HOLD
+
+
+def released_hold() -> Iterator[None]:
+ """Every hold a test takes on a tree's Python is released and its modules evicted, so the next test's tree can load."""
+ yield
+ TREE_PYTHON_HOLD.release_all()
+ TREE_PYTHON_HOLD.unload_if_unheld()
diff --git a/tests/unit/pipelex/core/method_trees/test_tree_files.py b/tests/unit/pipelex/core/method_trees/test_tree_files.py
new file mode 100644
index 000000000..97d2d14e5
--- /dev/null
+++ b/tests/unit/pipelex/core/method_trees/test_tree_files.py
@@ -0,0 +1,72 @@
+from pathlib import Path
+
+import pytest
+from mthds.method_tree.lister import list_method_tree
+
+from pipelex.core.method_trees.exceptions import TreePythonError
+from pipelex.core.method_trees.tree_files import Base64FileEntry, encode_tree_file, read_call_files, tree_file_bytes, write_call_files
+from pipelex.libraries.pipe_func_call_files import travels_with_call
+
+_PNG_BYTES = b"\x89PNG\r\n\x1a\n\x00\x00"
+
+
+class TestTreeFiles:
+ @pytest.mark.parametrize(
+ ("data", "is_text"),
+ [pytest.param(b"VAT_RATE = 0.2\n", True, id="utf-8-text"), pytest.param(_PNG_BYTES, False, id="binary")],
+ )
+ def test_a_file_travels_as_its_text_or_in_base64_and_round_trips(self, data: bytes, is_text: bool) -> None:
+ entry = encode_tree_file(data=data)
+
+ assert isinstance(entry, str) is is_text
+ assert isinstance(entry, Base64FileEntry) is not is_text
+ assert tree_file_bytes(entry=entry) == data
+
+ def test_a_call_receives_every_file_of_the_tree_but_its_bundles_and_what_the_tree_excludes(self, tmp_path: Path) -> None:
+ method_dir = tmp_path / "invoicing"
+ (method_dir / "funcs").mkdir(parents=True)
+ (method_dir / "invoicing.mthds").write_text('domain = "invoicing"\n', encoding="utf-8")
+ (method_dir / "funcs" / "totals.py").write_text("TOTAL = 1\n", encoding="utf-8")
+ (method_dir / "logo.png").write_bytes(_PNG_BYTES)
+ (method_dir / ".gitignore").write_text("scratch.py\n", encoding="utf-8")
+ (method_dir / "scratch.py").write_text("NOT_SHIPPED = True\n", encoding="utf-8")
+ (method_dir / "structures.py").write_text(
+ "# Generated by Pipelex\nfrom pipelex.core.stuffs.structured_content import StructuredContent\n", encoding="utf-8"
+ )
+
+ files = read_call_files(tree=list_method_tree(method_dir))
+
+ assert sorted(files) == ["funcs/totals.py", "logo.png", "structures.py"]
+ assert files["funcs/totals.py"] == "TOTAL = 1\n"
+ assert files["logo.png"] == Base64FileEntry.make_from_bytes(_PNG_BYTES)
+
+ def test_a_filter_keeps_a_file_out_of_the_call(self, tmp_path: Path) -> None:
+ method_dir = tmp_path / "filtered"
+ method_dir.mkdir()
+ (method_dir / "filtered.mthds").write_text('domain = "filtered"\n', encoding="utf-8")
+ (method_dir / "helpers.py").write_text("X = 1\n", encoding="utf-8")
+
+ def _keeps(*, tree_path: str, data: bytes) -> bool:
+ return tree_path != "helpers.py" and bool(data)
+
+ files = read_call_files(tree=list_method_tree(method_dir), keeps=_keeps)
+
+ assert files == {}
+ assert travels_with_call(tree_path="logo.png", data=_PNG_BYTES)
+ assert travels_with_call(tree_path="helpers.py", data=b"X = 1\n")
+
+ def test_the_files_a_call_received_are_written_at_their_tree_paths(self, tmp_path: Path) -> None:
+ write_call_files(files={"funcs/totals.py": "TOTAL = 1\n", "logo.png": Base64FileEntry.make_from_bytes(_PNG_BYTES)}, destination=tmp_path)
+
+ assert (tmp_path / "funcs" / "totals.py").read_text(encoding="utf-8") == "TOTAL = 1\n"
+ assert (tmp_path / "logo.png").read_bytes() == _PNG_BYTES
+
+ @pytest.mark.parametrize("tree_path", ["../escape.py", "/etc/escape.py", "funcs//totals.py"], ids=["climbing", "absolute", "empty-segment"])
+ def test_a_key_that_is_not_a_tree_path_is_refused_before_anything_is_written(self, tmp_path: Path, tree_path: str) -> None:
+ destination = tmp_path / "call"
+ destination.mkdir()
+
+ with pytest.raises(TreePythonError, match="which is not a tree path"):
+ write_call_files(files={tree_path: "X = 1\n"}, destination=destination)
+
+ assert not (tmp_path / "escape.py").exists()
diff --git a/tests/unit/pipelex/core/method_trees/test_tree_python.py b/tests/unit/pipelex/core/method_trees/test_tree_python.py
new file mode 100644
index 000000000..98acc9fa2
--- /dev/null
+++ b/tests/unit/pipelex/core/method_trees/test_tree_python.py
@@ -0,0 +1,246 @@
+import importlib.util
+import json
+import sys
+from pathlib import Path
+
+import pytest
+from mthds.method_tree.lister import list_method_tree
+from mthds.method_tree.models import MethodTree
+from pytest_mock import MockerFixture
+
+from pipelex.base_exceptions import DisclosureMode
+from pipelex.core.method_trees import tree_python
+from pipelex.core.method_trees.exceptions import TreeModuleExcludedError, TreeModuleShadowingError, TreePythonError, TreePythonHeldError
+from pipelex.core.method_trees.tree_python import (
+ TREE_PYTHON_HOLD,
+ check_tree_module_names,
+ import_held_tree_module,
+ sharing_module_name_checks,
+)
+from pipelex.tools.typing.module_inspector import import_module_from_file
+
+_LABEL = "the method tree holding probe.mthds"
+
+
+def _write_tree(*, root: Path, files: dict[str, str]) -> MethodTree:
+ for tree_path, text in files.items():
+ target = root.joinpath(*tree_path.split("/"))
+ target.parent.mkdir(parents=True, exist_ok=True)
+ target.write_text(text, encoding="utf-8")
+ return list_method_tree(root)
+
+
+def _host_paths(*, tmp_path: Path) -> list[str]:
+ """The spellings of the test's directory on this host, which no message may name (`/var` is `/private/var` on macOS)."""
+ return [str(tmp_path), str(tmp_path.resolve())]
+
+
+def _helper_tree(*, root: Path, value: str) -> MethodTree:
+ """A tree whose structure module imports a sibling helper by its top-level name, as a step's module does."""
+ return _write_tree(
+ root=root,
+ files={
+ "probe.mthds": 'domain = "probe"\n',
+ "tree_probe_helpers.py": f"VALUE = {value!r}\n",
+ "tree_probe_shapes.py": "import tree_probe_helpers\n\nSHAPE = tree_probe_helpers.VALUE\n",
+ "tree_probe_steps.py": "from tree_probe_helpers import VALUE\n",
+ },
+ )
+
+
[email protected]("released_hold")
+class TestTreePython:
+ def test_a_held_tree_imports_by_real_name_writes_no_bytecode_and_is_evicted(self, tmp_path: Path) -> None:
+ tree = _write_tree(
+ root=tmp_path / "alpha",
+ files={
+ "alpha.mthds": 'domain = "alpha"\n',
+ "tree_probe_alpha/__init__.py": "",
+ "tree_probe_alpha/rates.py": "VAT_PERCENT = 20\n",
+ "tree_probe_alpha/totals.py": "from tree_probe_alpha.rates import VAT_PERCENT\n\nTOTAL = 100 + VAT_PERCENT\n",
+ },
+ )
+
+ identity = TREE_PYTHON_HOLD.acquire(tree=tree, holder="run-alpha", label=_LABEL)
+ module = import_held_tree_module(identity=identity, module_name="tree_probe_alpha.totals")
+
+ assert module.TOTAL == 120
+ assert module.__name__ == "tree_probe_alpha.totals"
+ assert sys.path[0] == str(identity.root)
+ assert not list((tmp_path / "alpha").rglob("__pycache__"))
+
+ TREE_PYTHON_HOLD.release(holder="run-alpha")
+ TREE_PYTHON_HOLD.unload_if_unheld()
+
+ assert "tree_probe_alpha.totals" not in sys.modules
+ assert "tree_probe_alpha" not in sys.modules
+ assert str(identity.root) not in sys.path
+
+ def test_another_tree_is_refused_while_one_is_held_naming_both(self, tmp_path: Path) -> None:
+ first = _write_tree(root=tmp_path / "first", files={"first.mthds": 'domain = "first"\n', "tree_probe_first.py": "X = 1\n"})
+ second = _write_tree(root=tmp_path / "second", files={"second.mthds": 'domain = "second"\n', "tree_probe_second.py": "X = 2\n"})
+ TREE_PYTHON_HOLD.acquire(tree=first, holder="run-first", label=_LABEL)
+ TREE_PYTHON_HOLD.acquire(tree=first, holder="run-first-again", label=_LABEL)
+
+ with pytest.raises(TreePythonHeldError) as exc_info:
+ TREE_PYTHON_HOLD.acquire(tree=second, holder="run-second", label="the method tree holding second.mthds")
+
+ assert f"the method tree holding second.mthds ({second.digest[:19]})" in exc_info.value.message
+ assert f"({first.digest[:19]})" in exc_info.value.message
+
+ TREE_PYTHON_HOLD.release(holder="run-first")
+ with pytest.raises(TreePythonHeldError):
+ TREE_PYTHON_HOLD.acquire(tree=second, holder="run-second", label=_LABEL)
+
+ TREE_PYTHON_HOLD.release(holder="run-first-again")
+ second_identity = TREE_PYTHON_HOLD.acquire(tree=second, holder="run-second", label=_LABEL)
+ assert import_held_tree_module(identity=second_identity, module_name="tree_probe_second").X == 2
+
+ @pytest.mark.parametrize(
+ ("module_file", "expected_message"),
+ [
+ pytest.param("json.py", "the standard-library module 'json'", id="standard-library"),
+ pytest.param("pytest.py", "the module 'pytest', already imported in this process", id="imported-package"),
+ ],
+ )
+ def test_a_module_that_would_shadow_another_is_refused(self, tmp_path: Path, module_file: str, expected_message: str) -> None:
+ tree = _write_tree(root=tmp_path / "shadow", files={"shadow.mthds": 'domain = "shadow"\n', module_file: "X = 1\n"})
+
+ with pytest.raises(TreeModuleShadowingError) as exc_info:
+ TREE_PYTHON_HOLD.acquire(tree=tree, holder="run-shadow", label=_LABEL)
+
+ assert expected_message in exc_info.value.message
+ assert module_file in exc_info.value.message
+
+ def test_the_structures_module_is_the_one_a_tree_may_ship(self, tmp_path: Path) -> None:
+ tree = _write_tree(root=tmp_path / "structured", files={"structured.mthds": 'domain = "structured"\n', "structures.py": "X = 1\n"})
+
+ identity = TREE_PYTHON_HOLD.acquire(tree=tree, holder="run-structured", label=_LABEL)
+
+ assert TREE_PYTHON_HOLD.is_held(identity=identity)
+
+ def test_an_import_of_a_file_the_tree_keeps_out_is_refused_with_the_rule(self, tmp_path: Path) -> None:
+ tree = _write_tree(
+ root=tmp_path / "guarded",
+ files={
+ "guarded.mthds": 'domain = "guarded"\n',
+ ".gitignore": "tree_probe_scratch.py\n",
+ "tree_probe_scratch.py": "SECRET = 1\n",
+ "tree_probe_guarded.py": "import tree_probe_scratch\n",
+ },
+ )
+ identity = TREE_PYTHON_HOLD.acquire(tree=tree, holder="run-guarded", label=_LABEL)
+
+ with pytest.raises(TreeModuleExcludedError) as exc_info:
+ import_held_tree_module(identity=identity, module_name="tree_probe_guarded")
+
+ assert "tree_probe_scratch" in str(exc_info.value)
+ assert "the membership rule excludes it" in str(exc_info.value)
+ assert "tree_probe_scratch" not in sys.modules
+
+ def test_a_tree_read_without_a_directory_holds_no_python(self, tmp_path: Path) -> None:
+ bundle = tmp_path / "lone.mthds"
+ bundle.write_text('domain = "lone"\n', encoding="utf-8")
+
+ with pytest.raises(TreePythonError, match="read alone, as a one-file tree"):
+ TREE_PYTHON_HOLD.acquire(tree=list_method_tree(bundle), holder="run-lone", label=_LABEL)
+
+ def test_a_helper_a_structure_module_imported_while_loading_is_evicted_with_its_tree(self, tmp_path: Path) -> None:
+ first = _helper_tree(root=tmp_path / "first", value="first")
+ second = _helper_tree(root=tmp_path / "second", value="second")
+ # A library load imports a structure module by its file, its directory first on the import path, and the module
+ # imports its sibling helper with Python's own loader, under the helper's top-level name.
+ import_module_from_file((tmp_path / "first" / "tree_probe_shapes.py").resolve())
+ assert sys.modules["tree_probe_helpers"].VALUE == "first"
+ first_identity = TREE_PYTHON_HOLD.acquire(tree=first, holder="run-first", label=_LABEL)
+ assert import_held_tree_module(identity=first_identity, module_name="tree_probe_steps").VALUE == "first"
+ TREE_PYTHON_HOLD.release(holder="run-first")
+
+ second_identity = TREE_PYTHON_HOLD.acquire(tree=second, holder="run-second", label=_LABEL)
+ module = import_held_tree_module(identity=second_identity, module_name="tree_probe_steps")
+
+ assert module.VALUE == "second"
+ assert sys.modules["tree_probe_helpers"].VALUE == "second"
+ first_root = str((tmp_path / "first").resolve())
+ assert first_root not in sys.path
+
+ def test_a_released_tree_leaves_the_import_path_until_it_is_held_again(self, tmp_path: Path) -> None:
+ tree = _write_tree(root=tmp_path / "between", files={"between.mthds": 'domain = "between"\n', "tree_probe_between.py": "X = 1\n"})
+ identity = TREE_PYTHON_HOLD.acquire(tree=tree, holder="run-between", label=_LABEL)
+ assert sys.path[0] == str(identity.root)
+
+ TREE_PYTHON_HOLD.release(holder="run-between")
+
+ assert str(identity.root) not in sys.path
+ assert importlib.util.find_spec("tree_probe_between") is None
+ held_again = TREE_PYTHON_HOLD.acquire(tree=tree, holder="run-between-again", label=_LABEL)
+ assert sys.path[0] == str(identity.root)
+ assert import_held_tree_module(identity=held_again, module_name="tree_probe_between").X == 1
+
+ def test_a_module_under_the_root_that_the_tree_keeps_out_is_not_evicted(self, tmp_path: Path) -> None:
+ first = _write_tree(
+ root=tmp_path / "rooted",
+ files={"rooted.mthds": 'domain = "rooted"\n', "tree_probe_rooted.py": "X = 1\n", "venv/pyvenv.cfg": "home = /usr\n"},
+ )
+ site_module_file = tmp_path / "rooted" / "venv" / "tree_probe_site.py"
+ site_module_file.write_text("SITE = 1\n", encoding="utf-8")
+ spec = importlib.util.spec_from_file_location("tree_probe_site", site_module_file)
+ assert spec is not None
+ assert spec.loader is not None
+ site_module = importlib.util.module_from_spec(spec)
+ spec.loader.exec_module(site_module)
+ sys.modules["tree_probe_site"] = site_module
+ second = _write_tree(root=tmp_path / "other", files={"other.mthds": 'domain = "other"\n', "tree_probe_other.py": "X = 2\n"})
+ try:
+ first_identity = TREE_PYTHON_HOLD.acquire(tree=first, holder="run-rooted", label=_LABEL)
+ import_held_tree_module(identity=first_identity, module_name="tree_probe_rooted")
+ TREE_PYTHON_HOLD.release(holder="run-rooted")
+
+ TREE_PYTHON_HOLD.acquire(tree=second, holder="run-other", label=_LABEL)
+
+ assert "tree_probe_rooted" not in sys.modules
+ assert sys.modules["tree_probe_site"] is site_module
+ finally:
+ sys.modules.pop("tree_probe_site", None)
+
+ def test_the_errors_of_a_tree_name_no_path_of_this_host_under_strict_disclosure(self, tmp_path: Path) -> None:
+ shadowing = _write_tree(root=tmp_path / "shadowing", files={"shadowing.mthds": 'domain = "shadowing"\n', "json.py": "X = 1\n"})
+ with pytest.raises(TreeModuleShadowingError) as shadowing_info:
+ TREE_PYTHON_HOLD.acquire(tree=shadowing, holder="run-shadowing", label="the method tree holding shadowing.mthds")
+ strict_report = json.dumps(shadowing_info.value.to_error_report().to_dict(disclosure_mode=DisclosureMode.STRICT))
+ assert "the method tree holding shadowing.mthds" in strict_report
+ assert "json.py" in strict_report
+
+ guarded = _write_tree(
+ root=tmp_path / "guarded",
+ files={
+ "guarded.mthds": 'domain = "guarded"\n',
+ ".gitignore": "tree_probe_hidden.py\n",
+ "tree_probe_hidden.py": "X = 1\n",
+ "tree_probe_reaches.py": "import tree_probe_hidden\n",
+ },
+ )
+ guarded_identity = TREE_PYTHON_HOLD.acquire(tree=guarded, holder="run-guarded", label="the method tree holding guarded.mthds")
+ with pytest.raises(TreeModuleExcludedError) as excluded_info:
+ import_held_tree_module(identity=guarded_identity, module_name="tree_probe_reaches")
+ assert "the file tree_probe_hidden.py of the method tree holding guarded.mthds" in excluded_info.value.message
+
+ other = _write_tree(root=tmp_path / "other", files={"other.mthds": 'domain = "other"\n', "tree_probe_other.py": "X = 2\n"})
+ with pytest.raises(TreePythonHeldError) as held_info:
+ TREE_PYTHON_HOLD.acquire(tree=other, holder="run-other", label="the method tree holding other.mthds")
+
+ for message in (strict_report, excluded_info.value.message, held_info.value.message):
+ for host_path in _host_paths(tmp_path=tmp_path):
+ assert host_path not in message
+
+ def test_a_load_judges_a_tree_s_module_names_once_for_all_its_steps(self, tmp_path: Path, mocker: MockerFixture) -> None:
+ judged_spy = mocker.spy(tree_python, "_module_outside_tree")
+ top_level_modules = {"tree_probe_first": "tree_probe_first.py", "tree_probe_second": "tree_probe_second/rates.py"}
+
+ with sharing_module_name_checks():
+ for _step in range(3):
+ check_tree_module_names(top_level_modules=top_level_modules, tree_label=_LABEL, root=tmp_path, checks_this_process=True)
+ assert judged_spy.call_count == len(top_level_modules)
+
+ check_tree_module_names(top_level_modules=top_level_modules, tree_label=_LABEL, root=tmp_path, checks_this_process=True)
+ assert judged_spy.call_count == 2 * len(top_level_modules)
diff --git a/tests/unit/pipelex/core/test_data/pipes/operators/func/pipe_func.py b/tests/unit/pipelex/core/test_data/pipes/operators/func/pipe_func.py
index 85f1ed4b8..1e7905371 100644
--- a/tests/unit/pipelex/core/test_data/pipes/operators/func/pipe_func.py
+++ b/tests/unit/pipelex/core/test_data/pipes/operators/func/pipe_func.py
@@ -13,7 +13,7 @@ ProcessedData = "Processed data concept"
type = "PipeFunc"
description = "Process data with function"
output = "ProcessedData"
-function_name = "process_data_function"
+function_name = "data_funcs.process_data_function"
""",
PipelexBundleBlueprint(
domain="test_pipes",
@@ -24,7 +24,7 @@ function_name = "process_data_function"
type="PipeFunc",
description="Process data with function",
output="ProcessedData",
- function_name="process_data_function",
+ function_name="data_funcs.process_data_function",
),
},
),
diff --git a/tests/unit/pipelex/kernel/test_func_ops.py b/tests/unit/pipelex/kernel/test_func_ops.py
index 8bb849b86..a0765f0b8 100644
--- a/tests/unit/pipelex/kernel/test_func_ops.py
+++ b/tests/unit/pipelex/kernel/test_func_ops.py
@@ -2,7 +2,7 @@
`run_func` is the whole reason these tests exist: no interpreter test reaches it. `PipeFunc` rides
the *pluggable* executor — which may run the function in a sandbox or a Temporal activity — and then
-stores the result, so it exercises `call_registered_function` (through the direct executor) and
+stores the result, so it exercises `call_function` (through the direct executor) and
`store_result` separately, never their composition. A programmatic caller on a `RuntimeBoot`-only
process rides exactly that composition, so it is tested here or nowhere.
@@ -11,6 +11,9 @@ it has to reach a worker thread, and a rewrite that dropped the `iscoroutinefunc
still pass every async-function test.
"""
+from collections.abc import Callable
+from typing import Any
+
import pytest
from pipelex.core.concepts.concept_factory import ConceptFactory
@@ -18,12 +21,7 @@ from pipelex.core.concepts.native.concept_native import NativeConceptCode
from pipelex.core.memory.working_memory import WorkingMemory
from pipelex.core.memory.working_memory_factory import WorkingMemoryFactory
from pipelex.core.stuffs.text_content import TextContent
-from pipelex.kernel.func_ops import call_registered_function, run_func
-from pipelex.system.registries.func_registry import func_registry
-
-ASYNC_FUNCTION_NAME = "kernel_unit_test_async_func"
-
-SYNC_FUNCTION_NAME = "kernel_unit_test_sync_func"
+from pipelex.kernel.func_ops import call_function, run_func
ASYNC_TEXT = "produced by the async function"
@@ -41,34 +39,22 @@ def _sync_producer(working_memory: WorkingMemory) -> TextContent: # ruff: ignor
return TextContent(text=SYNC_TEXT)
-def registered_producers():
- func_registry.register_function(_async_producer, name=ASYNC_FUNCTION_NAME)
- func_registry.register_function(_sync_producer, name=SYNC_FUNCTION_NAME)
- try:
- yield
- finally:
- func_registry.unregister_function_by_name(ASYNC_FUNCTION_NAME)
- func_registry.unregister_function_by_name(SYNC_FUNCTION_NAME)
-
-
[email protected]("registered_producers")
@pytest.mark.asyncio(loop_scope="class")
class TestFuncOps:
@pytest.mark.parametrize(
- ("function_name", "expected_text", "expected_qualname"),
+ ("function", "expected_text", "expected_qualname"),
[
- pytest.param(ASYNC_FUNCTION_NAME, ASYNC_TEXT, _async_producer.__qualname__, id="async-function"),
- pytest.param(SYNC_FUNCTION_NAME, SYNC_TEXT, _sync_producer.__qualname__, id="sync-function-on-a-worker-thread"),
+ pytest.param(_async_producer, ASYNC_TEXT, _async_producer.__qualname__, id="async-function"),
+ pytest.param(_sync_producer, SYNC_TEXT, _sync_producer.__qualname__, id="sync-function-on-a-worker-thread"),
],
)
- async def test_call_registered_function_runs_both_dispatch_arms(
+ async def test_call_function_runs_both_dispatch_arms(
self,
- function_name: str,
+ function: Callable[..., Any],
expected_text: str,
expected_qualname: str,
) -> None:
- call_result = await call_registered_function(function_name=function_name, memory=WorkingMemoryFactory.make_empty())
+ call_result = await call_function(function=function, memory=WorkingMemoryFactory.make_empty())
assert isinstance(call_result.content, TextContent)
assert call_result.content.text == expected_text
@@ -83,7 +69,7 @@ class TestFuncOps:
result = await run_func(
memory=memory,
- function_name=ASYNC_FUNCTION_NAME,
+ function=_async_producer,
concept=ConceptFactory.make_native_concept(native_concept_code=NativeConceptCode.TEXT),
result_name=RESULT_NAME,
)
diff --git a/tests/unit/pipelex/kernel/test_kernel_boot_contract.py b/tests/unit/pipelex/kernel/test_kernel_boot_contract.py
index 2b60152e6..88ec91a8b 100644
--- a/tests/unit/pipelex/kernel/test_kernel_boot_contract.py
+++ b/tests/unit/pipelex/kernel/test_kernel_boot_contract.py
@@ -120,7 +120,6 @@ _KERNEL_CALL_SCRIPT = textwrap.dedent(
from pipelex.runtime_boot import RuntimeBoot
from pipelex.system.pipe_run_mode import PipeRunMode
from pipelex.system.registries.class_registry_access import get_class_registry
- from pipelex.system.registries.func_registry import func_registry
from pipelex.system.runtime import IntegrationMode
from pipelex.tools.jinja2.template_category import TemplateCategory
from pipelex.tools.log.log_context import get_log_context
@@ -413,27 +412,24 @@ _KERNEL_CALL_SCRIPT = textwrap.dedent(
fail(f"run_compose_template rendered {compose_result.rendered_text!r}, not the template it was given")
check_stored(compose_result, "run_compose_template", TextContent)
- # Registered here rather than imported, because the func path's whole point is that the function is
- # the caller's: a kernel-only process has no library to have registered one for it.
- FUNCTION_NAME = "kernel_boot_contract_function"
- FUNCTION_TEXT = "produced by a registered function"
+ # Defined here and handed over, because the func path's whole point is that the function is the
+ # caller's: a kernel-only process has no library to have imported one for it.
+ FUNCTION_TEXT = "produced by the caller's function"
def produce_text(working_memory: WorkingMemory) -> TextContent:
return TextContent(text=FUNCTION_TEXT)
- func_registry.register_function(produce_text, name=FUNCTION_NAME)
-
func_result = asyncio.run(
run_func(
memory=WorkingMemoryFactory.make_empty(),
- function_name=FUNCTION_NAME,
+ function=produce_text,
concept=text_concept,
result_name="called",
)
)
if not isinstance(func_result.content, TextContent) or func_result.content.text != FUNCTION_TEXT:
- fail("run_func did not return what the registered function produced")
+ fail("run_func did not return what the function it was handed produced")
check_stored(func_result, "run_func", TextContent)
# Swept last, so a function-local interpreter import taken during any of the calls is caught too.
diff --git a/tests/unit/pipelex/language/test_mthds_schema.py b/tests/unit/pipelex/language/test_mthds_schema.py
index 2eba1d62c..0471bdb3e 100644
--- a/tests/unit/pipelex/language/test_mthds_schema.py
+++ b/tests/unit/pipelex/language/test_mthds_schema.py
@@ -18,7 +18,7 @@ if TYPE_CHECKING:
# minimal schema-valid values. Keyed by the `type` discriminator value. Used to
# build a minimal table that validates against exactly one oneOf arm.
_PIPE_KIND_EXTRA_FIELDS: dict[str, dict[str, Any]] = {
- "PipeFunc": {"function_name": "do_it"},
+ "PipeFunc": {"function_name": "funcs.do_it"},
"PipeImgGen": {"prompt": "draw a cat"},
"PipeCompose": {},
"PipeLLM": {},
@@ -250,7 +250,7 @@ class TestMthdsSchemaGeneration:
# shared by PipeLLM/PipeImgGen and `function_name` uniquely looks like PipeFunc, but with
# no `type` neither matches any arm.
pytest.param({"description": "looks like an impl", "output": "Text", "prompt": "do it"}, id="stray-prompt"),
- pytest.param({"description": "looks like PipeFunc", "output": "Text", "function_name": "do_it"}, id="stray-function-name"),
+ pytest.param({"description": "looks like PipeFunc", "output": "Text", "function_name": "funcs.do_it"}, id="stray-function-name"),
],
)
def test_typeless_table_with_stray_field_is_rejected(self, schema: dict[str, Any], table: dict[str, Any]) -> None:
diff --git a/tests/unit/pipelex/libraries/test_bundle_resources.py b/tests/unit/pipelex/libraries/test_bundle_resources.py
index 3978ebbd4..785fc7040 100644
--- a/tests/unit/pipelex/libraries/test_bundle_resources.py
+++ b/tests/unit/pipelex/libraries/test_bundle_resources.py
@@ -175,7 +175,7 @@ class TestBundleResources:
assert relocated is crate
def test_resources_round_trip_and_stay_out_of_the_fingerprint(self) -> None:
- """Like python_sources, the files travel with the crate but are not part of its structure."""
+ """Like the files a PipeFunc call receives, the files travel with the crate but are not part of its structure."""
resources = make_bundle_resources(files={"templates/report.docx": b"docx bytes"})
bare = LibraryCrateFactory.make_from_blueprints([_blueprint()])
with_resources = LibraryCrateFactory.make_from_blueprints([_blueprint()], bundle_resources=resources)
diff --git a/tests/unit/pipelex/libraries/test_library_crate_pipe_func_files.py b/tests/unit/pipelex/libraries/test_library_crate_pipe_func_files.py
new file mode 100644
index 000000000..49b8cc07f
--- /dev/null
+++ b/tests/unit/pipelex/libraries/test_library_crate_pipe_func_files.py
@@ -0,0 +1,60 @@
+from typing import ClassVar
+
+from pipelex.core.method_trees.tree_files import Base64FileEntry, TreeFileEntry
+from pipelex.libraries.library_crate import SUBMITTED_PYTHON_ROOT_KEY, LibraryCrate
+from pipelex.libraries.library_crate_factory import LibraryCrateFactory
+from tests.unit.pipelex.libraries.test_library_crate_data import BlueprintSamples
+
+
+class TestLibraryCratePipeFuncFiles:
+ """The files a PipeFunc call receives, captured on the crate: they round-trip, but never touch the fingerprint."""
+
+ _FILES: ClassVar[dict[str, dict[str, TreeFileEntry]]] = {
+ SUBMITTED_PYTHON_ROOT_KEY: {
+ "funcs/score.py": "from pipelex.core.memory.working_memory import WorkingMemory\n\n\ndef compute_score(working_memory): ...\n",
+ "structures/weighted_score.py": "from pydantic import BaseModel\n\n\nclass WeightedScore(BaseModel):\n value: float\n",
+ "data/logo.png": Base64FileEntry(base64="iVBORw0KGgo="),
+ },
+ }
+
+ def test_pipe_func_files_default_empty(self):
+ """A crate built without files carries an empty mapping (local/direct mode)."""
+ crate = LibraryCrateFactory.make_from_blueprints(blueprints=[BlueprintSamples.SCORING_BUNDLE])
+ assert crate.pipe_func_files == {}
+
+ def test_pipe_func_files_round_trip(self):
+ """The files survive a JSON round-trip verbatim, a binary file as its base64, alongside the rest of the crate."""
+ crate = LibraryCrateFactory.make_from_blueprints(
+ blueprints=[BlueprintSamples.SCORING_BUNDLE],
+ pipe_func_files=self._FILES,
+ )
+ assert crate.pipe_func_files == self._FILES
+
+ restored = LibraryCrate.model_validate_json(crate.model_dump_json())
+ assert restored.pipe_func_files == self._FILES
+ # The rest of the crate is unaffected by carrying files.
+ assert restored.concepts == crate.concepts
+ assert restored.pipes == crate.pipes
+ assert restored.fingerprint == crate.fingerprint
+
+ def test_fingerprint_excludes_pipe_func_files(self):
+ """The structural fingerprint is identical whether or not files travel, and regardless of their content.
+
+ Folding files into the fingerprint would break load_from_crate's structural-dedupe contract,
+ so two crates that differ ONLY in pipe_func_files must hash the same.
+ """
+ bare = LibraryCrateFactory.make_from_blueprints(blueprints=[BlueprintSamples.SCORING_BUNDLE])
+ with_files = LibraryCrateFactory.make_from_blueprints(
+ blueprints=[BlueprintSamples.SCORING_BUNDLE],
+ pipe_func_files=self._FILES,
+ )
+ other_files = LibraryCrateFactory.make_from_blueprints(
+ blueprints=[BlueprintSamples.SCORING_BUNDLE],
+ pipe_func_files={SUBMITTED_PYTHON_ROOT_KEY: {"funcs/score.py": "def compute_score(working_memory): return 42\n"}},
+ )
+
+ assert bare.fingerprint != ""
+ assert bare.fingerprint == with_files.fingerprint
+ assert with_files.fingerprint == other_files.fingerprint
+ # And the instance method agrees: files are not part of compute_fingerprint.
+ assert with_files.compute_fingerprint() == bare.compute_fingerprint()
diff --git a/tests/unit/pipelex/libraries/test_library_crate_python_sources.py b/tests/unit/pipelex/libraries/test_library_crate_python_sources.py
deleted file mode 100644
index ffd3f99ae..000000000
--- a/tests/unit/pipelex/libraries/test_library_crate_python_sources.py
+++ /dev/null
@@ -1,56 +0,0 @@
-from typing import ClassVar
-
-from pipelex.libraries.library_crate import LibraryCrate
-from pipelex.libraries.library_crate_factory import LibraryCrateFactory
-from tests.unit.pipelex.libraries.test_library_crate_data import BlueprintSamples
-
-
-class TestLibraryCratePythonSources:
- """Sandbox-hosted source capture on the crate: round-trips, but never touches the fingerprint."""
-
- _SOURCES: ClassVar[dict[str, str]] = {
- "funcs/score.py": "from pipelex.core.memory.working_memory import WorkingMemory\n\n\ndef compute_score(working_memory): ...\n",
- "structures/weighted_score.py": "from pydantic import BaseModel\n\n\nclass WeightedScore(BaseModel):\n value: float\n",
- }
-
- def test_python_sources_default_empty(self):
- """A crate built without python_sources carries an empty dict (local/direct mode)."""
- crate = LibraryCrateFactory.make_from_blueprints(blueprints=[BlueprintSamples.SCORING_BUNDLE])
- assert crate.python_sources == {}
-
- def test_python_sources_round_trip(self):
- """python_sources survives a JSON round-trip verbatim, alongside the rest of the crate."""
- crate = LibraryCrateFactory.make_from_blueprints(
- blueprints=[BlueprintSamples.SCORING_BUNDLE],
- python_sources=self._SOURCES,
- )
- assert crate.python_sources == self._SOURCES
-
- restored = LibraryCrate.model_validate_json(crate.model_dump_json())
- assert restored.python_sources == self._SOURCES
- # The rest of the crate is unaffected by carrying source.
- assert restored.concepts == crate.concepts
- assert restored.pipes == crate.pipes
- assert restored.fingerprint == crate.fingerprint
-
- def test_fingerprint_excludes_python_sources(self):
- """The structural fingerprint is identical whether or not source travels, and regardless of its content.
-
- Folding source into the fingerprint would break load_from_crate's structural-dedupe contract,
- so two crates that differ ONLY in python_sources must hash the same.
- """
- bare = LibraryCrateFactory.make_from_blueprints(blueprints=[BlueprintSamples.SCORING_BUNDLE])
- with_sources = LibraryCrateFactory.make_from_blueprints(
- blueprints=[BlueprintSamples.SCORING_BUNDLE],
- python_sources=self._SOURCES,
- )
- other_sources = LibraryCrateFactory.make_from_blueprints(
- blueprints=[BlueprintSamples.SCORING_BUNDLE],
- python_sources={"funcs/score.py": "def compute_score(working_memory): return 42\n"},
- )
-
- assert bare.fingerprint != ""
- assert bare.fingerprint == with_sources.fingerprint
- assert with_sources.fingerprint == other_sources.fingerprint
- # And the instance method agrees: source is not part of compute_fingerprint.
- assert with_sources.compute_fingerprint() == bare.compute_fingerprint()
diff --git a/tests/unit/pipelex/libraries/test_method_trees.py b/tests/unit/pipelex/libraries/test_method_trees.py
index e5357cee4..c3e3988af 100644
--- a/tests/unit/pipelex/libraries/test_method_trees.py
+++ b/tests/unit/pipelex/libraries/test_method_trees.py
@@ -10,9 +10,12 @@ from pipelex.libraries.method_trees import (
LIMITS_ADVICE,
bundle_files_under,
list_library_bundle_files,
+ list_library_dir_trees,
list_package_method_tree,
reporting_host_library_tree_faults,
+ submitted_python_files,
tree_bundle_of_lone_file,
+ tree_python_sources,
)
_MANIFEST = '[package]\naddress = "github.com/pipelex-tests/nested"\nversion = "1.0.0"\ndescription = "A nested package"\n'
@@ -205,3 +208,32 @@ class TestMethodTrees:
found = bundle_files_under(library_dirs=[tmp_path / "lib", tmp_path / "missing"])
assert set(found) == {tmp_path / "lib" / "a.mthds", tmp_path / "lib" / ".hidden" / "b.mthds", tmp_path / "lib" / "link.mthds"}
+
+ def test_only_a_directory_whose_own_tree_holds_no_bundle_submits_its_files(self, tmp_path: Path) -> None:
+ """Its files are the Python submitted beside bundles parsed from strings; a tree holding a bundle carries its own."""
+ _touch(directory=tmp_path, tree_path="submitted/funcs/totals.py", content="X = 1\n")
+ _touch(directory=tmp_path, tree_path="submitted/notes.txt", content="note\n")
+ _touch(directory=tmp_path, tree_path="method/method.mthds")
+ _touch(directory=tmp_path, tree_path="method/funcs.py", content="X = 2\n")
+ _touch(directory=tmp_path, tree_path="rooted/METHODS.toml", content=_MANIFEST)
+ _touch(directory=tmp_path, tree_path="rooted/python/helpers.py", content="X = 3\n")
+
+ submitted = {
+ library_tree.library_dir.name: submitted_python_files(library_tree=library_tree)
+ for library_tree in list_library_dir_trees(library_dirs=[tmp_path / "submitted", tmp_path / "method", tmp_path / "rooted" / "python"])
+ }
+
+ assert submitted == {"submitted": {"funcs/totals.py": b"X = 1\n", "notes.txt": b"note\n"}, "method": {}, "python": {"helpers.py": b"X = 3\n"}}
+
+ def test_the_python_of_a_nested_tree_is_named_from_its_library_directory(self, tmp_path: Path) -> None:
+ _touch(directory=tmp_path, tree_path="top.mthds")
+ _touch(directory=tmp_path, tree_path="top_funcs.py", content="X = 1\n")
+ _touch(directory=tmp_path, tree_path="checkouts/reports/report.mthds")
+ _touch(directory=tmp_path, tree_path="checkouts/reports/models/report.py", content="X = 2\n")
+ (tmp_path / "checkouts" / "reports" / ".git").mkdir()
+
+ sources: dict[str, str] = {}
+ for library_tree in list_library_dir_trees(library_dirs=[tmp_path]):
+ sources.update(tree_python_sources(tree=library_tree.tree, library_dir=library_tree.library_dir))
+
+ assert sources == {"top_funcs.py": "X = 1\n", "checkouts/reports/models/report.py": "X = 2\n"}
diff --git a/tests/unit/pipelex/migration/test_function_names_migration.py b/tests/unit/pipelex/migration/test_function_names_migration.py
new file mode 100644
index 000000000..4f40646b9
--- /dev/null
+++ b/tests/unit/pipelex/migration/test_function_names_migration.py
@@ -0,0 +1,229 @@
+from pathlib import Path
+
+import pytest
+
+from pipelex.migration.exceptions import FunctionNamesMigrationError
+from pipelex.migration.function_names import migrate_function_names, migrate_function_names_in_directory
+from pipelex.pipe_operators.func.function_name import function_name_form_violation
+
+_MARKED_TOTALS = """\
+from pipelex.core.memory.working_memory import WorkingMemory
+from pipelex.core.stuffs.text_content import TextContent
+from pipelex.system.registries.func_registry import pipe_func
+
+
+@pipe_func()
+async def compute_totals(working_memory: WorkingMemory) -> TextContent:
+ return TextContent(text="total")
+
+
+def helper() -> int:
+ return 1
+"""
+
+_NAMED_MARKER = """\
+from pipelex.system.registries import func_registry
+
+
+@func_registry.pipe_func(name="invoice_summary")
+def summarize(working_memory):
+ return working_memory
+"""
+
+_BUNDLE = """\
+domain = "invoicing"
+
+# The totals, computed in Python
+[pipe.totals]
+type = "PipeFunc" # aligned on purpose
+description = "Compute the totals"
+output = "Text"
+function_name = "compute_totals"
+
+[pipe.summary]
+type = "PipeFunc"
+description = "Summarize the invoice"
+output = "Text"
+function_name = "invoice_summary"
+
+[pipe.already]
+type = "PipeFunc"
+description = "Already an import path"
+output = "Text"
+function_name = "funcs.totals.compute_totals"
+
+[pipe.write]
+type = "PipeLLM"
+description = "Not a PipeFunc"
+output = "Text"
+prompt = "compute_totals"
+"""
+
+
+class TestFunctionNamesMigration:
+ def test_flat_names_become_import_paths_and_markers_lose_their_name(self) -> None:
+ migration = migrate_function_names(
+ files={"invoicing.mthds": _BUNDLE, "funcs/totals.py": _MARKED_TOTALS, "funcs/summary/__init__.py": _NAMED_MARKER},
+ )
+
+ assert migration.is_complete
+ assert [(rewrite.pipe_code, rewrite.flat_name, rewrite.import_path) for rewrite in migration.rewrites] == [
+ ("totals", "compute_totals", "funcs.totals.compute_totals"),
+ ("summary", "invoice_summary", "funcs.summary.summarize"),
+ ]
+ assert [(removal.module, removal.function, removal.registered_name) for removal in migration.marker_removals] == [
+ ("funcs/summary/__init__.py", "summarize", "invoice_summary"),
+ ]
+ assert sorted(migration.changed_files) == ["funcs/summary/__init__.py", "invoicing.mthds"]
+ rewritten_bundle = migration.changed_files["invoicing.mthds"]
+ assert rewritten_bundle == _BUNDLE.replace('"compute_totals"\n', '"funcs.totals.compute_totals"\n', 1).replace(
+ '"invoice_summary"', '"funcs.summary.summarize"'
+ )
+ assert "# The totals, computed in Python" in rewritten_bundle
+ assert 'type = "PipeFunc" # aligned on purpose' in rewritten_bundle
+ assert migration.changed_files["funcs/summary/__init__.py"] == _NAMED_MARKER.replace('(name="invoice_summary")', "()")
+ for rewrite in migration.rewrites:
+ assert function_name_form_violation(function_name=rewrite.import_path) is None
+
+ @pytest.mark.parametrize(
+ ("files", "expected_reason", "expected_candidates"),
+ [
+ pytest.param(
+ {"funcs/totals.py": _MARKED_TOTALS.replace("@pipe_func()\n", "")},
+ "No module of the method tree defines a @pipe_func function registered as 'compute_totals'",
+ [],
+ id="no-module-registers-the-name",
+ ),
+ pytest.param(
+ {"funcs/totals.py": _MARKED_TOTALS, "funcs/other_totals.py": _MARKED_TOTALS},
+ "More than one @pipe_func function of the method tree is registered as 'compute_totals'",
+ ["funcs.other_totals.compute_totals", "funcs.totals.compute_totals"],
+ id="two-modules-register-the-name",
+ ),
+ pytest.param(
+ {"my-funcs/totals.py": _MARKED_TOTALS},
+ "not an import path",
+ [],
+ id="the-module-has-no-import-path",
+ ),
+ ],
+ )
+ def test_a_name_that_cannot_be_resolved_is_left_and_reported(
+ self, files: dict[str, str], expected_reason: str, expected_candidates: list[str]
+ ) -> None:
+ bundle = _BUNDLE.split("[pipe.summary]", maxsplit=1)[0]
+ migration = migrate_function_names(files={"invoicing.mthds": bundle, **files})
+
+ assert not migration.is_complete
+ assert migration.rewrites == []
+ assert "invoicing.mthds" not in migration.changed_files
+ (unresolved,) = migration.unresolved
+ assert unresolved.bundle == "invoicing.mthds"
+ assert unresolved.pipe_code == "totals"
+ assert unresolved.flat_name == "compute_totals"
+ assert expected_reason in unresolved.reason
+ assert unresolved.candidates == expected_candidates
+
+ @pytest.mark.parametrize(
+ "module_text",
+ [
+ pytest.param(f"\x0c\n{_NAMED_MARKER}", id="form-feed-line-above"),
+ pytest.param(f'"""Summaries\u2028of invoices\x0b."""\n\n{_NAMED_MARKER}', id="line-separator-in-docstring"),
+ pytest.param(_NAMED_MARKER.replace("\n", "\r\n"), id="crlf-line-endings"),
+ pytest.param(_NAMED_MARKER.replace("\n", "\r"), id="cr-line-endings"),
+ ],
+ )
+ def test_a_marker_loses_its_name_whatever_characters_python_does_not_count_as_line_breaks(self, module_text: str) -> None:
+ bundle = _BUNDLE.split("[pipe.already]", maxsplit=1)[0]
+ migration = migrate_function_names(
+ files={"invoicing.mthds": bundle, "funcs/totals.py": _MARKED_TOTALS, "funcs/summary.py": module_text},
+ )
+
+ assert migration.is_complete
+ assert migration.changed_files["funcs/summary.py"] == module_text.replace('(name="invoice_summary")', "()")
+
+ def test_a_marker_whose_name_is_not_a_literal_is_left_and_reported(self) -> None:
+ module_text = _NAMED_MARKER.replace('(name="invoice_summary")', "(name=SUMMARY_NAME)")
+ bundle = _BUNDLE.split("[pipe.summary]", maxsplit=1)[0]
+ migration = migrate_function_names(files={"invoicing.mthds": bundle, "funcs/totals.py": _MARKED_TOTALS, "funcs/summary.py": module_text})
+
+ assert not migration.is_complete
+ assert migration.unresolved == []
+ assert migration.marker_removals == []
+ (marker,) = migration.unresolved_markers
+ assert (marker.module, marker.function, marker.registered_name) == ("funcs/summary.py", "summarize", None)
+ assert "not a string literal" in marker.reason
+ assert "funcs/summary.py" not in migration.changed_files
+
+ def test_a_marker_keeps_its_name_while_a_step_naming_it_is_unresolved(self) -> None:
+ other_marker = _NAMED_MARKER.replace("def summarize(", "def summarize_again(")
+ bundle = _BUNDLE.split("[pipe.already]", maxsplit=1)[0]
+ migration = migrate_function_names(
+ files={
+ "invoicing.mthds": bundle,
+ "funcs/totals.py": _MARKED_TOTALS,
+ "funcs/summary.py": _NAMED_MARKER,
+ "funcs/other_summary.py": other_marker,
+ },
+ )
+
+ assert not migration.is_complete
+ assert [(rewrite.pipe_code, rewrite.import_path) for rewrite in migration.rewrites] == [("totals", "funcs.totals.compute_totals")]
+ (unresolved,) = migration.unresolved
+ assert (unresolved.pipe_code, unresolved.flat_name) == ("summary", "invoice_summary")
+ assert migration.marker_removals == []
+ assert [(marker.module, marker.registered_name) for marker in migration.unresolved_markers] == [
+ ("funcs/other_summary.py", "invoice_summary"),
+ ("funcs/summary.py", "invoice_summary"),
+ ]
+ assert all("'summary'" in marker.reason for marker in migration.unresolved_markers)
+ assert sorted(migration.changed_files) == ["invoicing.mthds"]
+
+ def test_a_bundle_that_does_not_parse_is_reported_without_a_step(self) -> None:
+ broken_bundle = '[pipe.totals]\ntype = "PipeFunc"\nfunction_name = "compute_totals\n'
+ migration = migrate_function_names(files={"broken.mthds": broken_bundle, "funcs/totals.py": _MARKED_TOTALS})
+
+ assert not migration.is_complete
+ (unresolved,) = migration.unresolved
+ assert unresolved.bundle == "broken.mthds"
+ assert unresolved.pipe_code is None
+ assert unresolved.flat_name is None
+ assert "does not parse as TOML" in unresolved.reason
+ assert migration.changed_files == {}
+
+ def test_a_bundle_that_does_not_parse_and_names_no_function_is_not_reported(self) -> None:
+ migration = migrate_function_names(files={"broken.mthds": "domain = \n", "funcs/totals.py": _MARKED_TOTALS})
+
+ assert migration.is_complete
+ assert migration.unresolved == []
+
+ def test_a_method_naming_only_import_paths_is_left_untouched(self) -> None:
+ bundle = _BUNDLE.split("[pipe.already]")[1]
+ migration = migrate_function_names(files={"invoicing.mthds": f"[pipe.already]{bundle}", "funcs/totals.py": _MARKED_TOTALS})
+
+ assert migration.is_complete
+ assert not migration.has_changes
+ assert migration.changed_files == {}
+
+ @pytest.mark.parametrize("is_dry_run", [True, False], ids=["dry-run", "write"])
+ def test_a_method_folder_is_rewritten_unless_it_is_a_dry_run(self, tmp_path: Path, is_dry_run: bool) -> None:
+ (tmp_path / "funcs").mkdir()
+ (tmp_path / "invoicing.mthds").write_text(_BUNDLE, encoding="utf-8")
+ (tmp_path / "funcs" / "totals.py").write_text(_MARKED_TOTALS, encoding="utf-8")
+ (tmp_path / "funcs" / "summary.py").write_text(_NAMED_MARKER, encoding="utf-8")
+
+ migration = migrate_function_names_in_directory(directory=tmp_path, is_dry_run=is_dry_run)
+
+ assert migration.is_complete
+ assert sorted(migration.changed_files) == ["funcs/summary.py", "invoicing.mthds"]
+ on_disk = (tmp_path / "invoicing.mthds").read_text(encoding="utf-8")
+ if is_dry_run:
+ assert on_disk == _BUNDLE
+ else:
+ assert on_disk == migration.changed_files["invoicing.mthds"]
+ assert (tmp_path / "funcs" / "summary.py").read_text(encoding="utf-8") == migration.changed_files["funcs/summary.py"]
+ assert not list(tmp_path.rglob("__pycache__"))
+
+ def test_a_folder_that_is_not_a_method_tree_is_refused(self, tmp_path: Path) -> None:
+ with pytest.raises(FunctionNamesMigrationError):
+ migrate_function_names_in_directory(directory=tmp_path / "missing", is_dry_run=True)
diff --git a/tests/unit/pipelex/mthds_parsing/test_bundle_elaborator.py b/tests/unit/pipelex/mthds_parsing/test_bundle_elaborator.py
index ccd1bd21b..b182e9051 100644
--- a/tests/unit/pipelex/mthds_parsing/test_bundle_elaborator.py
+++ b/tests/unit/pipelex/mthds_parsing/test_bundle_elaborator.py
@@ -28,7 +28,7 @@ class TestBundleElaborator:
type="PipeFunc",
description="Plain",
output="Foo",
- function_name="my_func",
+ function_name="funcs.my_func",
),
},
)
diff --git a/tests/unit/pipelex/pipe_machinery/test_data.py b/tests/unit/pipelex/pipe_machinery/test_data.py
index 92cb27138..ed73cfe46 100644
--- a/tests/unit/pipelex/pipe_machinery/test_data.py
+++ b/tests/unit/pipelex/pipe_machinery/test_data.py
@@ -11,7 +11,7 @@ class InputNameRefusalTestData:
description = "Totals an invoice"
inputs = { invoice = "Invoice" }
output = "Number"
-function_name = "total_invoice"
+function_name = "invoice_funcs.total_invoice"
""",
PipeType.PIPE_IMG_GEN: """type = "PipeImgGen"
description = "Draws the letterhead of an invoice"
diff --git a/tests/unit/pipelex/pipe_operators/pipe_func/data.py b/tests/unit/pipelex/pipe_operators/pipe_func/data.py
index d3b660a8e..7a01e3817 100644
--- a/tests/unit/pipelex/pipe_operators/pipe_func/data.py
+++ b/tests/unit/pipelex/pipe_operators/pipe_func/data.py
@@ -5,22 +5,24 @@ from pipelex.core.stuffs.text_content import TextContent
from pipelex.pipe_operators.func.pipe_func_blueprint import PipeFuncBlueprint
from pipelex.system.registries.func_registry import pipe_func
+# The functions the test steps name by import path, from this module
+_MODULE = "tests.unit.pipelex.pipe_operators.pipe_func.data"
-# Register test functions for validation tests
-@pipe_func(name="my_function")
+
+@pipe_func()
def my_function(working_memory: WorkingMemory) -> TextContent: # ruff: ignore[unused-function-argument] # pyright: ignore[reportUnusedParameter]
"""Test function with no inputs."""
return TextContent(text="test output")
-@pipe_func(name="process_text")
+@pipe_func()
def process_text(working_memory: WorkingMemory) -> TextContent:
"""Test function with single text input."""
input_data = working_memory.get_stuff_as_str("input_data")
return TextContent(text=f"processed: {input_data}")
-@pipe_func(name="combine_data")
+@pipe_func()
def combine_data(working_memory: WorkingMemory) -> TextContent:
"""Test function with multiple inputs."""
text_input = working_memory.get_stuff_as_str("text_input")
@@ -28,14 +30,14 @@ def combine_data(working_memory: WorkingMemory) -> TextContent:
return TextContent(text=f"combined: {text_input} and {number_input}")
-@pipe_func(name="process_image")
+@pipe_func()
def process_image(working_memory: WorkingMemory) -> TextContent:
"""Test function with image input."""
image = working_memory.get_stuff_as_image("image")
return TextContent(text=f"processed image: {image.url}")
-@pipe_func(name="process_all")
+@pipe_func()
def process_all(working_memory: WorkingMemory) -> TextContent:
"""Test function with mixed inputs."""
text = working_memory.get_stuff_as_str("text")
@@ -54,7 +56,7 @@ class PipeFuncInputTestCases:
description="Test case: valid_no_inputs",
inputs={},
output="native.Text",
- function_name="my_function",
+ function_name=f"{_MODULE}.my_function",
),
)
@@ -64,7 +66,7 @@ class PipeFuncInputTestCases:
description="Test case: valid_single_input",
inputs={"input_data": "native.Text"},
output="native.Text",
- function_name="process_text",
+ function_name=f"{_MODULE}.process_text",
),
)
@@ -74,7 +76,7 @@ class PipeFuncInputTestCases:
description="Test case: valid_multiple_inputs",
inputs={"text_input": "native.Text", "number_input": "native.Number"},
output="native.Text",
- function_name="combine_data",
+ function_name=f"{_MODULE}.combine_data",
),
)
@@ -84,7 +86,7 @@ class PipeFuncInputTestCases:
description="Test case: valid_image_input",
inputs={"image": "native.Image"},
output="native.Text",
- function_name="process_image",
+ function_name=f"{_MODULE}.process_image",
),
)
@@ -94,7 +96,7 @@ class PipeFuncInputTestCases:
description="Test case: valid_mixed_inputs",
inputs={"text": "native.Text", "image": "native.Image", "number": "native.Number"},
output="native.Text",
- function_name="process_all",
+ function_name=f"{_MODULE}.process_all",
),
)
diff --git a/tests/unit/pipelex/pipe_operators/pipe_func/test_direct_executor_workdir.py b/tests/unit/pipelex/pipe_operators/pipe_func/test_direct_executor_workdir.py
index ef043e4e7..e200c398c 100644
--- a/tests/unit/pipelex/pipe_operators/pipe_func/test_direct_executor_workdir.py
+++ b/tests/unit/pipelex/pipe_operators/pipe_func/test_direct_executor_workdir.py
@@ -1,5 +1,4 @@
import sys
-import tempfile
from pathlib import Path
import pytest
@@ -12,15 +11,16 @@ from pipelex.core.memory.working_memory_factory import WorkingMemoryFactory
from pipelex.interpreter_hub import get_library_manager, scoped_current_library, set_current_library
from pipelex.libraries.library_crate import LibraryCrate
from pipelex.pipe_operators.func import direct_pipe_func_executor
-from pipelex.pipe_operators.func.direct_pipe_func_executor import DirectPipeFuncExecutor, bundle_import_dirs
-from pipelex.pipe_operators.func.pipe_func_execution_dtos import PipeFuncExecutionRequest
+from pipelex.pipe_operators.func.direct_pipe_func_executor import DirectPipeFuncExecutor
+from pipelex.pipe_operators.func.pipe_func_execution_dtos import PipeFuncExecutionRequest, PipeFuncExecutionResponse
+from pipelex.pipe_operators.func.pipe_func_execution_transport import build_pipe_func_execution_request
from pipelex.pipe_run.pipe_run_params import PipeRunParams
from pipelex.system.job_metadata import JobMetadata, RunMetadata
from pipelex.system.pipe_run_mode import PipeRunMode
# A method whose ONLY custom source is the PipeFunc — its output structure (Greeting) is a concept
# declared in the .mthds, NOT a shipped .py. The box must regenerate `structures.py` from the crate's
-# concepts so the func's `from structures import Greeting` resolves and the func registers.
+# concepts so the func's `from structures import greet_demo__Greeting` resolves.
_GREET_MTHDS = """\
domain = "greet_demo"
description = "Greeting demo for structure regeneration"
@@ -34,58 +34,97 @@ text = { type = "text", description = "the greeting text", required = true }
type = "PipeFunc"
description = "Build a greeting via customer code"
output = "Greeting"
-function_name = "greet_it"
+function_name = "greet_demo_funcs.greeting.greet_it"
"""
_GREET_FUNC = """\
from pipelex.core.memory.working_memory import WorkingMemory
from pipelex.system.registries.func_registry import pipe_func
+from greet_demo_funcs.wording import GREETING_TEXT
from structures import greet_demo__Greeting
-@pipe_func(name="greet_it")
+@pipe_func()
async def greet_it(working_memory: WorkingMemory) -> greet_demo__Greeting:
- return greet_demo__Greeting(text="hello from generated structures")
+ return greet_demo__Greeting(text=GREETING_TEXT)
"""
+_GREET_WORDING = 'GREETING_TEXT = "hello from generated structures"\n'
-# A PipeFunc file, so the transported run imports it by path, which puts its resolved directory on sys.path.
-_SYMLINK_PROBE_FUNC = """\
-from pipelex.core.memory.working_memory import WorkingMemory
-from pipelex.core.stuffs.text_content import TextContent
-from pipelex.system.registries.func_registry import pipe_func
+def _job_metadata() -> JobMetadata:
+ return JobMetadata(run_metadata=RunMetadata(storage_scope="test/scope", read_scope=None, user_id="user", pipeline_run_id="run"))
-@pipe_func(name="symlink_probe_func")
-async def symlink_probe_func(working_memory: WorkingMemory) -> TextContent:
- return TextContent(text="probe")
-"""
+
+def _dry_run_params() -> PipeRunParams:
+ return PipeRunParams(run_mode=PipeRunMode.DRY, pipe_stack_limit=10, batch_max_concurrency=None)
+
+
+def _missing_pipe_request(*, files: dict[str, str]) -> PipeFuncExecutionRequest:
+ return PipeFuncExecutionRequest(
+ crate=LibraryCrate(),
+ files=dict(files),
+ working_memory_raw={},
+ pipe_code="missing_pipe",
+ function_name="missing_funcs.missing_func",
+ job_metadata=_job_metadata(),
+ pipe_run_params=_dry_run_params(),
+ )
+
+
+async def _run_transported(*, request: PipeFuncExecutionRequest, tree_root: Path | None = None) -> PipeFuncExecutionResponse:
+ library_id = direct_pipe_func_executor._TRANSPORTED_LIBRARY_ID # ruff: ignore[private-member-access] # pyright: ignore[reportPrivateUsage]
+ with scoped_current_library(library_id=library_id):
+ try:
+ if tree_root is None:
+ return await DirectPipeFuncExecutor().run_pipe_func_transported(request=request)
+ return await DirectPipeFuncExecutor().run_pipe_func_transported_at(request=request, tree_root=tree_root)
+ finally:
+ get_library_manager().teardown(library_id=library_id)
+
+
+def _greet_request(*, tmp_path: Path) -> PipeFuncExecutionRequest:
+ """The request a sandbox-hosted deployment builds for the greet step of a method folder holding its Python."""
+ method_dir = tmp_path / "greet_method"
+ (method_dir / "greet_demo_funcs").mkdir(parents=True)
+ (method_dir / "greet.mthds").write_text(_GREET_MTHDS, encoding="utf-8")
+ (method_dir / "greet_demo_funcs" / "greeting.py").write_text(_GREET_FUNC, encoding="utf-8")
+ (method_dir / "greet_demo_funcs" / "wording.py").write_text(_GREET_WORDING, encoding="utf-8")
+
+ pipe_func_config = get_config().interpreter.pipe_func
+ previous_mode = pipe_func_config.execution_mode
+ pipe_func_config.execution_mode = "local_sandbox"
+ library_manager = get_library_manager()
+ try:
+ build_library_id, _ = library_manager.open_library()
+ set_current_library(library_id=build_library_id)
+ try:
+ library_manager.load_libraries(library_id=build_library_id, library_dirs=[method_dir])
+ return build_pipe_func_execution_request(
+ job_metadata=_job_metadata(),
+ # Qualified, as PipeFunc sends it: the transported library is keyed by pipe_ref and its lookup is strict.
+ pipe_code="greet_demo.greet",
+ function_name="greet_demo_funcs.greeting.greet_it",
+ working_memory=WorkingMemoryFactory.make_empty(),
+ pipe_run_params=_dry_run_params(),
+ )
+ finally:
+ library_manager.teardown(library_id=build_library_id)
+ finally:
+ pipe_func_config.execution_mode = previous_mode
class TestDirectExecutorWorkdir:
@pytest.mark.asyncio
async def test_transported_run_cleans_up_workdir_on_failure(self, tmp_path: Path, mocker: MockerFixture):
- """The materialized source workdir must not leak, even when the transported run fails."""
+ """The directory the call's files are written into must not leak, even when the transported run fails."""
workdir = tmp_path / "transported_workdir"
workdir.mkdir()
mocker.patch("tempfile.mkdtemp", return_value=str(workdir))
- request = PipeFuncExecutionRequest(
- crate=LibraryCrate(python_sources={"funcs.py": "x = 1\n"}),
- working_memory_raw={},
- pipe_code="missing_pipe",
- function_name="missing_func",
- job_metadata=JobMetadata(run_metadata=RunMetadata(storage_scope="test/scope", read_scope=None, user_id="user", pipeline_run_id="run")),
- pipe_run_params=PipeRunParams(run_mode=PipeRunMode.DRY, pipe_stack_limit=10, batch_max_concurrency=None),
- )
-
- with scoped_current_library(library_id=direct_pipe_func_executor._TRANSPORTED_LIBRARY_ID): # ruff: ignore[private-member-access] # pyright: ignore[reportPrivateUsage]
- try:
- with pytest.raises((PipelexError, ValidationError)):
- await DirectPipeFuncExecutor().run_pipe_func_transported(request=request)
- finally:
- get_library_manager().teardown(library_id=direct_pipe_func_executor._TRANSPORTED_LIBRARY_ID) # ruff: ignore[private-member-access] # pyright: ignore[reportPrivateUsage]
+ with pytest.raises((PipelexError, ValidationError)):
+ await _run_transported(request=_missing_pipe_request(files={"missing_funcs.py": "x = 1\n"}))
assert not workdir.exists()
@@ -96,112 +135,58 @@ class TestDirectExecutorWorkdir:
workdir.mkdir()
mocker.patch("tempfile.mkdtemp", return_value=str(workdir))
- request = PipeFuncExecutionRequest(
- crate=LibraryCrate(python_sources={"funcs/helpers.py": "x = 1\n"}),
- working_memory_raw={},
- pipe_code="missing_pipe",
- function_name="missing_func",
- job_metadata=JobMetadata(run_metadata=RunMetadata(storage_scope="test/scope", read_scope=None, user_id="user", pipeline_run_id="run")),
- pipe_run_params=PipeRunParams(run_mode=PipeRunMode.DRY, pipe_stack_limit=10, batch_max_concurrency=None),
- )
-
- with scoped_current_library(library_id=direct_pipe_func_executor._TRANSPORTED_LIBRARY_ID): # ruff: ignore[private-member-access] # pyright: ignore[reportPrivateUsage]
- try:
- with pytest.raises((PipelexError, ValidationError)):
- await DirectPipeFuncExecutor().run_pipe_func_transported(request=request)
- finally:
- get_library_manager().teardown(library_id=direct_pipe_func_executor._TRANSPORTED_LIBRARY_ID) # ruff: ignore[private-member-access] # pyright: ignore[reportPrivateUsage]
+ with pytest.raises((PipelexError, ValidationError)):
+ await _run_transported(request=_missing_pipe_request(files={"missing_funcs/helpers.py": "x = 1\n"}))
stale_entries = [entry for entry in sys.path if Path(entry).is_relative_to(workdir)]
assert stale_entries == []
@pytest.mark.asyncio
async def test_transported_run_restores_sys_path_under_a_symlinked_temp_dir(self, tmp_path: Path, mocker: MockerFixture):
- """Importing a file puts its resolved directory on sys.path, so a temp dir reached through a symlink
- (`/var` on macOS) must still leave no entry behind.
- """
+ """The call's root is resolved, so a temp dir reached through a symlink (`/var` on macOS) leaves no entry behind."""
real_workdir = tmp_path / "real_workdir"
real_workdir.mkdir()
linked_workdir = tmp_path / "linked_workdir"
linked_workdir.symlink_to(real_workdir, target_is_directory=True)
mocker.patch("tempfile.mkdtemp", return_value=str(linked_workdir))
- request = PipeFuncExecutionRequest(
- crate=LibraryCrate(python_sources={"funcs/probe.py": _SYMLINK_PROBE_FUNC}),
- working_memory_raw={},
- pipe_code="missing_pipe",
- function_name="missing_func",
- job_metadata=JobMetadata(run_metadata=RunMetadata(storage_scope="test/scope", read_scope=None, user_id="user", pipeline_run_id="run")),
- pipe_run_params=PipeRunParams(run_mode=PipeRunMode.DRY, pipe_stack_limit=10, batch_max_concurrency=None),
- )
-
- with scoped_current_library(library_id=direct_pipe_func_executor._TRANSPORTED_LIBRARY_ID): # ruff: ignore[private-member-access] # pyright: ignore[reportPrivateUsage]
- try:
- with pytest.raises((PipelexError, ValidationError)):
- await DirectPipeFuncExecutor().run_pipe_func_transported(request=request)
- finally:
- get_library_manager().teardown(library_id=direct_pipe_func_executor._TRANSPORTED_LIBRARY_ID) # ruff: ignore[private-member-access] # pyright: ignore[reportPrivateUsage]
+ with pytest.raises((PipelexError, ValidationError)):
+ await _run_transported(request=_missing_pipe_request(files={"missing_funcs/probe.py": "x = 1\n"}))
stale_entries = [entry for entry in sys.path if Path(entry).is_relative_to(real_workdir) or Path(entry).is_relative_to(linked_workdir)]
assert stale_entries == []
- def test_import_dirs_search_the_root_first_then_sorted_subdirs(self, tmp_path: Path):
- """A module name two bundle directories share resolves to the same file in every process, root first."""
- for relpath in ("zeta/structures.py", "alpha/helpers.py", "funcs.py", "alpha/deep/more.py", "notes/readme.txt"):
- file_path = tmp_path / relpath
- file_path.parent.mkdir(parents=True, exist_ok=True)
- file_path.write_text("x = 1\n", encoding="utf-8")
+ @pytest.mark.asyncio
+ async def test_transported_run_imports_by_import_path_and_generates_concept_structures(self, tmp_path: Path):
+ """A PipeFunc that imports a sibling module and its output concept's structure runs in the box.
+
+ The call carries every file of the step's tree but the bundles: the function's module and the module it imports
+ travel at their tree paths, and the box regenerates `structures.py` from the crate's concepts, so the method
+ ships no copy of it that could drift from the .mthds.
+ """
+ request = _greet_request(tmp_path=tmp_path)
+ assert sorted(request.files) == ["greet_demo_funcs/greeting.py", "greet_demo_funcs/wording.py"]
+ assert request.crate.pipe_func_files == {}
+
+ response = await _run_transported(request=request)
- assert bundle_import_dirs(workdir=tmp_path) == [tmp_path, tmp_path / "alpha", tmp_path / "alpha" / "deep", tmp_path / "zeta"]
+ assert response.function_module == "greet_demo_funcs.greeting"
+ assert response.function_qualname == "greet_it"
+ assert "greet_demo_funcs.greeting" not in sys.modules
@pytest.mark.asyncio
- async def test_transported_run_generates_concept_structures(self):
- """A PipeFunc that imports its output concept structure runs even though the method ships no
- structure .py — the box regenerates `structures.py` from the crate's concepts.
-
- Regression for the atlas custom-PipeFunc failure: the method's `structures/*.py` never traveled,
- so `from structures import <Concept>` raised ModuleNotFoundError in the box and the func never
- registered ("Function not found in registry"). Generating the structures from the .mthds (the
- single source of truth) makes the import resolve without shipping — or drifting — a copy.
- """
- # Build a real crate from the .mthds (hosted mode: the loader tolerates the yet-unregistered
- # function), then attach ONLY the PipeFunc source — deliberately NO structures/ source.
- pipe_func_config = get_config().interpreter.pipe_func
- previous_mode = pipe_func_config.execution_mode
- pipe_func_config.execution_mode = "local_sandbox"
- library_manager = get_library_manager()
- try:
- with tempfile.TemporaryDirectory() as mthds_dir:
- (Path(mthds_dir) / "greet.mthds").write_text(_GREET_MTHDS, encoding="utf-8")
- build_library_id, _ = library_manager.open_library()
- set_current_library(library_id=build_library_id)
- try:
- library_manager.load_libraries(library_id=build_library_id, library_dirs=[Path(mthds_dir)])
- crate = library_manager.get_crate(library_id=build_library_id)
- assert crate is not None
- finally:
- library_manager.teardown(library_id=build_library_id)
- finally:
- pipe_func_config.execution_mode = previous_mode
-
- crate_with_func = crate.model_copy(update={"python_sources": {"funcs.py": _GREET_FUNC}})
- request = PipeFuncExecutionRequest(
- crate=crate_with_func,
- working_memory_raw=WorkingMemoryFactory.make_empty().dump_for_transport(),
- # Qualified, as PipeFunc now sends it: the transported library is keyed by pipe_ref and
- # its lookup is strict, so a bare code would not resolve on the far side.
- pipe_code="greet_demo.greet",
- function_name="greet_it",
- job_metadata=JobMetadata(run_metadata=RunMetadata(storage_scope="test/scope", read_scope=None, user_id="user", pipeline_run_id="run")),
- pipe_run_params=PipeRunParams(run_mode=PipeRunMode.DRY, pipe_stack_limit=10, batch_max_concurrency=None),
- )
-
- with scoped_current_library(library_id=direct_pipe_func_executor._TRANSPORTED_LIBRARY_ID): # ruff: ignore[private-member-access] # pyright: ignore[reportPrivateUsage]
- try:
- # No raise = the func registered = `from structures import Greeting` resolved against the
- # generated structures.py. Before the fix this raised "Function 'greet_it' not found in registry".
- response = await DirectPipeFuncExecutor().run_pipe_func_transported(request=request)
- finally:
- get_library_manager().teardown(library_id=direct_pipe_func_executor._TRANSPORTED_LIBRARY_ID) # ruff: ignore[private-member-access] # pyright: ignore[reportPrivateUsage]
+ async def test_transported_run_at_a_root_the_client_filled(self, tmp_path: Path):
+ """A sandbox client may unpack the call's files itself and hand the directory over; nothing is written beside them."""
+ request = _greet_request(tmp_path=tmp_path)
+ tree_root = (tmp_path / "unpacked").resolve()
+ for tree_path, entry in request.files.items():
+ target = tree_root.joinpath(*tree_path.split("/"))
+ target.parent.mkdir(parents=True, exist_ok=True)
+ assert isinstance(entry, str)
+ target.write_text(entry, encoding="utf-8")
+
+ response = await _run_transported(request=request.model_copy(update={"files": {}}), tree_root=tree_root)
assert response.function_qualname == "greet_it"
+ assert (tree_root / "structures.py").is_file()
+ assert not list(tree_root.rglob("__pycache__"))
diff --git a/tests/unit/pipelex/pipe_operators/pipe_func/test_pipe_func_blueprint.py b/tests/unit/pipelex/pipe_operators/pipe_func/test_pipe_func_blueprint.py
index e69de29bb..4562b33a4 100644
--- a/tests/unit/pipelex/pipe_operators/pipe_func/test_pipe_func_blueprint.py
+++ b/tests/unit/pipelex/pipe_operators/pipe_func/test_pipe_func_blueprint.py
@@ -0,0 +1,52 @@
+import pytest
+from pydantic import ValidationError
+
+from pipelex.pipe_operators.func.function_name import FunctionImportPath, split_function_name
+from pipelex.pipe_operators.func.pipe_func_blueprint import PipeFuncBlueprint
+
+
+def _blueprint(*, function_name: str) -> PipeFuncBlueprint:
+ return PipeFuncBlueprint(description="Compute the totals", inputs={}, output="Text", function_name=function_name)
+
+
+class TestPipeFuncBlueprint:
+ @pytest.mark.parametrize(
+ ("function_name", "expected"),
+ [
+ pytest.param("totals.compute_totals", FunctionImportPath(module_name="totals", function="compute_totals"), id="module-at-the-root"),
+ pytest.param(
+ "funcs.totals.compute_totals", FunctionImportPath(module_name="funcs.totals", function="compute_totals"), id="module-in-a-package"
+ ),
+ pytest.param("fonctions.résumé.calculer", FunctionImportPath(module_name="fonctions.résumé", function="calculer"), id="non-ascii"),
+ ],
+ )
+ def test_an_import_path_is_split_at_its_last_dot(self, function_name: str, expected: FunctionImportPath) -> None:
+ assert _blueprint(function_name=function_name).function_name == function_name
+ assert split_function_name(function_name=function_name) == expected
+
+ def test_a_decomposed_name_is_read_in_nfc(self) -> None:
+ decomposed = "fonctions.résumé.calculer"
+
+ assert split_function_name(function_name=decomposed) == FunctionImportPath(module_name="fonctions.résumé", function="calculer")
+
+ @pytest.mark.parametrize(
+ ("function_name", "expected_message"),
+ [
+ pytest.param("compute_totals", "holds no dot, so it names no module", id="flat-name"),
+ pytest.param("funcs..compute_totals", "holds the segment ''", id="empty-segment"),
+ pytest.param("funcs.class.compute_totals", "holds the segment 'class'", id="keyword"),
+ pytest.param("my-funcs.compute_totals", "holds the segment 'my-funcs'", id="not-an-identifier"),
+ pytest.param("funcs.five.compute_totals", "changes under NFKC normalization", id="ligature"),
+ ],
+ )
+ def test_a_name_that_is_not_an_import_path_is_refused_as_the_bundle_is_read(self, function_name: str, expected_message: str) -> None:
+ with pytest.raises(ValidationError, match=expected_message):
+ _blueprint(function_name=function_name)
+
+ def test_a_flat_name_is_refused_with_the_migration_command(self) -> None:
+ with pytest.raises(ValidationError) as exc_info:
+ _blueprint(function_name="compute_totals")
+
+ message = str(exc_info.value)
+ assert "pipelex fix function-names <method folder>" in message
+ assert "'funcs.totals.compute_totals'" in message
diff --git a/tests/unit/pipelex/pipe_operators/pipe_func/test_pipe_func_call_context.py b/tests/unit/pipelex/pipe_operators/pipe_func/test_pipe_func_call_context.py
new file mode 100644
index 000000000..25a0fe0c3
--- /dev/null
+++ b/tests/unit/pipelex/pipe_operators/pipe_func/test_pipe_func_call_context.py
@@ -0,0 +1,42 @@
+import asyncio
+
+import pytest
+
+from pipelex.pipe_operators.func.pipe_func_call_context import function_of_call_in_flight, pipe_func_call_in_flight
+
+
+def _compute_totals() -> str:
+ return "totals"
+
+
+def _compute_taxes() -> str:
+ return "taxes"
+
+
+class TestPipeFuncCallContext:
+ def test_the_function_is_named_for_its_own_step_and_only_for_the_scope(self) -> None:
+ assert function_of_call_in_flight(pipe_ref="invoicing.totals") is None
+
+ with pipe_func_call_in_flight(pipe_ref="invoicing.totals", function=_compute_totals):
+ assert function_of_call_in_flight(pipe_ref="invoicing.totals") is _compute_totals
+ assert function_of_call_in_flight(pipe_ref="invoicing.taxes") is None
+ with pipe_func_call_in_flight(pipe_ref="invoicing.taxes", function=_compute_taxes):
+ assert function_of_call_in_flight(pipe_ref="invoicing.taxes") is _compute_taxes
+ assert function_of_call_in_flight(pipe_ref="invoicing.totals") is _compute_totals
+
+ assert function_of_call_in_flight(pipe_ref="invoicing.totals") is None
+
+ @pytest.mark.asyncio
+ async def test_concurrent_calls_each_see_their_own_function(self) -> None:
+ async def _call(*, pipe_ref: str, function: object) -> object:
+ assert callable(function)
+ with pipe_func_call_in_flight(pipe_ref=pipe_ref, function=function):
+ await asyncio.sleep(0)
+ return function_of_call_in_flight(pipe_ref=pipe_ref)
+
+ seen = await asyncio.gather(
+ _call(pipe_ref="invoicing.totals", function=_compute_totals),
+ _call(pipe_ref="invoicing.taxes", function=_compute_taxes),
+ )
+
+ assert seen == [_compute_totals, _compute_taxes]
diff --git a/tests/unit/pipelex/pipe_operators/pipe_func/test_pipe_func_definition_check.py b/tests/unit/pipelex/pipe_operators/pipe_func/test_pipe_func_definition_check.py
new file mode 100644
index 000000000..431129c79
--- /dev/null
+++ b/tests/unit/pipelex/pipe_operators/pipe_func/test_pipe_func_definition_check.py
@@ -0,0 +1,77 @@
+import ast
+
+import pytest
+from pytest_mock import MockerFixture
+
+from pipelex.pipe_operators.func.pipe_func_definition_check import pipe_func_definition_violation
+
+_HEADER = "from pipelex.system.registries.func_registry import pipe_func\n\n\n"
+
+_BODY = "(working_memory):\n return working_memory\n"
+
+_INDENTED_BODY = "(working_memory):\n return working_memory\n"
+
+
+class TestPipeFuncDefinitionCheck:
+ @pytest.mark.parametrize(
+ "source",
+ [
+ pytest.param(f"{_HEADER}@pipe_func()\ndef compute_totals{_BODY}", id="def"),
+ pytest.param(f"{_HEADER}@pipe_func()\nasync def compute_totals{_BODY}", id="async-def"),
+ pytest.param(
+ f"from pipelex.system.registries.func_registry import pipe_func as marker\n\n\n@marker()\ndef compute_totals{_BODY}", id="aliased"
+ ),
+ pytest.param(
+ f"from pipelex.system.registries import func_registry\n\n\n@func_registry.pipe_func()\ndef compute_totals{_BODY}", id="module-import"
+ ),
+ pytest.param(f"import pipelex.system.registries.func_registry as reg\n\n\[email protected]_func()\ndef compute_totals{_BODY}", id="import-as"),
+ pytest.param(
+ f"import functools\n{_HEADER}@functools.cache\n@pipe_func()\ndef compute_totals{_BODY}",
+ id="among-other-decorators",
+ ),
+ ],
+ )
+ def test_a_marked_top_level_function_is_proven(self, source: str) -> None:
+ assert pipe_func_definition_violation(source=source, function="compute_totals", module_label="funcs/totals.py") is None
+
+ @pytest.mark.parametrize(
+ ("source", "expected_message"),
+ [
+ pytest.param("def broken(:\n", "does not parse as Python", id="syntax-error"),
+ pytest.param(f"{_HEADER}@pipe_func()\ndef other{_BODY}", "defines no top-level function 'compute_totals'", id="missing"),
+ pytest.param(f"{_HEADER}def compute_totals{_BODY}", "is not decorated with @pipe_func()", id="unmarked"),
+ pytest.param(f"{_HEADER}@pipe_func\ndef compute_totals{_BODY}", "without parentheses", id="marker-without-parentheses"),
+ pytest.param(f'{_HEADER}@pipe_func(name="totals")\ndef compute_totals{_BODY}', "passes arguments to @pipe_func()", id="marker-name"),
+ pytest.param(
+ f"{_HEADER}@pipe_func()\ndef compute_totals{_BODY}\n\n@pipe_func()\ndef compute_totals{_BODY}",
+ "more than once",
+ id="defined-twice",
+ ),
+ pytest.param(f"{_HEADER}from funcs.impl import compute_totals\n", "an import, which re-exports", id="re-export"),
+ pytest.param(f"{_HEADER}compute_totals = len\n", "an assignment", id="assignment"),
+ pytest.param(f"{_HEADER}class compute_totals:\n pass\n", "a class definition", id="class"),
+ pytest.param(
+ f"{_HEADER}import sys\nif sys.platform:\n @pipe_func()\n def compute_totals{_INDENTED_BODY}",
+ "under a condition",
+ id="conditional-definition",
+ ),
+ pytest.param(
+ f"from somewhere_else import pipe_func\n\n\n@pipe_func()\ndef compute_totals{_BODY}", "is not decorated", id="foreign-marker"
+ ),
+ ],
+ )
+ def test_what_cannot_be_proven_is_refused_naming_the_module(self, source: str, expected_message: str) -> None:
+ violation = pipe_func_definition_violation(source=source, function="compute_totals", module_label="funcs/totals.py")
+
+ assert violation is not None
+ assert expected_message in violation
+ assert "funcs/totals.py" in violation
+
+ def test_a_module_is_parsed_once_for_every_function_checked_in_it(self, mocker: MockerFixture) -> None:
+ source = f"{_HEADER}@pipe_func()\ndef parsed_once_first{_BODY}\n\n@pipe_func()\ndef parsed_once_second{_BODY}"
+ parse_spy = mocker.spy(ast, "parse")
+
+ for function in ("parsed_once_first", "parsed_once_second", "parsed_once_first"):
+ assert pipe_func_definition_violation(source=source, function=function, module_label="funcs/parsed_once.py") is None
+
+ assert parse_spy.call_count == 1
diff --git a/tests/unit/pipelex/pipe_operators/pipe_func/test_pipe_func_execution_transport.py b/tests/unit/pipelex/pipe_operators/pipe_func/test_pipe_func_execution_transport.py
index 94e8be542..f644210c5 100644
--- a/tests/unit/pipelex/pipe_operators/pipe_func/test_pipe_func_execution_transport.py
+++ b/tests/unit/pipelex/pipe_operators/pipe_func/test_pipe_func_execution_transport.py
@@ -7,8 +7,10 @@ must forbid extras (it is a wire contract).
"""
import pytest
+from mthds.method_tree.file_entry import Base64FileEntry
from pydantic import ValidationError
+from pipelex.core.method_trees.tree_files import tree_file_bytes
from pipelex.libraries.library_crate import LibraryCrate
from pipelex.pipe_operators.func.pipe_func_execution_transport import (
DEFAULT_PIPE_FUNC_TIMEOUT_SECONDS,
@@ -38,6 +40,22 @@ class TestPipeFuncExecutionTransport:
assert "working_memory_raw" in PipeFuncExecutionRequest.model_fields
assert PipeFuncExecutionRequest.model_fields["timeout_seconds"].default == DEFAULT_PIPE_FUNC_TIMEOUT_SECONDS
+ def test_request_carries_the_call_files_as_file_entries(self) -> None:
+ """The call's files travel at their tree paths, text as text and other bytes in Base64, and survive a JSON round-trip."""
+ binary = bytes([0, 159, 146, 150])
+ request = PipeFuncExecutionRequest(
+ crate=LibraryCrate(),
+ files={"funcs/totals.py": "TOTAL = 3\n", "assets/logo.bin": Base64FileEntry.make_from_bytes(binary)},
+ working_memory_raw={},
+ pipe_code="my_domain.my_pipe",
+ function_name="funcs.totals.compute_totals",
+ job_metadata=JobMetadata(run_metadata=RunMetadata(storage_scope="test/scope", read_scope=None, user_id="user", pipeline_run_id="run")),
+ pipe_run_params=PipeRunParams(run_mode=PipeRunMode.DRY, pipe_stack_limit=10, batch_max_concurrency=None),
+ )
+ again = PipeFuncExecutionRequest.model_validate_json(request.model_dump_json())
+ assert tree_file_bytes(entry=again.files["funcs/totals.py"]) == b"TOTAL = 3\n"
+ assert tree_file_bytes(entry=again.files["assets/logo.bin"]) == binary
+
def test_request_rejects_non_finite_timeout(self) -> None:
"""A non-finite timeout would disable the runaway-code guard, so the wire contract rejects it."""
with pytest.raises(ValidationError, match="timeout_seconds"):
@@ -45,7 +63,7 @@ class TestPipeFuncExecutionTransport:
crate=LibraryCrate(),
working_memory_raw={},
pipe_code="my_pipe",
- function_name="my_func",
+ function_name="my_funcs.my_func",
job_metadata=JobMetadata(
run_metadata=RunMetadata(storage_scope="test/scope", read_scope=None, user_id="user", pipeline_run_id="run")
),
diff --git a/tests/unit/pipelex/pipe_operators/pipe_func/test_pipe_func_hosted_mode.py b/tests/unit/pipelex/pipe_operators/pipe_func/test_pipe_func_hosted_mode.py
index cb4251e8f..4e0f8728b 100644
--- a/tests/unit/pipelex/pipe_operators/pipe_func/test_pipe_func_hosted_mode.py
+++ b/tests/unit/pipelex/pipe_operators/pipe_func/test_pipe_func_hosted_mode.py
@@ -5,87 +5,120 @@ from pydantic import ValidationError
from pytest_mock import MockerFixture
from pipelex.core.memory.working_memory_factory import WorkingMemoryFactory
+from pipelex.core.method_trees.tree_files import TreeFileEntry
+from pipelex.core.pipes.exceptions import PipeValidationError
from pipelex.core.stuffs.list_content import ListContent
+from pipelex.interpreter_hub import get_library_manager
+from pipelex.libraries.library_crate import SUBMITTED_PYTHON_ROOT_KEY
from pipelex.pipe_machinery.pipe_factory import PipeFactory
+from pipelex.pipe_operators.func import pipe_func as pipe_func_module
from pipelex.pipe_operators.func.pipe_func import PipeFunc
from pipelex.pipe_operators.func.pipe_func_blueprint import PipeFuncBlueprint
from pipelex.pipe_run.pipe_run_params import PipeRunParams
from pipelex.system.job_metadata import JobMetadata, RunMetadata
from pipelex.system.pipe_run_mode import PipeRunMode
+from pipelex.validation_error_types import PipeValidationErrorType
-_UNREGISTERED = PipeFuncBlueprint(
- description="Hosted-mode case: function lives only in the sandbox, never in this process.",
- inputs={},
- output="native.Text",
- function_name="a_function_that_is_not_registered_here",
-)
+_SANDBOX_FUNCTION_NAME = "sandbox_funcs.totals.compute_totals"
+_SANDBOX_MODULE = """\
+from pipelex.core.memory.working_memory import WorkingMemory
+from pipelex.core.stuffs.text_content import TextContent
+from pipelex.system.registries.func_registry import pipe_func
-def _make_pipe_func() -> PipeFunc:
- return PipeFactory[PipeFunc].make_from_blueprint(
- domain_code="hosted_test",
- pipe_code="hosted_pipe",
- blueprint=_UNREGISTERED,
+
+@pipe_func()
+def compute_totals(working_memory: WorkingMemory) -> TextContent:
+ return TextContent(text="total")
+"""
+
+
+def _make_pipe_func(*, output: str = "native.Text") -> PipeFunc:
+ blueprint = PipeFuncBlueprint(
+ description="Hosted-mode case: the function lives only in the files a sandbox receives, never in this process.",
+ inputs={},
+ output=output,
+ function_name=_SANDBOX_FUNCTION_NAME,
)
+ return PipeFactory[PipeFunc].make_from_blueprint(domain_code="hosted_test", pipe_code="hosted_pipe", blueprint=blueprint)
-class TestPipeFuncHostedMode:
- """In sandbox-hosted mode the PipeFunc validators accept a function that is absent from this
- process's func_registry; in local/direct mode the same input still raises.
- """
+def _capture_submitted_files(*, library_id: str, files: dict[str, TreeFileEntry] | None, mocker: MockerFixture) -> None:
+ library_manager = get_library_manager()
+ original = library_manager.get_pipe_func_files
- def test_local_mode_rejects_unregistered_function(self, load_empty_library: Callable[[], str]):
- """Default (local) mode is unchanged: an unregistered function_name fails construction."""
- load_empty_library()
- with pytest.raises(ValidationError, match="not found in registry"):
- _make_pipe_func()
+ def get_pipe_func_files(*, library_id: str, root_key: str) -> dict[str, TreeFileEntry] | None:
+ if root_key == SUBMITTED_PYTHON_ROOT_KEY:
+ return files
+ return original(library_id=library_id, root_key=root_key)
- def test_hosted_mode_accepts_unregistered_function_name(
- self,
- load_empty_library: Callable[[], str],
- mocker: MockerFixture,
- ):
- """Hosted mode skips the func_registry lookup, so the declared name is accepted verbatim."""
- load_empty_library()
+ mocker.patch.object(library_manager, "get_pipe_func_files", side_effect=get_pipe_func_files)
+ assert library_id
+
+
+class TestPipeFuncHostedMode:
+ def test_a_flat_function_name_is_refused_in_every_mode(self, mocker: MockerFixture):
+ """A flat name, the key of the retired registry, is refused as the blueprint is read, naming the migration."""
+ for is_hosted in (False, True):
+ mocker.patch("pipelex.pipe_operators.func.pipe_func.is_pipe_func_sandbox_hosted", return_value=is_hosted)
+ with pytest.raises(ValidationError, match="pipelex fix function-names"):
+ PipeFuncBlueprint(description="A flat name", inputs={}, output="native.Text", function_name="compute_totals")
+
+ def test_hosted_validation_reads_the_submitted_files(self, load_empty_library: Callable[[], str], mocker: MockerFixture):
+ """In sandbox-hosted mode a step parsed from a string is validated from the files submitted with it, never imported."""
+ library_id = load_empty_library()
mocker.patch("pipelex.pipe_operators.func.pipe_func.is_pipe_func_sandbox_hosted", return_value=True)
+ _capture_submitted_files(library_id=library_id, files={"sandbox_funcs/totals.py": _SANDBOX_MODULE}, mocker=mocker)
+ _make_pipe_func().validate_output_with_library()
+
+ def test_a_step_proves_its_function_once_however_often_it_is_validated(self, load_empty_library: Callable[[], str], mocker: MockerFixture):
+ """A dependency package's step is validated as it is built and again with its consumer's library: it reads its module once."""
+ library_id = load_empty_library()
+ mocker.patch("pipelex.pipe_operators.func.pipe_func.is_pipe_func_sandbox_hosted", return_value=True)
+ _capture_submitted_files(library_id=library_id, files={"sandbox_funcs/totals.py": _SANDBOX_MODULE}, mocker=mocker)
+ locate_spy = mocker.spy(pipe_func_module, "locate_in_files")
pipe_func = _make_pipe_func()
- assert pipe_func.function_name == "a_function_that_is_not_registered_here"
+ pipe_func.validate_once_built()
+ pipe_func.validate_with_libraries()
- def test_hosted_mode_skips_output_validation(
+ assert locate_spy.call_count == 1
+
+ @pytest.mark.parametrize(
+ ("files", "expected_message"),
+ [
+ (None, "sandbox_funcs/totals.py"),
+ ({"sandbox_funcs/totals.py": _SANDBOX_MODULE.replace("@pipe_func()\n", "")}, "not decorated with @pipe_func()"),
+ ({"json.py": "X = 1\n", "sandbox_funcs/totals.py": _SANDBOX_MODULE}, "standard-library module 'json'"),
+ ],
+ ids=["module_not_submitted", "function_unmarked", "module_shadows_stdlib"],
+ )
+ def test_hosted_validation_refuses_what_the_submitted_files_cannot_prove(
self,
+ files: dict[str, TreeFileEntry] | None,
+ expected_message: str,
load_empty_library: Callable[[], str],
mocker: MockerFixture,
):
- """validate_output_with_library is a no-op in hosted mode (the sandbox validates for real)."""
- load_empty_library()
+ """A missing module, an unmarked function or a shadowing module is refused at validation, naming the step."""
+ library_id = load_empty_library()
mocker.patch("pipelex.pipe_operators.func.pipe_func.is_pipe_func_sandbox_hosted", return_value=True)
+ _capture_submitted_files(library_id=library_id, files=files, mocker=mocker)
- pipe_func = _make_pipe_func()
- # Must not raise even though the function is not registered and its return type is unknown here.
- pipe_func.validate_output_with_library()
+ with pytest.raises(PipeValidationError) as raised:
+ _make_pipe_func().validate_output_with_library()
+
+ assert raised.value.error_type == PipeValidationErrorType.PIPE_FUNC_FUNCTION_REFUSED
+ assert raised.value.pipe_code == "hosted_pipe"
+ assert expected_message in str(raised.value)
@pytest.mark.asyncio
- async def test_hosted_dry_run_honors_output_multiplicity(
- self,
- load_empty_library: Callable[[], str],
- mocker: MockerFixture,
- ):
- """Hosted dry run must mock a list-shaped output for a multiplicity output like `Text[]`."""
+ async def test_hosted_dry_run_honors_output_multiplicity(self, load_empty_library: Callable[[], str], mocker: MockerFixture):
+ """Hosted dry run must mock a list-shaped output for a multiplicity output like `Text[]`, without importing the function."""
load_empty_library()
mocker.patch("pipelex.pipe_operators.func.pipe_func.is_pipe_func_sandbox_hosted", return_value=True)
- blueprint = PipeFuncBlueprint(
- description="Hosted-mode multiplicity case: the mock output must be a list, not a scalar.",
- inputs={},
- output="native.Text[]",
- function_name="a_function_that_is_not_registered_here",
- )
- pipe_func = PipeFactory[PipeFunc].make_from_blueprint(
- domain_code="hosted_test",
- pipe_code="hosted_multiplicity_pipe",
- blueprint=blueprint,
- )
+ pipe_func = _make_pipe_func(output="native.Text[]")
pipe_output = await pipe_func._dry_run_operator_pipe( # ruff: ignore[private-member-access] # pyright: ignore[reportPrivateUsage]
job_metadata=JobMetadata(run_metadata=RunMetadata(storage_scope="test/scope", read_scope=None, user_id="user", pipeline_run_id="run")),
diff --git a/tests/unit/pipelex/pipe_operators/pipe_func/test_pipe_func_input.py b/tests/unit/pipelex/pipe_operators/pipe_func/test_pipe_func_input.py
index c3f692e43..f02bf7b09 100644
--- a/tests/unit/pipelex/pipe_operators/pipe_func/test_pipe_func_input.py
+++ b/tests/unit/pipelex/pipe_operators/pipe_func/test_pipe_func_input.py
@@ -11,10 +11,10 @@ from tests.unit.pipelex.pipe_operators.pipe_func.data import PipeFuncInputTestCa
class TestPipeFuncValidation:
- """Test PipeFunc with valid, registered, and functioning functions.
+ """Test PipeFunc with valid functions named by import path.
- Error cases (function not found, invalid return types, etc.) are tested
- at the registry level in test_func_registry_utils.py.
+ Error cases (function not found, invalid return types, etc.) are tested where the function is located and
+ checked, in the tests of pipe_func_python and of the definition check.
"""
@pytest.mark.parametrize(
@@ -28,7 +28,7 @@ class TestPipeFuncValidation:
load_test_library: Callable[[list[Path]], None],
):
load_test_library([Path(Path(__file__).parent)])
- """Test that PipeFunc works correctly with valid, registered functions."""
+ """Test that PipeFunc works correctly with valid functions named by import path."""
log.verbose(f"Testing valid case: {test_id}")
pipe_func = PipeFactory[PipeFunc].make_from_blueprint(
diff --git a/tests/unit/pipelex/pipe_operators/pipe_func/test_pipe_func_transported_ref.py b/tests/unit/pipelex/pipe_operators/pipe_func/test_pipe_func_transported_ref.py
index 1329c00a5..df1520eb8 100644
--- a/tests/unit/pipelex/pipe_operators/pipe_func/test_pipe_func_transported_ref.py
+++ b/tests/unit/pipelex/pipe_operators/pipe_func/test_pipe_func_transported_ref.py
@@ -22,11 +22,12 @@ from pipelex.pipe_operators.func.pipe_func_executor_protocol import PipeFuncExec
from pipelex.pipe_run.pipe_run_params import PipeRunParams
from pipelex.system.job_metadata import JobMetadata, RunMetadata
from pipelex.system.pipe_run_mode import PipeRunMode
-from pipelex.system.registries.func_registry import func_registry
+from pipelex.system.registries.func_registry import pipe_func
+@pipe_func()
def transported_ref_probe_func(working_memory: WorkingMemory) -> TextContent: # ruff: ignore[unused-function-argument] # pyright: ignore[reportUnusedParameter]
- """Registered in the test so the PipeFunc validators accept the blueprint in local mode."""
+ """The function the step names by its import path, this test module's own."""
return TextContent(text="probe")
@@ -38,12 +39,11 @@ class TestPipeFuncTransportedRef:
mocker: MockerFixture,
):
load_empty_library()
- func_registry.register_function(transported_ref_probe_func, name="transported_ref_probe_func")
blueprint = PipeFuncBlueprint(
description="Producer-side pin: the executor must receive the qualified ref.",
inputs={},
output="native.Text",
- function_name="transported_ref_probe_func",
+ function_name=f"{__name__}.transported_ref_probe_func",
)
the_pipe_func = PipeFactory[PipeFunc].make_from_blueprint(
domain_code="func_probe_domain",
diff --git a/tests/unit/pipelex/test_serialization_json_schema.py b/tests/unit/pipelex/test_serialization_json_schema.py
index 4d38f097c..770522c8c 100644
--- a/tests/unit/pipelex/test_serialization_json_schema.py
+++ b/tests/unit/pipelex/test_serialization_json_schema.py
@@ -40,12 +40,10 @@ WRAP_SERIALIZER_MODELS = [
ConstructBlueprint,
]
-# `FuncRegistry` holds live callables, which have no JSON Schema at all — pydantic refuses to build
-# one. Pinned by name rather than skipped silently: a model that newly stops being describable drops
+# The models pydantic refuses to describe at all, such as one holding live callables: none today.
+# Pinned by name rather than skipped silently: a model that newly stops being describable drops
# out of the sweep, and this is what makes that visible instead of quietly shrinking the guard.
-MODELS_WITHOUT_A_JSON_SCHEMA = {
- "pipelex.system.registries.func_registry.FuncRegistry",
-}
+MODELS_WITHOUT_A_JSON_SCHEMA: set[str] = set()
REMEDY = (
"Drop the return annotation from the model's `@model_serializer` — pydantic turns it into the "
diff --git a/tests/unit/pipelex/tools/test.mthds b/tests/unit/pipelex/tools/test.mthds
index 7f227ffdf..5d190f480 100644
--- a/tests/unit/pipelex/tools/test.mthds
+++ b/tests/unit/pipelex/tools/test.mthds
@@ -10,4 +10,4 @@ type = "PipeFunc"
description = "Read the content of related codebase files"
inputs = { related_file_paths = "FilePath" }
output = "CodebaseFileContent[]"
-function_name = "read_file_content"
+function_name = "codebase_funcs.read_file_content"
diff --git a/tests/unit/pipelex/tools/test_file_func_registry.py b/tests/unit/pipelex/tools/test_file_func_registry.py
deleted file mode 100644
index 7813a9e69..000000000
--- a/tests/unit/pipelex/tools/test_file_func_registry.py
+++ /dev/null
@@ -1,46 +0,0 @@
-from pydantic import Field
-
-from pipelex.core.memory.working_memory import WorkingMemory
-from pipelex.core.stuffs.list_content import ListContent
-from pipelex.core.stuffs.structured_content import StructuredContent
-from pipelex.system.registries.func_registry import pipe_func
-
-
-class FilePath(StructuredContent):
- """A path to a file in the codebase."""
-
- path: str = Field(description="Path to the file")
-
-
-class CodebaseFileContent(StructuredContent):
- """Content of a codebase file."""
-
- file_path: str = Field(description="Path to the codebase file")
- file_content: str = Field(description="Content of the codebase file")
-
-
-@pipe_func()
-def read_file_content(working_memory: WorkingMemory) -> ListContent[CodebaseFileContent]:
- """Read the content of related codebase files.
-
- Args:
- working_memory: Working memory containing related_file_paths
-
- Returns:
- ListContent of CodebaseFileContent objects
-
- """
- file_paths_list = working_memory.get_stuff_as_list("related_file_paths", item_type=FilePath)
-
- codebase_files: list[CodebaseFileContent] = []
- for file_path in file_paths_list.items:
- try:
- with open(file_path.path, encoding="utf-8") as file:
- content = file.read()
- codebase_files.append(CodebaseFileContent(file_path=file_path.path, file_content=content))
- except Exception as exc:
- codebase_files.append(
- CodebaseFileContent(file_path=file_path.path, file_content=f"# File not found or unreadable: {file_path.path}\n# Error: {exc!s}"),
- )
-
- return ListContent[CodebaseFileContent](items=codebase_files)
diff --git a/tests/unit/pipelex/tools/test_func_registry.py b/tests/unit/pipelex/tools/test_func_registry.py
deleted file mode 100644
index dc352ae07..000000000
--- a/tests/unit/pipelex/tools/test_func_registry.py
+++ /dev/null
@@ -1,196 +0,0 @@
-from collections.abc import Callable
-from typing import Any
-
-import pytest
-
-from pipelex.core.memory.working_memory import WorkingMemory
-from pipelex.core.stuffs.text_content import TextContent
-from pipelex.system.registries.exceptions import FuncRegistryError
-from pipelex.system.registries.func_registry import FuncRegistry
-
-
-def sample_function():
- return "sample"
-
-
-def another_function(x: int):
- return x * 2
-
-
-# Valid function for testing
-def valid_function(working_memory: WorkingMemory) -> TextContent: # ruff: ignore[unused-function-argument] # pyright: ignore[reportUnknownParameterType,reportMissingParameterType, reportUnusedParameter]
- return TextContent(text="test")
-
-
-# Valid async function for testing (should also be eligible)
-async def valid_async_function(working_memory: WorkingMemory) -> TextContent: # ruff: ignore[unused-function-argument, unused-async] # pyright: ignore[reportUnknownParameterType,reportMissingParameterType, reportUnusedParameter]
- return TextContent(text="test")
-
-
-# Invalid functions for testing eligibility
-def wrong_param_name(other_param: WorkingMemory) -> TextContent: # ruff: ignore[unused-function-argument] # pyright: ignore[reportUnknownParameterType,reportMissingParameterType, reportUnusedParameter]
- return TextContent(text="test")
-
-
-def wrong_param_type(working_memory: str) -> TextContent: # ruff: ignore[unused-function-argument] # pyright: ignore[reportUnknownParameterType,reportMissingParameterType, reportUnusedParameter]
- return TextContent(text="test")
-
-
-def wrong_return_type(working_memory: WorkingMemory) -> str: # ruff: ignore[unused-function-argument] # pyright: ignore[reportUnknownParameterType,reportMissingParameterType, reportUnusedParameter]
- return "test"
-
-
-def too_many_params(working_memory: WorkingMemory, extra: str) -> TextContent: # ruff: ignore[unused-function-argument] # pyright: ignore[reportUnknownParameterType,reportMissingParameterType, reportUnusedParameter]
- return TextContent(text="test")
-
-
-def no_params() -> TextContent:
- return TextContent(text="test")
-
-
-def no_type_hints(working_memory): # ruff: ignore[unused-function-argument, missing-type-function-argument] # pyright: ignore[reportUnknownParameterType,reportMissingParameterType, reportUnusedParameter]
- return TextContent(text="test")
-
-
-# Type alias for test parameters
-EligibilityTestCase = tuple[str, Callable[..., Any], bool]
-
-# Test data
-TEST_CASES: list[EligibilityTestCase] = [
- # Valid cases
- ("valid_function", valid_function, True),
- ("valid_async_function", valid_async_function, True),
- # Invalid cases
- ("wrong_param_name", wrong_param_name, False),
- ("wrong_param_type", wrong_param_type, False),
- ("wrong_return_type", wrong_return_type, False),
- ("too_many_params", too_many_params, False),
- ("no_params", no_params, False),
- ("no_type_hints", no_type_hints, False),
- ("sample_function", sample_function, False),
- ("another_function", another_function, False),
-]
-
-
-def registry():
- # Create a new registry for each test
- reg = FuncRegistry()
- yield reg
- # Teardown: clear the registry after each test
- reg.teardown()
-
-
-class TestFuncRegistry:
- @pytest.mark.parametrize(("test_name", "func", "is_eligible"), TEST_CASES)
- def test_function_eligibility_and_registration(self, registry: FuncRegistry, test_name: str, func: Callable[..., Any], is_eligible: bool):
- # Test eligibility check directly
- actual_eligibility = registry.is_eligible_function(func)
- assert actual_eligibility == is_eligible, f"Eligibility check failed for {test_name}: expected {is_eligible}, got {actual_eligibility}"
-
- # Test registration behavior
- if is_eligible:
- # Should register successfully
- registry.register_function(func)
- assert registry.has_function(func.__name__), f"Eligible function {test_name} should be registered"
- assert registry.get_function(func.__name__) is func, f"Should be able to retrieve registered function {test_name}"
- else:
- # Should be silently skipped (no error, no registration)
- registry.register_function(func)
- assert not registry.has_function(func.__name__), f"Ineligible function {test_name} should not be registered"
-
- def test_register_function_with_custom_name(self, registry: FuncRegistry):
- """Test registering a function with a custom name."""
- registry.register_function(valid_function, name="custom_name")
- assert registry.get_function("custom_name") is valid_function
- assert registry.get_function("valid_function") is None
-
- def test_get_required_function_not_found(self, registry: FuncRegistry):
- with pytest.raises(FuncRegistryError, match="not found in registry"):
- registry.get_required_function("non_existent_function")
-
- def test_unregister_function(self, registry: FuncRegistry):
- registry.register_function(valid_function)
- assert registry.has_function("valid_function")
- registry.unregister_function(valid_function)
- assert not registry.has_function("valid_function")
-
- def test_unregister_function_not_found(self, registry: FuncRegistry):
- with pytest.raises(FuncRegistryError, match="not found in registry"):
- registry.unregister_function(sample_function)
-
- def test_unregister_function_by_name(self, registry: FuncRegistry):
- registry.register_function(valid_function, name="custom")
- registry.unregister_function_by_name("custom")
- assert not registry.has_function("custom")
-
- def test_unregister_function_by_name_not_found(self, registry: FuncRegistry):
- with pytest.raises(FuncRegistryError, match="not found in registry"):
- registry.unregister_function_by_name("non_existent")
-
- def test_register_functions_dict_with_ineligible_functions(self, registry: FuncRegistry):
- functions: dict[str, Callable[..., Any]] = {"func1": sample_function, "func2": another_function}
- registry.register_functions_dict(functions)
- # Ineligible functions should be silently skipped
- assert not registry.has_function("func1")
- assert not registry.has_function("func2")
- assert len(registry.root) == 0
-
- def test_register_functions_list_with_ineligible_functions(self, registry: FuncRegistry):
- functions: list[Callable[..., Any]] = [sample_function, another_function]
- registry.register_functions(functions)
- # Ineligible functions should be silently skipped
- assert not registry.has_function("sample_function")
- assert not registry.has_function("another_function")
- assert len(registry.root) == 0
-
- def test_register_functions_empty_list(self, registry: FuncRegistry):
- registry.register_functions([])
- assert len(registry.root) == 0
-
- def test_register_same_function_twice_is_idempotent(self, registry: FuncRegistry):
- """Re-registering the very same function object is a no-op: a folder scanned twice must not fail."""
- registry.register_function(valid_function)
- registry.register_function(valid_function)
- assert registry.get_function("valid_function") is valid_function
-
- def test_register_colliding_function_raises_and_names_both_sources(self, registry: FuncRegistry):
- """Two distinct functions under one name is an error, and the message must name both origins."""
- registry.register_function(valid_function, name="shared_name")
- with pytest.raises(FuncRegistryError) as exc_info:
- registry.register_function(valid_async_function, name="shared_name")
-
- message = str(exc_info.value)
- assert "shared_name" in message
- assert "valid_function" in message
- assert "valid_async_function" in message
- assert valid_function.__module__ in message
- assert '@pipe_func(name="..."' in message
- # The first registration wins: the collision is refused, not applied.
- assert registry.get_function("shared_name") is valid_function
-
- def test_ineligible_function_does_not_collide(self, registry: FuncRegistry):
- """Ineligible functions never enter the registry, so they cannot trigger collision detection."""
- registry.register_function(valid_function, name="shared_name")
- registry.register_function(sample_function, name="shared_name")
- assert registry.get_function("shared_name") is valid_function
-
- def test_teardown(self, registry: FuncRegistry):
- registry.register_function(valid_function)
- assert registry.has_function("valid_function")
- registry.teardown()
- assert not registry.has_function("valid_function")
-
- def test_get_required_function_with_signature(self, registry: FuncRegistry):
- registry.register_function(valid_function)
- func = registry.get_required_function_with_signature("valid_function")
- assert func is valid_function
-
- def test_get_required_function_with_signature_not_found(self, registry: FuncRegistry):
- with pytest.raises(FuncRegistryError, match="not found in registry"):
- registry.get_required_function_with_signature("non_existent")
-
- def test_get_required_function_with_signature_not_callable(self, registry: FuncRegistry):
- registry.root["not_a_function"] = "a string" # type: ignore[assignment]
- with pytest.raises(FuncRegistryError, match="is not a callable function"):
- registry.get_required_function_with_signature("not_a_function")
diff --git a/tests/unit/pipelex/tools/test_func_registry_utils.py b/tests/unit/pipelex/tools/test_func_registry_utils.py
deleted file mode 100644
index 2c25caa52..000000000
--- a/tests/unit/pipelex/tools/test_func_registry_utils.py
+++ /dev/null
@@ -1,327 +0,0 @@
-import tempfile
-from pathlib import Path
-from typing import ClassVar
-
-import pytest
-from pydantic import Field
-
-from pipelex.core.memory.working_memory import WorkingMemory
-from pipelex.core.stuffs.structured_content import StructuredContent
-from pipelex.core.stuffs.text_content import TextContent
-from pipelex.system.registries.exceptions import FuncRegistryError
-from pipelex.system.registries.func_registry import func_registry
-from pipelex.system.registries.func_registry_utils import FuncRegistryUtils
-
-
-class FilePath(StructuredContent):
- path: str = Field(description="Path to the file")
-
-
-class CodebaseFileContent(StructuredContent):
- file_path: str = Field(description="Path to the codebase file")
- file_content: str = Field(description="Content of the codebase file")
-
-
-class TestCases:
- VALID_ASYNC_FUNCTION = """
-from pipelex.system.registries.func_registry import pipe_func
-
-@pipe_func()
-async def read_file_content(working_memory: WorkingMemory) -> ListContent[CodebaseFileContent]:
- '''Read the content of related codebase files.'''
-
- file_paths_list = working_memory.get_stuff_as_list("related_file_paths", item_type=FilePath)
-
- codebase_files: List[CodebaseFileContent] = []
- for file_path in file_paths_list.items:
- try:
- with open(file_path.path, "r", encoding="utf-8") as file:
- content = file.read()
- codebase_files.append(CodebaseFileContent(file_path=file_path.path, file_content=content))
- except Exception as e:
- codebase_files.append(
- CodebaseFileContent(file_path=file_path.path, file_content=f"# File not found or unreadable: {file_path.path}\\n# Error: {str(e)}")
- )
-
- return ListContent[CodebaseFileContent](items=codebase_files)
-"""
-
- VALID_SYNC_FUNCTION = """
-from pipelex.system.registries.func_registry import pipe_func
-
-# Sync function - should be accepted
-@pipe_func()
-def sync_function(working_memory: WorkingMemory) -> StructuredContent:
- '''This should be registered - sync functions are now eligible.'''
- pass
-"""
-
- INVALID_FUNCTIONS = """
-# Wrong parameter name - should be rejected
-def invalid_function_wrong_param_name(other_param: WorkingMemory) -> StructuredContent:
- '''This should not be registered - wrong parameter name.'''
- pass
-
-# Wrong parameter type - should be rejected
-def invalid_function_wrong_param_type(working_memory: str) -> StructuredContent:
- '''This should not be registered - wrong parameter type.'''
- pass
-
-# Wrong return type - should be rejected
-def invalid_function_wrong_return_type(working_memory: WorkingMemory) -> str:
- '''This should not be registered - wrong return type.'''
- return "test"
-
-# No return type annotation - should be rejected
-async def invalid_function_no_return_type(working_memory: WorkingMemory):
- '''This should not be registered - no return type annotation.'''
- return TextContent(text="no return type")
-
-# Wrong return type (not StuffContent) - should be rejected
-async def invalid_function_returns_str(working_memory: WorkingMemory) -> str:
- '''This should not be registered - returns str instead of StuffContent.'''
- return "wrong type"
-
-# Too many parameters - should be rejected
-def invalid_function_too_many_params(working_memory: WorkingMemory, extra_param: str) -> StructuredContent:
- '''This should not be registered - too many parameters.'''
- pass
-
-# No parameters - should be rejected
-def invalid_function_no_params() -> StructuredContent:
- '''This should not be registered - no parameters.'''
- pass
-
-# Missing type hints - should be rejected
-def invalid_function_no_type_hints(working_memory):
- '''This should not be registered - missing type hints.'''
- pass
-"""
-
- TEST_CASES: ClassVar[list[tuple[str, str, list[str], list[str]]]] = [
- ("valid_async_function", VALID_ASYNC_FUNCTION, ["read_file_content"], []),
- ("valid_sync_function", VALID_SYNC_FUNCTION, ["sync_function"], []),
- (
- "invalid_functions",
- INVALID_FUNCTIONS,
- [],
- [
- "invalid_function_wrong_param_name",
- "invalid_function_wrong_param_type",
- "invalid_function_wrong_return_type",
- "invalid_function_no_return_type",
- "invalid_function_returns_str",
- "invalid_function_too_many_params",
- "invalid_function_no_params",
- "invalid_function_no_type_hints",
- ],
- ),
- ]
-
-
-class TestFuncRegistryUtils:
- @pytest.mark.parametrize(
- ("test_name", "function_code", "expected_registered", "expected_not_registered"),
- TestCases.TEST_CASES,
- )
- def test_function_registration_eligibility(
- self,
- test_name: str, # ruff: ignore[unused-method-argument]
- function_code: str,
- expected_registered: list[str],
- expected_not_registered: list[str],
- ):
- # Create a temporary directory and file with test functions
- with tempfile.TemporaryDirectory() as temp_dir:
- test_file = Path(temp_dir) / "test_functions.py"
-
- # Write the complete test file with imports and function code
- full_code = f"""
-from typing import List
-
-from pydantic import Field
-
-from pipelex.core.memory.working_memory import WorkingMemory
-from pipelex.core.stuffs.list_content import ListContent
-from pipelex.core.stuffs.structured_content import StructuredContent
-from pipelex.core.stuffs.text_content import TextContent
-
-
-class FilePath(StructuredContent):
- path: str = Field(description="Path to the file")
-
-
-class CodebaseFileContent(StructuredContent):
- file_path: str = Field(description="Path to the codebase file")
- file_content: str = Field(description="Content of the codebase file")
-
-{function_code}
-"""
-
- test_file.write_text(full_code)
-
- # Clear the registry to start fresh
- func_registry.teardown()
-
- # Test the registration
- FuncRegistryUtils.register_funcs_in_folder(folder_path=Path(temp_dir), is_recursive=False)
-
- # Check that expected functions were registered
- for func_name in expected_registered:
- assert func_registry.has_function(func_name), f"{func_name} should be registered"
-
- # Verify we can get the function
- registered_func = func_registry.get_function(func_name)
- assert registered_func is not None, f"Registered function {func_name} should be retrievable"
- assert registered_func.__name__ == func_name, f"Function name should match {func_name}"
-
- # Check that expected functions were NOT registered
- for func_name in expected_not_registered:
- assert not func_registry.has_function(func_name), f"{func_name} should NOT be registered"
-
- def test_recursive_folder_search(self):
- """Test that recursive folder search works correctly."""
- with tempfile.TemporaryDirectory() as temp_dir:
- # Create nested directory structure
- nested_dir = Path(temp_dir) / "subdir"
- nested_dir.mkdir()
-
- # Create valid async function in root
- root_file = Path(temp_dir) / "root_functions.py"
- root_file.write_text("""
-from pipelex.core.memory.working_memory import WorkingMemory
-from pipelex.core.stuffs.text_content import TextContent
-from pipelex.system.registries.func_registry import pipe_func
-
-@pipe_func()
-async def root_function(working_memory: WorkingMemory) -> TextContent:
- return TextContent(text="root")
-""")
-
- # Create valid async function in subdirectory
- nested_file = nested_dir / "nested_functions.py"
- nested_file.write_text("""
-from pipelex.core.memory.working_memory import WorkingMemory
-from pipelex.core.stuffs.text_content import TextContent
-from pipelex.system.registries.func_registry import pipe_func
-
-@pipe_func()
-async def nested_function(working_memory: WorkingMemory) -> TextContent:
- return TextContent(text="nested")
-""")
-
- # Clear the registry
- func_registry.teardown()
-
- # Test recursive search
- FuncRegistryUtils.register_funcs_in_folder(folder_path=Path(temp_dir), is_recursive=True)
-
- # Both functions should be registered
- assert func_registry.has_function("root_function"), "root_function should be registered"
- assert func_registry.has_function("nested_function"), "nested_function should be registered"
-
- # Clear and test non-recursive search
- func_registry.teardown()
- FuncRegistryUtils.register_funcs_in_folder(folder_path=Path(temp_dir), is_recursive=False)
-
- # Only root function should be registered
- assert func_registry.has_function("root_function"), "root_function should be registered"
- assert not func_registry.has_function("nested_function"), "nested_function should NOT be registered"
-
- def test_same_function_name_in_two_folders_raises(self):
- """Two library dirs defining the same @pipe_func name must fail loudly, not silently pick a winner."""
- source_template = """
-from pipelex.core.memory.working_memory import WorkingMemory
-from pipelex.core.stuffs.text_content import TextContent
-from pipelex.system.registries.func_registry import pipe_func
-
-@pipe_func()
-async def summarize(working_memory: WorkingMemory) -> TextContent:
- return TextContent(text="{flavor}")
-"""
- with tempfile.TemporaryDirectory() as first_dir, tempfile.TemporaryDirectory() as second_dir:
- (Path(first_dir) / "first_funcs.py").write_text(source_template.format(flavor="first"))
- (Path(second_dir) / "second_funcs.py").write_text(source_template.format(flavor="second"))
-
- func_registry.teardown()
- FuncRegistryUtils.register_funcs_in_folder(folder_path=Path(first_dir), is_recursive=False)
- with pytest.raises(FuncRegistryError, match="summarize"):
- FuncRegistryUtils.register_funcs_in_folder(folder_path=Path(second_dir), is_recursive=False)
- func_registry.teardown()
-
- def test_rescanning_the_same_folder_is_idempotent(self):
- """A folder scanned twice reuses the cached module, so the same function object re-registers cleanly."""
- with tempfile.TemporaryDirectory() as temp_dir:
- (Path(temp_dir) / "stable_funcs.py").write_text("""
-from pipelex.core.memory.working_memory import WorkingMemory
-from pipelex.core.stuffs.text_content import TextContent
-from pipelex.system.registries.func_registry import pipe_func
-
-@pipe_func()
-async def stable_function(working_memory: WorkingMemory) -> TextContent:
- return TextContent(text="stable")
-""")
-
- func_registry.teardown()
- FuncRegistryUtils.register_funcs_in_folder(folder_path=Path(temp_dir), is_recursive=False)
- first_registration = func_registry.get_function("stable_function")
- FuncRegistryUtils.register_funcs_in_folder(folder_path=Path(temp_dir), is_recursive=False)
- assert func_registry.get_function("stable_function") is first_registration
- func_registry.teardown()
-
- def test_eligibility_check_directly(self):
- # Valid async function
- async def valid_async_function(working_memory: WorkingMemory) -> TextContent: # ruff: ignore[unused-function-argument, unused-async] # pyright: ignore[reportUnknownParameterType,reportMissingParameterType, reportUnusedParameter]
- return TextContent(text="test")
-
- # Valid sync function
- def valid_sync_function(working_memory: WorkingMemory) -> TextContent: # ruff: ignore[unused-function-argument] # pyright: ignore[reportUnknownParameterType,reportMissingParameterType, reportUnusedParameter]
- return TextContent(text="test")
-
- # Test eligibility
- assert func_registry.is_eligible_function(valid_async_function), "Valid async function should be eligible"
- assert func_registry.is_eligible_function(valid_sync_function), "Valid sync function should be eligible"
-
- def test_register_function_checks_eligibility(self):
- func_registry.teardown()
-
- # Valid async function
- async def valid_async_function(working_memory: WorkingMemory) -> TextContent: # ruff: ignore[unused-function-argument, unused-async] # pyright: ignore[reportUnknownParameterType,reportMissingParameterType, reportUnusedParameter]
- return TextContent(text="valid")
-
- # Valid sync function
- def valid_sync_function(working_memory: WorkingMemory) -> TextContent: # ruff: ignore[unused-function-argument] # pyright: ignore[reportUnknownParameterType,reportMissingParameterType, reportUnusedParameter]
- return TextContent(text="valid")
-
- # Try to register both functions
- func_registry.register_function(valid_async_function)
- func_registry.register_function(valid_sync_function)
-
- # Both functions should be registered
- assert func_registry.has_function("valid_async_function"), "Valid async function should be registered"
- assert func_registry.has_function("valid_sync_function"), "Valid sync function should be registered"
-
- # Test register_functions method as well
- func_registry.teardown()
-
- async def another_valid_async_function(working_memory: WorkingMemory) -> TextContent: # ruff: ignore[unused-function-argument, unused-async] # pyright: ignore[reportUnknownParameterType,reportMissingParameterType, reportUnusedParameter]
- return TextContent(text="another_valid_async")
-
- def another_valid_sync_function(working_memory: WorkingMemory) -> TextContent: # ruff: ignore[unused-function-argument] # pyright: ignore[reportUnknownParameterType,reportMissingParameterType, reportUnusedParameter]
- return TextContent(text="another_valid_sync")
-
- # Invalid function (wrong parameter name)
- def invalid_function(other_param: WorkingMemory) -> TextContent: # ruff: ignore[unused-function-argument] # pyright: ignore[reportUnknownParameterType,reportMissingParameterType, reportUnusedParameter]
- return TextContent(text="invalid")
-
- # Register multiple functions at once - invalid functions should be silently skipped
- func_registry.register_functions([invalid_function, another_valid_async_function, another_valid_sync_function])
-
- # Valid functions should be registered, invalid function silently skipped
- assert func_registry.has_function("another_valid_async_function"), "Valid async function should be registered"
- assert func_registry.has_function("another_valid_sync_function"), "Valid sync function should be registered"
- assert not func_registry.has_function("invalid_function"), "Invalid function should NOT be registered"
-
-
-if __name__ == "__main__":
- pytest.main([__file__])
diff --git a/tests/unit/pipelex/tools/test_read_py_sources.py b/tests/unit/pipelex/tools/test_read_py_sources.py
deleted file mode 100644
index 25e39e98e..000000000
--- a/tests/unit/pipelex/tools/test_read_py_sources.py
+++ /dev/null
@@ -1,36 +0,0 @@
-import sys
-from pathlib import Path
-
-from pipelex.system.registries.func_registry_utils import FuncRegistryUtils
-
-
-class TestReadPySources:
- """read_py_sources captures every .py as text, recursively, WITHOUT ever importing it."""
-
- def test_captures_nested_sources_without_importing(self, tmp_path: Path):
- """Nested .py files are captured by POSIX relpath, and no file is imported.
-
- Each fixture file imports a module that does not exist: if read_py_sources imported any of
- them, the import would raise ModuleNotFoundError. It must not — it only reads the bytes.
- """
- top_source = "import this_module_does_not_exist_top # noqa\n\nTOP = 1\n"
- nested_source = "import this_module_does_not_exist_nested # noqa\n\nNESTED = 2\n"
- (tmp_path / "top.py").write_text(top_source, encoding="utf-8")
- nested_dir = tmp_path / "structures"
- nested_dir.mkdir()
- (nested_dir / "item.py").write_text(nested_source, encoding="utf-8")
-
- sources = FuncRegistryUtils.read_py_sources(folder_path=tmp_path)
-
- assert sources == {
- "top.py": top_source,
- "structures/item.py": nested_source,
- }
- # No import happened: importing either fixture file would have raised on its phantom import,
- # and the phantom module names are absent from sys.modules.
- assert "this_module_does_not_exist_top" not in sys.modules
- assert "this_module_does_not_exist_nested" not in sys.modules
-
- def test_empty_folder_returns_empty(self, tmp_path: Path):
- """A folder with no Python files yields an empty mapping."""
- assert FuncRegistryUtils.read_py_sources(folder_path=tmp_path) == {}