Skip to content

Build AI methods with your coding agent, then run them anywhere: as an MCP for chatbots, as a webapp for people, or via API for your software.

feature/Loads-leave-event-loop · L-261017-b7684d (#1467)

Louis Choquela84c88538442
diff --git a/.drift/acks/cli-docs.toml b/.drift/acks/cli-docs.toml
index c53336bc4..90c78c2f6 100644
--- a/.drift/acks/cli-docs.toml
+++ b/.drift/acks/cli-docs.toml
@@ -1,8 +1,8 @@
contract = "cli-docs"
-digest = "sha256:8e42deecf8cc65eebf459049b3eb92d8d3b00c711db16e359080f7630ef788c4"
-reviewed_by = "Claude Opus 5.5"
-reviewed_at = "2026-10-20T04:12:48Z"
-rationale = "real-catch: _validate_core.py now splats tree_files_read and tree_file_readers into the validate bundle and validate method envelopes; docs/tools/cli/agent-cli.md gained a 'The files each step reads' note and pipelex/cli/agent_cli/CLAUDE.md's validate bullet names both members, their Markdown section and that validate all and validate pipe carry neither; nothing else under docs/tools/cli/ describes the validate envelope's members."
+digest = "sha256:5a0be307b82eceee7cc18c0127e0e3df9db4a382c534b61bf96b0cda39bddc8c"
+reviewed_by = "Louis Choquel"
+reviewed_at = "2026-10-20T05:51:05Z"
+rationale = "clean-pass: since the last ack the CLI triggers only switch their off-the-loop helper from run_in_worker_thread to run_method_load, the bounded method-load pool; no flag, output, exit code or error format changes, and neither docs/tools/cli/ nor pipelex/cli/agent_cli/CLAUDE.md names the helper or describes where a command's load runs, so nothing there is stale."
[trigger_files]
"pipelex/cli/__init__.py" = "blob:8b137891791fe96927ad78e64b0aad7bded08bdc"
@@ -45,7 +45,7 @@ rationale = "real-catch: _validate_core.py now splats tree_files_read and tree_f
"pipelex/cli/agent_cli/commands/run/pipe_cmd.py" = "blob:1a5a2f86ac80fb86cb53376202bad05ffd19e9f6"
"pipelex/cli/agent_cli/commands/run/stdin_resolver.py" = "blob:db0176d7de1e074fd6eefd787c10e71b891eeb32"
"pipelex/cli/agent_cli/commands/validate/__init__.py" = "blob:e69de29bb2d1d6434b8b29ae775ad8c2e48c5391"
-"pipelex/cli/agent_cli/commands/validate/_validate_core.py" = "blob:b30afc76d88860d0d34c00eb8bebbad7d54bd498"
+"pipelex/cli/agent_cli/commands/validate/_validate_core.py" = "blob:f74178959782fe6e8edc2b7171b9967d82cf2076"
"pipelex/cli/agent_cli/commands/validate/app.py" = "blob:1982684a2aae81e0d634bb3b262457a5f187d979"
"pipelex/cli/agent_cli/commands/validate/bundle_cmd.py" = "blob:3c7045c77c15b2ab5fdb1629a4203e53231b764d"
"pipelex/cli/agent_cli/commands/validate/method_cmd.py" = "blob:19cbf94ca5027983e5fd95b73bc346efaf195c37"
@@ -56,13 +56,13 @@ rationale = "real-catch: _validate_core.py now splats tree_files_read and tree_f
"pipelex/cli/commands/build/__init__.py" = "blob:e69de29bb2d1d6434b8b29ae775ad8c2e48c5391"
"pipelex/cli/commands/build/app.py" = "blob:d3205e1dc0268a9d32a517e4f9a9a3b4647e8889"
"pipelex/cli/commands/build/inputs/__init__.py" = "blob:e69de29bb2d1d6434b8b29ae775ad8c2e48c5391"
-"pipelex/cli/commands/build/inputs/_inputs_core.py" = "blob:30246cd2cdafd7c885c5f635181d743d075c5362"
+"pipelex/cli/commands/build/inputs/_inputs_core.py" = "blob:537f644babc852328dfef0bfd744c5b9ecf889bd"
"pipelex/cli/commands/build/inputs/app.py" = "blob:553e285ae7d1a9bf356b98ec268ba44bb1c30244"
"pipelex/cli/commands/build/inputs/bundle_cmd.py" = "blob:bd26ee8bd660ae5c09a7fa8a00720930c9854749"
"pipelex/cli/commands/build/inputs/method_cmd.py" = "blob:2fcaea4f05df4947d6ec1a49b2fa23b52c23ca3c"
"pipelex/cli/commands/build/inputs/pipe_cmd.py" = "blob:00625740fd29837e3165d31be692e40e96065fc8"
"pipelex/cli/commands/build/output/__init__.py" = "blob:e69de29bb2d1d6434b8b29ae775ad8c2e48c5391"
-"pipelex/cli/commands/build/output/_output_core.py" = "blob:e8a78b33c7113afe192ab9ba4d2680c35be6bcb0"
+"pipelex/cli/commands/build/output/_output_core.py" = "blob:e7d4e762ea4c384ada3a39a82316b900f841d58f"
"pipelex/cli/commands/build/output/app.py" = "blob:2bd9575e2093f13a1b5240f49b6f5a6a88bd1fee"
"pipelex/cli/commands/build/output/bundle_cmd.py" = "blob:0209b48f1c221fae59cf5bede2fbd92558a87f56"
"pipelex/cli/commands/build/output/method_cmd.py" = "blob:242434d518aff0b9706b910e7f937b53a601fde1"
@@ -111,7 +111,7 @@ rationale = "real-catch: _validate_core.py now splats tree_files_read and tree_f
"pipelex/cli/commands/run/_inputs_file_loader.py" = "blob:8039ca6b29f4f9091d0ea7406c8b8a4590848131"
"pipelex/cli/commands/run/_inputs_path_resolver.py" = "blob:c8ec89d98718e18af7aaaded44c9f545be60adab"
"pipelex/cli/commands/run/_main_stuff_file.py" = "blob:ca806e55085aa2e5748492751521296ce1f42960"
-"pipelex/cli/commands/run/_run_core.py" = "blob:633b707d1ae9bf6a08735f3acbd0c2a9ba415329"
+"pipelex/cli/commands/run/_run_core.py" = "blob:56cff4673ad0e30e87b25dada61515f3188feb0d"
"pipelex/cli/commands/run/_run_hosted.py" = "blob:f61b3ea7fdc6c29823ecd6cb5b469ab271814c3f"
"pipelex/cli/commands/run/_run_output_folder.py" = "blob:5834889a5e35e6457acf24761b408bc6a7bbfd6c"
"pipelex/cli/commands/run/app.py" = "blob:e312b159625e0a72bc9a18d3bfd2c73e7ac59f24"
@@ -123,7 +123,7 @@ rationale = "real-catch: _validate_core.py now splats tree_files_read and tree_f
"pipelex/cli/commands/update_cmd.py" = "blob:b2db6170ddd9c67f17d88096ba1f1796a897557d"
"pipelex/cli/commands/usage_stats_cmd.py" = "blob:b3723c64f911d1b69e1e1aa126610a132a16e3da"
"pipelex/cli/commands/validate/__init__.py" = "blob:e69de29bb2d1d6434b8b29ae775ad8c2e48c5391"
-"pipelex/cli/commands/validate/_validate_core.py" = "blob:735fb75881bee14bb120a3a7f7fe7746307ad30a"
+"pipelex/cli/commands/validate/_validate_core.py" = "blob:01c16592bd98d4b03e1a36c92699bb25fe0f4a84"
"pipelex/cli/commands/validate/app.py" = "blob:457e110f0458bbe042a5489f58bba91acd885ebd"
"pipelex/cli/commands/validate/bundle_cmd.py" = "blob:1bf408a2d49b92476c87fa080b850b82cc870a88"
"pipelex/cli/commands/validate/method_cmd.py" = "blob:f12ee233fc985d04a4512f1fef6b9ea02a1b9644"
diff --git a/CHANGELOG.md b/CHANGELOG.md
index ce1513b6c..8bae3ab20 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -9,10 +9,12 @@
### Changed
- **The API server's examples send a method in `files`**: the server's README and the API Server pages send a method to `/v1/execute` and `/v1/validate` in `files`, a map from each file's path in the method's folder to its content, where they sent `mthds_contents`. The reference pages and the OpenAPI artifact now say that `mthds_contents`, with `bundle_b64`, `resources` and `/v1/validate`'s `mthds_sources`, is being retired and will be refused by a later release, so a client sends `files` now.
+- **A PipeFunc package-offer factory is never called on a thread running an event loop (Breaking)**: the factory a plugin registers with `PluginRegistrar.add_pipe_func_package_offer` may block for as long as its work takes and must not use an event loop of its caller's, and pipelex enforces it: a load that would call the factory on a thread running an event loop fails with `PipelexUnexpectedError` instead, so a host that calls a load itself from asynchronous code, such as a worker's activity, moves it off the loop with `run_method_load` (`pipelex.libraries.method_load_threads`) or with `acquire_library_async` (`pipelex.pipeline.execution_seams`), the twin of `acquire_library` that loads off the loop. The factory's signature is unchanged, and so is the plugin API version. `run_method_load` runs a blocking call in the method-load pool, a named pool of at most `METHOD_LOAD_MAX_THREADS` threads (four) apart from asyncio's default executor, in a copy of the caller's context, and when its caller is cancelled, the shutdown of `asyncio.run` included, withdraws a call no thread has taken yet or lets the cancellation through only once the running call has ended; the general mechanism is `WorkerThreads` (`pipelex.tools.misc.async_utils`).
- **The refusal of a package the request did not send**: the message of the `unresolved_package_dependency` refusal a static resolution gives, such as the one behind `/v1/resolve`, `/v1/codegen` and `/v1/pipe-io`, tells its reader to send the package's files in `dependencies`, keyed by its address, beside the method's own files in `files`, where it spoke of an in-memory resolve and the host filesystem and advised directory-based resolution; it still carries the words `address-based dependency '<address>'`.
### Fixed
+- **Method loads leave the event loop**: every method load pipelex reaches from asynchronous code, a run's, a validation's and those of the commands that run a loop, runs in the method-load pool, where it ran on the event loop, the reading and parsing of the bundles it loads included, so listing and hashing a method's files, a sandbox's judgment of its `requirements.txt` and a fetch of a package it calls by address no longer hold up the other requests or activities the loop serves; the API server's run routes and `/v1/validate` also write a request's method tree to disk and remove it there, and judge its requirements there, so a client that disconnects never has the tree removed while the check still reads it, the crate routes resolve and project there, and the in-process validator reads a tree's entry pipe for its graph there too. A module imported by file while a method loads, such as one defining a structure class, is published and executed under a lock that the load's class discovery also holds, so two loads in parallel never take a module half executed, and a module that fails to execute leaves `sys.modules` rather than stay there half imported. `/v1/validate` of a method tree calling a package nothing shipped still resolves it as a run does, from the methods installed on the server and by a fetch on a miss, so its verdict matches the run's, and the fetch now runs off the event loop.
- **Where a method's Python travels, in the docs**: the PipeFunc Requirements, PipeFunc and run pages say that the API server receives a method as a tree holding its bundles and its Python together and judges its `requirements.txt` from the tree's listing, that the runner never imports a method's Python, that a run's crate names each step's method tree by its digest and carries none of its files, that a `PipeFunc` call's files are read through the method file store the worker's host installs, and that a worker a method tree's validation is dispatched to checks the method's requirements again, where they said the server received a method's Python apart from its bundles, the Python was captured as text, the crate carried the method's Python sources and the worker never checked anything.
### Removed
diff --git a/api/pipelex_api/method_tree_request.py b/api/pipelex_api/method_tree_request.py
index 0f96d28c9..89888dc68 100644
--- a/api/pipelex_api/method_tree_request.py
+++ b/api/pipelex_api/method_tree_request.py
@@ -36,8 +36,8 @@ import base64
import json
import shutil
import tempfile
-from collections.abc import Generator, Mapping
-from contextlib import ExitStack, contextmanager
+from collections.abc import AsyncGenerator, Generator, Mapping
+from contextlib import ExitStack, asynccontextmanager, contextmanager
from pathlib import Path
from typing import Any, NamedTuple, NoReturn, cast
@@ -56,6 +56,7 @@ from pipelex.cli.installed_methods import PROVENANCE_FILENAME
from pipelex.config import is_pipe_func_sandbox_hosted
from pipelex.core.bundle_resources.bundle_resource import SUBMITTED_FILES_FIELD, root_escape_of
from pipelex.errors.registry.pipelex_codes import PipelexErrorCode
+from pipelex.libraries.method_load_threads import run_method_load
from pipelex.libraries.written_trees import WRITTEN_METHOD_TREES, write_method_tree
from pipelex.methods.package_reason import MethodPackageReason
from pipelex.methods.structures_check import ensure_no_structured_content_in_library_sources, ensure_no_structured_content_python
@@ -778,6 +779,29 @@ class WrittenMethod(NamedTuple):
tree: MethodTree
+def _write_method(*, request_method: RequestMethod, releases: ExitStack) -> WrittenMethod:
+ """Write a method's trees into a temporary directory and hold each as written, each release pushed on `releases`.
+
+ The removal of the directory is pushed before anything is written, and each holding as soon as it is taken, so
+ closing `releases` undoes exactly what was done, however far the writing got. Blocking work.
+ """
+ temp_dir = Path(tempfile.mkdtemp(prefix="pipelex-method-"))
+ releases.callback(shutil.rmtree, temp_dir, ignore_errors=True)
+ method_dir = temp_dir / "tree"
+ written = write_method_tree(contents=request_method.tree.contents, destination=method_dir, expected_digest=request_method.tree.tree.digest)
+ releases.enter_context(WRITTEN_METHOD_TREES.holding(tree=written))
+ methods_dirs: list[Path] = []
+ for index_dependency, (address, dependency) in enumerate(request_method.dependencies.items()):
+ methods_dir = temp_dir / f"methods-{index_dependency}"
+ dependency_dir = methods_dir / dependency_dir_name(address=address)
+ dependency_written = write_method_tree(contents=dependency.contents, destination=dependency_dir, expected_digest=dependency.tree.digest)
+ releases.enter_context(WRITTEN_METHOD_TREES.holding(tree=dependency_written))
+ if dependency.has_python and is_pipe_func_sandbox_hosted():
+ ensure_no_structured_content_python(package_dir=dependency_dir, package_address=address)
+ methods_dirs.append(methods_dir)
+ return WrittenMethod(method_dir=method_dir, methods_dirs=methods_dirs or None, tree=written)
+
+
@contextmanager
def written_method(*, request_method: RequestMethod) -> Generator[WrittenMethod, None, None]:
"""Write a method's trees into a temporary directory, hold each as written, and remove them all on the way out.
@@ -786,20 +810,25 @@ def written_method(*, request_method: RequestMethod) -> Generator[WrittenMethod,
`<name>/` in a methods directory of its own, the layout the engine's `methods_dirs` take, so two dependencies sharing
a name never meet. On a sandbox-hosted deployment, a dependency tree declaring a Python structure class is refused,
naming it by its address, as a fetched package is (`MethodStructuresRefusedError`, a 422).
+
+ Writing and removing the files is blocking work, done on the calling thread: asynchronous code uses
+ `written_method_async`.
"""
- with ExitStack() as stack:
- temp_dir = Path(tempfile.mkdtemp(prefix="pipelex-method-"))
- stack.callback(shutil.rmtree, temp_dir, ignore_errors=True)
- method_dir = temp_dir / "tree"
- written = write_method_tree(contents=request_method.tree.contents, destination=method_dir, expected_digest=request_method.tree.tree.digest)
- stack.enter_context(WRITTEN_METHOD_TREES.holding(tree=written))
- methods_dirs: list[Path] = []
- for index_dependency, (address, dependency) in enumerate(request_method.dependencies.items()):
- methods_dir = temp_dir / f"methods-{index_dependency}"
- dependency_dir = methods_dir / dependency_dir_name(address=address)
- dependency_written = write_method_tree(contents=dependency.contents, destination=dependency_dir, expected_digest=dependency.tree.digest)
- stack.enter_context(WRITTEN_METHOD_TREES.holding(tree=dependency_written))
- if dependency.has_python and is_pipe_func_sandbox_hosted():
- ensure_no_structured_content_python(package_dir=dependency_dir, package_address=address)
- methods_dirs.append(methods_dir)
- yield WrittenMethod(method_dir=method_dir, methods_dirs=methods_dirs or None, tree=written)
+ with ExitStack() as releases:
+ yield _write_method(request_method=request_method, releases=releases)
+
+
+@asynccontextmanager
+async def written_method_async(*, request_method: RequestMethod) -> AsyncGenerator[WrittenMethod, None]:
+ """`written_method` for asynchronous code: the trees are written, and removed, in the method-load pool, off the event loop.
+
+ The directory is removed and its trees released whatever happens, a refusal while writing, a failure of the block
+ and a cancellation included. `run_method_load` lets a cancellation through only once its call has ended, or at once
+ when no thread has taken the call yet, so the removal never runs while the writing is still at work, and a
+ cancellation repeated during the removal waits for it too.
+ """
+ releases = ExitStack()
+ try:
+ yield await run_method_load(_write_method, request_method=request_method, releases=releases)
+ finally:
+ await run_method_load(releases.close)
diff --git a/api/pipelex_api/routes/pipelex/crate_ops.py b/api/pipelex_api/routes/pipelex/crate_ops.py
index 7468530a2..398693213 100644
--- a/api/pipelex_api/routes/pipelex/crate_ops.py
+++ b/api/pipelex_api/routes/pipelex/crate_ops.py
@@ -22,7 +22,6 @@ worker thread (`project_resolved_method`), never on the event loop: a slow resol
requests, and the current library, a context variable, is bound, read and cleared in that one thread's context.
"""
-import asyncio
from collections.abc import Callable
from enum import StrEnum
from typing import Literal, NamedTuple, NoReturn, TypeVar
@@ -34,6 +33,7 @@ from mthds.protocol.problem.registry import CoreCode
from pipelex.errors.problem import ProblemView
from pipelex.interpreter_hub import clear_current_library, get_current_library_id_or_none, get_library_manager, get_required_entry_pipe
from pipelex.libraries.library_crate import LibraryCrate
+from pipelex.libraries.method_load_threads import run_method_load
from pipelex.libraries.pipe.exceptions import EntryPipeAmbiguousError, EntryPipeNotFoundError, PipeNotFoundError
from pipelex.libraries.pipe.pipe_suggestions import MAIN_PIPE_DECLARATION
from pipelex.methods.method_ref import looks_like_method_ref
@@ -160,11 +160,11 @@ async def project_resolved_method(
"""Resolve the method the request names into its normalized crate, and return what `project` makes of it.
The method is read as `/validate` reads it (`select_request_method`), then written out as a run writes it and loaded
- alone as the caller's own library directory, its dependency trees as the packages it ships, in a worker thread.
- `project` runs in that thread with the freshly opened library loaded and current, so it can read live pipes, and the
- written tree held; both are released when it returns or raises. A method that does not resolve returns its invalid
- verdict instead, the `ValidateBundleError` carrying the structured validation items, with `project` never called and
- nothing left loaded.
+ alone as the caller's own library directory, its dependency trees as the packages it ships, in a worker thread of the
+ method-load pool (`run_method_load`). `project` runs in that thread with the freshly opened library loaded and
+ current, so it can read live pipes, and the written tree held; both are released when it returns or raises. A
+ method that does not resolve returns its invalid verdict instead, the `ValidateBundleError` carrying the structured
+ validation items, with `project` never called and nothing left loaded.
Raises (as problem documents): a 422 `request.unsupported` for a registry-form `method_ref`; the refusals of
`select_request_method`; the address-form fetch and location failures (`MethodRefError` subclasses); and whatever
@@ -184,7 +184,7 @@ async def project_resolved_method(
selected = await select_request_method(
request=request, method_ref=request_data.method_ref, resources=request_data.resources, read_scope=request_data.read_scope
)
- return await asyncio.to_thread(
+ return await run_method_load(
_project_written_method,
request_method=selected.request_method,
method_label=selected.method_label,
diff --git a/api/pipelex_api/routes/pipelex/pipeline.py b/api/pipelex_api/routes/pipelex/pipeline.py
index 042697c6d..4693c0b11 100644
--- a/api/pipelex_api/routes/pipelex/pipeline.py
+++ b/api/pipelex_api/routes/pipelex/pipeline.py
@@ -16,6 +16,7 @@ from pipelex.core.pipes.pipe_output import PipeOutput
from pipelex.errors.bundle_validation_failure import BundleValidationFailure
from pipelex.errors.problem import ProblemView
from pipelex.errors.registry.pipelex_codes import PipelexErrorCode
+from pipelex.libraries.method_load_threads import run_method_load
from pipelex.libraries.method_requirements import ensure_served_method_requirements_met
from pipelex.pipe_run.delivery_assignment import DeliveryAssignment, StorageTarget, WebhookTarget
from pipelex.pipe_run.exceptions import AsyncExecutionNotEnabledError
@@ -52,7 +53,7 @@ from pipelex_api.method_tree_request import (
select_tree_entry,
tree_documents_of,
with_resources,
- written_method,
+ written_method_async,
)
from pipelex_api.middleware import request_id_of
from pipelex_api.openapi_responses import (
@@ -560,7 +561,7 @@ class ApiRunner(PipelexMTHDSProtocol):
This runner's `library_dirs` are the directories it wrote the request's method tree into, the caller's own,
whose load reports a requirements refusal as the caller's too; judging it here first refuses the run before
any library opens or any run is registered, naming no path on this host. The tree is held as written, so its
- listing is the tree the request sent. The listing reads and hashes files, so it runs off the event loop.
+ listing is the tree the request sent. The listing reads and hashes files, so it runs off the event loop, in the method-load pool.
Raises:
MethodRequirementsFileError: The method's requirements.txt holds a line the standard refuses.
@@ -571,7 +572,9 @@ class ApiRunner(PipelexMTHDSProtocol):
if not self.library_dirs:
return
library_dirs = [Path(library_dir) for library_dir in self.library_dirs]
- await asyncio.to_thread(ensure_served_method_requirements_met, library_dirs=library_dirs)
+ # In the method-load pool, whose cancellation waits for the check to end: the directories it lists are removed
+ # when the request's scope exits, which a client disconnect can make happen while the check still hashes them.
+ await run_method_load(ensure_served_method_requirements_met, library_dirs=library_dirs)
async def validate_verdict(
self,
@@ -876,7 +879,7 @@ async def _request_method_of(*, run_request: RunRequest, body: bytes, blob_store
async def _run_source(*, run_request: RunRequest, body: bytes, blob_scope: str) -> AsyncGenerator[_ResolvedRunSource, None]:
"""Resolve the request's run source: a method tree (`files`, `bundle_b64` or a `method_ref`), or the inline contents.
- A method tree is written into a temporary directory and held as written, then handed to the runner as its one
+ A method tree is written into a temporary directory in a worker thread and held as written, then handed to the runner as its one
library directory, the caller's own, with the dependency trees it carries as `methods_dirs`: the load reads the
tree whole, its `.mthds` files as the method's bundles and every other file as one a step may read, such as a
`PipeDocGen` template or a `PipeFunc` module, and names each by its tree path. A refusal while loading it is the
@@ -914,7 +917,7 @@ async def _run_source(*, run_request: RunRequest, body: bytes, blob_scope: str)
)
return
pipe_code = run_request.pipe_code or select_tree_entry(tree=request_method.tree, method_label=method_label).pipe_ref
- with written_method(request_method=request_method) as written:
+ async with written_method_async(request_method=request_method) as written:
yield _ResolvedRunSource(
mthds_contents=None,
library_dirs=[str(written.method_dir)],
diff --git a/api/pipelex_api/routes/pipelex/validate.py b/api/pipelex_api/routes/pipelex/validate.py
index 8bdce5fa1..21ff9c6bd 100644
--- a/api/pipelex_api/routes/pipelex/validate.py
+++ b/api/pipelex_api/routes/pipelex/validate.py
@@ -21,7 +21,7 @@ from pipelex_api.bundle_resources import decode_bundle_resources
from pipelex_api.exception_handlers import problem_response
from pipelex_api.exceptions import RequestFieldConflictError
from pipelex_api.json_body import JsonBodyRoute
-from pipelex_api.method_tree_request import RequestMethod, select_tree_entry, written_method
+from pipelex_api.method_tree_request import RequestMethod, select_tree_entry, written_method_async
from pipelex_api.middleware import request_id_of
from pipelex_api.openapi_responses import PROBLEM_403_READ_SCOPE, PROBLEM_404_METHOD_PACKAGE, PROBLEM_500_HELD_BLOBS, PROBLEM_501_RUN_POLICY
from pipelex_api.problem_view import problem_view_of
@@ -429,7 +429,7 @@ async def validate_mthds(request: Request, request_data: ValidateRequest) -> JSO
entry = _optional_tree_entry(request_method=request_method, method_label=method_label)
tree_default_pipe_ref = entry.pipe_ref if entry is not None else None
default_pipe_ref_known = True
- with written_method(request_method=request_method) as written:
+ async with written_method_async(request_method=request_method) as written:
verdict = await ApiRunner(
library_dirs=[str(written.method_dir)],
library_dirs_are_callers=True,
diff --git a/api/tests/unit/test_loads_off_the_event_loop.py b/api/tests/unit/test_loads_off_the_event_loop.py
new file mode 100644
index 000000000..8dd5e94fd
--- /dev/null
+++ b/api/tests/unit/test_loads_off_the_event_loop.py
@@ -0,0 +1,174 @@
+import threading
+from collections.abc import Generator, Mapping
+from pathlib import Path
+from typing import Any
+
+import pytest
+from fastapi import FastAPI
+from fastapi.testclient import TestClient
+from mthds.method_tree.models import MethodTree
+from mthds.protocol.problem.registry import CoreCode
+from pipelex.config import get_config
+from pipelex.libraries.method_load_threads import METHOD_LOAD_THREADS
+from pipelex.libraries.written_trees import WRITTEN_METHOD_TREES, write_method_tree
+from pipelex.methods.exceptions import MethodFetchError
+from pipelex.methods.fetching import FetchedMethodPackage
+from pipelex.methods.method_ref import MethodRef
+from pipelex.plugins.pipe_func_package_offer_registry import (
+ PipeFuncPackageOffer,
+ PipeFuncPackageOfferRegistry,
+ PipeFuncPackageOfferRequest,
+ PipeFuncPackageRefusal,
+)
+from pytest_mock import MockerFixture
+
+from pipelex_api.api_config import ApiConfig
+from pipelex_api.exception_handlers import register_exception_handlers
+from pipelex_api.routes import router as api_router
+from tests.unit._constants import VENDORED_PROBE_ADDRESS, VENDORED_PROBE_FILES
+
+_PIPELINE_NS = "pipelex_api.routes.pipelex.pipeline"
+
+_ECHO_MTHDS = """\
+domain = "echo_demo"
+main_pipe = "echo"
+
+[pipe.echo]
+type = "PipeCompose"
+description = "Echo the data"
+inputs = { data = "Text" }
+output = "Text"
+template = "$data"
+"""
+
+# A method whose tree holds Python and a requirements.txt, so its every load asks the package-offer factory.
+_TREE_WITH_REQUIREMENTS = {
+ "main.mthds": _ECHO_MTHDS,
+ "helpers/constants.py": "ANSWER = 42\n",
+ "requirements.txt": "pandas>=2\n",
+}
+
+
+def _in_the_method_load_pool(thread_name: str) -> bool:
+ """Whether a call ran in a thread of the method-load pool, which runs no event loop."""
+ return thread_name.startswith(METHOD_LOAD_THREADS.name)
+
+
+class _RecordingOffer:
+ """The package-offer factory of the neutral `sandbox` mode, noting, at each call, whether its thread runs an event loop."""
+
+ def __init__(self) -> None:
+ self.thread_names: list[str] = []
+
+ def __call__(self, request: PipeFuncPackageOfferRequest) -> PipeFuncPackageOffer | PipeFuncPackageRefusal | None: # noqa: ARG002
+ self.thread_names.append(threading.current_thread().name)
+ return PipeFuncPackageOffer(
+ environment="the sandbox under test",
+ distributions={"pandas": "2.2.3"},
+ marker_environment={"python_version": "3.12", "sys_platform": "linux"},
+ )
+
+
[email protected](name="recording_offer")
+def recording_offer_fixture(mocker: MockerFixture) -> Generator[_RecordingOffer, None, None]:
+ """Run PipeFunc steps in the neutral `sandbox` mode, whose plugin's factory records the thread of each call."""
+ offer = _RecordingOffer()
+ mocker.patch(
+ "pipelex.libraries.package_environments.get_pipe_func_package_offer_registry", return_value=PipeFuncPackageOfferRegistry({"sandbox": offer})
+ )
+ pipe_func_config = get_config().interpreter.pipe_func
+ previous = pipe_func_config.execution_mode
+ pipe_func_config.execution_mode = "sandbox"
+ try:
+ yield offer
+ finally:
+ pipe_func_config.execution_mode = previous
+
+
+def _build_client(mocker: MockerFixture, *, tmp_path: Path) -> TestClient:
+ """The routes behind the production handlers, validating and running in-process, with no installed method to find."""
+ mocker.patch("pipelex.cli.installed_methods.GLOBAL_METHODS_DIR", tmp_path / "global-methods")
+ mocker.patch("pipelex.cli.installed_methods.PROJECT_METHODS_DIR", tmp_path / "project-methods")
+ mocker.patch(
+ f"{_PIPELINE_NS}.get_api_config", return_value=ApiConfig(orchestration_mode="direct", allow_request_orchestration_mode_override=False)
+ )
+ app = FastAPI()
+ app.include_router(api_router, prefix="/v1")
+ register_exception_handlers(app)
+ return TestClient(app)
+
+
+class TestLoadsOffTheEventLoop:
+ @pytest.mark.parametrize(
+ ("route", "body"),
+ [
+ ("/v1/validate", {"files": {"main.mthds": _ECHO_MTHDS}}),
+ ("/v1/execute", {"files": {"main.mthds": _ECHO_MTHDS}, "inputs": {"data": "hello"}}),
+ ],
+ )
+ def test_the_request_tree_is_written_off_the_event_loop_and_removed(
+ self, mocker: MockerFixture, tmp_path: Path, route: str, body: dict[str, Any]
+ ):
+ """The tree a request sends is written to disk in a worker thread, and its directory is gone once the route answers."""
+ client = _build_client(mocker, tmp_path=tmp_path)
+ writes: list[tuple[Path, str]] = []
+
+ def write_after_noting_the_thread(*, contents: Mapping[str, bytes], destination: Path, expected_digest: str) -> MethodTree:
+ writes.append((destination, threading.current_thread().name))
+ return write_method_tree(contents=contents, destination=destination, expected_digest=expected_digest)
+
+ mocker.patch("pipelex_api.method_tree_request.write_method_tree", side_effect=write_after_noting_the_thread)
+
+ response = client.post(route, json=body)
+
+ assert response.status_code == 200, response.text
+ ((method_dir, writing_thread),) = writes
+ assert _in_the_method_load_pool(writing_thread), writing_thread
+ assert not method_dir.parent.exists()
+ assert WRITTEN_METHOD_TREES.tree_written_at(directory=method_dir) is None
+
+ def test_a_validation_asks_the_package_offer_off_the_event_loop(self, mocker: MockerFixture, tmp_path: Path, recording_offer: _RecordingOffer):
+ """The runner's check of the served method asks the factory first, then the in-process validator's load does."""
+ client = _build_client(mocker, tmp_path=tmp_path)
+
+ response = client.post("/v1/validate", json={"files": _TREE_WITH_REQUIREMENTS})
+
+ assert response.status_code == 200, response.text
+ assert response.json()["is_valid"] is True, response.text
+ assert len(recording_offer.thread_names) > 1, recording_offer.thread_names
+ assert all(_in_the_method_load_pool(thread_name) for thread_name in recording_offer.thread_names), recording_offer.thread_names
+
+ def test_a_run_asks_the_package_offer_off_the_event_loop(self, mocker: MockerFixture, tmp_path: Path, recording_offer: _RecordingOffer):
+ """The runner's check of the served method asks the factory first, then the run's own load does."""
+ client = _build_client(mocker, tmp_path=tmp_path)
+
+ response = client.post("/v1/execute", json={"files": _TREE_WITH_REQUIREMENTS, "inputs": {"data": "hello"}})
+
+ assert response.status_code == 200, response.text
+ assert len(recording_offer.thread_names) > 1, recording_offer.thread_names
+ assert all(_in_the_method_load_pool(thread_name) for thread_name in recording_offer.thread_names), recording_offer.thread_names
+
+ def test_a_validation_fetches_an_unshipped_dependency_off_the_event_loop(self, mocker: MockerFixture, tmp_path: Path):
+ """A tree calling a package nothing shipped resolves as its run would, by a fetch on a miss, in a worker thread."""
+ client = _build_client(mocker, tmp_path=tmp_path)
+ mocker.patch("pipelex.methods.fetch_on_miss.is_method_fetch_on_miss_enabled", return_value=True)
+ fetching_threads: list[str] = []
+
+ def fail_the_fetch_after_noting_the_thread(*, ref: MethodRef, dest_dir: Path, refuse_structures: bool) -> FetchedMethodPackage: # noqa: ARG001
+ fetching_threads.append(threading.current_thread().name)
+ msg = "The clone failed."
+ raise MethodFetchError(msg).for_method_ref(method_ref=ref.ref_str)
+
+ mocker.patch("pipelex.methods.fetch_on_miss.fetch_method_package", side_effect=fail_the_fetch_after_noting_the_thread)
+ files = {tree_path: content for tree_path, content in VENDORED_PROBE_FILES.items() if not tree_path.startswith(".mthds/")}
+
+ response = client.post("/v1/validate", json={"files": files})
+
+ assert response.status_code == 200, response.text
+ verdict = response.json()
+ assert verdict["is_valid"] is False, verdict
+ assert [(item["code"], item["params"]["name"]) for item in verdict["errors"]] == [
+ (CoreCode.METHOD_DEPENDENCY_MISSING, VENDORED_PROBE_ADDRESS)
+ ], verdict
+ assert len(fetching_threads) == 1
+ assert _in_the_method_load_pool(fetching_threads[0]), fetching_threads
diff --git a/api/tests/unit/test_written_method_async.py b/api/tests/unit/test_written_method_async.py
new file mode 100644
index 000000000..2de3c919c
--- /dev/null
+++ b/api/tests/unit/test_written_method_async.py
@@ -0,0 +1,145 @@
+import asyncio
+import threading
+from collections.abc import Mapping
+from pathlib import Path
+
+import pytest
+from mthds.method_tree.models import MethodTree
+from pipelex.libraries.method_load_threads import METHOD_LOAD_THREADS
+from pipelex.libraries.written_trees import WRITTEN_METHOD_TREES, write_method_tree
+from pytest_mock import MockerFixture
+
+from pipelex_api.method_tree_request import RequestMethod, RequestTree, written_method_async
+
+#: How long a test waits for a thread to reach a point it signals, which bounds a regression's run.
+_SIGNAL_TIMEOUT_SECONDS = 10.0
+
+_ECHO_MTHDS = b"""\
+domain = "echo_demo"
+main_pipe = "echo"
+
+[pipe.echo]
+type = "PipeCompose"
+description = "Echo the data"
+inputs = { data = "Text" }
+output = "Text"
+template = "$data"
+"""
+
+_REQUEST_METHOD = RequestMethod(tree=RequestTree.make_from_contents(contents={"main.mthds": _ECHO_MTHDS}), dependencies={})
+
+
+def _in_the_method_load_pool(thread_name: str) -> bool:
+ """Whether a call ran in a thread of the method-load pool, which runs no event loop."""
+ return thread_name.startswith(METHOD_LOAD_THREADS.name)
+
+
+class _RecordingWrite:
+ """`write_method_tree`, noting each destination and the thread it ran on, then writing for real.
+
+ Once ``hold`` is called, a write signals ``started`` and blocks until ``release`` is set, as a large tree's does.
+ """
+
+ def __init__(self) -> None:
+ self.destinations: list[Path] = []
+ self.writing_threads: list[str] = []
+ self.started = threading.Event()
+ self.release = threading.Event()
+ self.release.set()
+
+ def hold(self) -> None:
+ self.release.clear()
+
+ def __call__(self, *, contents: Mapping[str, bytes], destination: Path, expected_digest: str) -> MethodTree:
+ self.destinations.append(destination)
+ self.writing_threads.append(threading.current_thread().name)
+ self.started.set()
+ if not self.release.wait(timeout=_SIGNAL_TIMEOUT_SECONDS):
+ msg = "the held write was never released"
+ raise AssertionError(msg)
+ return write_method_tree(contents=contents, destination=destination, expected_digest=expected_digest)
+
+
[email protected](name="recording_write")
+def recording_write_fixture(mocker: MockerFixture) -> _RecordingWrite:
+ recording = _RecordingWrite()
+ mocker.patch("pipelex_api.method_tree_request.write_method_tree", side_effect=recording)
+ return recording
+
+
+def _assert_removed_and_released(*, method_dir: Path) -> None:
+ """The temporary directory the method was written into is gone, and no tree written there is held."""
+ assert not method_dir.parent.exists()
+ assert WRITTEN_METHOD_TREES.tree_written_at(directory=method_dir) is None
+
+
[email protected](loop_scope="class")
+class TestWrittenMethodAsync:
+ async def test_the_tree_is_written_off_the_event_loop_held_for_the_block_and_removed_after(self, recording_write: _RecordingWrite) -> None:
+ async with written_method_async(request_method=_REQUEST_METHOD) as written:
+ assert written.methods_dirs is None
+ assert (written.method_dir / "main.mthds").read_bytes() == _ECHO_MTHDS
+ assert WRITTEN_METHOD_TREES.tree_written_at(directory=written.method_dir) is written.tree
+
+ assert recording_write.destinations == [written.method_dir]
+ assert len(recording_write.writing_threads) == 1
+ assert _in_the_method_load_pool(recording_write.writing_threads[0]), recording_write.writing_threads
+ _assert_removed_and_released(method_dir=written.method_dir)
+
+ async def test_a_failing_block_still_removes_the_tree(self, recording_write: _RecordingWrite) -> None: # noqa: ARG002
+ method_dirs: list[Path] = []
+
+ async def fail_inside_the_block() -> None:
+ async with written_method_async(request_method=_REQUEST_METHOD) as written:
+ method_dirs.append(written.method_dir)
+ msg = "the block failed"
+ raise ValueError(msg)
+
+ with pytest.raises(ValueError, match="the block failed"):
+ await fail_inside_the_block()
+
+ (method_dir,) = method_dirs
+ _assert_removed_and_released(method_dir=method_dir)
+
+ async def test_a_cancelled_block_still_removes_the_tree(self, recording_write: _RecordingWrite) -> None: # noqa: ARG002
+ method_dirs: list[Path] = []
+ entered = asyncio.Event()
+
+ async def hold_the_method_until_cancelled() -> None:
+ async with written_method_async(request_method=_REQUEST_METHOD) as written:
+ method_dirs.append(written.method_dir)
+ entered.set()
+ await asyncio.Event().wait()
+
+ holder = asyncio.create_task(hold_the_method_until_cancelled())
+ await asyncio.wait_for(entered.wait(), timeout=_SIGNAL_TIMEOUT_SECONDS)
+ holder.cancel()
+ with pytest.raises(asyncio.CancelledError):
+ await holder
+
+ (method_dir,) = method_dirs
+ _assert_removed_and_released(method_dir=method_dir)
+
+ async def test_a_cancellation_during_the_write_removes_the_tree_once_the_write_has_ended(self, recording_write: _RecordingWrite) -> None:
+ """A thread cannot be interrupted, so the directory is removed only once the write is over, never under it."""
+ recording_write.hold()
+
+ async def hold_the_method() -> None:
+ async with written_method_async(request_method=_REQUEST_METHOD):
+ pytest.fail("the block ran although the write was cancelled")
+
+ holder = asyncio.create_task(hold_the_method())
+ assert await asyncio.to_thread(recording_write.started.wait, _SIGNAL_TIMEOUT_SECONDS), "the write never started"
+ (method_dir,) = recording_write.destinations
+ holder.cancel()
+ await asyncio.sleep(0.05)
+ assert not holder.done(), "the cancellation went through while the write was still running"
+ assert method_dir.parent.exists(), "the directory was removed while the write was still running"
+
+ recording_write.release.set()
+ with pytest.raises(asyncio.CancelledError):
+ await holder
+
+ assert len(recording_write.writing_threads) == 1
+ assert _in_the_method_load_pool(recording_write.writing_threads[0]), recording_write.writing_threads
+ _assert_removed_and_released(method_dir=method_dir)
diff --git a/docs/api-server/codegen.md b/docs/api-server/codegen.md
index 364f08041..fa2bfe87f 100644
--- a/docs/api-server/codegen.md
+++ b/docs/api-server/codegen.md
@@ -23,7 +23,7 @@ Providing neither or both is a request-shape `422`. Beside `files`, the request
**Python follows the run routes' policy.** A projection runs nothing, but loading a method's library on a deployment that is not sandbox-hosted imports each of its `.py` files that declares a structure class (a `StructuredContent` subclass), which runs that file's module-level code in the server's process. So there a tree, sent or fetched, that holds any `.py` file is refused with a `501` `pipelex.deployment.custom_code_requires_sandbox` before anything is written. On a sandbox-hosted deployment, a tree whose Python declares a structure class is refused with a `422` problem, `pipelex.deployment.python_structures_refused`, as on a [run](pipe-run.md#sending-a-method-as-a-tree).
-**A call by address resolves only from what was shipped.** A package the method calls by its address is loaded from the packages the request sends in `dependencies`, or that a fetched package ships under its `.mthds/methods/`, and from nowhere else: these routes never read a method installed on the runner and never fetch one. A call into a package nothing shipped is therefore the invalid verdict, one `method.dependency_missing` item per package, located on the pipe that calls it, naming the package's address as its `name` with the `reason` `not_found`. The method is written out, loaded, projected and released in a worker thread, so a slow load never holds up the server's other requests.
+**A call by address resolves only from what was shipped.** A package the method calls by its address is loaded from the packages the request sends in `dependencies`, or that a fetched package ships under its `.mthds/methods/`, and from nowhere else: these routes never read a method installed on the runner and never fetch one. A call into a package nothing shipped is therefore the invalid verdict, one `method.dependency_missing` item per package, located on the pipe that calls it, naming the package's address as its `name` with the `reason` `not_found`. The method is written out, loaded, projected and released in a worker thread of the bounded method-load pool, so a slow load never holds up the server's other requests.
## Resolve
diff --git a/docs/api-server/pipe-run.md b/docs/api-server/pipe-run.md
index f9484ae96..abd621ac6 100644
--- a/docs/api-server/pipe-run.md
+++ b/docs/api-server/pipe-run.md
@@ -248,7 +248,7 @@ The older `mthds_contents`, which carries only `.mthds` text, and `resources`, w
**One rule for every path.** The map is the tree: the runner reads it with the standard's reader and holds every key to the tree's one rule, whatever form the method came in. A key that is not a tree path (absolute, with a `.` or `..` segment, an empty segment, a backslash, a colon), a key the membership rule keeps out of every tree (a dotfile or a file under a dot directory such as `.mthds/`, `.git/` or `.env`, `__pycache__/`, `node_modules/`), one key given twice, two keys naming one file once normalized, or a file that is another entry's directory is a `422` `method.package_invalid` whose `reason` is the standard's refusal code, such as `invalid_tree_path`, `excluded_name`, `duplicate_key` or `path_collision`, and whose `path` is the first name it refuses; a key that would leave the tree's root, absolute or climbing with `..`, is the security refusal `security.input_path_refused`, naming the `rule` it broke, never the path. A malformed entry, such as a size written `13.0`, is a `422` `request.invalid` naming the field. A zip's member names are judged by the same rule, so a name given twice in an archive is a duplicate key.
-**The tree is written whole and loaded as the caller's.** The runner writes the tree into a temporary directory, exactly as the map holds it, and loads it as one library directory that belongs to the caller: every bundle is a member of the tree and is named by its tree path, every other file is one a step may read where the bundle sits, a `PipeDocGen` template beside its bundle or a `PipeFunc` module at the tree's import root, and a refusal of the load is the caller's `422`, or an `is_valid: false` verdict on `/validate`, naming files by their tree paths and never by the directory on the server. The directory is removed once the request returns, and the run's job names the tree by its digest, so an orchestrator that runs the job remotely reads its files from the deployment's method file store.
+**The tree is written whole and loaded as the caller's.** The runner writes the tree into a temporary directory, exactly as the map holds it, and loads it as one library directory that belongs to the caller: every bundle is a member of the tree and is named by its tree path, every other file is one a step may read where the bundle sits, a `PipeDocGen` template beside its bundle or a `PipeFunc` module at the tree's import root, and a refusal of the load is the caller's `422`, or an `is_valid: false` verdict on `/validate`, naming files by their tree paths and never by the directory on the server. The directory is written in a worker thread of the bounded method-load pool, off the server's event loop, and removed there once the request returns, whether it succeeded, failed or was cancelled. The run's job names the tree by its digest, so an orchestrator that runs the job remotely reads its files from the deployment's method file store.
**The entry pipe.** With no `pipe_code`, the run starts at the pipe the MTHDS protocol's rule names: the `main_pipe` of the `METHODS.toml` at the tree's root, in the bundle of a domain exporting it, or, in a tree with no manifest, the one `main_pipe` its bundles declare. A manifest declaring none, or bundles declaring none or several, is a `422` `request.pipe_unspecified`, which names the bundles' `main_pipe`s as its `candidates` when they declare several; name the pipe with `pipe_code` instead.
@@ -304,7 +304,7 @@ bundle.mthds
Two shipped packages a reference could not tell apart, whose manifests declare the same address and name compared without regard to case, the name being the directory's when the manifest has none, are a `422` `method.package_invalid` with the reason `dependency_address_collision`, whose `path` is the second package's directory, since nothing but their order would decide which one answers. In an archive, a package directory without a valid `METHODS.toml` is a `422` `method.package_invalid` with the reason `manifest_missing` or `manifest_invalid`, naming the manifest's path, and any other entry under `.mthds/`, such as a file directly under `.mthds/methods/`, one under a directory whose name no package's directory can take, such as a dot directory, or a package's own `.mthds/` store, is refused by the tree's rule; a package fetched by `method_ref` holds its `.mthds/methods/` to the same rule, its other entries left out of its tree. A package whose manifest declares no name takes the last segment of its address, which must then be a name a directory can take: a dependency keyed by an address ending in `.` or `..` with a nameless manifest is a `422` `method.package_invalid` with the reason `package_name_invalid`.
-The validation of a method sent as a tree, or fetched by `method_ref`, loads its shipped packages too, so `/validate` resolves a dependency the run would resolve.
+The validation of a method sent as a tree, or fetched by `method_ref`, resolves the packages it calls as the run does, its shipped packages first, then the methods installed on the server and a fetch on a miss, so `/validate` resolves a dependency the run would resolve (see [Validating a method tree](pipe-validate.md#validating-a-method-tree)). Both write the method to disk and load it in worker threads of the bounded method-load pool, off the server's event loop, so a large tree or a slow load never holds up the server's other requests.
---
diff --git a/docs/api-server/pipe-validate.md b/docs/api-server/pipe-validate.md
index b5b1269fc..e40f841e0 100644
--- a/docs/api-server/pipe-validate.md
+++ b/docs/api-server/pipe-validate.md
@@ -176,6 +176,8 @@ The `mthds_contents` path carries bundle text, not file paths, so by default the
A method sent in `files`, or fetched by `method_ref`, is validated whole, as it would run: the runner reads the tree under the standard's rule, writes it into a temporary directory, and validates it as one library directory of the caller's, with the packages it ships in `dependencies` (or, for a `method_ref` package, under its own `.mthds/methods/<name>/`) loaded beside it, so a call into one resolves as on a run. Every bundle is named by its tree path, so `errors[].location.file` names the owning file, and no item names a path on the server.
- **Python is checked by syntax.** Every `.py` file of the tree is read and compiled, never imported or run, whether or not a step names it: a file that does not parse, or is not UTF-8, makes the verdict `is_valid: false` with an item whose `location.file` is the file's tree path and whose `location.line` is the line of the error. A `PipeFunc` step's function is then located in the tree and its form checked statically. As on a run, a tree holding Python is refused with a `501` `pipelex.deployment.custom_code_requires_sandbox` on a deployment that is not sandbox-hosted.
+- **A call by address resolves as on a run.** A package the method calls by its address is searched for in the packages the request ships, then among the methods installed on the runner, and is fetched when it is in neither and the runner fetches on a miss, as a run of the same method finds it, so the verdict says what that run would do. This is deliberate, and unlike the crate routes, [`/resolve` and `/codegen`](codegen.md) and [`/pipe-io`](pipe-io.md), which resolve only from what was shipped: their artifacts describe the method as sent, while a verdict answers whether it runs here. A package found nowhere, or whose fetch fails, makes the verdict `is_valid: false`, with a `method.dependency_missing` item naming the package's address as its `name`.
+- **The tree is written and loaded off the event loop.** Writing the tree to disk, removing it afterwards, finding its entry pipe for the graph and every load of the validation run in worker threads of the bounded method-load pool, so a slow step, such as a large tree, a fetch on a miss or a sandbox's judgment of the method's `requirements.txt`, never holds up the server's other requests.
- **The default pipe and the graph follow the tree's entry pipe.** `default_pipe_ref` is the pipe the protocol's rule names, the `main_pipe` of the root `METHODS.toml` or the one `main_pipe` its bundles declare, and the graph is drawn from it. A tree with no entry pipe validates with neither `default_pipe_ref` nor `graph_spec`.
- **Refusals of the tree itself produce no verdict**: a key the tree's rule refuses is a **422** `method.package_invalid` naming the rule as its `reason`, or `security.input_path_refused` for a key leaving the tree's root, a malformed entry a **422** `request.invalid`, a hash the runner does not hold a **422** `request.blobs_missing`, and a tree past its limits a **422** `method.package_invalid` naming the limit, as on a [run](pipe-run.md#sending-a-method-as-a-tree).
- **The `mthds_contents` echo** holds the text of the tree's own bundles.
diff --git a/docs/under-the-hood/codegen-projections.md b/docs/under-the-hood/codegen-projections.md
index 5da15fc52..b06b02ac8 100644
--- a/docs/under-the-hood/codegen-projections.md
+++ b/docs/under-the-hood/codegen-projections.md
@@ -171,7 +171,7 @@ The remaining findings are artifacts of *how* you lint, not of file content. Non
The same engine backs the `pipelex-api` routes (`POST /v1/resolve`, `POST /v1/codegen` and `POST /v1/pipe-io` — the route envelopes are documented on the API server's [Resolve & Codegen](../api-server/codegen.md) and [Pipe I/O](../api-server/pipe-io.md) pages). Host-facing cores make that possible without any CLI plumbing:
-- **`pipelex.pipeline.resolve_bundle.resolve_crate_from_method_dir`** resolves a method's tree the host received and wrote out, which is how the three routes resolve the tree a request sends in `files` or a `method_ref` fetches: the directory is loaded alone as the caller's library directory, exactly as `validate_bundle` loads a `method_dir`, so each bundle is named by its tree path while the host holds the tree in `WRITTEN_METHOD_TREES`, the other files are the ones its steps read, and the packages it ships (laid out like `.mthds/methods/`) are the only place an address-based reference is looked up: the resolution is static, so it reads no method installed on the host and fetches nothing, and a call into a package nothing shipped is an `unresolved_package_dependency` item of the verdict. It shares `validate_bundle`'s `translate_to_validate_bundle_error` cascade, so an invalid library raises the one shared `ValidateBundleError` and a resolve verdict cannot drift from a validate verdict, and its **loaded-on-success contract**: the library is left loaded and current for the host to read live pipes from, and the host owns its teardown. Resolution is static (no dry-run sweep), matching `pipelex resolve`. The API server runs it in a worker thread, with the projection and the teardown.
+- **`pipelex.pipeline.resolve_bundle.resolve_crate_from_method_dir`** resolves a method's tree the host received and wrote out, which is how the three routes resolve the tree a request sends in `files` or a `method_ref` fetches: the directory is loaded alone as the caller's library directory, exactly as `validate_bundle` loads a `method_dir`, so each bundle is named by its tree path while the host holds the tree in `WRITTEN_METHOD_TREES`, the other files are the ones its steps read, and the packages it ships (laid out like `.mthds/methods/`) are the only place an address-based reference is looked up: the resolution is static, so it reads no method installed on the host and fetches nothing, and a call into a package nothing shipped is an `unresolved_package_dependency` item of the verdict. It shares `validate_bundle`'s `translate_to_validate_bundle_error` cascade, so an invalid library raises the one shared `ValidateBundleError` and a resolve verdict cannot drift from a validate verdict, and its **loaded-on-success contract**: the library is left loaded and current for the host to read live pipes from, and the host owns its teardown. Resolution is static (no dry-run sweep), matching `pipelex resolve`. The API server runs it in a worker thread of the method-load pool (`run_method_load`), with the projection and the teardown.
- **`build_stamped_projection`** (above) gives the host the stamped artifact set plus the lock as pure content. A client that writes the served files and lock verbatim reproduces a local run byte-for-byte — the offline `codegen check` passes on the written tree exactly as it would locally. There is deliberately no server-side check route: the check is offline by design.
## Surfacing imprecision, never guessing
diff --git a/docs/under-the-hood/error-model.md b/docs/under-the-hood/error-model.md
index c40ad42ea..fe882666a 100644
--- a/docs/under-the-hood/error-model.md
+++ b/docs/under-the-hood/error-model.md
@@ -209,7 +209,7 @@ Besides `category` and `message`, each item carries whatever identity fields its
**The unknown model is the worked example.** A pipe whose model field names a handle, an alias, a preset or a waterfall the model deck does not define is refused when the pipe is built: the operator's deck check raises `ModelChoiceNotFoundError`, and the operator raises it again as a `PipeOperatorModelChoiceError` located on the pipe and on the field, to which the load adds the file. Every pipe type that names a model (`PipeLLM`, `PipeStructure`, `PipeImgGen`, `PipeExtract`, `PipeSearch`) does this through the same `PipeOperator.locating_model_choice` wrapper, so each gives the same item: `category: pipe_validation`, `error_type: unknown_model`, the `pipe_code`, `domain_code` and `source`, the `field_name` and a `field_path` of `pipe.<code>.model` (`pipe.<code>.model_to_structure` for a `PipeLLM`'s structuring model), the `model_reference` exactly as written, the `model_type` (`llm`, `text_extractor`, `img_gen` or `search`), and the deck's `suggestions` of the same kind, each spelled as a reference the field accepts. The suggestions stay in the item's `message` too, for consumers that keep only the message. When the deck offers exactly one suggestion, the item carries an `unsafe` `rename-model` fix, a `remap_value` of the field from the reference as written to that suggestion. It is unsafe because the suggestion is a fuzzy match over the deck's names, which can be a different model altogether, so `pipelex fix bundle` never applies it on its own; an author or an agent applies it deliberately. An inline setting table (`model = { model = "…", temperature = 0.2 }`) is not looked up in the deck, so it is not refused here. A default the deck names itself, which a `PipeJudge` naming no model resolves as its method loads, is not the method's: the wrapper lets its `ModelDeckChoiceUnresolvedError` through unchanged, the deployment's fault and no verdict item. On a run the same refusal stops the bundle before any pipe runs, as the same `unknown_model` item (see the next paragraph).
-**A run refuses an invalid bundle with the same verdict.** A run loads its bundle before any pipe runs, and the run path's `acquire_library` performs that load inside the same `translate_to_validate_bundle_error`, so a bundle the load refuses raises the `ValidateBundleError` with exactly the items validating it gives, before any inference is spent. `ValidateBundleError` carries `method.invalid`, so a hosted run route answers it with a 422 whose problem carries one `errors` item per validation item, where each refusal used to reach the host in its raw class: a misspelled concept as a `ConceptLibraryError` answered 500, a wiring mismatch as a `PipeValidationError` answered by the catch-all 500, a TOML fault as a 422 with no items, and a check firing inside a pipe's pydantic validator as a `runtime` `PipeExecutionError` reading "Input validation failed". The translation's own exceptions hold on this path too: an unknown entry pipe still raises its `PipeNotFoundError`, and a fault of the tool or its environment still propagates as itself. Two things differ from the validate path, both about whose files are loaded. The run path threads no source of its own onto its contents, so their items carry no `source`, even on a local run of a bundle file. A package the bundle depends on by address is loaded from the host's install directory within the same translation, as it is on the in-memory validate path, and a refusal inside it names the package by its address, never that directory. And the library directories load translated only when they are the caller's own, as on a local CLI run (`library_dirs_are_callers`); a host's directories (installed libraries, a temporary directory of shipped Python) load untranslated, because a fault there is not the caller's to fix: a refusal there is the `HostLibraryLoadRefusalError` described above, `pipelex.config.invalid`. The local `pipelex run` renders the verdict with the grouped panel `pipelex validate` prints, and `pipelex-agent run` copies its `validation_errors` into the error envelope.
+**A run refuses an invalid bundle with the same verdict.** A run loads its bundle before any pipe runs, and the run path's `acquire_library_async`, the twin of `acquire_library` that loads in a worker thread, performs that load inside the same `translate_to_validate_bundle_error`, so a bundle the load refuses raises the `ValidateBundleError` with exactly the items validating it gives, before any inference is spent. `ValidateBundleError` carries `method.invalid`, so a hosted run route answers it with a 422 whose problem carries one `errors` item per validation item, where each refusal used to reach the host in its raw class: a misspelled concept as a `ConceptLibraryError` answered 500, a wiring mismatch as a `PipeValidationError` answered by the catch-all 500, a TOML fault as a 422 with no items, and a check firing inside a pipe's pydantic validator as a `runtime` `PipeExecutionError` reading "Input validation failed". The translation's own exceptions hold on this path too: an unknown entry pipe still raises its `PipeNotFoundError`, and a fault of the tool or its environment still propagates as itself. Two things differ from the validate path, both about whose files are loaded. The run path threads no source of its own onto its contents, so their items carry no `source`, even on a local run of a bundle file. A package the bundle depends on by address is loaded from the host's install directory within the same translation, as it is on the in-memory validate path, and a refusal inside it names the package by its address, never that directory. And the library directories load translated only when they are the caller's own, as on a local CLI run (`library_dirs_are_callers`); a host's directories (installed libraries, a temporary directory of shipped Python) load untranslated, because a fault there is not the caller's to fix: a refusal there is the `HostLibraryLoadRefusalError` described above, `pipelex.config.invalid`. The local `pipelex run` renders the verdict with the grouped panel `pipelex validate` prints, and `pipelex-agent run` copies its `validation_errors` into the error envelope.
**Signatures are never an error.** An unimplemented `PipeSignature` reached during validation is a *runnability fact*, not a validation failure: the validator no longer raises on it. The assembled library's outstanding signatures ride the validation report's `pending_signatures`, and `is_runnable = not pending_signatures`. `allow_signatures` is a sweep-mechanics flag only (whether signature pipes are mock-run and listed in `validated_pipes`) — it does not change the verdict, so strict ≡ lenient in the report body. The "is this a failure?" decision moves to the consumer: the CLI exits non-zero on `not is_runnable` unless `--allow-signatures`; the HTTP caller reads `is_runnable`. (The **execute/run** path is different: running a stub still raises `PipeSignatureNotExecutableError`.)
diff --git a/docs/under-the-hood/pipe-func-requirements.md b/docs/under-the-hood/pipe-func-requirements.md
index b58e9bb8d..f50251688 100644
--- a/docs/under-the-hood/pipe-func-requirements.md
+++ b/docs/under-the-hood/pipe-func-requirements.md
@@ -17,6 +17,10 @@ The check is `pipelex/libraries/method_requirements.py`, and a load runs it at t
- `LibraryManager.load_from_blueprints` checks the tree of every bundle read from this host's disk, on every load: a blueprint an earlier load placed in its tree, a refused one included, is judged again rather than taken as judged.
- A dependency package, one a bundle calls by its address, is checked as soon as its tree is listed, before any of its bundles loads.
+**A load never runs on the event loop.** It is blocking work: it lists and hashes the method's files, asks the package-offer factory to judge its requirements, reads the methods installed on this host and may fetch a package a bundle calls by its address. So every load Pipelex reaches from asynchronous code, a run's (`acquire_library_async` in `pipelex/pipeline/execution_seams.py`), a validation's (`validate_bundle`) and those of the commands that run a loop, runs in a worker thread of the method-load pool through `run_method_load` (`pipelex/libraries/method_load_threads.py`), the reading and parsing of the bundles it loads included. The thread runs in a copy of its caller's context, so it reads the current library and every other context variable its caller set, and what it sets stays in its copy. When its caller is cancelled, the shutdown of `asyncio.run` included, a load no thread has taken yet is withdrawn and never runs, and a running one is waited for until it ends before the cancellation goes through, since a thread cannot be interrupted, so a library is torn down only when nothing still loads into it.
+
+**The method-load pool is bounded.** At most `METHOD_LOAD_MAX_THREADS` loads, four, run at once in a process, together with the blocking work around them, such as the API server writing a request's tree to disk and judging its requirements. A load asked for while every thread is busy waits for one, its caller awaiting on the event loop without blocking it. The pool is its own rather than asyncio's default executor, so slow loads, each of which can hold its thread for seconds while a package is fetched or a sandbox resolves requirements, never take the threads every other blocking call of the process borrows, such as a DNS lookup. Loads running in parallel import the modules defining their structure classes under one lock, which their class discovery also holds, so no load takes another's module half executed.
+
A requirement whose marker is false in the environment is not needed there and is skipped. One is met when the environment holds a distribution of its name, compared normalized, at a version its specifier contains, pre-releases included, and when, for each extra it names, the requirements the distribution's metadata declares under that extra are met in turn. A marker that cannot be evaluated, such as one comparing as versions a value that is not one (which releases of `packaging` before 22 refuse), refuses its requirement rather than the load.
## The environment the check reads
@@ -91,7 +95,7 @@ A mode with one fixed package set ignores the requirements and answers its packa
- **The markers are the factory's to evaluate.** The requirements arrive with their markers unevaluated, since Pipelex cannot evaluate a marker before it knows the environment the mode would run the method in. A requirement whose marker is false in that environment is not needed there, and the check skips it, so the factory evaluates the markers against the marker environment of the place it would offer, builds nothing for a requirement the place does not need, and never refuses for one: a refusal rests only on requirements whose markers hold there. `requirements_needed_in` (`pipelex/libraries/method_requirements.py`) picks them out as the check would, keeping a requirement whose marker reads a variable the marker environment does not state, or cannot be evaluated, which the check refuses rather than skips.
- **`PipeFuncPackageOffer`** is closed, frozen plain data: `environment`, the place as a refusal names it to the method's author; `distributions`, each distribution's name with its exact installed version, the names compared normalized; and `marker_environment`, the PEP 508 marker values of the sandbox's interpreter. A variable left out of `marker_environment`, such as `python_full_version` when the image pins only its minor version, makes every requirement whose marker reads it refused. Pipelex judges every requirement against the offer, one built for them included, so an offer that does not meet them refuses the method as a fixed package set would, naming each requirement.
- **`PipeFuncPackageRefusal`** is the answer when no environment of the mode can hold the requirements the place needs, such as a version conflicting with a package the sandbox's base pins, or a package with no wheel for the sandbox: `environment`, the place as the refusal names it, and `reason`, the plugin's own words, such as its resolver's explanation. The method is refused with `MethodRequirementsNotMetError`, a fault of the method's carrying `method.dependency_missing`, whose message, "The method needs Python packages that the Example sandbox cannot offer, as its requirements.txt declares: …", carries the reason, with one item on the file and no line, since the reason is about the requirements together. The reason reaches the method's author, a server's caller included, so it names no path on the host.
-- **The factory is synchronous, and caching by the requirements is the plugin's job.** The check runs inside the library load, which is synchronous and runs on whatever thread its caller runs it on: a worker thread for the API server's check of a served method, and the thread of a running event loop for a load inside an asynchronous run or validation. An asynchronous factory could not be awaited there without blocking that same thread, so it would buy nothing short of making every load asynchronous. Pipelex calls the factory for each tree declaring requirements each time a check judges it, never at registration, and keeps none of its answers, so a load that judges a tree at more than one of its steps asks again. A factory whose answer takes time, such as one resolving the requirements with `uv`, which takes seconds the first time, bounds the work with a timeout of its own and caches its answers by the requirements' texts, so only a process's first check of a requirement set pays for it: on the API server, the check of a served method runs first, off the event loop, and the load after it reads the cache. A factory may be called on a thread running an event loop, so it never calls `asyncio.run`.
+- **The factory is synchronous and never called on a thread running an event loop, and caching by the requirements is the plugin's job.** The check runs inside the library load, which is synchronous, and [every load Pipelex reaches from asynchronous code runs in a worker thread of the method-load pool](#what-is-checked-and-when), as the API server's check of a served method does, while a synchronous caller, such as a command loading a method before it starts a loop, calls it on its own thread, which runs none. So a factory may block for as long as its work takes, and it must not use an event loop of its caller's, since none runs on its thread and the one its caller's code runs on, if any, belongs to another thread. Pipelex enforces the rule: a factory about to be called on a thread running an event loop is not called, and the load fails with `PipelexUnexpectedError`, so a host that calls a load itself from asynchronous code, such as a worker's activity loading a library directly, moves it off the loop the same way, with `run_method_load`. An asynchronous factory would buy nothing short of making every load asynchronous, so the seam stays synchronous. Pipelex calls the factory for each tree declaring requirements each time a check judges it, never at registration, and keeps none of its answers, so a load that judges a tree at more than one of its steps asks again. A factory whose answer takes time, such as one resolving the requirements with `uv`, which takes seconds the first time, bounds the work with a timeout of its own and caches its answers by the requirements' texts, so only a process's first check of a requirement set pays for it: on the API server, the check of a served method runs first, and the load after it reads the cache.
- **A fault of the plugin's own is not a refusal.** A resolver the plugin cannot reach, or one outlasting its timeout, says nothing about the method: the factory raises it as an error of the plugin's, which Pipelex lets through, rather than answering a `PipeFuncPackageRefusal`, which would blame the method's author.
- **It fails closed.** A mode with no registered offer, or whose factory answers `None`, meets no requirement: a method declaring one is refused with `PipeFuncPackageSetUnstatedError`, a fault of the deployment carrying `runtime.misconfigured`, whose message only the operator view carries, rather than loading and failing inside the sandbox at its first import. Its message says which of the two happened: no plugin registers an offer for the mode, or the registered one could not state the packages, whose remedy is in the plugin's configuration. A method declaring none still loads.
- **It is interpreter-layer**, like `add_pipe_func_executor`: a plugin published under `pipelex.plugins.kernel` that calls it is refused with `PluginLayerViolationError`. A second plugin stating the same mode is refused with `DuplicatePipeFuncPackageOfferError`, naming both.
diff --git a/pipelex/cli/agent_cli/commands/validate/_validate_core.py b/pipelex/cli/agent_cli/commands/validate/_validate_core.py
index b30afc76d..f74178959 100644
--- a/pipelex/cli/agent_cli/commands/validate/_validate_core.py
+++ b/pipelex/cli/agent_cli/commands/validate/_validate_core.py
@@ -14,11 +14,12 @@ from pipelex.interpreter_hub import (
resolve_library_dirs,
set_current_library,
)
+from pipelex.libraries.method_load_threads import run_method_load
from pipelex.pipeline.advisory_warnings import collect_advisory_warnings, collect_current_library_entry_pipe_refs
from pipelex.pipeline.blueprint_selection import collect_entry_pipe_refs
from pipelex.pipeline.bundle_validator import BundleValidator
from pipelex.pipeline.dry_run_warnings import build_dry_run_warnings
-from pipelex.pipeline.execution_seams import acquire_library
+from pipelex.pipeline.execution_seams import acquire_library_async
from pipelex.pipeline.tree_files_read import tree_files_envelope_fields
from pipelex.pipeline.usage_stats_reporting import MethodValidationReport
from pipelex.pipeline.validate_bundle import build_pending_signatures, build_validated_pipes, validate_bundle
@@ -50,19 +51,19 @@ async def validate_all_core(*, library_dirs: list[Path] | None = None, allow_sig
prev_library_id = get_current_library_id_or_none()
# A refusal of the libraries while loading them is the invalid verdict, through the shared
# bundle-loading cascade, as on `validate bundle`. The directories are the caller's own, so
- # acquire_library translates their load itself, while the library is still current: the translation
+ # acquire_library_async translates their load itself, while the library is still current: the translation
# reads the library's pipe sources to locate an item, and a translation around the call would run
- # only after acquire_library had torn the failed library down, leaving the item without its file.
- # The sweep below stays untranslated on purpose: a failing dry run is answered as the DryRunError it
- # is, as on `validate pipe`.
- acquired_id, _ = acquire_library(
+ # only after the failed library was torn down, leaving the item without its file. The load runs in a
+ # worker thread, off the event loop. The sweep below stays untranslated on purpose: a failing dry run
+ # is answered as the DryRunError it is, as on `validate pipe`.
+ acquired_id, _ = await acquire_library_async(
library_id="",
library_dirs=[str(library_dir) for library_dir in library_dirs] if library_dirs else None,
library_dirs_are_callers=True,
)
try:
validation_report.describe(pipes=get_own_pipes())
- # acquire_library left the freshly-acquired library current, so the inner sweep targets it
+ # acquire_library_async left the freshly-acquired library current, so the inner sweep targets it
# (it filters signatures in strict mode itself). The returned map is keyed by namespaced pipe_ref.
# A pipe whose dry run fails is the invalid verdict, one located dry_run item per failing pipe,
# through the same cascade and item builder as `validate bundle`.
@@ -246,9 +247,9 @@ async def validate_pipe_core(
if effective_dirs:
# A refusal of the libraries while loading them is the invalid verdict, through the shared
- # bundle-loading cascade, as on `validate bundle`.
+ # bundle-loading cascade, as on `validate bundle`. The load runs in a worker thread, off the event loop.
with translate_to_validate_bundle_error():
- library_manager.load_libraries(library_id=library_id, library_dirs=effective_dirs)
+ await run_method_load(library_manager.load_libraries, library_id=library_id, library_dirs=effective_dirs)
# The entry-pipe lookup stays outside the cascade: a code naming no pipe is no verdict. A pipe whose
# dry run fails is the invalid verdict, one located dry_run item per failing pipe, as on `validate bundle`.
diff --git a/pipelex/cli/commands/build/inputs/_inputs_core.py b/pipelex/cli/commands/build/inputs/_inputs_core.py
index 30246cd2c..537f644ba 100644
--- a/pipelex/cli/commands/build/inputs/_inputs_core.py
+++ b/pipelex/cli/commands/build/inputs/_inputs_core.py
@@ -14,6 +14,7 @@ from pipelex.cli.error_handlers import (
)
from pipelex.core.pipes.exceptions import PipeOperatorModelChoiceError
from pipelex.interpreter_hub import get_library_manager, get_required_entry_pipe, resolve_library_dirs, set_current_library
+from pipelex.libraries.method_load_threads import run_method_load
from pipelex.pipe_machinery.rendering.input_renderer import InputsTemplateFormat, NoInputsRequiredError, render_inputs, render_inputs_toml
from pipelex.pipe_operators.exceptions import PipeOperatorModelAvailabilityError
from pipelex.pipelex import PACKAGE_VERSION
@@ -82,7 +83,8 @@ async def _generate_inputs_core(
set_current_library(library_id=library_id)
effective_dirs, _ = resolve_library_dirs(library_dir)
if effective_dirs:
- library_manager.load_libraries(library_id=library_id, library_dirs=effective_dirs)
+ # In a worker thread, off the event loop, as every load reached from asynchronous code.
+ await run_method_load(library_manager.load_libraries, library_id=library_id, library_dirs=effective_dirs)
if not pipe_code:
typer.secho("Failed to build inputs: no pipe code specified", fg=typer.colors.RED, err=True)
raise typer.Exit(1)
diff --git a/pipelex/cli/commands/build/output/_output_core.py b/pipelex/cli/commands/build/output/_output_core.py
index e8a78b33c..e7d4e762e 100644
--- a/pipelex/cli/commands/build/output/_output_core.py
+++ b/pipelex/cli/commands/build/output/_output_core.py
@@ -15,6 +15,7 @@ from pipelex.cli.error_handlers import (
from pipelex.core.concepts.concept_representation_generator import ConceptRepresentationFormat
from pipelex.core.pipes.exceptions import PipeOperatorModelChoiceError
from pipelex.interpreter_hub import get_library_manager, get_required_entry_pipe, resolve_library_dirs, set_current_library
+from pipelex.libraries.method_load_threads import run_method_load
from pipelex.pipe_machinery.rendering.output_renderer import render_output
from pipelex.pipe_operators.exceptions import PipeOperatorModelAvailabilityError
from pipelex.pipelex import PACKAGE_VERSION
@@ -81,7 +82,8 @@ async def _generate_output_core(
set_current_library(library_id=library_id)
effective_dirs, _ = resolve_library_dirs(library_dir)
if effective_dirs:
- library_manager.load_libraries(library_id=library_id, library_dirs=effective_dirs)
+ # In a worker thread, off the event loop, as every load reached from asynchronous code.
+ await run_method_load(library_manager.load_libraries, library_id=library_id, library_dirs=effective_dirs)
if not pipe_code:
typer.secho("Failed to build output: no pipe code specified", fg=typer.colors.RED, err=True)
raise typer.Exit(1)
diff --git a/pipelex/cli/commands/run/_run_core.py b/pipelex/cli/commands/run/_run_core.py
index 633b707d1..56cff4673 100644
--- a/pipelex/cli/commands/run/_run_core.py
+++ b/pipelex/cli/commands/run/_run_core.py
@@ -37,6 +37,7 @@ from pipelex.core.stuffs.stuff_viewer import render_stuff_viewer
from pipelex.errors.failure_problem import failure_log_fields
from pipelex.graph.graph_factory import generate_graph_outputs, save_graph_outputs_to_dir
from pipelex.interpreter_hub import clear_current_library, get_concept_library, get_library_manager
+from pipelex.libraries.method_load_threads import run_method_load
from pipelex.mthds_parsing.parser import MthdsParser
from pipelex.pipe_operators.exceptions import PipeOperatorModelAvailabilityError
from pipelex.pipelex import Pipelex
@@ -463,7 +464,9 @@ async def _execute_run(
if csv_list_content.items:
csv_row_model = type(csv_list_content.items[0])
else:
- csv_row_model = _resolve_row_model_for_empty_result(
+ # The reload is a whole library load, opened, read and torn down in a worker thread, off the event loop.
+ csv_row_model = await run_method_load(
+ _resolve_row_model_for_empty_result,
concept=csv_main_stuff.concept,
bundle_path=bundle_path,
mthds_content=mthds_content,
diff --git a/pipelex/cli/commands/validate/_validate_core.py b/pipelex/cli/commands/validate/_validate_core.py
index 735fb7588..01c16592b 100644
--- a/pipelex/cli/commands/validate/_validate_core.py
+++ b/pipelex/cli/commands/validate/_validate_core.py
@@ -22,6 +22,7 @@ from pipelex.interpreter_hub import (
resolve_library_dirs,
set_current_library,
)
+from pipelex.libraries.method_load_threads import run_method_load
from pipelex.libraries.pipe.exceptions import PipeLibraryError, PipeNotFoundError
from pipelex.pipe_operators.exceptions import PipeOperatorModelAvailabilityError
from pipelex.pipe_signature.signature_walk import collect_signature_refs
@@ -243,8 +244,9 @@ async def _validate_pipe_or_bundle(
# other invalid bundle. The entry-pipe lookup between them stays outside it: a code that names
# no pipe, or several, is an unresolvable target — no verdict, handled by execute_validate.
if effective_dirs:
+ # In a worker thread, off the event loop, as every load reached from asynchronous code.
with translate_to_validate_bundle_error():
- library_manager.load_libraries(library_id=library_id, library_dirs=effective_dirs)
+ await run_method_load(library_manager.load_libraries, library_id=library_id, library_dirs=effective_dirs)
pipe = get_required_entry_pipe(pipe_code=pipe_code)
validation_report.describe(pipes=get_own_pipes())
diff --git a/pipelex/libraries/library_manager.py b/pipelex/libraries/library_manager.py
index e0195a029..56267e59f 100644
--- a/pipelex/libraries/library_manager.py
+++ b/pipelex/libraries/library_manager.py
@@ -93,6 +93,7 @@ from pipelex.tools.log.error_fields import error_fields
from pipelex.tools.log.log_fields import USER_ACTION_FIELD
from pipelex.tools.misc.exceptions import SemVerError
from pipelex.tools.misc.semver import parse_constraint, parse_version, version_satisfies
+from pipelex.tools.typing.module_inspector import MODULE_IMPORT_LOCK
from pipelex.validation_error_types import PipeValidationErrorType
if TYPE_CHECKING:
@@ -766,7 +767,8 @@ class LibraryManager(LibraryManagerAbstract):
# Import modules and register in global registries
# Import from user directories
- if is_pipe_func_sandbox_hosted():
+ sandbox_hosted = is_pipe_func_sandbox_hosted()
+ if sandbox_hosted:
# Sandbox-hosted mode imports no Python from a library directory, whatever its origin (an
# inline bundle, a stored method, a host directory): importing a file runs its module-level
# code in this process. A structure class cannot travel to a sandbox: it would have to be
@@ -778,21 +780,24 @@ class LibraryManager(LibraryManagerAbstract):
ensure_no_structured_content_in_library_sources(
sources_by_dir=_python_sources_by_tree(library_trees=library_trees, bundle_files=all_bundle_files),
)
- else:
- for library_dir in all_dirs:
- # Only import files that contain StructuredContent subclasses (uses AST pre-check), so
- # concepts declared as `structure = "ClassName"` resolve that class from the registry.
- # A PipeFunc's function is not imported here: its step imports it by its import path when
- # it first runs.
- ClassRegistryUtils.import_modules_in_folder(
- folder_path=library_dir,
- base_class_names=[StructuredContent.__name__],
- )
+ # Loads run in parallel worker threads, so the imports and the class discovery that reads them hold the import
+ # lock together: no other load's import is half executed while the discovery reads `sys.modules`.
+ with MODULE_IMPORT_LOCK:
+ if not sandbox_hosted:
+ for library_dir in all_dirs:
+ # Only import files that contain StructuredContent subclasses (uses AST pre-check), so
+ # concepts declared as `structure = "ClassName"` resolve that class from the registry.
+ # A PipeFunc's function is not imported here: its step imports it by its import path when
+ # it first runs.
+ ClassRegistryUtils.import_modules_in_folder(
+ folder_path=library_dir,
+ base_class_names=[StructuredContent.__name__],
+ )
- # Auto-discover and register all StructuredContent classes from sys.modules
- num_registered = ClassRegistryUtils.auto_register_all_subclasses(
- base_class=StructuredContent,
- )
+ # Auto-discover and register all StructuredContent classes from sys.modules
+ num_registered = ClassRegistryUtils.auto_register_all_subclasses(
+ base_class=StructuredContent,
+ )
log.verbose("Auto-registered the StructuredContent classes of the loaded modules", fields={"class_count": num_registered})
# Load MTHDS files into the specific library
diff --git a/pipelex/libraries/method_load_threads.py b/pipelex/libraries/method_load_threads.py
new file mode 100644
index 000000000..a009e24dc
--- /dev/null
+++ b/pipelex/libraries/method_load_threads.py
@@ -0,0 +1,39 @@
+"""The pool of worker threads that method loads run in, off the event loop.
+
+A method load is blocking work that can take long: it lists and hashes the method's files, judges its
+`requirements.txt` through the package-offer factory of the deployment's PipeFunc sandbox, which may resolve the
+requirements, and reads the methods installed on this host or fetches a package a bundle calls by its address. On the
+event loop it would hold up every other request or activity the loop serves, so every load pipelex reaches from
+asynchronous code runs in this pool, through `run_method_load`, and so does the blocking work around one, such as
+writing a request's method tree to disk and removing it, judging its requirements before it loads, and listing a tree to
+find its entry pipe. A host that calls a load itself from asynchronous code runs it here the same way.
+
+The pool is bounded: at most `METHOD_LOAD_MAX_THREADS` of these calls run at once in a process, and one asked for while
+every thread is busy waits for a thread to free, its caller awaiting on the event loop without blocking it. The bound,
+and the pool being this one rather than asyncio's default executor, keep slow loads, each of which can hold its thread
+for seconds while a package is fetched or a sandbox resolves requirements, from taking the threads the default executor
+lends every other blocking call of the process, such as a DNS lookup, so a burst of them never starves unrelated
+requests on a small container.
+"""
+
+from collections.abc import Callable
+from typing import ParamSpec, TypeVar
+
+from pipelex.tools.misc.async_utils import WorkerThreads
+
+_T = TypeVar("_T")
+_P = ParamSpec("_P")
+
+#: How many method loads, and calls of the blocking work around them, run at once in a process.
+METHOD_LOAD_MAX_THREADS = 4
+
+METHOD_LOAD_THREADS = WorkerThreads(name="pipelex-method-load", max_workers=METHOD_LOAD_MAX_THREADS)
+
+
+async def run_method_load(func: Callable[_P, _T], /, *args: _P.args, **kwargs: _P.kwargs) -> _T:
+ """Run a method load, or the blocking work around one, in the method-load pool, off the event loop.
+
+ The call runs in a copy of the caller's context, and a cancellation of the caller goes through only once the call
+ has ended, or at once when no thread has taken it yet (`WorkerThreads.run`).
+ """
+ return await METHOD_LOAD_THREADS.run(func, *args, **kwargs)
diff --git a/pipelex/libraries/package_environments.py b/pipelex/libraries/package_environments.py
index 09383e642..ce353ee99 100644
--- a/pipelex/libraries/package_environments.py
+++ b/pipelex/libraries/package_environments.py
@@ -25,11 +25,13 @@ from packaging.markers import default_environment
from packaging.utils import canonicalize_name
from typing_extensions import override
+from pipelex.base_exceptions import PipelexUnexpectedError
from pipelex.config import get_config, get_pipe_func_execution_mode, is_pipe_func_sandbox_hosted
from pipelex.core.method_trees.requirements_file import RequirementLine
from pipelex.interpreter_hub import get_pipe_func_package_offer_registry
from pipelex.libraries.exceptions import PipeFuncPackageSetUnstatedError
from pipelex.plugins.pipe_func_package_offer_registry import PipeFuncPackageOffer, PipeFuncPackageOfferRequest, PipeFuncPackageRefusal
+from pipelex.tools.misc.async_utils import is_event_loop_running_on_this_thread
class PythonPackageEnvironment(Protocol):
@@ -160,6 +162,8 @@ def resolve_pipe_func_package_environment(
Raises:
PipeFuncPackageSetUnstatedError: The mode runs PipeFunc steps out of this process and no plugin states, or the
stating plugin could not state, the packages found there.
+ PipelexUnexpectedError: The factory would be called on a thread running an event loop, which its contract
+ forbids: the load reaching it was called on the loop rather than through `run_method_load`.
"""
if not is_pipe_func_sandbox_hosted():
return InterpreterPackageEnvironment(names_host_paths=names_host_paths)
@@ -167,6 +171,16 @@ def resolve_pipe_func_package_environment(
offer_factory = get_pipe_func_package_offer_registry().get_optional(mode=execution_mode)
if offer_factory is None:
raise PipeFuncPackageSetUnstatedError(execution_mode=execution_mode, offer_registered=False)
+ if is_event_loop_running_on_this_thread():
+ # The factory's contract: it is never called on a thread running an event loop, so it may block. A load reached
+ # from asynchronous code runs in the method-load pool (`run_method_load`); one called on the loop itself is a host
+ # wired wrongly, refused here rather than let the factory hold up every task the loop serves.
+ msg = (
+ f"The package-offer factory of the PipeFunc execution mode '{execution_mode}' was about to be called on a thread "
+ "running an event loop: a method load reached from asynchronous code must run off the loop, through "
+ "`run_method_load` (pipelex.libraries.method_load_threads)."
+ )
+ raise PipelexUnexpectedError(msg)
answer = offer_factory(PipeFuncPackageOfferRequest(pipe_func_config=get_config().interpreter.pipe_func, requirements=tuple(requirements)))
if answer is None:
raise PipeFuncPackageSetUnstatedError(execution_mode=execution_mode, offer_registered=True)
diff --git a/pipelex/pipeline/bundle_validator.py b/pipelex/pipeline/bundle_validator.py
index 6eee7540e..1ad4ad02f 100644
--- a/pipelex/pipeline/bundle_validator.py
+++ b/pipelex/pipeline/bundle_validator.py
@@ -65,7 +65,7 @@ from pipelex.pipe_run.pipe_job import PipeJob
from pipelex.pipe_run.pipe_router import PipeRouter
from pipelex.pipe_run.pipe_run import PipeRun
from pipelex.pipeline.exceptions import withhold_names_in_text
-from pipelex.pipeline.execution_seams import acquire_library, prepare_pipe_job
+from pipelex.pipeline.execution_seams import acquire_library_async, prepare_pipe_job
from pipelex.pipeline.pipeline_factory import PipelineFactory
from pipelex.pipeline.validate_bundle_translation import HOST_LIBRARY_FILE_PLACEHOLDER
from pipelex.runtime_hub import get_telemetry_manager, scoped_content_generator
@@ -286,14 +286,15 @@ class BundleValidator:
CLIs) own their library and use :meth:`validate_pipes` instead.
"""
prev_library_id = get_current_library_id_or_none()
- acquired_id, _ = acquire_library(
+ # Loaded in a worker thread, off the event loop, and left current in this context for the sweep.
+ acquired_id, _ = await acquire_library_async(
library_id=library_id,
library_dirs=library_dirs,
mthds_contents=mthds_contents,
bundle_uris=bundle_uris,
)
try:
- # acquire_library left the freshly-acquired library current, so the inner sweep over the
+ # acquire_library_async left the freshly-acquired library current, so the inner sweep over the
# current library targets exactly acquired_id (it filters signatures in strict mode itself).
return await self.validate_current_library(allow_signatures=allow_signatures, caller_identity=caller_identity)
finally:
diff --git a/pipelex/pipeline/execution_seams.py b/pipelex/pipeline/execution_seams.py
index 492a163bd..02bfd200c 100644
--- a/pipelex/pipeline/execution_seams.py
+++ b/pipelex/pipeline/execution_seams.py
@@ -11,7 +11,8 @@ Two pure-ish building blocks composed by both the single-run wrapper
bundle while it loads is the ``ValidateBundleError`` verdict, as on the
validate path. Returns the ``library_id`` plus the bundle's qualified
``main_pipe`` (when an ``mthds_contents`` bundle declares one), leaving pipe
- resolution to the caller.
+ resolution to the caller. :func:`acquire_library_async` is its twin for
+ asynchronous code, which loads in a worker thread, never on the event loop.
- :func:`prepare_pipe_job` — build a :class:`PipeJob` against an already-open
library: working memory (user inputs, mock inputs, file-input preparation),
run params, job metadata, and the library crate. **Pure**: no pipeline-manager
@@ -19,8 +20,8 @@ Two pure-ish building blocks composed by both the single-run wrapper
library mutation.
"""
-from collections.abc import Mapping, Sequence
-from contextlib import AbstractContextManager, nullcontext
+from collections.abc import Generator, Mapping, Sequence
+from contextlib import AbstractContextManager, contextmanager, nullcontext
from pathlib import Path
from typing import TYPE_CHECKING
@@ -42,6 +43,7 @@ from pipelex.interpreter_hub import (
)
from pipelex.kernel.memory_ops import shape_inputs
from pipelex.libraries.bundle_resources import check_submitted_files
+from pipelex.libraries.method_load_threads import run_method_load
from pipelex.libraries.pipe.pipe_suggestions import declared_pipe_refs
from pipelex.mthds_parsing.parser import MthdsParser
from pipelex.pipe_machinery.pipe_abstract import PipeAbstract
@@ -139,10 +141,75 @@ def acquire_library(
Returns the ``library_id`` and the bundle's qualified ``main_pipe`` ref
(``domain.pipe_code``) when an ``mthds_contents`` bundle declares one, else
``None``. Pipe resolution is the caller's responsibility.
+
+ The load runs on the calling thread, so asynchronous code calls :func:`acquire_library_async` instead.
"""
+ _ensure_submitted_files_accompany_contents(mthds_contents=mthds_contents, submitted_files=submitted_files)
+ with _acquiring_library(library_id=library_id) as acquired_id:
+ qualified_main_pipe = _load_acquired_library(
+ library_id=acquired_id,
+ library_dirs=library_dirs,
+ mthds_contents=mthds_contents,
+ bundle_uris=bundle_uris,
+ library_dirs_are_callers=library_dirs_are_callers,
+ methods_dirs=methods_dirs,
+ submitted_files=submitted_files,
+ )
+ return acquired_id, qualified_main_pipe
+
+
+async def acquire_library_async(
+ library_id: str,
+ *,
+ library_dirs: list[str] | None = None,
+ mthds_contents: list[str] | None = None,
+ bundle_uris: list[str] | None = None,
+ library_dirs_are_callers: bool = False,
+ methods_dirs: list[Path] | None = None,
+ submitted_files: Mapping[str, bytes] | None = None,
+) -> tuple[str, str | None]:
+ """:func:`acquire_library` for asynchronous code: the same library, loaded in a worker thread, off the event loop.
+
+ A load is blocking work that can take long: it reads and hashes the method's files, judges its ``requirements.txt``
+ through the package-offer factory of the deployment's PipeFunc sandbox, which may resolve the requirements, and
+ reads the methods installed on this host or fetches a package the bundles call by address. On the event loop, it
+ would hold up every other request or activity it serves for as long as it takes.
+
+ The library is opened and made current here, in the caller's context, where it stays current on success, and the
+ loads run in the worker thread, which reads that binding from its copy of the context. Their refusals are translated
+ there, while the library is still open and current, as :func:`acquire_library` translates them. On a failure, a
+ cancellation included, the caller's outer current library is restored and the library torn down, once the worker
+ thread has ended.
+ """
+ _ensure_submitted_files_accompany_contents(mthds_contents=mthds_contents, submitted_files=submitted_files)
+ with _acquiring_library(library_id=library_id) as acquired_id:
+ qualified_main_pipe = await run_method_load(
+ _load_acquired_library,
+ library_id=acquired_id,
+ library_dirs=library_dirs,
+ mthds_contents=mthds_contents,
+ bundle_uris=bundle_uris,
+ library_dirs_are_callers=library_dirs_are_callers,
+ methods_dirs=methods_dirs,
+ submitted_files=submitted_files,
+ )
+ return acquired_id, qualified_main_pipe
+
+
+def _ensure_submitted_files_accompany_contents(*, mthds_contents: list[str] | None, submitted_files: Mapping[str, bytes] | None) -> None:
if submitted_files and not mthds_contents:
msg = "submitted_files accompany mthds_contents: a bundle loaded from a directory reads the files of its method tree"
raise PipelexUnexpectedError(msg)
+
+
+@contextmanager
+def _acquiring_library(*, library_id: str) -> Generator[str, None, None]:
+ """Open the library and make it current for the block, which loads it; on any failure, restore and tear it down.
+
+ Yields the id the library was opened under. A failure of the block, ``BaseException`` such as
+ ``asyncio.CancelledError`` included, restores the caller's outer current library and tears the library down before
+ it propagates, so a failed load never leaks a ``Library``. On success the library is left open and current.
+ """
library_manager = get_library_manager()
prev_library_id = get_current_library_id_or_none()
# Adopt the canonical id open_library keyed the Library under: when given a falsy library_id it
@@ -152,76 +219,8 @@ def acquire_library(
success = False
try:
set_current_library(library_id=library_id)
-
- effective_dirs, source_label = resolve_library_dirs(library_dirs)
- if effective_dirs:
- effective_dirs_desc = ", ".join(str(dir_path) for dir_path in effective_dirs)
- log.verbose(f"Loading libraries from {len(effective_dirs)} directory(ies) ({source_label}): {effective_dirs_desc}")
- # The caller's directories refuse as their invalid bundle; a host's report a tree it cannot list, or a method
- # that refuses to load, as its own fault.
- library_dirs_translation: AbstractContextManager[None] = (
- translate_to_validate_bundle_error() if library_dirs_are_callers else reporting_host_library_faults()
- )
- with library_dirs_translation:
- library_manager.load_libraries(library_id=library_id, library_dirs=effective_dirs, methods_dirs=methods_dirs)
- else:
- log.verbose(f"No library directories to load ({source_label})")
-
- qualified_main_pipe: str | None = None
- if mthds_contents:
- # Beside a host's library directories, the verdict names none of their files, which are paths on the host.
- host_library_withholding: AbstractContextManager[None] = (
- nullcontext()
- if library_dirs_are_callers
- else withholding_host_library_files(library_dirs=effective_dirs, caller_contents=mthds_contents)
- )
- with host_library_withholding, translate_to_validate_bundle_error():
- # The file on this host each content was read from, where the caller names one that exists: that bundle
- # is the file's, whose steps find the files they name in its method tree. Every other content belongs to
- # no tree, and its steps read only the files submitted with it, so its blueprint validates under that rule.
- resolved_uris = _resolved_bundle_uris(bundle_uris=bundle_uris or [])
- host_bundle_files: list[Path | None] = [uri if uri.is_file() else None for uri in resolved_uris] or [None] * len(mthds_contents)
- all_blueprints = [
- MthdsParser.make_pipelex_bundle_blueprint(
- mthds_content=content,
- file_origin=BundleFileOrigin.SUBMITTED_FILES if host_bundle_file is None else BundleFileOrigin.METHOD_TREE,
- )
- for content, host_bundle_file in zip(mthds_contents, host_bundle_files, strict=True)
- ]
- if submitted_files:
- check_submitted_files(blueprints=all_blueprints, submitted_files=submitted_files)
- library_manager.add_submitted_files(library_id=library_id, submitted_files=submitted_files)
- if not library_dirs_are_callers:
- # A refusal the caller reads offers the pipes of the bundles it sent, never the host's.
- library_manager.get_library(library_id=library_id).pipe_library.set_caller_pipe_refs(
- pipe_refs=declared_pipe_refs(blueprints=all_blueprints)
- )
-
- # Filter out blueprints whose URIs are already loaded (e.g. via PIPELEXPATH), and record the file on this
- # host each of the others was read from, so its steps read beside it as when its directory loads.
- blueprints_to_load: list[PipelexBundleBlueprint] = list(all_blueprints)
- if bundle_uris:
- current_library = library_manager.get_library(library_id=library_id)
- blueprints_to_load = []
- for blueprint, uri, resolved_uri, host_bundle_file in zip(
- all_blueprints, bundle_uris, resolved_uris, host_bundle_files, strict=True
- ):
- if resolved_uri in current_library.loaded_mthds_paths:
- log.verbose(f"Bundle '{uri}' was skipped, already loaded from library directories")
- else:
- if host_bundle_file is not None:
- blueprint.mark_read_from_host_file(bundle_file=host_bundle_file)
- blueprints_to_load.append(blueprint)
-
- if blueprints_to_load:
- library_manager.load_from_blueprints(library_id=library_id, blueprints=blueprints_to_load, methods_dirs=methods_dirs)
-
- # Qualify main_pipe with domain to avoid ambiguity when multiple domains define pipes with
- # the same code — via the one shared selection rule (first declaring main_pipe, else first).
- qualified_main_pipe = select_primary_blueprint(all_blueprints).main_pipe_ref
-
+ yield library_id
success = True
- return library_id, qualified_main_pipe
finally:
if not success:
# Restore the caller's outer current-library FIRST so the safety guarantee holds even when
@@ -233,6 +232,87 @@ def acquire_library(
library_manager.teardown(library_id=library_id)
+def _load_acquired_library(
+ *,
+ library_id: str,
+ library_dirs: list[str] | None,
+ mthds_contents: list[str] | None,
+ bundle_uris: list[str] | None,
+ library_dirs_are_callers: bool,
+ methods_dirs: list[Path] | None,
+ submitted_files: Mapping[str, bytes] | None,
+) -> str | None:
+ """Load the directories and the contents into the library :func:`acquire_library` opened and made current.
+
+ Blocking work, run on the calling thread by :func:`acquire_library` and in a worker thread by
+ :func:`acquire_library_async`. Returns the bundle's qualified ``main_pipe``, or ``None``.
+ """
+ library_manager = get_library_manager()
+ effective_dirs, source_label = resolve_library_dirs(library_dirs)
+ if effective_dirs:
+ effective_dirs_desc = ", ".join(str(dir_path) for dir_path in effective_dirs)
+ log.verbose(f"Loading libraries from {len(effective_dirs)} directory(ies) ({source_label}): {effective_dirs_desc}")
+ # The caller's directories refuse as their invalid bundle; a host's report a tree it cannot list, or a method
+ # that refuses to load, as its own fault.
+ library_dirs_translation: AbstractContextManager[None] = (
+ translate_to_validate_bundle_error() if library_dirs_are_callers else reporting_host_library_faults()
+ )
+ with library_dirs_translation:
+ library_manager.load_libraries(library_id=library_id, library_dirs=effective_dirs, methods_dirs=methods_dirs)
+ else:
+ log.verbose(f"No library directories to load ({source_label})")
+
+ qualified_main_pipe: str | None = None
+ if mthds_contents:
+ # Beside a host's library directories, the verdict names none of their files, which are paths on the host.
+ host_library_withholding: AbstractContextManager[None] = (
+ nullcontext() if library_dirs_are_callers else withholding_host_library_files(library_dirs=effective_dirs, caller_contents=mthds_contents)
+ )
+ with host_library_withholding, translate_to_validate_bundle_error():
+ # The file on this host each content was read from, where the caller names one that exists: that bundle
+ # is the file's, whose steps find the files they name in its method tree. Every other content belongs to
+ # no tree, and its steps read only the files submitted with it, so its blueprint validates under that rule.
+ resolved_uris = _resolved_bundle_uris(bundle_uris=bundle_uris or [])
+ host_bundle_files: list[Path | None] = [uri if uri.is_file() else None for uri in resolved_uris] or [None] * len(mthds_contents)
+ all_blueprints = [
+ MthdsParser.make_pipelex_bundle_blueprint(
+ mthds_content=content,
+ file_origin=BundleFileOrigin.SUBMITTED_FILES if host_bundle_file is None else BundleFileOrigin.METHOD_TREE,
+ )
+ for content, host_bundle_file in zip(mthds_contents, host_bundle_files, strict=True)
+ ]
+ if submitted_files:
+ check_submitted_files(blueprints=all_blueprints, submitted_files=submitted_files)
+ library_manager.add_submitted_files(library_id=library_id, submitted_files=submitted_files)
+ if not library_dirs_are_callers:
+ # A refusal the caller reads offers the pipes of the bundles it sent, never the host's.
+ library_manager.get_library(library_id=library_id).pipe_library.set_caller_pipe_refs(
+ pipe_refs=declared_pipe_refs(blueprints=all_blueprints)
+ )
+
+ # Filter out blueprints whose URIs are already loaded (e.g. via PIPELEXPATH), and record the file on this
+ # host each of the others was read from, so its steps read beside it as when its directory loads.
+ blueprints_to_load: list[PipelexBundleBlueprint] = list(all_blueprints)
+ if bundle_uris:
+ current_library = library_manager.get_library(library_id=library_id)
+ blueprints_to_load = []
+ for blueprint, uri, resolved_uri, host_bundle_file in zip(all_blueprints, bundle_uris, resolved_uris, host_bundle_files, strict=True):
+ if resolved_uri in current_library.loaded_mthds_paths:
+ log.verbose(f"Bundle '{uri}' was skipped, already loaded from library directories")
+ else:
+ if host_bundle_file is not None:
+ blueprint.mark_read_from_host_file(bundle_file=host_bundle_file)
+ blueprints_to_load.append(blueprint)
+
+ if blueprints_to_load:
+ library_manager.load_from_blueprints(library_id=library_id, blueprints=blueprints_to_load, methods_dirs=methods_dirs)
+
+ # Qualify main_pipe with domain to avoid ambiguity when multiple domains define pipes with
+ # the same code — via the one shared selection rule (first declaring main_pipe, else first).
+ qualified_main_pipe = select_primary_blueprint(all_blueprints).main_pipe_ref
+ return qualified_main_pipe
+
+
def load_libraries_and_activate(library_dirs: Sequence[str | Path] | None = None) -> str:
"""Open a fresh library, set it current, resolve + load ``library_dirs``, and leave it loaded.
diff --git a/pipelex/pipeline/inputs_template.py b/pipelex/pipeline/inputs_template.py
index 5ba4dff7b..712fb859e 100644
--- a/pipelex/pipeline/inputs_template.py
+++ b/pipelex/pipeline/inputs_template.py
@@ -8,6 +8,7 @@ from typing import TYPE_CHECKING, Any
from pipelex.core.pipes.inputs.exceptions import NoInputsRequiredError
from pipelex.core.qualified_ref import QualifiedRef
from pipelex.interpreter_hub import get_library_manager, get_required_entry_pipe, resolve_library_dirs, set_current_library
+from pipelex.libraries.method_load_threads import run_method_load
from pipelex.pipe_machinery.pipe_factory import PipeFactory
from pipelex.pipe_machinery.rendering.input_renderer import build_concept_comments, no_inputs_message, render_inputs
from pipelex.pipeline.blueprint_selection import select_primary_blueprint
@@ -77,7 +78,8 @@ async def build_inputs_for_pipe(
set_current_library(library_id=library_id)
effective_dirs, _ = resolve_library_dirs(library_dirs)
if effective_dirs:
- library_manager.load_libraries(library_id=library_id, library_dirs=effective_dirs)
+ # In a worker thread, off the event loop, as every load reached from asynchronous code.
+ await run_method_load(library_manager.load_libraries, library_id=library_id, library_dirs=effective_dirs)
if not pipe_code:
msg = "No pipe code specified"
diff --git a/pipelex/pipeline/pipeline_run_setup.py b/pipelex/pipeline/pipeline_run_setup.py
index e86260d37..a717ca8e1 100644
--- a/pipelex/pipeline/pipeline_run_setup.py
+++ b/pipelex/pipeline/pipeline_run_setup.py
@@ -26,7 +26,7 @@ from pipelex.pipe_run.pipe_run_params import (
VariableMultiplicity,
)
from pipelex.pipeline.exceptions import PipeExecutionError
-from pipelex.pipeline.execution_seams import acquire_library, prepare_pipe_job
+from pipelex.pipeline.execution_seams import acquire_library_async, prepare_pipe_job
from pipelex.runtime_hub import (
get_event_log_override,
get_otel_tracer,
@@ -328,8 +328,9 @@ async def pipeline_run_setup(
event_log: EventLogProtocol | None = None
success = False
try:
- # Seam 1: open the library and load dirs + blueprints. Owns its own load-failure teardown.
- library_id, qualified_main_pipe = acquire_library(
+ # Seam 1: open the library and load dirs + blueprints, in a worker thread, off the event loop. Owns its own
+ # load-failure teardown, and leaves the library current in this context.
+ library_id, qualified_main_pipe = await acquire_library_async(
library_id=library_id,
library_dirs=library_dirs,
mthds_contents=mthds_contents,
diff --git a/pipelex/pipeline/validate_bundle.py b/pipelex/pipeline/validate_bundle.py
index b81f318b7..625af3b9f 100644
--- a/pipelex/pipeline/validate_bundle.py
+++ b/pipelex/pipeline/validate_bundle.py
@@ -1,5 +1,4 @@
import ast
-import asyncio
from collections.abc import Mapping
from contextlib import AbstractContextManager, nullcontext
from pathlib import Path
@@ -27,6 +26,7 @@ from pipelex.interpreter_hub import (
)
from pipelex.libraries.bundle_resources import check_submitted_files
from pipelex.libraries.library_utils import get_pipelex_mthds_files_from_dirs
+from pipelex.libraries.method_load_threads import run_method_load
from pipelex.libraries.method_trees import list_local_method_tree
from pipelex.libraries.pipe.exceptions import EntryPipeNotFoundError
from pipelex.libraries.pipe.pipe_suggestions import close_pipe_refs, declared_pipe_refs
@@ -172,6 +172,14 @@ async def validate_bundle(
``collect_dry_run_warnings`` is for a caller that puts what the dry runs find without failing in a
verdict of its own, as the validate surfaces do (see ``BundleValidator.validate_pipes``); any other
caller, a fix loop or a builder, leaves it ``False`` and the dry runs log what they find.
+
+ **Every load runs in a worker thread of the method-load pool, never on the event loop** (``run_method_load``), the
+ reading and parsing of the bundles it loads included. A load is blocking work that can take long: it lists and
+ hashes the method's files, judges a ``requirements.txt`` through the package-offer factory of the deployment's
+ PipeFunc sandbox, which may resolve the requirements, and resolves a package the bundles call by its address as a
+ run does, among the packages shipped in ``methods_dirs``, then the methods installed on this host, then by a fetch
+ on a miss when fetch-on-miss is on. The library is opened and made current here, in the caller's context, and each
+ load reads that binding from the worker thread's copy of it; the dry runs stay on the loop.
"""
provided_params = sum(
[
@@ -225,7 +233,6 @@ async def validate_bundle(
set_current_library(library_id=library_id)
if method_dir is not None:
- await asyncio.sleep(0) # Yield to event loop (keeps function async-compatible)
result = await _validate_method_dir(
method_dir=method_dir,
methods_dirs=methods_dirs,
@@ -242,8 +249,6 @@ async def validate_bundle(
# Load libraries from resolved directories before loading the bundle
effective_dirs, source_label = resolve_library_dirs(library_dirs)
- loaded_pipes: list[PipeAbstract] | None = None
- loaded_blueprints: list[PipelexBundleBlueprint] | None = None
# Beside submitted content, a host's library directories are withheld from the verdict: their files are paths
# on the host. The caller's own directories, and every directory of a bundle file, keep their paths.
host_library_withholding: AbstractContextManager[None]
@@ -260,33 +265,24 @@ async def validate_bundle(
else:
host_library_withholding = nullcontext()
host_library_tree_faults = nullcontext()
- await asyncio.sleep(0) # Yield to event loop (keeps function async-compatible)
+ # Every load below, the reading and parsing of the bundles it loads included, runs in a worker thread, off the
+ # event loop (see `validate_bundle`), its refusals translated here, while the library is still open and current.
with host_library_withholding, translate_to_validate_bundle_error():
if effective_dirs:
log.verbose(f"Loading libraries from {len(effective_dirs)} directory(ies) ({source_label}) for validation")
with host_library_tree_faults:
- library_manager.load_libraries(
- library_id=library_id,
- library_dirs=effective_dirs,
- )
+ await run_method_load(library_manager.load_libraries, library_id=library_id, library_dirs=effective_dirs)
else:
log.verbose(f"No library directories to load ({source_label})")
if mthds_contents is not None:
- content_sources: list[str | None] = list(mthds_sources) if mthds_sources is not None else [None] * len(mthds_contents)
- # Contents belong to no method tree: their steps read only the files submitted with them.
- loaded_blueprints = [
- MthdsParser.make_pipelex_bundle_blueprint(
- mthds_content=content, mthds_source=source, file_origin=BundleFileOrigin.SUBMITTED_FILES
- )
- for content, source in zip(mthds_contents, content_sources, strict=True)
- ]
- if submitted_files:
- check_submitted_files(blueprints=loaded_blueprints, submitted_files=submitted_files)
- library_manager.add_submitted_files(library_id=library_id, submitted_files=submitted_files)
- if not library_dirs_are_callers:
- # A refusal the caller reads offers the pipes of the bundles it sent, never the host's.
- library.pipe_library.set_caller_pipe_refs(pipe_refs=declared_pipe_refs(blueprints=loaded_blueprints))
- loaded_pipes = library_manager.load_from_blueprints(library_id=library_id, blueprints=loaded_blueprints)
+ loaded_blueprints, loaded_pipes = await run_method_load(
+ _load_contents,
+ library_id=library_id,
+ mthds_contents=mthds_contents,
+ mthds_sources=mthds_sources,
+ submitted_files=submitted_files,
+ library_dirs_are_callers=library_dirs_are_callers,
+ )
dry_run_results = await BundleValidator().validate_pipes(
pipes=_pipes_to_dry_run(loaded_pipes, dry_run_pipe_codes=dry_run_pipe_codes),
library_id=library_id,
@@ -307,19 +303,9 @@ async def validate_bundle(
else:
assert mthds_file_path is not None
- blueprint = MthdsParser.make_pipelex_bundle_blueprint(bundle_path=mthds_file_path)
+ blueprint, loaded_pipes = await run_method_load(_load_bundle_file, library_id=library_id, mthds_file_path=mthds_file_path)
loaded_blueprints = [blueprint]
- if mthds_file_path.resolve() not in library.loaded_mthds_paths:
- # File not yet loaded - load it from the blueprint
- loaded_pipes = library_manager.load_from_blueprints(library_id=library_id, blueprints=[blueprint])
- else:
- # File already loaded - get existing pipes from library by their refs. A bundle's
- # `[pipe.<code>]` keys are bare and belong to the bundle's own domain, so they are
- # qualified here rather than searched for: the library is keyed by qualified ref.
- pipe_codes = list(blueprint.pipe.keys()) if blueprint.pipe else []
- loaded_pipes = [library.pipe_library.get_required_pipe(pipe_code=f"{blueprint.domain}.{code}") for code in pipe_codes]
-
dry_run_results = await BundleValidator().validate_pipes(
pipes=_pipes_to_dry_run(loaded_pipes, dry_run_pipe_codes=dry_run_pipe_codes),
library_id=library_id,
@@ -434,6 +420,85 @@ def dirs_beside_method(*, method_dir: Path, library_dirs: Sequence[Path] | None)
return [library_dir for library_dir in library_dirs or [] if library_dir.resolve() != resolved_method_dir]
+def _load_contents(
+ *,
+ library_id: str,
+ mthds_contents: list[str],
+ mthds_sources: Sequence[str | None] | None,
+ submitted_files: Mapping[str, bytes] | None,
+ library_dirs_are_callers: bool,
+) -> tuple[list[PipelexBundleBlueprint], list[PipeAbstract]]:
+ """Parse submitted contents and load them, with the files submitted beside them, into the library.
+
+ Blocking work, run in the method-load pool by ``validate_bundle``.
+
+ Returns:
+ The blueprint of each content, and the pipes the load built.
+ """
+ library_manager = get_library_manager()
+ content_sources: list[str | None] = list(mthds_sources) if mthds_sources is not None else [None] * len(mthds_contents)
+ # Contents belong to no method tree: their steps read only the files submitted with them.
+ blueprints = [
+ MthdsParser.make_pipelex_bundle_blueprint(mthds_content=content, mthds_source=source, file_origin=BundleFileOrigin.SUBMITTED_FILES)
+ for content, source in zip(mthds_contents, content_sources, strict=True)
+ ]
+ if submitted_files:
+ check_submitted_files(blueprints=blueprints, submitted_files=submitted_files)
+ library_manager.add_submitted_files(library_id=library_id, submitted_files=submitted_files)
+ if not library_dirs_are_callers:
+ # A refusal the caller reads offers the pipes of the bundles it sent, never the host's.
+ library_manager.get_library(library_id=library_id).pipe_library.set_caller_pipe_refs(pipe_refs=declared_pipe_refs(blueprints=blueprints))
+ loaded_pipes = library_manager.load_from_blueprints(library_id=library_id, blueprints=blueprints)
+ return blueprints, loaded_pipes
+
+
+def _load_bundle_file(*, library_id: str, mthds_file_path: Path) -> tuple[PipelexBundleBlueprint, list[PipeAbstract]]:
+ """Read and parse a bundle file, and load it unless its library directories already did.
+
+ Blocking work, run in the method-load pool by ``validate_bundle``.
+
+ Returns:
+ The bundle's blueprint, and its pipes, whichever load built them.
+ """
+ library_manager = get_library_manager()
+ library = library_manager.get_library(library_id=library_id)
+ blueprint = MthdsParser.make_pipelex_bundle_blueprint(bundle_path=mthds_file_path)
+ if mthds_file_path.resolve() not in library.loaded_mthds_paths:
+ # File not yet loaded - load it from the blueprint
+ return blueprint, library_manager.load_from_blueprints(library_id=library_id, blueprints=[blueprint])
+ # File already loaded - get existing pipes from library by their refs. A bundle's `[pipe.<code>]` keys are bare and
+ # belong to the bundle's own domain, so they are qualified here rather than searched for: the library is keyed by
+ # qualified ref.
+ pipe_codes = list(blueprint.pipe.keys()) if blueprint.pipe else []
+ return blueprint, [library.pipe_library.get_required_pipe(pipe_code=f"{blueprint.domain}.{code}") for code in pipe_codes]
+
+
+def _load_method_dir(
+ *,
+ method_dir: Path,
+ methods_dirs: Sequence[Path] | None,
+ library_dirs: Sequence[Path] | None,
+ library_id: str,
+) -> tuple[MethodTree, list[PipeAbstract]]:
+ """List the method's tree, check its Python parses, and load it with the caller's directories beside it.
+
+ Blocking work, run in a worker thread by ``_validate_method_dir``. The tree's packages called by address resolve as
+ a run resolves them: the packages shipped in ``methods_dirs`` first, then the methods installed on this host and a
+ fetch on a miss, so the verdict is the one the run of the same method meets.
+
+ Returns:
+ The method's tree, and the pipes the load built, those of the directories beside it included.
+ """
+ tree = list_local_method_tree(handed_over=method_dir)
+ _ensure_tree_python_parses(tree=tree)
+ loaded_pipes = get_library_manager().load_libraries(
+ library_id=library_id,
+ library_dirs=[method_dir, *dirs_beside_method(method_dir=method_dir, library_dirs=library_dirs)],
+ methods_dirs=list(methods_dirs) if methods_dirs is not None else None,
+ )
+ return tree, loaded_pipes
+
+
async def _validate_method_dir(
*,
method_dir: Path,
@@ -452,14 +517,16 @@ async def _validate_method_dir(
"""
library_manager = get_library_manager()
library = library_manager.get_library(library_id=library_id)
- beside_dirs = dirs_beside_method(method_dir=method_dir, library_dirs=library_dirs)
with translate_to_validate_bundle_error():
- tree = list_local_method_tree(handed_over=method_dir)
- _ensure_tree_python_parses(tree=tree)
- loaded_pipes = library_manager.load_libraries(
+ # Listing the tree, reading its Python and loading it are blocking work, which a fetch on a miss, or the
+ # package-offer factory of the deployment's PipeFunc sandbox, can make long: it runs in a worker thread, off the
+ # event loop, its refusals translated here, while the library is still open and current.
+ tree, loaded_pipes = await run_method_load(
+ _load_method_dir,
+ method_dir=method_dir,
+ methods_dirs=methods_dirs,
+ library_dirs=library_dirs,
library_id=library_id,
- library_dirs=[method_dir, *beside_dirs],
- methods_dirs=list(methods_dirs) if methods_dirs is not None else None,
)
blueprints = library_manager.get_accumulated_blueprints(library_id)
# Only the tree's own bundles are the method. The directories beside it, and the packages nested in it that the
@@ -488,11 +555,22 @@ async def _validate_method_dir(
)
+def _load_directory(*, directory: Path, mthds_files: list[Path], library_id: str) -> tuple[list[PipelexBundleBlueprint], list[PipeAbstract]]:
+ """Parse a directory's bundles and load the directory: blocking work, run in the method-load pool.
+
+ Returns:
+ The blueprint of each bundle of the directory, and the pipes the load built.
+ """
+ blueprints = [MthdsParser.make_pipelex_bundle_blueprint(bundle_path=mthds_file) for mthds_file in mthds_files]
+ loaded_pipes = get_library_manager().load_libraries(library_id=library_id, library_dirs=[Path(directory)])
+ return blueprints, loaded_pipes
+
+
async def validate_bundles_from_directory(directory: Path, *, allow_signatures: bool = False) -> ValidateBundleResult:
- # A directory whose tree cannot be listed is the invalid verdict, as a bundle of it that does not load is.
+ # A directory whose tree cannot be listed is the invalid verdict, as a bundle of it that does not load is, and it is
+ # given before any library opens. The listing, like the parsing and the load below, runs off the event loop.
with translate_to_validate_bundle_error():
- mthds_files = get_pipelex_mthds_files_from_dirs(dirs={directory})
- all_blueprints: list[PipelexBundleBlueprint] = []
+ mthds_files = await run_method_load(get_pipelex_mthds_files_from_dirs, dirs={directory})
library_manager = get_library_manager()
library_id, library = library_manager.open_library()
@@ -501,11 +579,7 @@ async def validate_bundles_from_directory(directory: Path, *, allow_signatures:
try:
set_current_library(library_id=library_id)
with translate_to_validate_bundle_error():
- for mthds_file in mthds_files:
- blueprint = MthdsParser.make_pipelex_bundle_blueprint(bundle_path=mthds_file)
- all_blueprints.append(blueprint)
-
- loaded_pipes = library_manager.load_libraries(library_id=library_id, library_dirs=[Path(directory)])
+ all_blueprints, loaded_pipes = await run_method_load(_load_directory, directory=directory, mthds_files=mthds_files, library_id=library_id)
dry_run_results = await BundleValidator().validate_pipes(pipes=loaded_pipes, library_id=library_id, allow_signatures=allow_signatures)
result = ValidateBundleResult(
blueprints=all_blueprints,
diff --git a/pipelex/pipeline/validate_in_process.py b/pipelex/pipeline/validate_in_process.py
index 3490a7e9a..cf003dcce 100644
--- a/pipelex/pipeline/validate_in_process.py
+++ b/pipelex/pipeline/validate_in_process.py
@@ -26,6 +26,7 @@ from typing import TYPE_CHECKING
from pipelex import log
from pipelex.base_exceptions import PipelexError, ValidationErrorItem
from pipelex.interpreter_hub import clear_current_library, get_current_library_id_or_none, get_library_manager, set_current_library
+from pipelex.libraries.method_load_threads import run_method_load
from pipelex.libraries.method_trees import list_local_method_tree
from pipelex.mthds_parsing.pipelex_bundle_blueprint import PipelexBundleBlueprint
from pipelex.pipe_run.dry_run_in_process import best_effort_graph_spec
@@ -53,7 +54,8 @@ def _graph_target_ref(*, graph_pipe_code: str | None, method_dir: Path | None, b
"""The pipe the graph arm draws: the explicit target, else a method tree's entry pipe, else the batch's `main_pipe`.
A method tree starts at its entry pipe by the MTHDS protocol's rule (`select_method_entry`), so its graph is that
- pipe's, and a tree with no entry pipe, which a run naming no pipe refuses, gets no graph.
+ pipe's, and a tree with no entry pipe, which a run naming no pipe refuses, gets no graph. Listing and reading the tree
+ is blocking work, so `validate_bundles_in_process` runs this in a worker thread.
"""
if graph_pipe_code is not None:
return graph_pipe_code
@@ -206,7 +208,10 @@ async def _validate_bundles_in_scope(
dry_run_result=result.dry_run_result,
)
validation_report.count_warnings(count=len(warnings))
- graph_target_ref = _graph_target_ref(graph_pipe_code=graph_pipe_code, method_dir=method_dir, blueprints=result.blueprints)
+ # A method tree's entry pipe is read from its files on this host's disk: blocking work, done in a worker thread.
+ graph_target_ref = await run_method_load(
+ _graph_target_ref, graph_pipe_code=graph_pipe_code, method_dir=method_dir, blueprints=result.blueprints
+ )
# The graph arm dry-runs its target once more, and finds again what the sweep found, which `warnings`
# already carries: it is collected and dropped here rather than logged as if nothing reported it.
with collecting_dry_run_warnings():
diff --git a/pipelex/plugins/pipe_func_package_offer_registry.py b/pipelex/plugins/pipe_func_package_offer_registry.py
index 991d39db4..786651f60 100644
--- a/pipelex/plugins/pipe_func_package_offer_registry.py
+++ b/pipelex/plugins/pipe_func_package_offer_registry.py
@@ -90,12 +90,20 @@ class PipeFuncPackageOfferRequest(NamedTuple):
# packages at all. The requirements arrive with their markers unevaluated: the factory evaluates them against the marker
# environment of the place it would offer, and a refusal rests only on those whose markers hold there, never on one the
# place does not need (``requirements_needed_in``). Called synchronously, once for each method tree declaring
-# requirements each time a check judges it, never at registration, from whatever thread runs the load, which may be
-# running an event loop. It is therefore
-# synchronous, and caching its answers by the requirements is the plugin's job: a factory whose answer takes time to
-# compute, such as one resolving the requirements with a resolver, bounds the work with a timeout of its own and caches
-# by the requirements' texts, so only a process's first check of a requirement set pays for it. A fault of its own,
-# such as a resolver it cannot reach, is raised as an error of the plugin's, never answered as a refusal.
+# requirements each time a check judges it, never at registration.
+#
+# Its threading contract: a factory is never called on a thread running an event loop. Every method load pipelex reaches
+# from asynchronous code, such as a validation or a run on the API server or in a worker's activity, runs in the
+# method-load pool's worker threads (`run_method_load`, `acquire_library_async`); a synchronous caller, such as a CLI
+# command loading a method before it starts a loop, calls it on its own thread, which runs none; and a host calling a
+# load itself from asynchronous code moves it off the loop the same way. Pipelex enforces it: a factory about to be
+# called on a thread running an event loop is not called, and the load fails with `PipelexUnexpectedError`. A factory
+# may therefore block for as long as its work takes, and must not use an event loop of its caller's: none runs on its
+# thread, and the one its caller's code runs on, if any, belongs to another thread. Caching its answers
+# by the requirements is still the plugin's job: a factory whose answer takes time to compute, such as one resolving
+# the requirements with a resolver, bounds the work with a timeout of its own and caches by the requirements' texts, so
+# only a process's first check of a requirement set pays for it. A fault of its own, such as a resolver it cannot
+# reach, is raised as an error of the plugin's, never answered as a refusal.
PipeFuncPackageOfferFactoryFn: TypeAlias = Callable[[PipeFuncPackageOfferRequest], PipeFuncPackageOffer | PipeFuncPackageRefusal | None]
diff --git a/pipelex/plugins/registrar.py b/pipelex/plugins/registrar.py
index 29ea3d5c7..b86531c34 100644
--- a/pipelex/plugins/registrar.py
+++ b/pipelex/plugins/registrar.py
@@ -491,8 +491,9 @@ class PluginRegistrar:
the pipe_func config and one method's requirements, each time a method declaring requirements loads, never
here. It answers the offer meeting them, such as an environment built for them, a ``PipeFuncPackageRefusal``
carrying its reason when no environment of the mode holds them, or ``None`` when it cannot state the offer; a
- mode with no offer, or a ``None`` answer, meets no requirement. The call is synchronous, so a factory caches its
- answers by the requirements. Fail-loud on a duplicate mode, naming both plugins.
+ mode with no offer, or a ``None`` answer, meets no requirement. The call is synchronous and never made on a
+ thread running an event loop, which pipelex enforces, so a factory may block and must not use an event loop of
+ its caller's; it caches its answers by the requirements. Fail-loud on a duplicate mode, naming both plugins.
"""
self._require_interpreter_layer(capability=f"pipe_func package offer {mode}")
self._add(
diff --git a/pipelex/tools/misc/async_utils.py b/pipelex/tools/misc/async_utils.py
index 699b7578b..fcdd89701 100644
--- a/pipelex/tools/misc/async_utils.py
+++ b/pipelex/tools/misc/async_utils.py
@@ -1,8 +1,96 @@
import asyncio
+import contextlib
+import contextvars
+import os
+import threading
from collections.abc import Awaitable, Callable, Sequence
-from typing import TypeVar, cast
+from concurrent.futures import ThreadPoolExecutor
+from typing import Any, ParamSpec, TypeVar, cast
_T = TypeVar("_T")
+_P = ParamSpec("_P")
+
+
+def is_event_loop_running_on_this_thread() -> bool:
+ """Whether the calling thread is running an event loop, which a blocking call would hold up."""
+ try:
+ asyncio.get_running_loop()
+ except RuntimeError:
+ return False
+ return True
+
+
+class WorkerThreads:
+ """A named pool of at most ``max_workers`` worker threads, which runs blocking calls off the event loop.
+
+ The pool starts its threads as calls need them. A call made while every thread is busy waits for one to free, its
+ caller awaiting on the event loop meanwhile, never blocking it. A process forked from one whose pool had started
+ threads starts a pool of its own, since a forked child holds none of its parent's threads.
+ """
+
+ def __init__(self, *, name: str, max_workers: int) -> None:
+ self.name = name
+ self.max_workers = max_workers
+ self._executor: ThreadPoolExecutor | None = None
+ self._executor_lock = threading.Lock()
+ if hasattr(os, "register_at_fork"):
+ os.register_at_fork(after_in_child=self._forget_parent_executor)
+
+ def _forget_parent_executor(self) -> None:
+ self._executor = None
+ self._executor_lock = threading.Lock()
+
+ def _executor_of_this_process(self) -> ThreadPoolExecutor:
+ with self._executor_lock:
+ if self._executor is None:
+ self._executor = ThreadPoolExecutor(max_workers=self.max_workers, thread_name_prefix=self.name)
+ return self._executor
+
+ async def run(self, func: Callable[_P, _T], /, *args: _P.args, **kwargs: _P.kwargs) -> _T:
+ """Run a blocking call in one of the pool's threads and return its result.
+
+ The call runs in a copy of the caller's context, as ``asyncio.to_thread`` runs one, so it reads the caller's
+ context variables, such as the current library, the caller's identity or the log context. What the call sets in
+ them stays in that copy: a binding the caller reads afterwards is one the caller makes itself.
+
+ A thread cannot be interrupted, so a cancellation of the caller never counts the call as over while its thread
+ still runs. The call is submitted to the pool's executor, and what this waits for is the executor's own future of
+ it, never a task of the event loop, which the shutdown of ``asyncio.run`` would cancel along with the caller. A
+ call no thread has taken yet is withdrawn and never runs; a call already running is waited for until it ends
+ before the cancellation goes through, a cancellation repeated meanwhile included. The caller's own cleanup, such
+ as tearing down the library the call was loading into, therefore never runs while the call is still at work on
+ what it cleans up. The call's result or error is then dropped, since no caller is left to take it.
+ """
+ call_context = contextvars.copy_context()
+
+ def call_in_the_callers_context() -> _T:
+ return call_context.run(func, *args, **kwargs)
+
+ thread_future = self._executor_of_this_process().submit(call_in_the_callers_context)
+ outcome = asyncio.wrap_future(thread_future)
+ try:
+ return await asyncio.shield(outcome)
+ except asyncio.CancelledError:
+ # A call no thread has taken yet is withdrawn: it never runs, so its future is done at once.
+ thread_future.cancel()
+ outcome.add_done_callback(_drop_outcome)
+ while not thread_future.done():
+ # A cancellation repeated while the call ends changes nothing: the call cannot be stopped either way.
+ # `outcome` mirrors the executor's future and is done only once that future is.
+ with contextlib.suppress(asyncio.CancelledError):
+ await asyncio.wait({outcome})
+ raise
+
+
+def _drop_outcome( # kw-only: ignore
+ outcome: asyncio.Future[Any],
+) -> None:
+ """Retrieve a dropped call's error, which keeps asyncio from logging it as never retrieved.
+
+ A future's done-callback, which asyncio calls with the future as its one positional argument.
+ """
+ if not outcome.cancelled():
+ outcome.exception()
async def _invoke(factory: Callable[[], Awaitable[_T]]) -> _T:
diff --git a/pipelex/tools/typing/module_inspector.py b/pipelex/tools/typing/module_inspector.py
index befb3d8a3..d92f2f356 100644
--- a/pipelex/tools/typing/module_inspector.py
+++ b/pipelex/tools/typing/module_inspector.py
@@ -2,6 +2,7 @@ import ast
import importlib.util
import inspect
import sys
+import threading
from pathlib import Path
from typing import Any
@@ -12,6 +13,13 @@ from pipelex.tools.typing.exceptions import ModuleFileError
_inserted_module_dirs: list[str] = []
+#: Held while `import_module_from_file` publishes a module in `sys.modules` and executes it, and by a library load around
+#: its imports and the class discovery that reads them (`LibraryManager.load_libraries`). Method loads run in parallel
+#: worker threads, and a module is published before it executes, so without it a second load could take a module half
+#: executed, or discover the classes of one, as complete. Reentrant, since a module executing may import another by file.
+MODULE_IMPORT_LOCK = threading.RLock()
+
+
def import_module_from_file(file_path: Path) -> Any:
"""Imports a module from a file path.
@@ -38,37 +46,47 @@ def import_module_from_file(file_path: Path) -> Any:
# Convert file path to module-style path to use as the actual module name
module_name = convert_file_path_to_module_path(file_path)
- # Check if module is already loaded to avoid duplicate loading
- if module_name in sys.modules:
- return sys.modules[module_name]
-
- # Use importlib.util to load the module from file path
- spec = importlib.util.spec_from_file_location(module_name, str(file_path))
- if spec is None or spec.loader is None:
- msg = f"Could not load module from {file_path}"
- raise ModuleFileError(msg)
-
- module = importlib.util.module_from_spec(spec)
-
- # Add the module to sys.modules to ensure proper imports within the module
- sys.modules[module_name] = module
-
- # Put the file's OWN directory on sys.path so a module imported by path can import its siblings
- # (e.g. `from helpers import ...` between files in the same bundle). Importing via
- # spec_from_file_location does NOT do this automatically — unlike running a script, where the
- # script's directory lands on sys.path[0]. Without it, a top-level sibling import raises
- # ModuleNotFoundError and the module never finishes importing. Deduped, so repeated scans don't
- # grow sys.path unbounded.
- module_dir = str(file_path.resolve().parent)
- if module_dir not in sys.path:
- sys.path.insert(0, module_dir)
- if module_dir not in _inserted_module_dirs:
- _inserted_module_dirs.append(module_dir)
-
- # Execute the module
- spec.loader.exec_module(module)
-
- return module
+ # The module is published before it executes, so another thread must not read it until it has: the check, the
+ # publication and the execution hold the lock together.
+ with MODULE_IMPORT_LOCK:
+ # Check if module is already loaded to avoid duplicate loading
+ if module_name in sys.modules:
+ return sys.modules[module_name]
+
+ # Use importlib.util to load the module from file path
+ spec = importlib.util.spec_from_file_location(module_name, str(file_path))
+ if spec is None or spec.loader is None:
+ msg = f"Could not load module from {file_path}"
+ raise ModuleFileError(msg)
+
+ module = importlib.util.module_from_spec(spec)
+
+ # Add the module to sys.modules to ensure proper imports within the module
+ sys.modules[module_name] = module
+
+ # Put the file's OWN directory on sys.path so a module imported by path can import its siblings
+ # (e.g. `from helpers import ...` between files in the same bundle). Importing via
+ # spec_from_file_location does NOT do this automatically — unlike running a script, where the
+ # script's directory lands on sys.path[0]. Without it, a top-level sibling import raises
+ # ModuleNotFoundError and the module never finishes importing. Deduped, so repeated scans don't
+ # grow sys.path unbounded.
+ module_dir = str(file_path.resolve().parent)
+ if module_dir not in sys.path:
+ sys.path.insert(0, module_dir)
+ if module_dir not in _inserted_module_dirs:
+ _inserted_module_dirs.append(module_dir)
+
+ # Execute the module. One that fails to execute leaves `sys.modules`, as the import system's own imports do, so a
+ # later import executes it again rather than taking the half-executed module as imported.
+ executed = False
+ try:
+ spec.loader.exec_module(module)
+ executed = True
+ finally:
+ if not executed and sys.modules.get(module_name) is module:
+ del sys.modules[module_name]
+
+ return module
def drop_module_dirs_inserted_under(*, directory: Path) -> None:
diff --git a/subject_grants.toml b/subject_grants.toml
index 6c8094a50..d1ae791b5 100644
--- a/subject_grants.toml
+++ b/subject_grants.toml
@@ -3820,6 +3820,10 @@ rationale = "Verb-object: scans the given dirs; single operand."
param = "traversable"
rationale = "Nested helper: scans the traversable; single operand."
+["pipelex/libraries/method_load_threads.py::run_method_load"]
+param = "func"
+rationale = "Verb-object: runs the given load callable in the method-load pool, forwarding its own arguments as asyncio.to_thread does; positional-only, so no keyword of the callable is shadowed."
+
["pipelex/libraries/pipe/pipe_library.py::PipeLibrary.pretty_list_pipes._format_concept_code"]
param = "concept_code"
rationale = "Nested helper: formats the concept code; single operand."
@@ -4396,6 +4400,10 @@ rationale = "Projects the sweep's per-pipe status map into warning items; reads
param = "library_id"
rationale = "Verb-object: acquires the library named by the id; single operand."
+["pipelex/pipeline/execution_seams.py::acquire_library_async"]
+param = "library_id"
+rationale = "Verb-object: acquires the library named by the id; single operand, the same call shape as acquire_library."
+
["pipelex/pipeline/execution_seams.py::load_libraries_and_activate"]
param = "library_dirs"
rationale = "Verb-object: loads the given library dirs; options stay keyword."
@@ -5828,6 +5836,10 @@ rationale = "Jinja op on the mermaid_code; single operand (context keyword-only)
param = "node_id"
rationale = "Transforms the node_id; single operand."
+["pipelex/tools/misc/async_utils.py::WorkerThreads.run"]
+param = "func"
+rationale = "Verb-object: runs the given callable in one of the pool's threads, forwarding its own arguments as asyncio.to_thread does; positional-only, so no keyword of the callable is shadowed."
+
["pipelex/tools/misc/async_utils.py::_invoke"]
param = "factory"
rationale = "Operates on the factory; single operand (qualifiers keyword-only)."
diff --git a/tests/integration/pipelex/libraries/test_method_loads_off_the_event_loop.py b/tests/integration/pipelex/libraries/test_method_loads_off_the_event_loop.py
new file mode 100644
index 000000000..8e5c5dfcb
--- /dev/null
+++ b/tests/integration/pipelex/libraries/test_method_loads_off_the_event_loop.py
@@ -0,0 +1,193 @@
+import asyncio
+import threading
+from collections.abc import Generator
+from pathlib import Path
+
+import pytest
+from mthds.method_tree.digest import digest_of_contents
+from mthds.method_tree.models import MethodTree
+from pytest_mock import MockerFixture
+
+from pipelex.base_exceptions import PipelexUnexpectedError
+from pipelex.interpreter_hub import (
+ clear_current_library,
+ get_current_library_id_or_none,
+ get_library_manager,
+ scoped_current_library,
+ set_current_library,
+)
+from pipelex.libraries.method_load_threads import METHOD_LOAD_THREADS
+from pipelex.libraries.method_trees import list_local_method_tree
+from pipelex.libraries.written_trees import WRITTEN_METHOD_TREES, write_method_tree
+from pipelex.pipeline.execution_seams import acquire_library_async
+from pipelex.pipeline.validate_bundle import validate_bundle
+from pipelex.pipeline.validate_in_process import validate_bundles_in_process
+from pipelex.plugins.pipe_func_package_offer_registry import (
+ PipeFuncPackageOffer,
+ PipeFuncPackageOfferRegistry,
+ PipeFuncPackageOfferRequest,
+ PipeFuncPackageRefusal,
+)
+
+#: How long a test waits for a thread to reach a point it signals, which bounds a regression's run.
+_SIGNAL_TIMEOUT_SECONDS = 10.0
+
+_ECHO_MTHDS = b"""\
+domain = "off_the_loop"
+main_pipe = "echo"
+
+[pipe.echo]
+type = "PipeCompose"
+description = "Echo the data"
+inputs = { data = "Text" }
+output = "Text"
+template = "$data"
+"""
+
+# A method whose tree holds Python and a requirements.txt, so its every load asks the package-offer factory.
+_TREE_WITH_REQUIREMENTS = {
+ "main.mthds": _ECHO_MTHDS,
+ "helpers/constants.py": b"ANSWER = 42\n",
+ "requirements.txt": b"pandas>=2\n",
+}
+
+
+def _all_in_the_method_load_pool(thread_names: list[str]) -> bool:
+ """Whether every one of these calls ran, and at least one did, in a thread of the method-load pool, which runs no loop."""
+ return bool(thread_names) and all(thread_name.startswith(METHOD_LOAD_THREADS.name) for thread_name in thread_names)
+
+
+class _RecordingOffer:
+ """The package-offer factory of the neutral `sandbox` mode, noting, at each call, the thread it was called on.
+
+ Once ``hold`` is called, a call signals ``started`` and blocks until ``release`` is set, as a factory resolving the
+ requirements does while it works.
+ """
+
+ def __init__(self) -> None:
+ self.thread_names: list[str] = []
+ self.started = threading.Event()
+ self.release = threading.Event()
+ self.release.set()
+
+ def hold(self) -> None:
+ self.release.clear()
+
+ def __call__(self, request: PipeFuncPackageOfferRequest) -> PipeFuncPackageOffer | PipeFuncPackageRefusal | None: # ruff: ignore[unused-method-argument]
+ self.thread_names.append(threading.current_thread().name)
+ self.started.set()
+ if not self.release.wait(timeout=_SIGNAL_TIMEOUT_SECONDS):
+ msg = "the held factory was never released"
+ raise AssertionError(msg)
+ return PipeFuncPackageOffer(
+ environment="the sandbox under test",
+ distributions={"pandas": "2.2.3"},
+ marker_environment={"python_version": "3.12", "sys_platform": "linux"},
+ )
+
+
[email protected](name="recording_offer")
+def recording_offer_fixture(mocker: MockerFixture, sandbox_hosted_mode: None) -> _RecordingOffer: # ruff: ignore[unused-function-argument]
+ """Run PipeFunc steps in the neutral `sandbox` mode, whose plugin's factory records the thread of each call."""
+ offer = _RecordingOffer()
+ mocker.patch(
+ "pipelex.libraries.package_environments.get_pipe_func_package_offer_registry", return_value=PipeFuncPackageOfferRegistry({"sandbox": offer})
+ )
+ return offer
+
+
[email protected](name="held_tree")
+def held_tree_fixture(tmp_path: Path) -> Generator[Path, None, None]:
+ """The tree with requirements, written and held as a host holds a request's tree, its directory yielded."""
+ written = write_method_tree(
+ contents=_TREE_WITH_REQUIREMENTS, destination=tmp_path / "tree", expected_digest=digest_of_contents(_TREE_WITH_REQUIREMENTS)
+ )
+ with WRITTEN_METHOD_TREES.holding(tree=written):
+ yield tmp_path / "tree"
+
+
[email protected](loop_scope="class")
+class TestMethodLoadsOffTheEventLoop:
+ async def test_a_validation_of_a_method_tree_asks_the_package_offer_off_the_event_loop(
+ self, held_tree: Path, recording_offer: _RecordingOffer
+ ) -> None:
+ result = await validate_bundle(method_dir=held_tree)
+
+ assert [pipe.code for pipe in result.pipes] == ["echo"]
+ assert _all_in_the_method_load_pool(recording_offer.thread_names), recording_offer.thread_names
+
+ async def test_the_in_process_validator_reads_the_tree_entry_for_its_graph_off_the_event_loop(
+ self, mocker: MockerFixture, held_tree: Path, recording_offer: _RecordingOffer
+ ) -> None:
+ """Beside the load, the validator lists the tree once more to find the entry pipe its graph draws, off the loop too."""
+ listing_threads: list[str] = []
+
+ def list_after_noting_the_thread(*, handed_over: Path) -> MethodTree:
+ listing_threads.append(threading.current_thread().name)
+ return list_local_method_tree(handed_over=handed_over)
+
+ mocker.patch("pipelex.pipeline.validate_in_process.list_local_method_tree", side_effect=list_after_noting_the_thread)
+
+ report = await validate_bundles_in_process(method_dir=held_tree)
+
+ assert report.graph_spec is not None
+ assert len(listing_threads) == 1
+ assert _all_in_the_method_load_pool(listing_threads), listing_threads
+ assert _all_in_the_method_load_pool(recording_offer.thread_names), recording_offer.thread_names
+
+ async def test_an_asynchronous_acquisition_loads_off_the_event_loop_and_leaves_the_library_current(
+ self, held_tree: Path, recording_offer: _RecordingOffer
+ ) -> None:
+ library_manager = get_library_manager()
+ outer_library_id = get_current_library_id_or_none()
+
+ library_id, qualified_main_pipe = await acquire_library_async("off-the-loop-acquisition", library_dirs=[str(held_tree)])
+ try:
+ assert library_id == "off-the-loop-acquisition"
+ assert qualified_main_pipe is None
+ assert get_current_library_id_or_none() == library_id
+ assert _all_in_the_method_load_pool(recording_offer.thread_names), recording_offer.thread_names
+ finally:
+ library_manager.teardown(library_id=library_id)
+ if outer_library_id is not None:
+ set_current_library(library_id=outer_library_id)
+ else:
+ clear_current_library()
+
+ assert outer_library_id != library_id
+
+ async def test_a_load_called_on_the_event_loop_is_refused_before_the_package_offer_is_asked(
+ self, held_tree: Path, recording_offer: _RecordingOffer
+ ) -> None:
+ """The factory is never called on a thread running an event loop: a load called on the loop itself fails loudly."""
+ library_manager = get_library_manager()
+ library_id, _ = library_manager.open_library()
+ try:
+ with scoped_current_library(library_id=library_id), pytest.raises(PipelexUnexpectedError, match="thread running an event loop"):
+ library_manager.load_libraries(library_id=library_id, library_dirs=[held_tree])
+ finally:
+ library_manager.teardown(library_id=library_id)
+
+ assert recording_offer.thread_names == []
+
+ async def test_a_cancelled_acquisition_tears_its_library_down_once_the_load_has_ended(
+ self, mocker: MockerFixture, held_tree: Path, recording_offer: _RecordingOffer
+ ) -> None:
+ """A load cannot be interrupted in its thread, so the library is torn down only once the load is over."""
+ library_manager = get_library_manager()
+ teardown_spy = mocker.spy(library_manager, "teardown")
+ recording_offer.hold()
+
+ acquisition = asyncio.create_task(acquire_library_async("off-the-loop-cancelled", library_dirs=[str(held_tree)]))
+ assert await asyncio.to_thread(recording_offer.started.wait, _SIGNAL_TIMEOUT_SECONDS), "the load never asked the factory"
+ acquisition.cancel()
+ await asyncio.sleep(0.05)
+ assert not acquisition.done(), "the cancellation went through while the load was still running"
+ assert teardown_spy.call_count == 0
+
+ recording_offer.release.set()
+ with pytest.raises(asyncio.CancelledError):
+ await acquisition
+
+ teardown_spy.assert_called_once_with(library_id="off-the-loop-cancelled")
+ assert _all_in_the_method_load_pool(recording_offer.thread_names), recording_offer.thread_names
diff --git a/tests/integration/pipelex/pipeline/test_pipeline_run_setup_storage_scope_gate.py b/tests/integration/pipelex/pipeline/test_pipeline_run_setup_storage_scope_gate.py
index a3d6d497e..f0368c98d 100644
--- a/tests/integration/pipelex/pipeline/test_pipeline_run_setup_storage_scope_gate.py
+++ b/tests/integration/pipelex/pipeline/test_pipeline_run_setup_storage_scope_gate.py
@@ -53,7 +53,7 @@ class TestPipelineRunSetupStorageScopeGate:
execution_config = get_config().interpreter.pipeline_execution.with_execution_overrides(generate_graph=False)
pipeline_manager = mocker.spy(pipeline_run_setup_module, "get_pipeline_manager")
telemetry_manager = mocker.spy(pipeline_run_setup_module, "get_telemetry_manager")
- acquire_library = mocker.spy(pipeline_run_setup_module, "acquire_library")
+ acquire_library = mocker.spy(pipeline_run_setup_module, "acquire_library_async")
with pytest.raises(ValueError, match="storage_scope"):
await pipeline_run_setup(
@@ -80,7 +80,7 @@ class TestPipelineRunSetupStorageScopeGate:
"""
execution_config = get_config().interpreter.pipeline_execution.with_execution_overrides(generate_graph=False)
telemetry_manager = mocker.spy(pipeline_run_setup_module, "get_telemetry_manager")
- acquire_library = mocker.spy(pipeline_run_setup_module, "acquire_library")
+ acquire_library = mocker.spy(pipeline_run_setup_module, "acquire_library_async")
with pytest.raises(ValueError, match="storage_scope"):
await pipeline_run_setup(
diff --git a/tests/integration/pipelex/pipeline/test_read_scope_routes.py b/tests/integration/pipelex/pipeline/test_read_scope_routes.py
index 13164b98b..f094e6efa 100644
--- a/tests/integration/pipelex/pipeline/test_read_scope_routes.py
+++ b/tests/integration/pipelex/pipeline/test_read_scope_routes.py
@@ -150,7 +150,7 @@ class TestReadScopeRoutes:
async def test_a_mismatched_host_registers_no_run(self, mocker: MockerFixture, storage_scope: str, match: str) -> None:
execution_config = get_config().interpreter.pipeline_execution.with_execution_overrides(generate_graph=False)
pipeline_manager = mocker.spy(pipeline_run_setup_module, "get_pipeline_manager")
- acquire_library = mocker.spy(pipeline_run_setup_module, "acquire_library")
+ acquire_library = mocker.spy(pipeline_run_setup_module, "acquire_library_async")
with pytest.raises(ValueError, match=match):
await pipeline_run_setup(
diff --git a/tests/unit/pipelex/tools/misc/test_worker_threads.py b/tests/unit/pipelex/tools/misc/test_worker_threads.py
new file mode 100644
index 000000000..a8356d42e
--- /dev/null
+++ b/tests/unit/pipelex/tools/misc/test_worker_threads.py
@@ -0,0 +1,155 @@
+import asyncio
+import threading
+from contextvars import ContextVar
+
+import pytest
+
+from pipelex.tools.misc.async_utils import WorkerThreads, is_event_loop_running_on_this_thread
+
+_PROBE: ContextVar[str | None] = ContextVar("worker_threads_probe", default=None)
+
+#: How long a test waits for a thread to reach a point it signals, which bounds a regression's run.
+_SIGNAL_TIMEOUT_SECONDS = 10.0
+#: How long a test lets a cancellation settle before it checks that nothing went through.
+_SETTLE_SECONDS = 0.05
+
+
+def _describe_thread(*, label: str) -> tuple[str, bool, str | None, str]:
+ """What the call saw where it ran: its label, whether a loop ran there, the probe's value, and its thread's name."""
+ return label, is_event_loop_running_on_this_thread(), _PROBE.get(), threading.current_thread().name
+
+
+def _set_the_probe(*, value: str) -> None:
+ _PROBE.set(value)
+
+
+def _refuse(*, message: str) -> None:
+ raise ValueError(message)
+
+
+class _HeldCall:
+ """A blocking call that signals once it runs, then blocks until released, recording the order of what happens."""
+
+ def __init__(self, *, events: list[str], name: str) -> None:
+ self.events = events
+ self.name = name
+ self.started = threading.Event()
+ self.release = threading.Event()
+
+ def __call__(self) -> str:
+ self.started.set()
+ if not self.release.wait(timeout=_SIGNAL_TIMEOUT_SECONDS):
+ msg = f"the held call '{self.name}' was never released"
+ raise AssertionError(msg)
+ self.events.append(f"{self.name} ended")
+ return self.name
+
+
+async def _abandon_a_running_call(*, threads: WorkerThreads, held_call: _HeldCall, events: list[str]) -> None:
+ """Start a caller whose call blocks, then return while it runs, so `asyncio.run` cancels every task to shut down."""
+
+ async def call_then_clean_up() -> None:
+ try:
+ await threads.run(held_call)
+ finally:
+ events.append("cleanup")
+
+ caller = asyncio.create_task(call_then_clean_up())
+ assert await asyncio.to_thread(held_call.started.wait, _SIGNAL_TIMEOUT_SECONDS), "the call never started"
+ # Released from another thread a moment after the shutdown began cancelling the caller and every other task.
+ threading.Timer(0.2, held_call.release.set).start()
+ events.append(f"abandoned {caller.get_name()}")
+
+
[email protected](loop_scope="class")
+class TestWorkerThreads:
+ async def test_the_call_runs_in_a_named_thread_off_the_loop_with_the_callers_context(self) -> None:
+ threads = WorkerThreads(name="pipelex-test-threads", max_workers=2)
+ token = _PROBE.set("caller's value")
+ try:
+ label, ran_on_a_loop, probe_value, thread_name = await threads.run(_describe_thread, label="described")
+ finally:
+ _PROBE.reset(token)
+
+ assert label == "described"
+ assert ran_on_a_loop is False
+ assert probe_value == "caller's value"
+ assert thread_name.startswith("pipelex-test-threads")
+
+ async def test_what_the_call_binds_stays_in_its_thread(self) -> None:
+ await WorkerThreads(name="pipelex-test-threads", max_workers=1).run(_set_the_probe, value="the call's value")
+
+ assert _PROBE.get() is None
+
+ async def test_the_calls_error_reaches_the_caller(self) -> None:
+ with pytest.raises(ValueError, match="refused in the worker thread"):
+ await WorkerThreads(name="pipelex-test-threads", max_workers=1).run(_refuse, message="refused in the worker thread")
+
+ async def test_a_call_waits_for_a_free_thread_beyond_the_bound(self) -> None:
+ threads = WorkerThreads(name="pipelex-test-threads", max_workers=1)
+ events: list[str] = []
+ first = _HeldCall(events=events, name="first")
+ second = _HeldCall(events=events, name="second")
+ second.release.set()
+
+ first_caller = asyncio.create_task(threads.run(first))
+ assert await asyncio.to_thread(first.started.wait, _SIGNAL_TIMEOUT_SECONDS), "the first call never started"
+ second_caller = asyncio.create_task(threads.run(second))
+ await asyncio.sleep(_SETTLE_SECONDS)
+ assert not second.started.is_set(), "a call ran beyond the pool's bound"
+
+ first.release.set()
+ assert await first_caller == "first"
+ assert await second_caller == "second"
+ assert events == ["first ended", "second ended"]
+
+ async def test_a_cancelled_caller_waits_for_the_running_call_to_end(self) -> None:
+ """A thread cannot be interrupted: the cancellation goes through once the call ends, a second one meanwhile included.
+
+ So the caller's cleanup never runs while the call is still at work.
+ """
+ events: list[str] = []
+ held_call = _HeldCall(events=events, name="held")
+ caller = asyncio.create_task(WorkerThreads(name="pipelex-test-threads", max_workers=1).run(held_call))
+ assert await asyncio.to_thread(held_call.started.wait, _SIGNAL_TIMEOUT_SECONDS), "the call never started"
+ caller.cancel()
+ await asyncio.sleep(_SETTLE_SECONDS)
+ caller.cancel()
+ await asyncio.sleep(_SETTLE_SECONDS)
+ assert not caller.done(), "the cancellation went through while the call was still running"
+
+ held_call.release.set()
+ with pytest.raises(asyncio.CancelledError):
+ await caller
+
+ assert events == ["held ended"]
+
+ async def test_a_cancelled_caller_withdraws_a_call_no_thread_has_taken(self) -> None:
+ threads = WorkerThreads(name="pipelex-test-threads", max_workers=1)
+ events: list[str] = []
+ running = _HeldCall(events=events, name="running")
+ queued = _HeldCall(events=events, name="queued")
+ queued.release.set()
+
+ running_caller = asyncio.create_task(threads.run(running))
+ assert await asyncio.to_thread(running.started.wait, _SIGNAL_TIMEOUT_SECONDS), "the running call never started"
+ queued_caller = asyncio.create_task(threads.run(queued))
+ await asyncio.sleep(_SETTLE_SECONDS)
+ queued_caller.cancel()
+ with pytest.raises(asyncio.CancelledError):
+ await asyncio.wait_for(queued_caller, timeout=_SIGNAL_TIMEOUT_SECONDS)
+
+ running.release.set()
+ assert await running_caller == "running"
+ assert not queued.started.is_set(), "a withdrawn call ran"
+ assert events == ["running ended"]
+
+ async def test_the_shutdown_of_asyncio_run_waits_for_the_running_call_before_the_cleanup(self) -> None:
+ """`asyncio.run` cancels the caller and every other task at once; the call's end still comes before the cleanup."""
+ threads = WorkerThreads(name="pipelex-test-threads", max_workers=1)
+ events: list[str] = []
+ held_call = _HeldCall(events=events, name="held")
+
+ await asyncio.to_thread(asyncio.run, _abandon_a_running_call(threads=threads, held_call=held_call, events=events))
+
+ assert events[1:] == ["held ended", "cleanup"], events
diff --git a/tests/unit/pipelex/tools/typing/test_module_import_lock.py b/tests/unit/pipelex/tools/typing/test_module_import_lock.py
new file mode 100644
index 000000000..976116c79
--- /dev/null
+++ b/tests/unit/pipelex/tools/typing/test_module_import_lock.py
@@ -0,0 +1,91 @@
+import sys
+import threading
+import time
+import types
+from collections.abc import Generator
+from concurrent.futures import ThreadPoolExecutor
+from pathlib import Path
+from typing import Any
+
+import pytest
+
+from pipelex.tools.typing.module_inspector import convert_file_path_to_module_path, drop_module_dirs_inserted_under, import_module_from_file
+
+#: How long a test waits for a thread to reach a point it signals, which bounds a regression's run.
+_SIGNAL_TIMEOUT_SECONDS = 10.0
+#: How long a test lets a second import run before it checks that it is still waiting.
+_SETTLE_SECONDS = 0.1
+#: The `sys.modules` key of the module through which a test hands the module it imports the events that pace its execution.
+_SIGNALS_MODULE = "_pipelex_test_module_import_lock_signals"
+
+_SLOW_MODULE = f"""\
+import sys
+
+_signals = sys.modules[{_SIGNALS_MODULE!r}]
+_signals.started.set()
+if not _signals.release.wait(timeout={_SIGNAL_TIMEOUT_SECONDS}):
+ raise AssertionError("the slow module was never released")
+
+
+class LateClass:
+ pass
+"""
+
+_FAILING_MODULE = """\
+class EarlyClass:
+ pass
+
+
+raise RuntimeError("the module fails while it executes")
+"""
+
+
[email protected](name="module_dir")
+def module_dir_fixture(tmp_path: Path) -> Generator[Path, None, None]:
+ """A directory for the modules a test imports by file, whose modules and `sys.path` entries are dropped after it."""
+ signals = types.ModuleType(_SIGNALS_MODULE)
+ vars(signals).update(started=threading.Event(), release=threading.Event())
+ sys.modules[_SIGNALS_MODULE] = signals
+ try:
+ yield tmp_path
+ finally:
+ signals.release.set()
+ sys.modules.pop(_SIGNALS_MODULE, None)
+ for python_file in tmp_path.glob("*.py"):
+ sys.modules.pop(convert_file_path_to_module_path(python_file), None)
+ drop_module_dirs_inserted_under(directory=tmp_path.resolve())
+
+
+def _signals() -> Any:
+ return sys.modules[_SIGNALS_MODULE]
+
+
+class TestModuleImportLock:
+ def test_two_concurrent_imports_of_one_file_both_get_the_complete_module(self, module_dir: Path) -> None:
+ """The module is published before it executes: the second import waits for the execution, never takes it half done."""
+ module_file = module_dir / "slow_structures.py"
+ module_file.write_text(_SLOW_MODULE, encoding="utf-8")
+ signals = _signals()
+
+ with ThreadPoolExecutor(max_workers=2) as executor:
+ first = executor.submit(import_module_from_file, module_file)
+ assert signals.started.wait(timeout=_SIGNAL_TIMEOUT_SECONDS), "the first import never executed the module"
+ second = executor.submit(import_module_from_file, module_file)
+ time.sleep(_SETTLE_SECONDS)
+ assert not second.done(), "the second import returned while the module was still executing"
+ signals.release.set()
+ first_module = first.result(timeout=_SIGNAL_TIMEOUT_SECONDS)
+ second_module = second.result(timeout=_SIGNAL_TIMEOUT_SECONDS)
+
+ assert first_module is second_module
+ assert hasattr(first_module, "LateClass")
+
+ def test_a_module_failing_to_execute_leaves_sys_modules(self, module_dir: Path) -> None:
+ """As with the import system's own imports, a later import executes it again rather than take it half executed."""
+ module_file = module_dir / "failing_structures.py"
+ module_file.write_text(_FAILING_MODULE, encoding="utf-8")
+
+ with pytest.raises(RuntimeError, match="the module fails while it executes"):
+ import_module_from_file(module_file)
+
+ assert convert_file_path_to_module_path(module_file) not in sys.modules