Build AI methods with your coding agent, then run them anywhere: as an MCP for chatbots, as a webapp for people, or via API for your software.
refactor/One-listing-per-run · L-261016-486c27 (#1471)
Louis Choquela7149181595d
diff --git a/.drift/acks/cli-docs.toml b/.drift/acks/cli-docs.tomlindex 55e920d0e..69c7b9410 100644--- a/.drift/acks/cli-docs.toml+++ b/.drift/acks/cli-docs.toml@@ -1,8 +1,8 @@contract = "cli-docs"-digest = "sha256:99b717b4eca9c5e27f0f7246a5017d2a2d6e3bc90b9cc96c6e07c5bcae112440"-reviewed_by = "Claude Opus 5.5"-reviewed_at = "2026-10-20T08:41:28Z"-rationale = "real-catch: this branch makes pipelex-agent validate bundle and fix bundle exit 2, no verdict, on a configuration refused before the boot (report_agent_init_failure takes exit_code, which the agent resolver passes as 2), as the human pipelex validate/fix bundle already did, while run and inputs keep exit 1 and a lone .mthds file, not listed before the boot, keeps the boot's exit 1; docs/tools/cli/agent-cli.md (validate and fix), validate.md, fix.md (exit codes), docs/under-the-hood/error-model.md and pipelex/cli/agent_cli/CLAUDE.md say so, and the agent validate, run and inputs command docstrings state the exit codes each command gives. Merged with dev's validate bundle --graph output folder (its run_output folder under the directory run bundle writes to), whose agent-cli.md and CLAUDE.md changes are kept beside this branch's; the graph and view failures still exit 2, as the validate bundle docstring says."+digest = "sha256:7d61432bb26b53a3fac3a61531332f19c0b516bd40e85a345072226b9706dc75"+reviewed_by = "Louis Choquel"+reviewed_at = "2026-10-20T09:11:42Z"+rationale = "real-catch: merged origin/dev's agent validate and fix exit 2 on a configuration refused before the boot, and its validate bundle --graph output folder, with this branch's CLI changes, where run, validate and fix hand the MethodTreeListing that resolved their target down to their load, so the tree they list is the tree loaded, and a file of it edited after that listing is refused as an invalid method naming the file; docs/tools/cli/run.md, agent-cli.md and pipelex/cli/agent_cli/CLAUDE.md now say both sides, and no flag, output or error format changed beyond what each side documents."[trigger_files]"pipelex/cli/__init__.py" = "blob:8b137891791fe96927ad78e64b0aad7bded08bdc"@@ -37,20 +37,20 @@ rationale = "real-catch: this branch makes pipelex-agent validate bundle and fix"pipelex/cli/agent_cli/commands/plxt_passthrough.py" = "blob:dc4916d28a5e83e18c9c59cc7e81191ea7869646""pipelex/cli/agent_cli/commands/run/__init__.py" = "blob:e69de29bb2d1d6434b8b29ae775ad8c2e48c5391""pipelex/cli/agent_cli/commands/run/_output_helpers.py" = "blob:40ed1700e0d5e86d7696eadd7f9887baeb8059f6"-"pipelex/cli/agent_cli/commands/run/_run_core.py" = "blob:44036b8b1e344e52880896d4c8edc2bfd35fe60d"+"pipelex/cli/agent_cli/commands/run/_run_core.py" = "blob:6190638f8542dd39db5458bc47a2da332fb39726""pipelex/cli/agent_cli/commands/run/_run_hosted.py" = "blob:8d2e819c44d5669756e931b9f3fc83741d54a1df""pipelex/cli/agent_cli/commands/run/app.py" = "blob:7546d9b31ea2dc22b1ddae538edfbc8e7f7d08df"-"pipelex/cli/agent_cli/commands/run/bundle_cmd.py" = "blob:1dc97782d3bc94e2a06337a5f143d22cf77d4d4b"+"pipelex/cli/agent_cli/commands/run/bundle_cmd.py" = "blob:a461d40dc34a1f8e273358eb6b6f2e6ea084f391""pipelex/cli/agent_cli/commands/run/method_cmd.py" = "blob:0df77aa1e35c22ceffdcefd7f4c12f28fda366c7""pipelex/cli/agent_cli/commands/run/pipe_cmd.py" = "blob:6bacf08886c38463828d9432b2cae4730e9004a3""pipelex/cli/agent_cli/commands/run/stdin_resolver.py" = "blob:db0176d7de1e074fd6eefd787c10e71b891eeb32""pipelex/cli/agent_cli/commands/validate/__init__.py" = "blob:e69de29bb2d1d6434b8b29ae775ad8c2e48c5391"-"pipelex/cli/agent_cli/commands/validate/_validate_core.py" = "blob:f74178959782fe6e8edc2b7171b9967d82cf2076"+"pipelex/cli/agent_cli/commands/validate/_validate_core.py" = "blob:11e4ee205e309a9aa89e0aced56ae52059f32eba""pipelex/cli/agent_cli/commands/validate/app.py" = "blob:1982684a2aae81e0d634bb3b262457a5f187d979"-"pipelex/cli/agent_cli/commands/validate/bundle_cmd.py" = "blob:3df822724c3320a2a6251cf1abe9a495a7e6c2c4"+"pipelex/cli/agent_cli/commands/validate/bundle_cmd.py" = "blob:92d87a0b3f1187b72d6584bc5f29169e998f46ee""pipelex/cli/agent_cli/commands/validate/method_cmd.py" = "blob:19cbf94ca5027983e5fd95b73bc346efaf195c37""pipelex/cli/agent_cli/commands/validate/pipe_cmd.py" = "blob:ca0b7bdce8e9a023e664289907f9db129c77b2cc"-"pipelex/cli/bundle_target_resolution.py" = "blob:fd4c1da386c33f4542c11a68ad4ce5bed64bc4aa"+"pipelex/cli/bundle_target_resolution.py" = "blob:c59957d96c974a0bc1e435b768010391b4c74143""pipelex/cli/cli_factory.py" = "blob:9768ead842eecbfc9509dc68aafca31957a27439""pipelex/cli/commands/__init__.py" = "blob:e69de29bb2d1d6434b8b29ae775ad8c2e48c5391""pipelex/cli/commands/build/__init__.py" = "blob:e69de29bb2d1d6434b8b29ae775ad8c2e48c5391"@@ -78,7 +78,7 @@ rationale = "real-catch: this branch makes pipelex-agent validate bundle and fix"pipelex/cli/commands/doctor_cmd.py" = "blob:a991d210787c0df68d9cc6bac29361df89e8e9c7""pipelex/cli/commands/fix/__init__.py" = "blob:e69de29bb2d1d6434b8b29ae775ad8c2e48c5391""pipelex/cli/commands/fix/_diff_sandbox.py" = "blob:d05eb4351f7065eaa45f63089f2daae0b02457dd"-"pipelex/cli/commands/fix/_fix_core.py" = "blob:3e8246119b9e89d695b3a8383094d65b89aa9ed0"+"pipelex/cli/commands/fix/_fix_core.py" = "blob:d7b3b6038ddf9735c6d6d38afd36f398e508459f""pipelex/cli/commands/fix/app.py" = "blob:61ed114633c679a252fa7d9367d4cf8e800f86ee""pipelex/cli/commands/fix/bundle_cmd.py" = "blob:83bf3fda431fb5d81f1763be28835730df0a9872""pipelex/cli/commands/fix/function_names_cmd.py" = "blob:b645a3d7687f01f18474446d29e23df4f1acc751"@@ -111,11 +111,11 @@ rationale = "real-catch: this branch makes pipelex-agent validate bundle and fix"pipelex/cli/commands/run/_inputs_file_loader.py" = "blob:8039ca6b29f4f9091d0ea7406c8b8a4590848131""pipelex/cli/commands/run/_inputs_path_resolver.py" = "blob:c8ec89d98718e18af7aaaded44c9f545be60adab""pipelex/cli/commands/run/_main_stuff_file.py" = "blob:ca806e55085aa2e5748492751521296ce1f42960"-"pipelex/cli/commands/run/_run_core.py" = "blob:56cff4673ad0e30e87b25dada61515f3188feb0d"+"pipelex/cli/commands/run/_run_core.py" = "blob:c70b6f445b2961cbd0eda878f23900ece290dfd8""pipelex/cli/commands/run/_run_hosted.py" = "blob:f61b3ea7fdc6c29823ecd6cb5b469ab271814c3f""pipelex/cli/commands/run/_run_output_folder.py" = "blob:6f3b237849d9b6498a4dbf5cb81ceea948c91782""pipelex/cli/commands/run/app.py" = "blob:e312b159625e0a72bc9a18d3bfd2c73e7ac59f24"-"pipelex/cli/commands/run/bundle_cmd.py" = "blob:faff9a7cba8027bb7b13f7f5a3b94666f3742b80"+"pipelex/cli/commands/run/bundle_cmd.py" = "blob:b4803104d485fd60be7092ea6608875c65ee4953""pipelex/cli/commands/run/exceptions.py" = "blob:cebeb5950c87a1e113ed717565ae8cb783bb5001""pipelex/cli/commands/run/method_cmd.py" = "blob:924bca6e242d8490e32eeb9ed2329db3d1b58b8d""pipelex/cli/commands/run/pipe_cmd.py" = "blob:c967729a039518a718d7bc28d77797423ce20fc4"@@ -123,9 +123,9 @@ rationale = "real-catch: this branch makes pipelex-agent validate bundle and fix"pipelex/cli/commands/update_cmd.py" = "blob:b2db6170ddd9c67f17d88096ba1f1796a897557d""pipelex/cli/commands/usage_stats_cmd.py" = "blob:b3723c64f911d1b69e1e1aa126610a132a16e3da""pipelex/cli/commands/validate/__init__.py" = "blob:e69de29bb2d1d6434b8b29ae775ad8c2e48c5391"-"pipelex/cli/commands/validate/_validate_core.py" = "blob:01c16592bd98d4b03e1a36c92699bb25fe0f4a84"+"pipelex/cli/commands/validate/_validate_core.py" = "blob:c90fee5164c0e0357f9a2f4723e3e1bed55b2feb""pipelex/cli/commands/validate/app.py" = "blob:457e110f0458bbe042a5489f58bba91acd885ebd"-"pipelex/cli/commands/validate/bundle_cmd.py" = "blob:1bf408a2d49b92476c87fa080b850b82cc870a88"+"pipelex/cli/commands/validate/bundle_cmd.py" = "blob:37db76207cb135ae813055b3cb4be2945ef25d3d""pipelex/cli/commands/validate/method_cmd.py" = "blob:f12ee233fc985d04a4512f1fef6b9ea02a1b9644""pipelex/cli/commands/validate/pipe_cmd.py" = "blob:ab8cca9c5c8c5d7dce005deba6d2d8b32dc2e81e""pipelex/cli/commands/which_cmd.py" = "blob:5febd14510603f09ef0fb29d5ef97c35b7a6692f"@@ -133,7 +133,7 @@ rationale = "real-catch: this branch makes pipelex-agent validate bundle and fix"pipelex/cli/error_handlers.py" = "blob:6f208c35d91cecaef82927634df9c58553dd181e""pipelex/cli/exceptions.py" = "blob:5c66f20b0a06c8ae8c20b94f92e2c30b78f5b6c5""pipelex/cli/installed_methods.py" = "blob:7e7f289d50ca6bc446b6891cbc846dc4a8cee6e2"-"pipelex/cli/judged_target.py" = "blob:8d3f3b3359992feeeca2bd4acd1d404262cbaf87"+"pipelex/cli/judged_target.py" = "blob:626ccf65bc3b15e06422d9f16c0b1a8083041b59""pipelex/cli/method_resolver.py" = "blob:54fe2e6036b637c2e298f18bbfd10cfc616ef4a8""pipelex/cli/readiness.py" = "blob:6d185324ed0784663e1dc78d1194a28312c54798""pipelex/cli/usage_stats_cli.py" = "blob:c4714885f6bae4ff30609c0db5463bd1b056ce61"diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 5c4757a37..a6f319fa5 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -13,10 +13,13 @@- **The API server's examples send a method in `files`**: the server's README and the API Server pages send a method to `/v1/execute` and `/v1/validate` in `files`, a map from each file's path in the method's folder to its content, where they sent `mthds_contents`. The reference pages and the OpenAPI artifact now say that `mthds_contents`, with `bundle_b64`, `resources` and `/v1/validate`'s `mthds_sources`, is being retired and will be refused by a later release, so a client sends `files` now.- **A PipeFunc package-offer factory is never called on a thread running an event loop (Breaking)**: the factory a plugin registers with `PluginRegistrar.add_pipe_func_package_offer` may block for as long as its work takes and must not use an event loop of its caller's, and pipelex enforces it: a load that would call the factory on a thread running an event loop fails with `PipelexUnexpectedError` instead, so a host that calls a load itself from asynchronous code, such as a worker's activity, moves it off the loop with `run_method_load` (`pipelex.libraries.method_load_threads`) or with `acquire_library_async` (`pipelex.pipeline.execution_seams`), the twin of `acquire_library` that loads off the loop. The factory's signature is unchanged, and so is the plugin API version. `run_method_load` runs a blocking call in the method-load pool, a named pool of at most `METHOD_LOAD_MAX_THREADS` threads (four) apart from asyncio's default executor, in a copy of the caller's context, and when its caller is cancelled, the shutdown of `asyncio.run` included, withdraws a call no thread has taken yet or lets the cancellation through only once the running call has ended; the general mechanism is `WorkerThreads` (`pipelex.tools.misc.async_utils`).- **The refusal of a package the request did not send**: the message of the `unresolved_package_dependency` refusal a static resolution gives, such as the one behind `/v1/resolve`, `/v1/codegen` and `/v1/pipe-io`, tells its reader to send the package's files in `dependencies`, keyed by its address, beside the method's own files in `files`, where it spoke of an in-memory resolve and the host filesystem and advised directory-based resolution; it still carries the words `address-based dependency '<address>'`.+- **A run lists each method tree once (Breaking)**: the method trees a run reads, the method's own, its library directories' and its dependency packages', are each listed once, and the run loads the trees its caller listed, so the `method_tree_digest` a run records is the digest of the tree it loaded: `PipelexMTHDSProtocol.execute`, `pipeline_run_setup`, `acquire_library`, `acquire_library_async`, `validate_bundle`, `validate_bundles_in_process`, `fix_bundle_file`, `fix_method_folder`, and `LibraryManagerAbstract.load_libraries` and `load_from_blueprints` take `tree_listing`, a `MethodTreeListing` (`pipelex.libraries.method_tree_listing`) holding the trees already listed, which `run bundle`, `validate bundle` and `fix bundle` on both CLIs hand down from the listing that resolved their target. The listing reads a path as the standard's lister does before it takes anything from what it holds: a folder and a symbolic link naming it are one listing, since the lister follows such a link, a bundle file named alone inside a library directory shares the listing of the tree that directory roots, and a symbolic link to a bundle file, named alone, is refused as the lister refuses it, even once the run listed the file it points to. Loads of one run running at the same time list each tree once and list different trees side by side. The API server's run routes and `/v1/validate` load the trees their check of the method's requirements listed, which `ensure_served_method_requirements_met` now lists through `tree_listing` in place of its `limits`, and which reaches the in-process validator through `scoped_tree_listing`, so the bundle-validator plugin contract is unchanged. `list_library_trees` and `list_library_dir_trees` (`pipelex.libraries.method_trees`) take the lister of a directory's own tree or of its trees (`own_tree_of`, `trees_of`), through which the listing lists them, and `tree_bundle_of_lone_file` is removed: a bundle file named alone is placed in its tree with `MethodTreeListing.lone_file_bundle`.+- **A run asks the PipeFunc package-offer factory once per tree**: the run's `MethodTreeListing` keeps each method tree and each dependency package whose `requirements.txt` a check found met, with the execution mode and the factory it was judged against (`PipeFuncEnvironmentSource`, `current_pipe_func_environment_source` in `pipelex.libraries.package_environments`), so a run judges each tree once against them however many of its steps and loads read it, two loads judging one tree at the same moment ask once (`MethodTreeListing.judge_requirements_once`), and on the API server the load serving a request takes the verdict of the runner's check of the served method instead of asking the factory again. A check against another mode or factory, a refused tree and every new run ask again: pipelex keeps a verdict for one run alone, so a factory whose answer takes time keeps its own cache across runs, by the requirements' texts. `ensure_method_requirements_met` and `ensure_package_requirements_met` take the listing as `tree_listing`.- **`pipelex-agent validate bundle` and `fix bundle` exit 2 on a configuration refused before the boot**: a configuration that does not validate, which these commands read before the boot for the limits a method's folder is listed under, exits 2, no verdict, where it exited 1, as `pipelex validate bundle` and `pipelex fix bundle` exit on it; the problem is unchanged, and `run bundle` and `inputs bundle` still exit 1.### Fixed+- **A method edited between its listing and its load is refused**: a run read its method's files again after listing them, so a file edited in between was loaded under a digest that named other bytes. Every file a load reads from a tree listed on this host, a bundle it parses, the `requirements.txt` it judges and the Python it scans, is now read against the hash the listing recorded, a load reads every file of a tree the run listed before it again, once per run, as it takes the tree, and the bundle text a CLI read for its run is checked against the listed file, so a bundle, a template or any other file of the tree edited, replaced or removed since the listing is refused before anything of the method is parsed, imported or run, with `MethodTreeChangedError` (`pipelex.libraries.exceptions`), coded `method.package_invalid` with the new reason `tree_changed` and the file's `path` in the tree; `run`, `validate` and `fix` report it as an invalid method whose item names that file. A `requirements.txt` that can no longer be read is this refusal too, where it was `requirements_invalid`. The checked reads are `read_tree_file_as_listed`, `read_tree_bundle_as_listed`, `ensure_tree_files_as_listed` and `ensure_bundle_text_as_listed` in `pipelex.libraries.method_trees`, `MethodTreeListing.loading()` marks a load reading a listing and `ensure_unchanged()` checks a tree a caller placed a bundle in, and `MthdsParser.make_pipelex_bundle_blueprint` takes the bytes it parses as `bundle_data`, beside the `bundle_path` that names them.- **Method loads leave the event loop**: every method load pipelex reaches from asynchronous code, a run's, a validation's and those of the commands that run a loop, runs in the method-load pool, where it ran on the event loop, the reading and parsing of the bundles it loads included, so listing and hashing a method's files, a sandbox's judgment of its `requirements.txt` and a fetch of a package it calls by address no longer hold up the other requests or activities the loop serves; the API server's run routes and `/v1/validate` also write a request's method tree to disk and remove it there, and judge its requirements there, so a client that disconnects never has the tree removed while the check still reads it, the crate routes resolve and project there, and the in-process validator reads a tree's entry pipe for its graph there too. A module imported by file while a method loads, such as one defining a structure class, is published and executed under a lock that the load's class discovery also holds, so two loads in parallel never take a module half executed, and a module that fails to execute leaves `sys.modules` rather than stay there half imported. `/v1/validate` of a method tree calling a package nothing shipped still resolves it as a run does, from the methods installed on the server and by a fetch on a miss, so its verdict matches the run's, and the fetch now runs off the event loop.- **Where a method's Python travels, in the docs**: the PipeFunc Requirements, PipeFunc and run pages say that the API server receives a method as a tree holding its bundles and its Python together and judges its `requirements.txt` from the tree's listing, that the runner never imports a method's Python, that a run's crate names each step's method tree by its digest and carries none of its files, that a `PipeFunc` call's files are read through the method file store the worker's host installs, and that a worker a method tree's validation is dispatched to checks the method's requirements again, where they said the server received a method's Python apart from its bundles, the Python was captured as text, the crate carried the method's Python sources and the worker never checked anything.diff --git a/api/pipelex_api/routes/pipelex/pipeline.py b/api/pipelex_api/routes/pipelex/pipeline.pyindex 4693c0b11..d994ff36d 100644--- a/api/pipelex_api/routes/pipelex/pipeline.py+++ b/api/pipelex_api/routes/pipelex/pipeline.py@@ -18,6 +18,7 @@ from pipelex.errors.problem import ProblemViewfrom pipelex.errors.registry.pipelex_codes import PipelexErrorCodefrom pipelex.libraries.method_load_threads import run_method_loadfrom pipelex.libraries.method_requirements import ensure_served_method_requirements_met+from pipelex.libraries.method_tree_listing import MethodTreeListing, scoped_tree_listingfrom pipelex.pipe_run.delivery_assignment import DeliveryAssignment, StorageTarget, WebhookTargetfrom pipelex.pipe_run.exceptions import AsyncExecutionNotEnabledErrorfrom pipelex.pipe_run.pipe_run_protocol import PipeRunProtocol@@ -352,6 +353,7 @@ class ApiRunner(PipelexMTHDSProtocol):extra: dict[str, Any] | None = None,delivery_assignment: DeliveryAssignment | None = None,request_id: str | None = None,+ tree_listing: MethodTreeListing | None = None,requested_orchestration_mode: str | None = None,) -> PipelexRunResultExecute:"""Execute a method synchronously, dispatching by the resolved `orchestration_mode`.@@ -375,6 +377,9 @@ class ApiRunner(PipelexMTHDSProtocol):`RunMetadata.request_id` for log correlation, so it reaches the job a worker is handed.`requested_orchestration_mode` is the optional per-request backend override(`PipelineApiExtras.orchestration_mode`).++ The method's trees are listed once for the request: the check of its requirements lists them through+ `tree_listing`, or a listing of its own when none is given, and the run loads them from that same listing."""# Resolve the effective orchestration mode FIRST — a per-request override the deployment# policy forbids is refused (422) here, before any library load / run registration. Mirrors start().@@ -383,7 +388,8 @@ class ApiRunner(PipelexMTHDSProtocol):orchestrator = get_orchestrator_registry().get_optional(mode=orchestration_mode)if orchestrator is None:raise _missing_orchestrator_error(mode=orchestration_mode, api_config=api_config)- await self._ensure_served_method_requirements_met()+ listing = tree_listing if tree_listing is not None else MethodTreeListing()+ await self._ensure_served_method_requirements_met(tree_listing=listing)# Dispatch the run through the mode-selected orchestrator by injecting it as this runner's# PipeRun, then delegate to the base execute, which owns the full run lifecycle. The@@ -400,6 +406,7 @@ class ApiRunner(PipelexMTHDSProtocol):extra=extra,delivery_assignment=delivery_assignment,request_id=request_id,+ tree_listing=listing,)@override@@ -473,7 +480,10 @@ class ApiRunner(PipelexMTHDSProtocol):f"async-capable orchestration, and this deployment has none. Use /execute (synchronous) instead.")raise AsyncExecutionNotEnabledError(msg)- await self._ensure_served_method_requirements_met()+ # The method's trees are listed once for the request: the requirements check lists them, and the job's load+ # takes them from the same listing.+ tree_listing = MethodTreeListing()+ await self._ensure_served_method_requirements_met(tree_listing=tree_listing)created_at = get_current_iso_timestamp()pipelex_inputs: PipelineInputs | WorkingMemory | None = cast("PipelineInputs | WorkingMemory | None", inputs)@@ -507,6 +517,7 @@ class ApiRunner(PipelexMTHDSProtocol):submitted_files=self.submitted_files,pipeline_run_id=pipeline_run_id,request_id=request_id,+ tree_listing=tree_listing,)delivery_assignment = DeliveryAssignment(@@ -555,13 +566,15 @@ class ApiRunner(PipelexMTHDSProtocol):crate_fingerprint=pipe_job.job_metadata.run_metadata.crate_fingerprint,)- async def _ensure_served_method_requirements_met(self) -> None:+ async def _ensure_served_method_requirements_met(self, *, tree_listing: MethodTreeListing) -> None:"""Refuse the request's method when its requirements.txt is not met, before anything loads or is dispatched.This runner's `library_dirs` are the directories it wrote the request's method tree into, the caller's own,whose load reports a requirements refusal as the caller's too; judging it here first refuses the run beforeany library opens or any run is registered, naming no path on this host. The tree is held as written, so its- listing is the tree the request sent. The listing reads and hashes files, so it runs off the event loop, in the method-load pool.+ listing is the tree the request sent. The listing reads and hashes files, so it runs off the event loop, in the method-load pool,+ and it is made through `tree_listing`, which the caller hands to the request's load, so the load reads the trees+ checked here without listing them again.Raises:MethodRequirementsFileError: The method's requirements.txt holds a line the standard refuses.@@ -574,7 +587,7 @@ class ApiRunner(PipelexMTHDSProtocol):library_dirs = [Path(library_dir) for library_dir in self.library_dirs]# In the method-load pool, whose cancellation waits for the check to end: the directories it lists are removed# when the request's scope exits, which a client disconnect can make happen while the check still hashes them.- await run_method_load(ensure_served_method_requirements_met, library_dirs=library_dirs)+ await run_method_load(ensure_served_method_requirements_met, library_dirs=library_dirs, tree_listing=tree_listing)async def validate_verdict(self,@@ -624,6 +637,10 @@ class ApiRunner(PipelexMTHDSProtocol):arm hands the method's trees to its worker through. The route builds a validating runner with the`read_scope` its request names, `None` when it names none, and never with the caller's id a runwould fall back to: nothing lists or erases what a validation would leave under that prefix.++ The method's trees are listed once for the request: the requirements check lists them, and the validation is+ called with that listing in scope (`scoped_tree_listing`), which the in-process validator loads the trees from,+ while a validator dispatching to a worker never reads it."""# Resolve the effective mode FIRST — a per-request override the deployment policy forbids# is refused (422) here, before any validator dispatch / library load. Mirrors start().@@ -638,31 +655,35 @@ class ApiRunner(PipelexMTHDSProtocol):# worker a method tree's validation is dispatched to never reads these directories: it receives the method's# trees through its method file store and judges their requirements again as it loads them. A refusal is the# method's invalid verdict, as a run of it is refused before its files load.+ tree_listing = MethodTreeListing()try:with translate_to_validate_bundle_error():- await self._ensure_served_method_requirements_met()+ await self._ensure_served_method_requirements_met(tree_listing=tree_listing)except ValidateBundleError as invalid_method:return BundleValidationFailure(problem=invalid_method.to_problem(view=ProblemView.OPERATOR),validation_errors=invalid_method.validation_error_items(),)- verdict = await validator.validate_bundles(- mthds_contents=mthds_contents,- mthds_sources=mthds_sources,- allow_signatures=allow_signatures,- # A method directory is the whole method: no library directory loads beside it.- library_dirs=None if method_dir is not None else library_dirs,- method_dir=method_dir,- methods_dirs=self.methods_dirs if method_dir is not None else None,- # A validation is not a run, but its dry runs and its `pipe_dry_run` event are still- # done for the caller this runner was built for — the user and groups a run would state.- caller_identity=self.caller_identity,- graph_pipe_code=graph_pipe_code,- # The files submitted with the contents, which the validation's dry run reads, the template check included.- submitted_files=self.submitted_files,- # The scope the host named, under which a dispatched validator's worker reads the method's trees.- read_scope=self.read_scope,- )+ # The validator seam takes no listing: the one the requirements check made is in scope around the call, so the+ # in-process validator loads the trees it checked without listing them again.+ with scoped_tree_listing(tree_listing=tree_listing):+ verdict = await validator.validate_bundles(+ mthds_contents=mthds_contents,+ mthds_sources=mthds_sources,+ allow_signatures=allow_signatures,+ # A method directory is the whole method: no library directory loads beside it.+ library_dirs=None if method_dir is not None else library_dirs,+ method_dir=method_dir,+ methods_dirs=self.methods_dirs if method_dir is not None else None,+ # A validation is not a run, but its dry runs and its `pipe_dry_run` event are still+ # done for the caller this runner was built for — the user and groups a run would state.+ caller_identity=self.caller_identity,+ graph_pipe_code=graph_pipe_code,+ # The files submitted with the contents, which the validation's dry run reads, the template check included.+ submitted_files=self.submitted_files,+ # The scope the host named, under which a dispatched validator's worker reads the method's trees.+ read_scope=self.read_scope,+ )# The core seam types its valid arm at the protocol-level ValidationReport (a leaf type)# to stay import-acyclic in core; every registered validator in fact produces the canonical# PipelexValidationReport. Recover the precise type here — the single narrowing point — sodiff --git a/api/tests/unit/test_loads_off_the_event_loop.py b/api/tests/unit/test_loads_off_the_event_loop.pyindex 8dd5e94fd..1abb04bb0 100644--- a/api/tests/unit/test_loads_off_the_event_loop.py+++ b/api/tests/unit/test_loads_off_the_event_loop.py@@ -6,10 +6,11 @@ from typing import Anyimport pytestfrom fastapi import FastAPIfrom fastapi.testclient import TestClient-from mthds.method_tree.models import MethodTree+from mthds.method_tree.models import MethodTree, TreeLimitsfrom mthds.protocol.problem.registry import CoreCodefrom pipelex.config import get_configfrom pipelex.libraries.method_load_threads import METHOD_LOAD_THREADS+from pipelex.libraries.method_trees import list_library_dir_tree, list_local_method_treefrom pipelex.libraries.written_trees import WRITTEN_METHOD_TREES, write_method_treefrom pipelex.methods.exceptions import MethodFetchErrorfrom pipelex.methods.fetching import FetchedMethodPackage@@ -128,26 +129,59 @@ class TestLoadsOffTheEventLoop:assert WRITTEN_METHOD_TREES.tree_written_at(directory=method_dir) is Nonedef test_a_validation_asks_the_package_offer_off_the_event_loop(self, mocker: MockerFixture, tmp_path: Path, recording_offer: _RecordingOffer):- """The runner's check of the served method asks the factory first, then the in-process validator's load does."""+ """The runner's check of the served method asks the factory, once, and the in-process validator's load takes its verdict."""client = _build_client(mocker, tmp_path=tmp_path)response = client.post("/v1/validate", json={"files": _TREE_WITH_REQUIREMENTS})assert response.status_code == 200, response.textassert response.json()["is_valid"] is True, response.text- assert len(recording_offer.thread_names) > 1, recording_offer.thread_names+ assert len(recording_offer.thread_names) == 1, recording_offer.thread_namesassert all(_in_the_method_load_pool(thread_name) for thread_name in recording_offer.thread_names), recording_offer.thread_namesdef test_a_run_asks_the_package_offer_off_the_event_loop(self, mocker: MockerFixture, tmp_path: Path, recording_offer: _RecordingOffer):- """The runner's check of the served method asks the factory first, then the run's own load does."""+ """The runner's check of the served method asks the factory, once, and the run's own load takes its verdict."""client = _build_client(mocker, tmp_path=tmp_path)response = client.post("/v1/execute", json={"files": _TREE_WITH_REQUIREMENTS, "inputs": {"data": "hello"}})assert response.status_code == 200, response.text- assert len(recording_offer.thread_names) > 1, recording_offer.thread_names+ assert len(recording_offer.thread_names) == 1, recording_offer.thread_namesassert all(_in_the_method_load_pool(thread_name) for thread_name in recording_offer.thread_names), recording_offer.thread_names+ @pytest.mark.parametrize(+ ("route", "body"),+ [+ ("/v1/validate", {"files": _TREE_WITH_REQUIREMENTS}),+ ("/v1/execute", {"files": _TREE_WITH_REQUIREMENTS, "inputs": {"data": "hello"}}),+ ],+ ids=["validate", "execute"],+ )+ def test_a_request_lists_each_tree_of_its_method_once(+ self, mocker: MockerFixture, tmp_path: Path, recording_offer: _RecordingOffer, route: str, body: dict[str, Any]+ ):+ """The runner's check of the served method lists its trees, and the validation or the run loads them from that listing."""+ client = _build_client(mocker, tmp_path=tmp_path)+ listed: list[Path] = []++ def list_directory_after_noting_it(*, library_dir: Path, limits: TreeLimits | None = None) -> MethodTree | None:+ listed.append(library_dir.resolve())+ return list_library_dir_tree(library_dir=library_dir, limits=limits)++ def list_handed_over_after_noting_it(*, handed_over: Path, limits: TreeLimits | None = None) -> MethodTree:+ listed.append(handed_over.resolve())+ return list_local_method_tree(handed_over=handed_over, limits=limits)++ mocker.patch("pipelex.libraries.method_tree_listing.list_library_dir_tree", side_effect=list_directory_after_noting_it)+ mocker.patch("pipelex.libraries.method_tree_listing.list_local_method_tree", side_effect=list_handed_over_after_noting_it)++ response = client.post(route, json=body)++ assert response.status_code == 200, response.text+ assert recording_offer.thread_names, "the check never judged the method"+ assert listed, "the request listed no tree"+ assert len(listed) == len(set(listed)), listed+def test_a_validation_fetches_an_unshipped_dependency_off_the_event_loop(self, mocker: MockerFixture, tmp_path: Path):"""A tree calling a package nothing shipped resolves as its run would, by a fetch on a miss, in a worker thread."""client = _build_client(mocker, tmp_path=tmp_path)diff --git a/api/tests/unit/test_method_requirements_refusal.py b/api/tests/unit/test_method_requirements_refusal.pyindex e05010f7e..1838d1792 100644--- a/api/tests/unit/test_method_requirements_refusal.py+++ b/api/tests/unit/test_method_requirements_refusal.py@@ -21,6 +21,7 @@ from pipelex.errors.bundle_validation_failure import BundleValidationFailurefrom pipelex.errors.registry.pipelex_codes import PipelexErrorCodefrom pipelex.interpreter_hub import get_interpreter_hubfrom pipelex.libraries.exceptions import MethodRequirementsNotMetError+from pipelex.libraries.method_tree_listing import MethodTreeListingfrom pipelex.pipe_run.delivery_assignment import DeliveryAssignmentfrom pipelex.pipe_run.pipe_job import PipeJobfrom pipelex.plugins.bundle_validator_registry import BundleValidatorRegistry@@ -256,7 +257,7 @@ class TestMethodRequirementsRefusal:"""The check lists and hashes the method's files, so it runs in a worker thread, never on the loop serving requests."""threads_holding_a_loop: list[bool] = []- def refuse_after_noting_the_thread(*, library_dirs: Sequence[Path]) -> None: # noqa: ARG001+ def refuse_after_noting_the_thread(*, library_dirs: Sequence[Path], tree_listing: MethodTreeListing) -> None: # noqa: ARG001try:asyncio.get_running_loop()except RuntimeError:diff --git a/docs/building-methods/method-trees.md b/docs/building-methods/method-trees.mdindex 0c5b1fcb7..bd4359574 100644--- a/docs/building-methods/method-trees.md+++ b/docs/building-methods/method-trees.md@@ -81,7 +81,9 @@ max_total_bytes = 4294967296These limits are not the hosted API's. A deployment sets its own, which a local run cannot know, and applying them locally would refuse a method rooted in a cluttered folder that loads today; a sender warned of the hosted limits before it sends is the hosted run's concern, not a local load's.-Every listing computes the tree's digest, the SHA-256 of its canonical listing of paths, hashes and sizes, which names the exact files of a method: two folders with the same files have the same digest. A local run records the digest of the tree it ran. `pipelex run bundle` prints it with the run's recap and writes `method_tree.json` into the run's output folder, holding the digest, the tree's root on this machine, its entry and each file's path, hash and size. `pipelex-agent run bundle` adds `method_tree_digest` to the run's output file and, with `--with-memory`, to its output.+Every listing computes the tree's digest, the SHA-256 of its canonical listing of paths, hashes and sizes, which names the exact files of a method: two folders with the same files have the same digest. A local run records the digest of the tree it ran, and lists that tree once: the command lists the folder or the file it was handed to find the pipe to start at, and the run loads the tree that listing returned rather than listing it again, so the digest recorded is the digest of the tree loaded. `validate bundle` and `fix bundle` likewise load the tree they listed to resolve their target, and so does the API server, which lists a request's method once to check its requirements and to load it. `pipelex run bundle` prints the digest with the run's recap and writes `method_tree.json` into the run's output folder, holding the digest, the tree's root on this machine, its entry and each file's path, hash and size. `pipelex-agent run bundle` adds `method_tree_digest` to the run's output file and, with `--with-memory`, to its output. A run lists each tree once whichever way it is named: a folder and a symbolic link naming it are one listing, as the lister follows such a link, and a bundle file named alone inside a library directory shares the listing of the tree the directory roots, while a symbolic link to a bundle file, named alone, is refused as the lister refuses it, even after the run listed the file it points to.++The files of a tree may change after it is listed, so a run never trusts them blindly. The load reads each bundle it parses, the `requirements.txt` it judges and the Python it scans against the hash the listing recorded, and reads every file of a tree listed before the load again, once per run, as the load takes the tree, such as the folder the command resolved: a bundle, a template or any other file of the tree edited, replaced or removed since the listing is refused before anything of the method is parsed or run, with a `MethodTreeChangedError` (`method.package_invalid`, reason `tree_changed`) naming the file by its path in the tree, which `run`, `validate` and `fix` report as an invalid method whose item names that file. Run again once the files stop changing. A file a step reads during the run, such as a template, is read against its hash as well (see [Inside a run](#inside-a-run)).## Inside a rundiff --git a/docs/tools/cli/agent-cli.md b/docs/tools/cli/agent-cli.mdindex cc784c7cf..3b82c345f 100644--- a/docs/tools/cli/agent-cli.md+++ b/docs/tools/cli/agent-cli.md@@ -51,7 +51,7 @@ pipelex-agent run method <NAME> [OPTIONS]For `bundle` and `method`, use `--pipe` to target a specific pipe.-`run bundle` reads its method through its [method tree](../../building-methods/method-trees.md), as `pipelex run bundle` does. A folder is the method rooted at the nearest `METHODS.toml` at or above it, else at the folder itself, and with no `--pipe` the run starts at the manifest's `main_pipe`, or, with no manifest, at the one `main_pipe` its bundles declare; none or several is refused with the problem `request.pipe_unspecified`, whose message names each bundle by its path in the tree and whose `params.candidates` names the declared ones when there are several, with no `errors`, since the method may be valid and only the pipe to start at is missing: pass `--pipe`. A bundle file named alone starts at its own `main_pipe`, and with no `METHODS.toml` at or above its folder its method is that one file, so a step naming a file beside it, such as a `template_file`, is refused with the advice to run the folder or to add a `METHODS.toml`. The run's output file (`live_run.json` or `dry_run.json`, in the run's output folder) carries `method_tree_digest`, the digest of the tree the run read, and so does the output under `--with-memory`.+`run bundle` reads its method through its [method tree](../../building-methods/method-trees.md), as `pipelex run bundle` does. A folder is the method rooted at the nearest `METHODS.toml` at or above it, else at the folder itself, and with no `--pipe` the run starts at the manifest's `main_pipe`, or, with no manifest, at the one `main_pipe` its bundles declare; none or several is refused with the problem `request.pipe_unspecified`, whose message names each bundle by its path in the tree and whose `params.candidates` names the declared ones when there are several, with no `errors`, since the method may be valid and only the pipe to start at is missing: pass `--pipe`. A bundle file named alone starts at its own `main_pipe`, and with no `METHODS.toml` at or above its folder its method is that one file, so a step naming a file beside it, such as a `template_file`, is refused with the advice to run the folder or to add a `METHODS.toml`. The run's output file (`live_run.json` or `dry_run.json`, in the run's output folder) carries `method_tree_digest`, the digest of the tree the run read, and so does the output under `--with-memory`: the tree is listed once, before its bundle is read, and the run loads the tree that listing returned, so the digest is the digest of the files the run loaded. A file of the tree edited after that listing is refused as an invalid method whose item names the file, rather than loaded.#### Running on the hosted APIdiff --git a/docs/tools/cli/run.md b/docs/tools/cli/run.mdindex 9bc88e926..a8f7060a5 100644--- a/docs/tools/cli/run.md+++ b/docs/tools/cli/run.md@@ -82,7 +82,7 @@ pipelex run bundle <PATH> [OPTIONS]Runs a method from a bundle file (`.mthds`) or a method's folder, read through its [method tree](../../building-methods/method-trees.md). A folder is the method rooted at the nearest `METHODS.toml` at or above it, else at the folder itself, and with no `--pipe` the run starts at the manifest's `main_pipe`, or, with no manifest, at the one `main_pipe` its bundles declare; none or several is refused with `Failed to run:` and a message naming each bundle by its path in the tree, which is no verdict on the method, since only the pipe to start at is missing: pass `--pipe`. An inputs file (`inputs.json` or `inputs.toml`) in the folder is auto-detected when `--inputs` is not passed (see [Auto-detection and ambiguity](#auto-detection-and-ambiguity)). A bundle file named alone starts at its own `main_pipe`, and with no `METHODS.toml` at or above its folder its method is that one file, so a step naming a file beside it, such as a `template_file`, is refused: run the folder instead, or add a `METHODS.toml`.-The run prints the digest of the tree it read with its recap, and records the tree as `method_tree.json` in its output folder (see [The results directory](#the-results-directory)).+The run prints the digest of the tree it read with its recap, listed once, before its bundle is read, and loaded from that same listing, and records the tree as `method_tree.json` in its output folder (see [The results directory](#the-results-directory)). A file of the tree edited after that listing is refused as an invalid method naming the file, rather than loaded.**Arguments:**diff --git a/docs/under-the-hood/pipe-func-requirements.md b/docs/under-the-hood/pipe-func-requirements.mdindex f50251688..7c1c2cff9 100644--- a/docs/under-the-hood/pipe-func-requirements.md+++ b/docs/under-the-hood/pipe-func-requirements.md@@ -14,8 +14,12 @@ A method tree is checked when it lies on this host's disk, holds a `requirementsThe check is `pipelex/libraries/method_requirements.py`, and a load runs it at three places, each before any Python of the tree is imported, or scanned for structure classes on a sandbox-hosted deployment, and before any of its pipes is built. A load never imports a `PipeFunc`'s module anyway: the step imports its function's module by its import path when it first binds it, in a validation's dry run or its first run, which comes after the load and its check, and the only Python a direct-mode load imports, the modules defining a concept's structure class, is imported after the check too.- `LibraryManager.load_libraries` checks every tree of its library directories, and the tree of each bundle file it is handed, once all of them are listed.-- `LibraryManager.load_from_blueprints` checks the tree of every bundle read from this host's disk, on every load: a blueprint an earlier load placed in its tree, a refused one included, is judged again rather than taken as judged.-- A dependency package, one a bundle calls by its address, is checked as soon as its tree is listed, before any of its bundles loads.+- `LibraryManager.load_from_blueprints` checks the tree of every bundle read from this host's disk, on every load, taking as judged only a tree the run's listing holds as met (below): a blueprint an earlier load placed in its tree is judged again when that load's listing was another, and a refused one always is.+- A dependency package, one a bundle calls by its address, is checked as soon as its tree is listed, before any of its bundles loads. The run's listing lists the package's tree once and keeps its verdict as it keeps a method's, so a package loaded again in the run, by another load or another bundle calling it, is neither listed nor judged again.++**Each tree is judged once per run.** The listing a run reads its method trees through, a `MethodTreeListing` (`pipelex/libraries/method_tree_listing.py`), keeps each tree a check of the run found met, by its directory and its digest, with what the verdict was judged against: the PipeFunc execution mode and the plugin factory stating its packages (`PipeFuncEnvironmentSource` in `pipelex/libraries/package_environments.py`). A later check of the same run against the same mode and factory takes the verdict as given: `load_from_blueprints` does not judge again a tree `load_libraries` just judged, and a load does not judge again a tree the API server's check of the request found met. Two loads of the run judging one tree at the same moment, in two threads of the method-load pool, judge it once, the second waiting for the first and taking its verdict (`MethodTreeListing.judge_requirements_once`). A check against another mode or another factory judges the tree again, a refused tree is never kept, so a check that comes after a refusal judges it again, and a load handed no listing makes one of its own, shared by its steps. The verdict does not depend on whether a refusal may name host paths, which changes only its wording, so a server's check and the load after it share it. It lasts one run, so a factory whose answer takes time keeps its own cache across runs, by the requirements' texts.++**The `requirements.txt` judged is the one the run listed.** The check reads the file against the hash its tree's listing recorded, so a file edited, replaced or removed since the listing is refused with `method.package_invalid`, reason `tree_changed`, rather than judged in place of the file the run's digest names (see [Method trees](../building-methods/method-trees.md#limits-and-the-tree-digest)).**A load never runs on the event loop.** It is blocking work: it lists and hashes the method's files, asks the package-offer factory to judge its requirements, reads the methods installed on this host and may fetch a package a bundle calls by its address. So every load Pipelex reaches from asynchronous code, a run's (`acquire_library_async` in `pipelex/pipeline/execution_seams.py`), a validation's (`validate_bundle`) and those of the commands that run a loop, runs in a worker thread of the method-load pool through `run_method_load` (`pipelex/libraries/method_load_threads.py`), the reading and parsing of the bundles it loads included. The thread runs in a copy of its caller's context, so it reads the current library and every other context variable its caller set, and what it sets stays in its copy. When its caller is cancelled, the shutdown of `asyncio.run` included, a load no thread has taken yet is withdrawn and never runs, and a running one is waited for until it ends before the cancellation goes through, since a thread cannot be interrupted, so a library is torn down only when nothing still loads into it.@@ -50,13 +54,13 @@ A server loads its own library directories (`PIPELEXPATH`, the instance defaults## Where the check runs on a sandbox-hosted deployment-**On the runner first, and never in the sandbox.** The runner is the process that holds the method's files, and the place a refusal reaches the caller before anything is dispatched. The runner loads the directory it makes for a request's files as one of its own library directories, where a refusal is the host's fault, so it judges the caller's method first, through `ensure_served_method_requirements_met`, run off the event loop since it lists and hashes the method's files:+**On the runner first, and never in the sandbox.** The runner is the process that holds the method's files, and the place a refusal reaches the caller before anything is dispatched. The runner loads the directory it makes for a request's files as one of its own library directories, where a refusal is the host's fault, so it judges the caller's method first, through `ensure_served_method_requirements_met`, run off the event loop since it lists and hashes the method's files. It lists them through a `MethodTreeListing` (`pipelex/libraries/method_tree_listing.py`) it hands to the run or the validation serving the request, which loads the trees from it, so a request lists and hashes the method's files once:- **A run** (`/execute`, `/start`) with a `files` or `bundle_b64` bundle, or by `method_ref`, is refused there as a `422` naming each requirement, before the method's files load and before any dispatch.- **A validation of a method tree** (`/validate`), sent in `files` or fetched by `method_ref`, is judged there too, whether its validator runs in the runner's process or on a worker it is dispatched to, a Temporal worker for instance: `ApiRunner.validate_verdict` returns a refusal as the invalid verdict, one item per line, before the validator is reached. Inline `mthds_contents` carry no Python and are not checked.- **A directory of the request that cannot be listed** stays the host's fault, a `500` `HostLibraryTreeError` carrying `pipelex.config.invalid`, as the load reports it.- **The worker running a run** loads the library crate the runner built, which names each step's method tree by its digest and carries none of the tree's files, its Python and its `requirements.txt` included, so loading it checks nothing. A `PipeFunc` call's files are read on the worker, through the method file store its host installs (`HubSlot.METHOD_FILE_STORE`), and handed to the sandbox, which loads the same crate and checks nothing either.-- **A worker a method tree's validation is dispatched to** receives the method's trees rather than a crate (`BundleValidatorProtocol`) and loads them, so it judges the requirements again, as every load of a tree does, against the package set the plugins of its own process state for the mode. Its deployment therefore registers the same offer as the runner's: a worker stating none refuses every method declaring requirements with `PipeFuncPackageSetUnstatedError`.+- **A worker a method tree's validation is dispatched to** receives the method's trees rather than a crate (`BundleValidatorProtocol`) and loads them, so it judges the requirements again, in its own process and through its own listing, against the package set the plugins of its own process state for the mode. Its deployment therefore registers the same offer as the runner's: a worker stating none refuses every method declaring requirements with `PipeFuncPackageSetUnstatedError`.## The seam: stating the sandbox's packages@@ -95,7 +99,7 @@ A mode with one fixed package set ignores the requirements and answers its packa- **The markers are the factory's to evaluate.** The requirements arrive with their markers unevaluated, since Pipelex cannot evaluate a marker before it knows the environment the mode would run the method in. A requirement whose marker is false in that environment is not needed there, and the check skips it, so the factory evaluates the markers against the marker environment of the place it would offer, builds nothing for a requirement the place does not need, and never refuses for one: a refusal rests only on requirements whose markers hold there. `requirements_needed_in` (`pipelex/libraries/method_requirements.py`) picks them out as the check would, keeping a requirement whose marker reads a variable the marker environment does not state, or cannot be evaluated, which the check refuses rather than skips.- **`PipeFuncPackageOffer`** is closed, frozen plain data: `environment`, the place as a refusal names it to the method's author; `distributions`, each distribution's name with its exact installed version, the names compared normalized; and `marker_environment`, the PEP 508 marker values of the sandbox's interpreter. A variable left out of `marker_environment`, such as `python_full_version` when the image pins only its minor version, makes every requirement whose marker reads it refused. Pipelex judges every requirement against the offer, one built for them included, so an offer that does not meet them refuses the method as a fixed package set would, naming each requirement.- **`PipeFuncPackageRefusal`** is the answer when no environment of the mode can hold the requirements the place needs, such as a version conflicting with a package the sandbox's base pins, or a package with no wheel for the sandbox: `environment`, the place as the refusal names it, and `reason`, the plugin's own words, such as its resolver's explanation. The method is refused with `MethodRequirementsNotMetError`, a fault of the method's carrying `method.dependency_missing`, whose message, "The method needs Python packages that the Example sandbox cannot offer, as its requirements.txt declares: …", carries the reason, with one item on the file and no line, since the reason is about the requirements together. The reason reaches the method's author, a server's caller included, so it names no path on the host.-- **The factory is synchronous and never called on a thread running an event loop, and caching by the requirements is the plugin's job.** The check runs inside the library load, which is synchronous, and [every load Pipelex reaches from asynchronous code runs in a worker thread of the method-load pool](#what-is-checked-and-when), as the API server's check of a served method does, while a synchronous caller, such as a command loading a method before it starts a loop, calls it on its own thread, which runs none. So a factory may block for as long as its work takes, and it must not use an event loop of its caller's, since none runs on its thread and the one its caller's code runs on, if any, belongs to another thread. Pipelex enforces the rule: a factory about to be called on a thread running an event loop is not called, and the load fails with `PipelexUnexpectedError`, so a host that calls a load itself from asynchronous code, such as a worker's activity loading a library directly, moves it off the loop the same way, with `run_method_load`. An asynchronous factory would buy nothing short of making every load asynchronous, so the seam stays synchronous. Pipelex calls the factory for each tree declaring requirements each time a check judges it, never at registration, and keeps none of its answers, so a load that judges a tree at more than one of its steps asks again. A factory whose answer takes time, such as one resolving the requirements with `uv`, which takes seconds the first time, bounds the work with a timeout of its own and caches its answers by the requirements' texts, so only a process's first check of a requirement set pays for it: on the API server, the check of a served method runs first, and the load after it reads the cache.+- **The factory is synchronous and never called on a thread running an event loop, and caching by the requirements is the plugin's job.** The check runs inside the library load, which is synchronous, and [every load Pipelex reaches from asynchronous code runs in a worker thread of the method-load pool](#what-is-checked-and-when), as the API server's check of a served method does, while a synchronous caller, such as a command loading a method before it starts a loop, calls it on its own thread, which runs none. So a factory may block for as long as its work takes, and it must not use an event loop of its caller's, since none runs on its thread and the one its caller's code runs on, if any, belongs to another thread. Pipelex enforces the rule: a factory about to be called on a thread running an event loop is not called, and the load fails with `PipelexUnexpectedError`, so a host that calls a load itself from asynchronous code, such as a worker's activity loading a library directly, moves it off the loop the same way, with `run_method_load`. An asynchronous factory would buy nothing short of making every load asynchronous, so the seam stays synchronous. Pipelex calls the factory for each tree declaring requirements the first time a check of a run judges it, never at registration, and keeps the tree's verdict for the rest of the run only when the tree meets its requirements: a load asks once per tree, however many of its steps judge the tree, and on the API server the check of a served method asks for the request, the load after it asking nothing. Pipelex keeps no answer across runs, so the next run, or the next request, asks again for the same requirements. A factory whose answer takes time, such as one resolving the requirements with `uv`, which takes seconds the first time, bounds the work with a timeout of its own and caches its answers by the requirements' texts, so only a process's first check of a requirement set pays for it.- **A fault of the plugin's own is not a refusal.** A resolver the plugin cannot reach, or one outlasting its timeout, says nothing about the method: the factory raises it as an error of the plugin's, which Pipelex lets through, rather than answering a `PipeFuncPackageRefusal`, which would blame the method's author.- **It fails closed.** A mode with no registered offer, or whose factory answers `None`, meets no requirement: a method declaring one is refused with `PipeFuncPackageSetUnstatedError`, a fault of the deployment carrying `runtime.misconfigured`, whose message only the operator view carries, rather than loading and failing inside the sandbox at its first import. Its message says which of the two happened: no plugin registers an offer for the mode, or the registered one could not state the packages, whose remedy is in the plugin's configuration. A method declaring none still loads.- **It is interpreter-layer**, like `add_pipe_func_executor`: a plugin published under `pipelex.plugins.kernel` that calls it is refused with `PluginLayerViolationError`. A second plugin stating the same mode is refused with `DuplicatePipeFuncPackageOfferError`, naming both.diff --git a/pipelex/cli/agent_cli/CLAUDE.md b/pipelex/cli/agent_cli/CLAUDE.mdindex fe1d37bb2..001c83717 100644--- a/pipelex/cli/agent_cli/CLAUDE.md+++ b/pipelex/cli/agent_cli/CLAUDE.md@@ -105,7 +105,7 @@ commands/- **Async core**: Run and validate are async — commands use `asyncio.run()`.- **File convention**: Generated outputs go to `mthds-wip/` with incremental naming (`pipeline_01/`, `pipeline_02/`). A `run` writes into a new numbered folder, `results/<pipe_code>_output_<NN>/`, made by `make_run_output_folder` (`pipelex/cli/commands/run/_run_output_folder.py`) beside the bundle it ran, else under `mthds-wip/`; the folder holds a `.gitignore` of `*`, so a run's outputs never join the method tree and two runs read the same tree, with the same `method_tree_digest`. `run_pipeline_core`'s `output_dir_override` replaces the directory the folder is made under: `run method` on a fetched target (a method address or GitHub URL) passes `results/` under the caller's CWD, since the fetched clone is an ephemeral temp dir deleted at process exit, and `run bundle` passes the directory `bundle_run_outputs_dir` settles for its target, `results/` beside the bundle file named or the bundle defining a folder's entry pipe, or the folder's own `results/` when `--pipe` names the pipe, which `validate bundle --graph` asks too.- **Validate refusals of the method are verdicts**: the `validate` commands run their library load inside `translate_to_validate_bundle_error` (`pipelex/pipeline/validate_bundle_translation.py`), so every refusal raised while loading a bundle whose code is in the `method` domain, the unknown model (`PipeOperatorModelChoiceError`, an `unknown_model` item) included, reaches the command as a `ValidateBundleError` and leaves with the invalid verdict's problem and exit 1. A refusal of another actor, the request's or the input's included, keeps its own class and code, and so does a dry run none of whose failures is the method's. No `validate` command keeps an arm of its own for a model-choice error; exit 2 stays for a failure of the tool or its environment (config, runtime, security).-- **Run refusals are verdicts too**: a run loads its bundle inside the same translation (`acquire_library`), so an invalid bundle leaves `run pipe` / `run bundle` / `run method` as a `ValidateBundleError` with exit 1, through the generic `agent_error` arm (`run bundle`'s own reading of its bundle through `agent_error_method_refusal()`, in the caller's view on a runner), and its problem (`method.invalid`) carries the verdict's items as `errors`, and the Markdown lists the same errors one per line, so both formats list the one problem's errors and a foreign exception raised while handling an earlier verdict lists none of its items. `run_pipeline_core` sets `library_dirs_are_callers=True`, so a refusal in a `-L` directory (or the bundle's own directory) is the same verdict; `run bundle`'s own parse of a bundle without `--pipe` goes through the translation too, and so does its reading of the method tree: a folder resolves through `pipelex/cli/bundle_target_resolution.py` (`resolve_method_folder_run`, the protocol's entry rule in `pipelex/pipeline/method_entry.py`) and a lone file through `list_bundle_file_tree`, listed before the file is read, so a tree past its limits, a symbolic link or a special file such as a FIFO is a `ValidateBundleError` too, refused unread, and a bundle file the listing cannot read is `client.file_unreadable`, while a folder with no single `main_pipe` is the run's own refusal, a `MethodEntryError` coded `request.pipe_unspecified` or `request.pipe_ambiguous`, no verdict on a method that may be valid. So no `run` command keeps an arm of its own for a model-choice error either: an unknown model reaches it as the verdict's `unknown_model` item.+- **Run refusals are verdicts too**: a run loads its bundle inside the same translation (`acquire_library`), so an invalid bundle leaves `run pipe` / `run bundle` / `run method` as a `ValidateBundleError` with exit 1, through the generic `agent_error` arm (`run bundle`'s own reading of its bundle through `agent_error_method_refusal()`, in the caller's view on a runner), and its problem (`method.invalid`) carries the verdict's items as `errors`, and the Markdown lists the same errors one per line, so both formats list the one problem's errors and a foreign exception raised while handling an earlier verdict lists none of its items. `run_pipeline_core` sets `library_dirs_are_callers=True`, so a refusal in a `-L` directory (or the bundle's own directory) is the same verdict; `run bundle`'s own parse of a bundle without `--pipe` goes through the translation too, and so does its reading of the method tree: a folder resolves through `pipelex/cli/bundle_target_resolution.py` (`resolve_method_folder_run`, the protocol's entry rule in `pipelex/pipeline/method_entry.py`) and a lone file through `list_bundle_file_tree`, listed before the file is read, through the `MethodTreeListing` the command hands to `run_pipeline_core` so the run loads the tree it listed rather than listing it again (`pipelex/libraries/method_tree_listing.py`; `validate bundle` and `fix bundle` hand theirs down through `JudgedBundleTarget.tree_listing`), so a tree past its limits, a symbolic link or a special file such as a FIFO is a `ValidateBundleError` too, refused unread, and so is a tree a file of which changed after that listing (`MethodTreeChangedError`, reason `tree_changed`, whose item names the file), and a bundle file the listing cannot read is `client.file_unreadable`, while a folder with no single `main_pipe` is the run's own refusal, a `MethodEntryError` coded `request.pipe_unspecified` or `request.pipe_ambiguous`, no verdict on a method that may be valid. So no `run` command keeps an arm of its own for a model-choice error either: an unknown model reaches it as the verdict's `unknown_model` item.- **Two readings of a folder, one resolution path**: `run bundle`, `validate bundle`, `fix bundle` and `inputs bundle` read a folder through `pipelex/cli/bundle_target_resolution.py` (`run` through `resolve_method_folder_run`, the others through `resolve_bundle_target_core`, validate and fix by way of `pipelex/cli/judged_target.py`, whose `JudgedBundleTarget.judged_path` is the path the verdict names and whose `fix_judged_target` is the one fix dispatch both CLIs call, and of `commands/bundle_path_resolver.py`), as its method tree, but with the `FolderReading` their job asks for. `validate` and `fix` take `WHOLE_TREE`, the MTHDS protocol's validation (`mthds/docs/spec/protocol.md`): every bundle of the tree is judged (`validate_bundle(method_dir=..., library_dirs_are_callers=True)`, `fix_method_folder`), with no pipe to start at, so a folder whose bundles declare several `main_pipe`s, or none, is validated like any other; mthds-js's local runner calls `pipelex validate bundle <dir>` to validate a whole method at install and relies on this. `--pipe` there only narrows the dry run, and `--graph` / `--view` draw from the pipe named, else from the entry (`locate_folder_pipe`), or, when the folder decides none, put `graph_unavailable` in the envelope and leave the verdict alone. `run` and `inputs` take `ENTRY_PIPE`, acting on one pipe: the one `--pipe` names, else the protocol's entry (`select_method_entry` in `pipelex/pipeline/method_entry.py`); no single `main_pipe` is the `MethodEntryError` with exit 1 on both, coded `request.pipe_unspecified` or `request.pipe_ambiguous`, the message the same, and no verdict on a method that may be valid. A folder holding no bundle is `EmptyMethodTreeError` with exit 2 on `validate` and `fix`, and a `MethodEntryError` saying it has no pipe to start at on the others, each problem coded `method.invalid`. A folder whose tree is refused at resolution (past its limits, a symbolic link) is the invalid verdict: `ValidateBundleError` on `validate`, and on `fix` the still-invalid `method.invalid` problem of a fix that could not start (`unfixable_method_result`, through `agent_error_fix_bundle`), exit 1 both. A fix of a folder writes anywhere in its method tree, from the root (`method_tree_root`) down, and the human `--diff` preview mirrors that root, so a fix landing in a bundle above the folder named is proposed and written alike. Only the tree's own pipes are judged: `validate_bundle(method_dir=...)` keeps the blueprints its tree lists (`_is_member_blueprint`), so a nested package, a dependency or a `-L` directory loads without being judged; `dirs_beside_method` is the one place the `-L` directories naming the folder itself are dropped, compared resolved. Reading the tree's limits loads the configuration before the boot, so each resolution site catches `PipelexConfigError` and reports it through `report_agent_init_failure` (`agent_cli_factory.py`), the problem a failed boot gives: with exit 2 on `validate` and `fix`, whose resolver passes `exit_code=2` since the refusal gives no verdict, as the human `pipelex validate bundle` and `fix bundle` exit, and with the boot's exit 1 on `run` and `inputs`. `validate` and `fix` do not list a lone `.mthds` file before the boot, so for a file the same configuration surfaces only at the boot, which exits 1.- **Hosted runs**: every `run` subcommand takes `--runner local|hosted`, `--hosted/--local` and `--base-url`; `resolve_agent_run_execution()` (`run/_run_hosted.py`) settles them through `pipelex.hosted.execution.resolve_run_execution` (the flag, then `[run] execution`, then local), refusing a `--runner` that contradicts the flag pair and a `--base-url` on a local run as `client.argument_invalid` problems; a configuration that cannot be read there reports its own problem, as a failed boot does (`agent_cli_factory.py`). Every refusal made before anything is sent is a problem in the operator's view, except a refusal of the method the run would send; what the run itself fails with is the problem the runner or the client produced, in the caller's view (below). `run bundle` reads its bundle before it runs it, wherever it runs, so it settles where the run executes first (`resolve_agent_run_execution()`, then `refuse_local_only_flags()`), and reports a bundle that does not parse or validate (`ValidateBundleError`) or a bundle file or folder with no pipe to start at (`MethodEntryError`, `request.pipe_unspecified`, which a lone file with no `main_pipe` raises too, naming its pipes as `candidates`) through `agent_error_method_refusal()` (`run/_run_hosted.py`): on this machine's runtime through `agent_error`, in the operator's view with the command's sentence; on a runner as the refusal the runner would answer for the same method, `to_problem(view=ProblemView.CALLER)` printed through `agent_error_problem()`, with no `diagnostics` and no `message`, since both carry the raise site's classes and text, the items, their locations, the caller's step and the Markdown tip kept. A hosted run boots nothing: it builds a `HostedRunRequest` from what `pipelex/cli/commands/run/_hosted_sources.py` resolves (the bundle and its library's `.mthds` files, a pipe's library, or a method's files; an address or a catalog id as `method_ref` / `method_id`, never fetched) and `run_hosted_for_agent()` runs it through `pipelex.hosted.hosted_run.run_hosted`, which reads the signature and uploads local input files (only when some input string names an existing local file, so a method with an address-based dependency runs with any other inputs), then drives the lifecycle `start_and_wait` drives step by step (the `/v1/version` handshake, `start`, `wait_for_result`, and the blocking `execute` on a server with no run store), so a failure while following an acknowledged run is a `HostedRunPollingError` naming the run, and a transport failure after the run request was sent is a `HostedRunOutcomeUnknownError` rather than an `ApiUnreachableError`. `run_hosted`'s `on_started` callback records the acknowledged run's id, so Ctrl-C (`KeyboardInterrupt`, or the `CancelledError` of a cancelled task) leaves as the problem of a `HostedRunInterruptedError`, `client.follow_failed` carrying `pipeline_run_id` once the run was acknowledged and `client.outcome_unknown` before, with exit code 130. `run bundle` on a file with no `-L` sends `PIPELEXPATH`'s files (`hosted_library_dirs`), and an installed method named like a catalog id is that method. An exception no hosted arm names is this process's own failure and leaves through the command-boundary catch-all as a local run's does, in the operator's view: a Pipelex error under its own code, any other exception as `runtime.unexpected`. Its success envelope is `build_run_output()`'s, with the main output's content as `json`, an empty `markdown`, the working memory dump and `pipeline_run_id`; nothing is written to disk. `--dry-run` / `--mock-inputs` are refused on it, while `--graph` and `--costs` are accepted and ignored. There is no global `--runner` any more, and no `ctx.obj`.- **A run on a runner that fails** reports the problem the runner or the client produced, in the caller's view, as it came: `agent_error_hosted()` (`run/_run_hosted.py`) builds it with `hosted_failure_problem()` and prints it through `agent_error_problem()` (`agent_output.py`), which neither rebuilds nor reclassifies it, in Markdown by default and JSON with `--error-format json`. A refusal the runner answered (pipelex-sdk's `ApiResponseError`), a runner that cannot be reached (`ApiUnreachableError`, `client.unreachable` naming `params.base_url`) and every other request error of pipelex-sdk or the MTHDS client carries the standard client's problem as `exc.problem`: the runner's own problem document when the answer is one, its `errors` items, its next step and its `request_id` included (`received_problem()`, which keeps only the members the standard and Pipelex declare, at every level, leaves out alone one of Pipelex's members that does not read, fills `request_id` from the answer when the document names none, and projects an answer carrying diagnostics onto the caller's view, deciding it from the document as it came, and a nested `run_error` always, since it is the run's stored record), else the code the client reads from the answer, such as `client.response_unreadable` naming the status, or `client.outcome_unknown` for an answer that leaves a run possibly started, which pipelex-sdk reads for every 5xx on the run request but a 501 and a 503 coded `pipeline_start_unavailable`, whatever the body. An answer in a shape that predates the standard's, which the client reads by its status alone, goes through `hosted_refusal_problem()` (`pipelex/hosted/error_rendering.py`): a refusal of the hosted plane reads as the core code its status and the plane's own `code` name, built by `make_problem` in the caller's view, `auth.missing_credential` for a `401` or `403` with no key sent (`HOSTED_MISSING_KEY_NEXT_STEP`), `auth.invalid_credential` for a `401` and for the edge's bare `403`, `auth.forbidden` for a `403` the platform codes with one of its authorization codes, `forbidden` or `role_not_permitted` (`HOSTED_FORBIDDEN_NEXT_STEP`), and `auth.invalid_credential` for a `403` with any other code, `run.not_found` or `method.not_found` for a `404` coded `run_not_found` or `method_version_not_found` and `request.not_found` for any other, `account.rate_limited` with its `retry_after` for a `429`, with the plane's own `detail` and the next step the human CLI gives (`HOSTED_REFUSAL_NEXT_STEPS_BY_STATUS`); a runner's own refusal in the former shape that lists validation items, the `422` of a method that does not validate on `/v1/start` or `/v1/execute`, is read first, by `_former_refusal_problem()` in `run/_run_hosted.py`, as `method.invalid` with those items, its reason, its advice and its status, through the builder a stored report in the same shape goes through (`_former_problem()`); any other such answer that gives its reason, a runner's own `422` for a reserved input key included, keeps the client's `client.response_unreadable`, read by the standard's members alone so a member beside them that does not read never stops it (`_client_reading()`), with that reason as its `detail` and the human CLI's step (`hosted_refusal_next_step()`), of the kind its advice names through `FORMER_ADVICE_KINDS` (`pipelex/hosted/error_rendering.py`), else `change_input` for a `4xx`; an answer that may have started a run keeps its `client.outcome_unknown`. A run that started and failed (`RunFailedError`) is `run.failed` naming the run and its status, with the client's sentence as `message`, which quotes the stored report's reason, and nests the run's own problem as `run_error`: a stored report that is a problem document is read as a runner's answer is, so a member it holds beside the problem is left out, and one in the former `RunErrorReport` shape is `method.invalid` with its validation items (`hosted_method_problem_items()`) or `runtime.unexpected`, its message, its advice (kind through `FORMER_ADVICE_KINDS`) and its `retryable` carried (`_former_run_problem()`). Markdown renders the nested problem with the head `render_problem_markdown` uses, one heading level down (`_problem_head_lines`). Every error of pipelex-sdk's own but an answer's refusal and an unreachable runner adds its sentence as `message`, since it says what its code's parameters cannot (how long the client waited, that a completed run delivered no main output), and every one but an answer's refusal and a failed run takes the next step the human CLI tailors for it (`tailored_next_step()`, the step `describe_hosted_error()` gives when it is not `HOSTED_RUN_REQUEST_NEXT_STEP`), keeping its code's kind. pipelex's own `HostedRunError`s report their problem in the caller's view with their sentence as `message` and their advice as the Markdown tip, `pipeline_run_id` included for `HostedRunPollingError` and `HostedRunInterruptedError`; a `HostedMethodInvalidError` lists pipe-io's items as `errors`, each coded as a local verdict codes it (`hosted_method_problem_items()`, through `validation_item_code` and `problem_item_of`, `method.schema_violation` with the item's sentence as its finding for an item of a type this version does not read, each item read by the fields this version declares, so a field it does not keeps nothing from being typed, and a declared field whose value it cannot read, such as a newer runner's fix, is left out alone, so the item keeps its type, its locators and its dependency facts), its file in `location.file`. Nothing of the envelope this CLI wrote before it reported problems is left (`error`, `error_type`, `error_source`, `hint`, `error_domain`, `retryable`, `error_code`, `http_status`, …), which `tests/unit/pipelex/cli/test_agent_output_drift.py` guards in the source and in the emitted document.diff --git a/pipelex/cli/agent_cli/commands/run/_run_core.py b/pipelex/cli/agent_cli/commands/run/_run_core.pyindex 44036b8b1..6190638f8 100644--- a/pipelex/cli/agent_cli/commands/run/_run_core.py+++ b/pipelex/cli/agent_cli/commands/run/_run_core.py@@ -26,6 +26,7 @@ from pipelex.tools.misc.json_utils import clean_json_dumpsif TYPE_CHECKING:from mthds.method_tree.models import MethodTree+ from pipelex.libraries.method_tree_listing import MethodTreeListingfrom pipelex.system.configuration.configs import PipelineExecutionConfig@@ -56,6 +57,7 @@ async def run_pipeline_core(inputs_base_dir: Path | None = None,output_dir_override: Path | None = None,method_tree: MethodTree | None = None,+ tree_listing: MethodTreeListing | None = None,) -> dict[str, Any]:"""Core logic for running a pipeline and returning JSON-serializable output.@@ -80,6 +82,8 @@ async def run_pipeline_core(ephemeral clone.method_tree: The method tree the run reads, when the command listed one: its digest is recorded as``method_tree_digest`` beside the run's other side effects, on disk and under ``with_memory``.+ tree_listing: The listing the command listed ``method_tree`` through, which the run loads its trees from, so+ the digest recorded is the digest of the tree the run loaded. None lists the trees for the run.Returns:Dictionary with execution results suitable for JSON serialization.@@ -118,6 +122,7 @@ async def run_pipeline_core(pipe_code=pipe_code,mthds_contents=mthds_contents,inputs=inputs,+ tree_listing=tree_listing,)pipe_output = response.pipe_outputdiff --git a/pipelex/cli/agent_cli/commands/run/bundle_cmd.py b/pipelex/cli/agent_cli/commands/run/bundle_cmd.pyindex 1dc97782d..a461d40dc 100644--- a/pipelex/cli/agent_cli/commands/run/bundle_cmd.py+++ b/pipelex/cli/agent_cli/commands/run/bundle_cmd.py@@ -39,6 +39,7 @@ from pipelex.cli.commands.run._run_output_folder import bundle_run_outputs_dirfrom pipelex.hosted.exceptions import HostedRunSourceErrorfrom pipelex.hosted.hosted_run import HostedRunRequestfrom pipelex.hosted.run_config import RunExecution # ruff: ignore[typing-only-first-party-import] - typer reads the --runner annotation at runtime+from pipelex.libraries.method_tree_listing import MethodTreeListingfrom pipelex.mthds_parsing.helpers import is_pipelex_filefrom pipelex.mthds_parsing.parser import MthdsParser, decode_mthds_textfrom pipelex.pipe_operators.exceptions import PipeOperatorModelAvailabilityError@@ -147,8 +148,10 @@ def run_bundle_cmd(auto_inputs_dir: Path | None = Nonetarget_path = Path(path)- # The method tree the run reads, whose digest the run records: the folder's, or the bundle file's own.+ # The method tree the run reads, whose digest the run records: the folder's, or the bundle file's own. It is listed+ # once, through the listing the run's load takes it from.method_tree: MethodTree | None = None+ tree_listing = MethodTreeListing()# The bundle defining a folder's entry pipe, which the run's output folders are made beside.entry_bundle_file: Path | None = None# The pipe a hosted run names: `--pipe`, else the main pipe the local run starts at, qualified by its domain, so the@@ -157,7 +160,7 @@ def run_bundle_cmd(if target_path.is_dir():# Directory mode: the method rooted at or above the folder, started where the protocol's rule saystry:- folder_run = resolve_method_folder_run(directory=target_path, pipe_override=pipe)+ folder_run = resolve_method_folder_run(directory=target_path, pipe_override=pipe, tree_listing=tree_listing)except ValidateBundleError as exc:agent_error_method_refusal(error=exc, message=exc.message, execution=execution)except MethodEntryError as exc:@@ -205,7 +208,7 @@ def run_bundle_cmd(# before the file is read, since the listing refuses a symbolic link, a special file such as a FIFO and a# file past this machine's limits without reading it.try:- method_tree = list_bundle_file_tree(bundle_file=target_path)+ method_tree = list_bundle_file_tree(bundle_file=target_path, tree_listing=tree_listing)except ValidateBundleError as exc:# A file this machine would not let the listing read is the file the command was handed# (`client.file_unreadable`), never a fault of the method.@@ -285,6 +288,7 @@ def run_bundle_cmd(inputs_base_dir=parsed_inputs.inputs_base_dir,output_dir_override=output_dir_override,method_tree=method_tree,+ tree_listing=tree_listing,))agent_success_formatted(diff --git a/pipelex/cli/agent_cli/commands/validate/_validate_core.py b/pipelex/cli/agent_cli/commands/validate/_validate_core.pyindex f74178959..11e4ee205 100644--- a/pipelex/cli/agent_cli/commands/validate/_validate_core.py+++ b/pipelex/cli/agent_cli/commands/validate/_validate_core.py@@ -28,6 +28,8 @@ from pipelex.pipeline.validate_bundle_translation import translate_to_validate_bif TYPE_CHECKING:from pathlib import Path+ from pipelex.libraries.method_tree_listing import MethodTreeListing+async def validate_all_core(*, library_dirs: list[Path] | None = None, allow_signatures: bool = False) -> dict[str, Any]:"""Validate all pipes in all libraries.@@ -164,6 +166,7 @@ async def validate_method_folder_core(library_dirs: list[Path] | None,allow_signatures: bool,pipe_code: str | None,+ tree_listing: MethodTreeListing | None = None,) -> dict[str, Any]:"""Validate a method's folder whole: every bundle of its method tree, as the MTHDS protocol's validation judges a method.@@ -176,6 +179,8 @@ async def validate_method_folder_core(library_dirs: The folder and the ``-L`` directories, the caller's own.allow_signatures: Whether to accept PipeSignature placeholders in the dependency graph.pipe_code: The one pipe to dry-run, or None for every pipe of the method.+ tree_listing: The listing the command resolved the folder through, which the validation loads its tree from, so+ the tree is listed once; None lists it for the validation.Returns:Dictionary with validation results suitable for JSON serialization, ``bundle_path`` naming the folder.@@ -192,6 +197,7 @@ async def validate_method_folder_core(allow_signatures=allow_signatures,dry_run_pipe_codes=[pipe_code] if pipe_code else None,collect_dry_run_warnings=True,+ tree_listing=tree_listing,)warnings = collect_advisory_warnings(pipes=result.pipes,diff --git a/pipelex/cli/agent_cli/commands/validate/bundle_cmd.py b/pipelex/cli/agent_cli/commands/validate/bundle_cmd.pyindex 3df822724..92d87a0b3 100644--- a/pipelex/cli/agent_cli/commands/validate/bundle_cmd.py+++ b/pipelex/cli/agent_cli/commands/validate/bundle_cmd.py@@ -184,7 +184,11 @@ def validate_bundle_cmd(# The whole method, every pipe of it, or only the pipe named while the whole tree loadsresult = asyncio.run(validate_method_folder_core(- method_dir=target.method_dir, library_dirs=library_dirs, allow_signatures=allow_signatures, pipe_code=pipe+ method_dir=target.method_dir,+ library_dirs=library_dirs,+ allow_signatures=allow_signatures,+ pipe_code=pipe,+ tree_listing=target.tree_listing,))elif pipe:diff --git a/pipelex/cli/bundle_target_resolution.py b/pipelex/cli/bundle_target_resolution.pyindex fd4c1da38..c59957d96 100644--- a/pipelex/cli/bundle_target_resolution.py+++ b/pipelex/cli/bundle_target_resolution.py@@ -15,6 +15,10 @@ command:- A `.mthds` file named alone is the member of its own file's tree (see `list_bundle_file_tree`), and acts on its own`main_pipe` unless a pipe is named.+The tree is listed once, through a `MethodTreeListing` the command hands down to its load+(`pipelex.libraries.method_tree_listing`): the load takes the tree from it rather than listing the folder or the file+again, so the tree whose digest a run records is the tree it loaded.+The shared core decides; each command renders the refusals as its own surface renders them."""@@ -27,7 +31,7 @@ from mthds.method_tree.models import MethodTreefrom mthds.protocol.problem.registry import CoreCodefrom pipelex.libraries.exceptions import EmptyMethodTreeError, MethodTreeLoadError-from pipelex.libraries.method_trees import list_local_method_tree+from pipelex.libraries.method_tree_listing import MethodTreeListingfrom pipelex.mthds_parsing.helpers import is_pipelex_filefrom pipelex.pipeline.exceptions import MethodEntryErrorfrom pipelex.pipeline.method_entry import (@@ -63,8 +67,8 @@ class MethodFolderTarget(NamedTuple):entry_bundle_file: Path | None-def list_method_folder(*, directory: Path) -> MethodTree:- """The method tree of a folder handed to a command.+def list_method_folder(*, directory: Path, tree_listing: MethodTreeListing) -> MethodTree:+ """The method tree of a folder handed to a command, listed through the listing the command hands to its load.Raises:EmptyMethodTreeError: The tree holds no bundle: the membership rule keeps every file of the folder out of it, or@@ -75,7 +79,7 @@ def list_method_folder(*, directory: Path) -> MethodTree:"""msg = f"The folder '{directory}' holds no .mthds bundle."try:- tree = list_local_method_tree(handed_over=directory)+ tree = tree_listing.method_tree(handed_over=directory)except EmptyMethodTreeError as exc:raise EmptyMethodTreeError(msg) from excexcept MethodTreeLoadError:@@ -86,8 +90,8 @@ def list_method_folder(*, directory: Path) -> MethodTree:return tree-def resolve_method_folder(*, directory: Path, pipe_override: str | None) -> MethodFolderTarget:- """List a method folder's tree and settle the one pipe a command acts on.+def resolve_method_folder(*, directory: Path, pipe_override: str | None, tree_listing: MethodTreeListing) -> MethodFolderTarget:+ """List a method folder's tree, through the listing the command hands to its load, and settle the one pipe it acts on.Raises:MethodEntryError: The folder holds no bundle, or no pipe was named and the method has no single pipe to start@@ -97,7 +101,7 @@ def resolve_method_folder(*, directory: Path, pipe_override: str | None) -> Methread, each as the invalid verdict `validate` gives."""try:- tree = list_method_folder(directory=directory)+ tree = list_method_folder(directory=directory, tree_listing=tree_listing)except EmptyMethodTreeError as exc:msg = f"The folder '{directory}' holds no .mthds bundle, so it has no pipe to start at."raise MethodEntryError(msg).with_error_code(CoreCode.METHOD_INVALID) from exc@@ -110,9 +114,11 @@ def resolve_method_folder(*, directory: Path, pipe_override: str | None) -> Methreturn MethodFolderTarget(tree=tree, pipe_code=entry.pipe_ref, entry=entry, entry_bundle_file=tree.file_location(entry.tree_path))-def resolve_method_folder_run(*, directory: Path, pipe_override: str | None) -> MethodFolderTarget:+def resolve_method_folder_run(*, directory: Path, pipe_override: str | None, tree_listing: MethodTreeListing) -> MethodFolderTarget:"""A method folder as `run bundle` reads it: every refusal of the method is the verdict.+ The tree is listed through `tree_listing`, which the run hands to its load, so the run loads the tree listed here.+Raises:ValidateBundleError: The tree is refused or past this machine's limits, or a bundle does not parse, each as theinvalid verdict `validate` gives.@@ -120,7 +126,7 @@ def resolve_method_folder_run(*, directory: Path, pipe_override: str | None) ->on the method, coded as the refusal of a run that selects no pipe."""with translate_to_validate_bundle_error():- return resolve_method_folder(directory=directory, pipe_override=pipe_override)+ return resolve_method_folder(directory=directory, pipe_override=pipe_override, tree_listing=tree_listing)def locate_folder_pipe(*, tree: MethodTree, pipe_override: str | None) -> MethodEntry:@@ -141,17 +147,18 @@ def locate_folder_pipe(*, tree: MethodTree, pipe_override: str | None) -> Methodreturn select_entry(candidates=candidates, manifest=read_method_manifest(tree=tree), method_label=method_label)-def list_bundle_file_tree(*, bundle_file: Path) -> MethodTree:+def list_bundle_file_tree(*, bundle_file: Path, tree_listing: MethodTreeListing) -> MethodTree:"""The method tree of a `.mthds` file named alone, as the run loads it.It is the member of the tree rooted at the nearest `METHODS.toml` at or above its folder, else the one-file tree of- that file, from which a reference to any other file is refused.+ that file, from which a reference to any other file is refused. It is listed through `tree_listing`, which the run+ hands to its load, so the run places the file in the tree listed here.Raises:ValidateBundleError: The tree is refused or past this machine's limits, as the invalid verdict `validate` gives."""with translate_to_validate_bundle_error():- return list_local_method_tree(handed_over=bundle_file)+ return tree_listing.method_tree(handed_over=bundle_file)class BundleTargetResolutionSuccess(NamedTuple):@@ -171,6 +178,8 @@ class BundleTargetResolutionSuccess(NamedTuple):method_tree: MethodTree | None#: The pipe the protocol's rule selected for a folder whose command acts on one pipe and named none, else None.entry: MethodEntry | None+ #: The listing the folder's tree was listed through, for the command to hand to its load; it holds nothing for a file.+ tree_listing: MethodTreeListingclass BundleTargetResolutionError(NamedTuple):@@ -203,13 +212,15 @@ def resolve_bundle_target_core("""library_dirs = [Path(library_dir_item).expanduser() for library_dir_item in library_dir] if library_dir else Nonetarget_path = Path(path).expanduser()+ # The command's one listing: the folder's tree is listed through it here, and its load takes the tree from it.+ tree_listing = MethodTreeListing()if target_path.is_dir():# The folder first, then the -L directories that are not the folder, however either is spelled.library_dirs = [target_path, *dirs_beside_method(method_dir=target_path, library_dirs=library_dirs)]match reading:case FolderReading.ENTRY_PIPE:- folder = resolve_method_folder(directory=target_path, pipe_override=pipe_override)+ folder = resolve_method_folder(directory=target_path, pipe_override=pipe_override, tree_listing=tree_listing)return BundleTargetResolutionSuccess(bundle_path=folder.entry_bundle_file,pipe_code=folder.pipe_code,@@ -217,6 +228,7 @@ def resolve_bundle_target_core(method_dir=target_path,method_tree=folder.tree,entry=folder.entry,+ tree_listing=tree_listing,)case FolderReading.WHOLE_TREE:return BundleTargetResolutionSuccess(@@ -224,8 +236,9 @@ def resolve_bundle_target_core(pipe_code=pipe_override,library_dirs=library_dirs,method_dir=target_path,- method_tree=list_method_folder(directory=target_path),+ method_tree=list_method_folder(directory=target_path, tree_listing=tree_listing),entry=None,+ tree_listing=tree_listing,)if is_pipelex_file(target_path):@@ -236,6 +249,7 @@ def resolve_bundle_target_core(method_dir=None,method_tree=None,entry=None,+ tree_listing=tree_listing,)return BundleTargetResolutionError(input_path=path, target_path=target_path)diff --git a/pipelex/cli/commands/fix/_fix_core.py b/pipelex/cli/commands/fix/_fix_core.pyindex 3e8246119..d7b3b6038 100644--- a/pipelex/cli/commands/fix/_fix_core.py+++ b/pipelex/cli/commands/fix/_fix_core.py@@ -21,6 +21,7 @@ from pipelex.cli.error_handlers import ()from pipelex.cli.judged_target import fix_judged_targetfrom pipelex.core.pipes.exceptions import PipeOperatorModelChoiceError+from pipelex.libraries.method_tree_listing import MethodTreeListingfrom pipelex.libraries.method_trees import method_tree_rootfrom pipelex.pipe_operators.exceptions import PipeOperatorModelAvailabilityErrorfrom pipelex.pipelex import Pipelex@@ -243,13 +244,15 @@ def _run_fix_preview(with tempfile.TemporaryDirectory(prefix="pipelex-fix-preview-") as sandbox_root:sandbox = mirror_bundle_for_preview(target.bundle_path, library_dirs=_mirrored_dirs(target=target), sandbox_root=Path(sandbox_root))# The target in the sandbox: a method's folder is found inside its tree root's copy, and the directories it- # loads with are the copies of the ones it loads with for real, so the copy is spelled as the original is.+ # loads with are the copies of the ones it loads with for real, so the copy is spelled as the original is. The+ # copies are other files than those the target's listing listed, so the sandbox's loads list them anew.sandbox_target = target._replace(bundle_path=sandbox.entry_path,method_dir=sandbox.to_copy(original=target.method_dir) if target.method_dir is not None else None,library_dirs=[sandbox.to_copy(original=library_dir) for library_dir in target.library_dirs]if target.method_dir is not None and target.library_dirs is not Noneelse sandbox.library_dirs,+ tree_listing=MethodTreeListing(),)result = asyncio.run(fix_judged_target(diff --git a/pipelex/cli/commands/run/_run_core.py b/pipelex/cli/commands/run/_run_core.pyindex 56cff4673..c70b6f445 100644--- a/pipelex/cli/commands/run/_run_core.py+++ b/pipelex/cli/commands/run/_run_core.py@@ -63,6 +63,7 @@ if TYPE_CHECKING:from pipelex.core.concepts.concept import Conceptfrom pipelex.core.stuffs.stuff_content import StuffContent+ from pipelex.libraries.method_tree_listing import MethodTreeListingCOMMAND = "run"_WORKING_MEMORY_FILENAME = "working_memory.json"@@ -168,6 +169,7 @@ def _resolve_row_model_for_empty_result(bundle_path: str | None,mthds_content: str | None,library_dir: list[str] | None,+ tree_listing: MethodTreeListing | None,) -> type[StuffContent]:"""Resolve the CSV row model for a run that produced no rows, by reloading its bundle.@@ -175,14 +177,16 @@ def _resolve_row_model_for_empty_result(run's own library — which owned the ClassRegistry holding its generated structure classes — istorn down by the time ``--save-csv`` runs. Reloading the same bundle into a throwaway libraryregenerates the declared concept's structure class under the same name and fields, which is what- the header needs. The load inputs are the ones handed to the runner just above, so the two see- the same bundle. The library is torn down again immediately; only the resolved class survives.+ the header needs. The load inputs are the ones handed to the runner just above, and so is the run's+ tree listing, so the two load the same trees without listing them again. The library is torn+ down again immediately; only the resolved class survives."""library_id, _main_pipe = acquire_library(library_id="",library_dirs=library_dir,mthds_contents=[mthds_content] if mthds_content else None,bundle_uris=[bundle_path] if bundle_path else None,+ tree_listing=tree_listing,)try:return get_concept_library().get_structure_class(concept=concept)@@ -211,11 +215,14 @@ async def _execute_run(dynamic_output_concept_ref: str | None = None,save_csv: str | None = None,method_tree: MethodTree | None = None,+ tree_listing: MethodTreeListing | None = None,) -> None:"""Core async execution logic for running a pipe.Shared between the ``method``, ``pipe`` and ``bundle`` subcommands. ``method_tree`` is the method tree the run- reads, when the command listed one, recorded as ``method_tree.json`` in the run's output folder.+ reads, when the command listed one, recorded as ``method_tree.json`` in the run's output folder, and+ ``tree_listing`` the listing the command listed it through, which the run loads its trees from, so the tree recorded+ is the tree loaded."""# Fail fast on an unusable --save-csv target BEFORE running the pipeline, so a bad path (empty,# or an unsupported suffix like .xlsx) doesn't waste a full inference run. The output-shape and@@ -294,6 +301,7 @@ async def _execute_run(mthds_contents=[mthds_content] if mthds_content else None,inputs=pipeline_inputs,dynamic_output_concept_ref=dynamic_output_concept_ref,+ tree_listing=tree_listing,)pipe_output = response.pipe_outputexcept ValidateBundleError as exc:@@ -471,6 +479,7 @@ async def _execute_run(bundle_path=bundle_path,mthds_content=mthds_content,library_dir=library_dir,+ tree_listing=tree_listing,)# Validate the row flatness BEFORE touching the filesystem, so a non-flat output leaves no# directory behind (the suffix was already validated up front).@@ -540,12 +549,14 @@ def execute_run(dynamic_output_concept_ref: str | None = None,save_csv: str | None = None,method_tree: MethodTree | None = None,+ tree_listing: MethodTreeListing | None = None,) -> None:"""Synchronous entry point that wraps the async execution with Pipelex setup/teardown.Shared between the ``method``, ``pipe`` and ``bundle`` subcommands, for a run that executes on this machine. Itprints the deck staleness notice, which the root callback leaves to it: only a local run reads the deck.``inputs`` is ``--inputs`` as given, or the inputs file ``run bundle`` auto-detected (see ``load_run_inputs``).+ ``tree_listing`` is the listing the command listed ``method_tree`` through, which the run loads its trees from."""warn_if_deck_stale()# A dry run makes no inference call, so it must not demand credentials: `needs_inference=False`@@ -585,6 +596,7 @@ def execute_run(dynamic_output_concept_ref=dynamic_output_concept_ref,save_csv=save_csv,method_tree=method_tree,+ tree_listing=tree_listing,))diff --git a/pipelex/cli/commands/run/bundle_cmd.py b/pipelex/cli/commands/run/bundle_cmd.pyindex faff9a7cb..b4803104d 100644--- a/pipelex/cli/commands/run/bundle_cmd.py+++ b/pipelex/cli/commands/run/bundle_cmd.py@@ -15,6 +15,7 @@ from pipelex.cli.commands.run._run_output_folder import run_output_labelfrom pipelex.cli.commands.run.exceptions import AmbiguousInputsFilesError, AutoDetectedInputsFileErrorfrom pipelex.cli.error_handlers import handle_pre_boot_config_error, handle_validate_bundle_errorfrom pipelex.hosted.exceptions import HostedRunSourceError+from pipelex.libraries.method_tree_listing import MethodTreeListingfrom pipelex.mthds_parsing.helpers import is_pipelex_filefrom pipelex.pipeline.exceptions import MethodEntryError, ValidateBundleError@@ -149,15 +150,17 @@ def run_bundle_cmd(run_inputs: str | DetectedInputsFile | None = inputstarget_path = Path(path)- # The method tree the run reads, whose digest the run records: the folder's, or the bundle file's own.+ # The method tree the run reads, whose digest the run records: the folder's, or the bundle file's own. It is listed+ # once, through the listing the run's load takes it from.method_tree: MethodTree+ tree_listing = MethodTreeListing()# The pipe a hosted run names: `--pipe`, else the main pipe the local run starts at, qualified by its domain, so the# hosted API starts at the same pipe when two domains of the files it is sent define the same code.hosted_pipe_code: str | None = pipeif target_path.is_dir():# Directory mode: the method rooted at or above the folder, started where the protocol's rule saystry:- folder_run = resolve_method_folder_run(directory=target_path, pipe_override=pipe)+ folder_run = resolve_method_folder_run(directory=target_path, pipe_override=pipe, tree_listing=tree_listing)except ValidateBundleError as exc:handle_validate_bundle_error(exc, library_dirs=[Path(one_dir) for one_dir in library_dir] if library_dir else None)except MethodEntryError as entry_exc:@@ -197,7 +200,7 @@ def run_bundle_cmd(elif is_pipelex_file(target_path):bundle_path = pathtry:- method_tree = list_bundle_file_tree(bundle_file=target_path)+ method_tree = list_bundle_file_tree(bundle_file=target_path, tree_listing=tree_listing)except ValidateBundleError as exc:handle_validate_bundle_error(exc, bundle_path=target_path, library_dirs=[Path(one_dir) for one_dir in library_dir] if library_dir else None@@ -276,4 +279,5 @@ def run_bundle_cmd(dynamic_output_concept_ref=dynamic_output_concept_ref,save_csv=save_csv,method_tree=method_tree,+ tree_listing=tree_listing,)diff --git a/pipelex/cli/commands/validate/_validate_core.py b/pipelex/cli/commands/validate/_validate_core.pyindex 01c16592b..c90fee516 100644--- a/pipelex/cli/commands/validate/_validate_core.py+++ b/pipelex/cli/commands/validate/_validate_core.py@@ -47,6 +47,7 @@ if TYPE_CHECKING:from pathlib import Pathfrom pipelex.base_exceptions import ValidationErrorItem+ from pipelex.libraries.method_tree_listing import MethodTreeListingfrom pipelex.pipe_machinery.pipe_abstract import PipeAbstractfrom pipelex.pipeline.bundle_validator import DryRunOutput@@ -163,10 +164,12 @@ async def _validate_pipe_or_bundle(method_dir: Path | None = None,library_dirs: list[Path] | None = None,allow_signatures: bool = False,+ tree_listing: MethodTreeListing | None = None,) -> None:"""Core async validation logic shared between the bundle, method and pipe subcommands.- A method's folder is judged whole, every bundle of its method tree, with ``library_dirs`` loaded beside it.+ A method's folder is judged whole, every bundle of its method tree, with ``library_dirs`` loaded beside it, its tree+ loaded from ``tree_listing``, the listing the command resolved the folder through, when given."""with MethodValidationReport() as validation_report:judged_path = method_dir or bundle_path@@ -181,6 +184,7 @@ async def _validate_pipe_or_bundle(library_dirs_are_callers=True,allow_signatures=allow_signatures,collect_dry_run_warnings=True,+ tree_listing=tree_listing,)else:bundle_result = await validate_bundle(@@ -188,6 +192,7 @@ async def _validate_pipe_or_bundle(library_dirs=library_dirs,allow_signatures=allow_signatures,collect_dry_run_warnings=True,+ tree_listing=tree_listing,)# Advisory lints, printed BEFORE the signature gate below exits non-zero — same ordering# as `validate --all`. A method with an unimplemented placeholder is the state an author@@ -289,6 +294,7 @@ def execute_validate(telemetry_command_label: str = COMMAND,allow_signatures: bool = False,orchestrator: str | None = None,+ tree_listing: MethodTreeListing | None = None,) -> None:"""Synchronous entry point wrapping the async validation with Pipelex setup/teardown.@@ -297,6 +303,9 @@ def execute_validate(its own in-process router so nested controller sub-pipes do not dispatch to the orchestratorruntime — so this flag does not change *what* validation does; it controls how Pipelex boots,which is the lever for exercising the "validation stays in-process on an orchestrator backend" contract.++ ``tree_listing`` is the listing the command resolved its target through, which the validation loads the method's+ trees from, so the tree is listed once."""make_pipelex_for_cli(context=ErrorContext.VALIDATION, needs_inference=False, boot_orchestrator=orchestrator)@@ -315,6 +324,7 @@ def execute_validate(method_dir=method_dir,library_dirs=library_dirs,allow_signatures=allow_signatures,+ tree_listing=tree_listing,))except PipeNotFoundError as exc:diff --git a/pipelex/cli/commands/validate/bundle_cmd.py b/pipelex/cli/commands/validate/bundle_cmd.pyindex 1bf408a2d..37db76207 100644--- a/pipelex/cli/commands/validate/bundle_cmd.py+++ b/pipelex/cli/commands/validate/bundle_cmd.py@@ -79,4 +79,5 @@ def validate_bundle_cmd(telemetry_command_label=f"{COMMAND} bundle",allow_signatures=allow_signatures,orchestrator=orchestrator,+ tree_listing=target.tree_listing,)diff --git a/pipelex/cli/judged_target.py b/pipelex/cli/judged_target.pyindex 8d3f3b335..626ccf65b 100644--- a/pipelex/cli/judged_target.py+++ b/pipelex/cli/judged_target.py@@ -3,6 +3,9 @@Both commands read their `path` argument the same way (`pipelex.cli.bundle_target_resolution`, `FolderReading.WHOLE_TREE`):a `.mthds` file is judged as a bundle file, a folder as its whole method tree. `fix` must patch exactly what `validate`judged, so the two CLIs share the target this module defines, the path it names, and the dispatch to the fix loop.++The target carries the listing its folder's tree was listed through (`pipelex.libraries.method_tree_listing`), which the+validation or the fix loop's first validation loads the tree from, so a command lists the tree it judges once."""from collections.abc import Sequence@@ -13,6 +16,7 @@ from mthds.method_tree.models import MethodTreefrom pipelex.base_exceptions import PipelexUnexpectedErrorfrom pipelex.cli.bundle_target_resolution import BundleTargetResolutionError, FolderReading, resolve_bundle_target_core+from pipelex.libraries.method_tree_listing import MethodTreeListingfrom pipelex.pipeline.fixes.fix_loop import FixBundleResult, fix_bundle_file, fix_method_folder@@ -27,6 +31,8 @@ class JudgedBundleTarget(NamedTuple):method_tree: MethodTree | None#: The folder, then the `-L` directories, `~` expanded; None for a file named with no `-L`.library_dirs: list[Path] | None+ #: The listing the folder's tree was listed through, which the command's load takes it from.+ tree_listing: MethodTreeListing@propertydef judged_path(self) -> Path:@@ -63,6 +69,7 @@ def resolve_judged_target(*, path: str, library_dir: list[str] | None) -> Judgedmethod_dir=result.method_dir,method_tree=result.method_tree,library_dirs=result.library_dirs,+ tree_listing=result.tree_listing,)@@ -98,6 +105,7 @@ async def fix_judged_target(max_iterations=max_iterations,select_codes=select_codes,ignore_codes=ignore_codes,+ tree_listing=target.tree_listing,)return await fix_bundle_file(target.judged_path,@@ -107,4 +115,5 @@ async def fix_judged_target(max_iterations=max_iterations,select_codes=select_codes,ignore_codes=ignore_codes,+ tree_listing=target.tree_listing,)diff --git a/pipelex/libraries/exceptions.py b/pipelex/libraries/exceptions.pyindex 3dadfad4f..c312a9633 100644--- a/pipelex/libraries/exceptions.py+++ b/pipelex/libraries/exceptions.py@@ -124,6 +124,23 @@ def _tree_limit_reason(*, kind: TreeLimitKind) -> MethodPackageReason:return MethodPackageReason.FILE_TOO_LARGE+class MethodTreeChangedError(MethodTreeLoadError):+ """Raised when a file of a method tree on this host's disk is not the file the tree's listing hashed when it is read.++ A run lists each tree once and records its digest, then reads the tree's files: the load its bundles, its+ `requirements.txt` and its Python, the run the files its steps name. Each read is checked against the hash the+ listing recorded, so a file edited, replaced or removed in between is refused rather than read as part of a tree it+ no longer belongs to. Its ``reason`` is ``tree_changed`` and its ``path`` the file's tree path, given where it is+ raised. Running again lists the tree as it now stands.+ """++ error_code = CoreCode.METHOD_PACKAGE_INVALID++ @override+ def problem_params(self) -> dict[str, Any]:+ return {"reason": MethodPackageReason.TREE_CHANGED}++class EmptyMethodTreeError(MethodTreeLoadError):"""Raised when a folder handed over as a method holds no bundle.diff --git a/pipelex/libraries/library_manager.py b/pipelex/libraries/library_manager.pyindex 360876b84..f00d6b6ed 100644--- a/pipelex/libraries/library_manager.py+++ b/pipelex/libraries/library_manager.py@@ -8,7 +8,7 @@ from typing import TYPE_CHECKING, Any, ForwardRef, Literal, get_args, get_originfrom kajson.class_registry import ClassRegistryfrom kajson.kajson_manager import KajsonManager-from mthds.method_tree.models import MethodTree, TreeLimits+from mthds.method_tree.models import MethodTreefrom mthds.method_tree.tree_path import path_collisions, tree_path_violationfrom mthds.package.dependency_resolver import ResolvedDependency, determine_exported_pipes, resolve_all_dependenciesfrom mthds.package.discovery import find_package_manifest@@ -55,16 +55,15 @@ from pipelex.libraries.library_crate_factory import LibraryCrateFactoryfrom pipelex.libraries.library_factory import LibraryFactoryfrom pipelex.libraries.library_manager_abstract import LibraryManagerAbstractfrom pipelex.libraries.method_requirements import ensure_method_requirements_met, ensure_package_requirements_met+from pipelex.libraries.method_tree_listing import MethodTreeListingfrom pipelex.libraries.method_trees import (LibraryBundleFile,LibraryTree,+ decoded_as_text_file,library_bundle_files_of,- list_library_dir_trees,- list_package_method_tree,listing_identity,- local_tree_limits,+ read_tree_bundle_as_listed,submitted_tree_files,- tree_bundle_of_lone_file,tree_bundles_of,tree_python_sources,)@@ -143,16 +142,17 @@ def find_methods_dirs_from_blueprints(blueprints: list[PipelexBundleBlueprint])return result-def _parse_dependency_bundle(*, mthds_path: Path, bundle_source: str, package_address: str) -> PipelexBundleBlueprint:+def _parse_dependency_bundle(*, tree_bundle: TreeBundle, bundle_source: str, package_address: str) -> PipelexBundleBlueprint:"""Parse one bundle of a dependency package under its caller-facing source, refusing the load when it does not parse.The bundle is parsed from its text under ``bundle_source`` rather than from its path, so the blueprint, every itema parse failure gives and every message the parser writes name the bundle by the package's address and its pathinside the package: parsing from the path would put the host's path into a TOML error's message and into the- blueprint's own messages. A file discovery has just listed that cannot be read is the host's fault and propagates.+ blueprint's own messages. Its bytes are read against the hash the package's listing recorded, and decoded as a text+ file is read, each line ending made a newline (`decoded_as_text_file`).Args:- mthds_path: The bundle's file on this host.+ tree_bundle: The bundle as a member of the package's tree.bundle_source: The bundle's caller-facing name, ``<package_address>/<path inside the package>``.package_address: The package's address, which the refusal's summary names.@@ -162,9 +162,11 @@ def _parse_dependency_bundle(*, mthds_path: Path, bundle_source: str, package_adRaises:MthdsParserError: The bundle is not UTF-8 or does not parse; each item carries ``bundle_source`` as its source,and the summary names the bundle and the package.+ MethodTreeChangedError: The bundle is no longer the file the package's listing hashed."""+ data = read_tree_bundle_as_listed(tree_bundle=tree_bundle)try:- mthds_content = mthds_path.read_text(encoding="utf-8")+ mthds_content = decoded_as_text_file(data=data)except UnicodeDecodeError as exc:msg = f"Bundle '{bundle_source}' of method package '{package_address}' is not valid UTF-8."raise (@@ -706,6 +708,7 @@ class LibraryManager(LibraryManagerAbstract):library_file_paths: list[Path] | None = None,methods_dirs: list[Path] | None = None,address_dependency_sources: AddressDependencySources = AddressDependencySources.ANYWHERE,+ tree_listing: MethodTreeListing | None = None,) -> list[PipeAbstract]:# `methods_dirs` are laid out like `.mthds/methods/` and hold the packages a caller ships with the method these# directories hold, searched first for an address-based reference of their bundles and never written.@@ -716,25 +719,27 @@ class LibraryManager(LibraryManagerAbstract):msg = f"Trying to load a library that does not exist: '{library_id}'"raise LibraryError(msg)+ # Every bundle is read as a member of its method tree: each library directory's tree and the trees nested in it,+ # then the tree of each file named alone. A file the membership rule keeps out is never read. Each tree is taken+ # from the run's listing, which lists it, under this machine's limits read once, only when nobody listed it yet:+ # the tree loaded is the tree the run listed, its digest the one the run records. A tree the run listed before+ # this load is read again as the load begins, and refused when a file of it changed since (`loading`).+ listing = tree_listing if tree_listing is not None else MethodTreeListing()# Bind the target library as current for the whole load: class registration and the# blueprint loads beneath resolve through the ambient current library, so a load into# a non-current library must not read the caller's binding.- with scoped_current_library(library_id=library_id):+ with scoped_current_library(library_id=library_id), listing.loading():if not library_dirs:library_dirs = []all_dirs: list[Path] = []all_dirs.extend(library_dirs)- # Every bundle is read as a member of its method tree: each library directory's tree and the trees nested- # in it, then the tree of each file named alone. A file the membership rule keeps out is never read.- # This machine's limits are read once for the whole load.- tree_limits = local_tree_limits()- library_trees = list_library_dir_trees(library_dirs=library_dirs, limits=tree_limits)+ library_trees = listing.library_dir_trees(library_dirs=library_dirs)all_bundle_files: list[LibraryBundleFile] = library_bundle_files_of(library_trees=library_trees)for library_file_path in library_file_paths or []:all_bundle_files.append(LibraryBundleFile(- tree_bundle=tree_bundle_of_lone_file(bundle_file=library_file_path, limits=tree_limits),+ tree_bundle=listing.lone_file_bundle(bundle_file=library_file_path),path=library_file_path,))@@ -745,12 +750,15 @@ class LibraryManager(LibraryManagerAbstract):# holding no bundle included, such as a library directory whose files are submitted beside the bundles parsed# from content strings (below). A refusal may name host paths: these are the caller's own# directories, or the host's, whose refusal `reporting_host_library_tree_faults` withholds as its own fault.+ # The run's listing keeps each tree found met, so the blueprint load below, or a server's check of the same+ # method before this load, judges each tree once and asks a sandbox's package-offer factory once for it.ensure_method_requirements_met(trees=[*(library_tree.tree for library_tree in library_trees),*(bundle_file.tree_bundle.tree for bundle_file in all_bundle_files),],names_host_paths=True,+ tree_listing=listing,)# A library directory whose own tree holds no bundle, such as the directory a server writes a request's@@ -816,6 +824,7 @@ class LibraryManager(LibraryManagerAbstract):bundle_files=bundle_files,methods_dirs=methods_dirs,address_dependency_sources=address_dependency_sources,+ tree_listing=listing,)@override@@ -1047,6 +1056,7 @@ class LibraryManager(LibraryManagerAbstract):blueprints: list[PipelexBundleBlueprint],methods_dirs: list[Path] | None = None,address_dependency_sources: AddressDependencySources = AddressDependencySources.ANYWHERE,+ tree_listing: MethodTreeListing | None = None,) -> list[PipeAbstract]:"""Load domains, concepts, and pipes from a list of blueprints.@@ -1061,6 +1071,9 @@ class LibraryManager(LibraryManagerAbstract):package found there is never installed anywhere.address_dependency_sources: Whether a reference ``methods_dirs`` do not answer is looked up among the methodsinstalled on this host and fetched on a miss (``ANYWHERE``), or refused (``SHIPPED``).+ tree_listing: The run's listing of the method trees it reads: a bundle read from a file named alone is placed+ in the tree the run listed for that file, such as the bundle file a CLI runs, rather than in a second+ listing of it. None lists such a file's tree for this load alone.Returns:List of all pipes that were loaded@@ -1068,23 +1081,26 @@ class LibraryManager(LibraryManagerAbstract):# Bind the target library as current for the whole load: the factories beneath# (concept resolution, class-registry access) resolve through the ambient current# library, so a load into a non-current library must not read the caller's binding.- with scoped_current_library(library_id=library_id), sharing_module_name_checks():+ listing = tree_listing if tree_listing is not None else MethodTreeListing()+ with scoped_current_library(library_id=library_id), sharing_module_name_checks(), listing.loading():# A bundle read from a file on this host that no loader placed in a method tree, such as a bundle file a CLI- # names alone, is read as a member of its own file's tree: the steps of every bundle read from this host's- # disk read only the files of their tree.- unplaced_host_blueprints = [- blueprint for blueprint in blueprints if blueprint.tree_bundle is None and blueprint.host_bundle_file is not None- ]- if unplaced_host_blueprints:- tree_limits = local_tree_limits()- for blueprint in unplaced_host_blueprints:- if blueprint.host_bundle_file is not None:- blueprint.place_in_tree(tree_bundle=tree_bundle_of_lone_file(bundle_file=blueprint.host_bundle_file, limits=tree_limits))- # Every bundle read from this host's disk loads under its tree's requirements.txt, on every load: a blueprint- # placed by an earlier load, a refused one included, is judged again rather than taken as judged.+ # names alone, is read as a member of its own file's tree, taken from the run's listing: the steps of every+ # bundle read from this host's disk read only the files of their tree. A tree a caller placed a bundle in+ # before the load is read again, once per run, as a tree the listing hands to the load is: a file of it+ # changed since its listing is refused here, before anything of it is imported or built.+ for blueprint in blueprints:+ if blueprint.tree_bundle is None and blueprint.host_bundle_file is not None:+ blueprint.place_in_tree(tree_bundle=listing.lone_file_bundle(bundle_file=blueprint.host_bundle_file))+ elif blueprint.tree_bundle is not None:+ listing.ensure_unchanged(tree=blueprint.tree_bundle.tree)+ # Every bundle read from this host's disk loads under its tree's requirements.txt, judged once per run: a tree+ # the run's listing holds as met, such as one `load_libraries` judged just before this load, is not judged+ # again, while a refused tree, or a blueprint an earlier load placed under another listing, is judged again+ # rather than taken as judged.ensure_method_requirements_met(trees=[blueprint.tree_bundle.tree for blueprint in blueprints if blueprint.tree_bundle is not None],names_host_paths=True,+ tree_listing=listing,)# Bundles parsed from content strings sit at the root of the tree of the files submitted beside them, when# there are any, so a step resolves a file it names there.@@ -1108,6 +1124,7 @@ class LibraryManager(LibraryManagerAbstract):blueprints=blueprints,methods_dirs=methods_dirs,address_dependency_sources=address_dependency_sources,+ tree_listing=listing,)# Build the crate (merges, qualifies, detects duplicates)@@ -1331,6 +1348,7 @@ class LibraryManager(LibraryManagerAbstract):bundle_files: list[LibraryBundleFile],methods_dirs: list[Path] | None = None,address_dependency_sources: AddressDependencySources = AddressDependencySources.ANYWHERE,+ tree_listing: MethodTreeListing | None = None,) -> list[PipeAbstract]:"""Load MTHDS files into a specific library.@@ -1344,6 +1362,7 @@ class LibraryManager(LibraryManagerAbstract):bundle_files: The bundle files to load, each the member of its method treemethods_dirs: Directories laid out like ``.mthds/methods/`` holding packages for this load aloneaddress_dependency_sources: Where a package the bundles call by address may be found (``load_libraries``)+ tree_listing: The listing the bundle files' trees were taken from, which the blueprint load goes on with"""valid_mthds_paths = [bundle_file.path for bundle_file in bundle_files]blueprints: list[PipelexBundleBlueprint] = []@@ -1358,12 +1377,14 @@ class LibraryManager(LibraryManagerAbstract):else str(mthds_file_path))bundle_sources.append(bundle_source)+ # The bundle's bytes are read against the hash its tree's listing recorded, so the bundle parsed is the one the+ # run's digest names: a file changed since the listing is refused rather than parsed.+ bundle_data = read_tree_bundle_as_listed(tree_bundle=bundle_file.tree_bundle)try:- blueprint = MthdsParser.make_pipelex_bundle_blueprint(bundle_path=mthds_file_path, mthds_source=bundle_source)+ blueprint = MthdsParser.make_pipelex_bundle_blueprint(+ bundle_path=mthds_file_path, bundle_data=bundle_data, mthds_source=bundle_source+ )blueprint.place_in_tree(tree_bundle=bundle_file.tree_bundle)- except FileNotFoundError as file_not_found_error:- msg = f"Could not find MTHDS bundle at '{bundle_source}'"- raise LibraryLoadingError(msg).with_error_code(CoreCode.METHOD_NOT_FOUND) from file_not_found_errorexcept MthdsParserError as parser_error:# Forward BLUEPRINT validation errors from interpretermsg = f"Could not load MTHDS bundle from '{bundle_source}' because of: {parser_error.message}"@@ -1391,6 +1412,7 @@ class LibraryManager(LibraryManagerAbstract):library_id=library_id,manifest=manifest,package_root=package_root,+ tree_listing=tree_listing,)# Store resolved absolute paths for duplicate detection in the library@@ -1408,6 +1430,7 @@ class LibraryManager(LibraryManagerAbstract):blueprints=blueprints,methods_dirs=methods_dirs,address_dependency_sources=address_dependency_sources,+ tree_listing=tree_listing,)except ValidationError as validation_error:# Categorize before wrapping. `translate_to_validate_bundle_error` has its own@@ -1565,6 +1588,7 @@ class LibraryManager(LibraryManagerAbstract):library_id: str,manifest: MethodsManifest,package_root: Path,+ tree_listing: MethodTreeListing | None,) -> None:"""Load dependency packages into the library.@@ -1576,6 +1600,7 @@ class LibraryManager(LibraryManagerAbstract):library_id: The library to load intomanifest: The consuming package's manifestpackage_root: The root directory of the consuming package+ tree_listing: The run's listing, which lists each package's tree once and judges its requirements once"""try:resolved_deps = resolve_all_dependencies(manifest=manifest, package_root=package_root)@@ -1585,15 +1610,15 @@ class LibraryManager(LibraryManagerAbstract):library = self.get_library(library_id=library_id)- # This machine's limits are read once for every package of the manifest.- tree_limits = local_tree_limits()+ # One listing for every package of the manifest, which reads this machine's limits once.+ listing = tree_listing if tree_listing is not None else MethodTreeListing()for resolved_dep in resolved_deps:self._load_single_dependency(library=library,library_id=library_id,resolved_dep=resolved_dep,package_address=resolved_dep.address,- tree_limits=tree_limits,+ tree_listing=listing,)# Wire concept resolver after all deps are loaded so cross-package@@ -1607,7 +1632,7 @@ class LibraryManager(LibraryManagerAbstract):library_id: str,resolved_dep: ResolvedDependency,package_address: str,- tree_limits: TreeLimits | None = None,+ tree_listing: MethodTreeListing | None = None,fetched_copy: FetchedCopy | None = None,caller_trees: tuple[MethodTree, ...] = (),) -> None:@@ -1631,7 +1656,9 @@ class LibraryManager(LibraryManagerAbstract):library_id: The id the manager holds the main library under, which holds the package's tree for its runsresolved_dep: The resolved dependency infopackage_address: The dependency's address as a reference names it, without any ``@<tag>``- tree_limits: The limits the package's tree is listed under, read from the configuration when not given+ tree_listing: The run's listing, which lists the package's tree once per run and keeps its requirements+ verdict, so a package loaded again in the run is neither hashed nor judged again; None lists it for+ this load alonefetched_copy: When the package is a copy this runtime fetched, that copy, whose listed tree is checked againstthe locks of ``caller_trees`` before any of its files is read, and recorded for a later load batch callingit; None for a package shipped or installed otherwise, which no lock is checked against here@@ -1652,8 +1679,10 @@ class LibraryManager(LibraryManagerAbstract):dep_blueprints: list[PipelexBundleBlueprint] = []# Each bundle as the member of the package's tree, by its caller-facing source, for the pipes reading a file it names.tree_bundles_by_source: dict[str, TreeBundle] = {}- # Listed under the package's address: a refusal never names the directory it was installed or fetched into.- package_tree = list_package_method_tree(root=resolved_dep.package_root, package_address=package_address, limits=tree_limits)+ # Listed under the package's address: a refusal never names the directory it was installed or fetched into. The+ # run's listing lists it once and keeps its verdict, so the package is hashed and judged once per run.+ listing = tree_listing if tree_listing is not None else MethodTreeListing()+ package_tree = listing.package_tree(root=resolved_dep.package_root, package_address=package_address)fetched_tree: FetchedTree | None = Noneif fetched_copy is not None:# The tree this load lists, and so loads, is the one checked: a fetched copy that is not the tree its callers'@@ -1661,14 +1690,14 @@ class LibraryManager(LibraryManagerAbstract):fetched_tree = FetchedTree(copy=fetched_copy, tree_digest=package_tree.digest)ensure_fetched_tree_locked(fetched_tree=fetched_tree, caller_trees=caller_trees)# The package's own requirements.txt is judged before any of its bundles loads, as the consumer's tree is.- ensure_package_requirements_met(tree=package_tree, package_address=package_address)+ ensure_package_requirements_met(tree=package_tree, package_address=package_address, tree_listing=listing)# Held for the consumer's runs, which read a file of the package's tree, such as one of its steps' template# files, by the tree's digest.LOCAL_METHOD_FILES.hold_tree(holder=library_id, tree=package_tree)for tree_bundle in tree_bundles_of(tree=package_tree):# The package's address and the bundle's tree path: never the host's path to the package.bundle_source = f"{package_address}/{tree_bundle.tree_path}"- blueprint = _parse_dependency_bundle(mthds_path=tree_bundle.location, bundle_source=bundle_source, package_address=package_address)+ blueprint = _parse_dependency_bundle(tree_bundle=tree_bundle, bundle_source=bundle_source, package_address=package_address)blueprint.place_in_tree(tree_bundle=tree_bundle)tree_bundles_by_source[bundle_source] = tree_bundledep_blueprints.append(blueprint)@@ -1847,6 +1876,7 @@ class LibraryManager(LibraryManagerAbstract):blueprints: list[PipelexBundleBlueprint],methods_dirs: list[Path] | None = None,address_dependency_sources: AddressDependencySources = AddressDependencySources.ANYWHERE,+ tree_listing: MethodTreeListing | None = None,) -> None:"""Scan blueprints for cross-package pipe refs with address-based aliases and load them.@@ -1879,6 +1909,7 @@ class LibraryManager(LibraryManagerAbstract):blueprints: The parsed bundle blueprints to scanmethods_dirs: The load's own methods directories, searched before everything else and never writtenaddress_dependency_sources: Whether an address ``methods_dirs`` do not answer may be looked up on this host+ tree_listing: The run's listing, which lists each package's tree once and judges its requirements onceRaises:LibraryLoadingError: One or more referenced packages could not be resolved.@@ -1929,6 +1960,7 @@ class LibraryManager(LibraryManagerAbstract):methods_dirs=methods_dirs,address_dependency_sources=address_dependency_sources,caller_trees=tuple(caller_trees.get(full_address, [])),+ tree_listing=tree_listing,)except (MethodFetchDisabledError, MethodDependencyFetchError, MethodDependencyNotShippedError) as exc:# The item carries the sentence its code renders; the host's log keeps the whole cause chain, git's raw output@@ -1966,6 +1998,7 @@ class LibraryManager(LibraryManagerAbstract):methods_dirs: list[Path] | None = None,address_dependency_sources: AddressDependencySources = AddressDependencySources.ANYWHERE,caller_trees: tuple[MethodTree, ...] = (),+ tree_listing: MethodTreeListing | None = None,) -> None:"""Load a method package on demand using its full address, fetching it on a miss.@@ -1985,6 +2018,7 @@ class LibraryManager(LibraryManagerAbstract):installed on this host and fetching nothing; ``ANYWHERE`` falls back to the installed methods and a fetchcaller_trees: The trees of the bundles calling the package, whose locks a copy this runtime fetched, forthis load or an earlier one, is checked against+ tree_listing: The run's listing, which lists the package's tree once and judges its requirements onceRaises:MethodRefError: The address resolves to no vendored or installed method and could not be@@ -2037,6 +2071,7 @@ class LibraryManager(LibraryManagerAbstract):package_address=f"{installed.manifest.address}/{installed.name}",fetched_copy=fetched_copy,caller_trees=caller_trees,+ tree_listing=tree_listing,)def _remove_pipes_from_blueprint(self, blueprint: PipelexBundleBlueprint) -> None:diff --git a/pipelex/libraries/library_manager_abstract.py b/pipelex/libraries/library_manager_abstract.pyindex dab00680f..29bc60525 100644--- a/pipelex/libraries/library_manager_abstract.py+++ b/pipelex/libraries/library_manager_abstract.py@@ -14,6 +14,7 @@ if TYPE_CHECKING:from mthds.method_tree.models import MethodTreefrom pipelex.libraries.library import Library+ from pipelex.libraries.method_tree_listing import MethodTreeListingclass LibraryManagerAbstract(ABC):@@ -169,6 +170,7 @@ class LibraryManagerAbstract(ABC):blueprints: list[PipelexBundleBlueprint],methods_dirs: list[Path] | None = None,address_dependency_sources: AddressDependencySources = AddressDependencySources.ANYWHERE,+ tree_listing: "MethodTreeListing | None" = None,) -> list[PipeAbstract]:"""Load the blueprints into the library, resolving their address-based dependencies.@@ -179,6 +181,8 @@ class LibraryManagerAbstract(ABC):for an address-based dependency before the installed stores and fetch-on-miss, and never writtenaddress_dependency_sources: ``SHIPPED`` searches ``methods_dirs`` alone, reading nothing installed on this hostand fetching nothing; ``ANYWHERE`` falls back to the installed stores and fetch-on-miss+ tree_listing: The run's listing of the method trees it reads, which places a bundle read from a file named+ alone in the tree the run listed for that file rather than listing it again; None lists it for this load"""@abstractmethod@@ -198,5 +202,17 @@ class LibraryManagerAbstract(ABC):library_file_paths: list[Path] | None = None,methods_dirs: list[Path] | None = None,address_dependency_sources: AddressDependencySources = AddressDependencySources.ANYWHERE,+ tree_listing: "MethodTreeListing | None" = None,) -> list[PipeAbstract]:- pass+ """Load the bundles of the library directories' method trees, and of the bundle files named alone, into the library.++ Args:+ library_id: The library to load into+ library_dirs: The library directories, each loaded as its own method tree and the trees nested in it+ library_file_paths: Bundle files named alone, each loaded as the member of its own file's tree+ methods_dirs: Directories laid out like ``.mthds/methods/`` holding packages for this load alone+ address_dependency_sources: Where a package the bundles call by address may be found+ tree_listing: The run's listing of the method trees it reads, such as the one a CLI made when it resolved+ the folder it runs: every tree is taken from it, and listed through it when nobody listed it yet, so+ the run lists each tree once. None lists the trees for this load alone+ """diff --git a/pipelex/libraries/method_requirements.py b/pipelex/libraries/method_requirements.pyindex 19eb68d3c..5434d1b6e 100644--- a/pipelex/libraries/method_requirements.py+++ b/pipelex/libraries/method_requirements.py@@ -15,6 +15,13 @@ never installs anything:for the tree's requirements, asked for each tree, so trees judged together each get the offer stated for their own.A deployment that cannot hold a tree's requirements in any environment refuses them with its own reason, which themethod's refusal carries.+- A tree is judged once per run: the run's listing (`pipelex.libraries.method_tree_listing`) keeps each tree a check+ found met, with the execution mode and the plugin factory it was judged against, so the later steps of a load, a load+ after a server's check of the same request, or a dependency package loaded again ask nothing more while those are+ unchanged, and two loads judging the tree at once ask once. A refused tree is never kept, and is judged again by the+ next check that reads it.+- The `requirements.txt` judged is read against the hash the tree's listing recorded, so the file judged is the file the+ run's digest names.- A requirement whose marker is false in that environment is not needed there and is not checked. One is met when theenvironment holds a distribution of its name at a version its specifier contains, pre-releases included, and, foreach extra it names, when what the extra needs is met in turn.@@ -26,12 +33,13 @@ the caller. A dependency package is named by its address, and a server checkingdirectory of its own names no path on the host at all (`ensure_served_method_requirements_met`)."""+import functoolsimport refrom collections.abc import Iterable, Mapping, Sequencefrom pathlib import Pathfrom typing import NamedTuple-from mthds.method_tree.models import MethodTree, TreeLimits+from mthds.method_tree.models import MethodTreefrom mthds.protocol.problem.registry import MethodDependencyMissingKind, MethodDependencyMissingReasonfrom packaging.markers import Marker, UndefinedComparison, UndefinedEnvironmentNamefrom packaging.requirements import InvalidRequirement, Requirement@@ -46,8 +54,13 @@ from pipelex.core.method_trees.requirements_file import (parse_requirements_text,)from pipelex.libraries.exceptions import MethodRequirementsFileError, MethodRequirementsNotMetError-from pipelex.libraries.method_trees import list_library_dir_trees, listing_identity, reporting_host_library_tree_faults-from pipelex.libraries.package_environments import PythonPackageEnvironment, resolve_pipe_func_package_environment+from pipelex.libraries.method_tree_listing import MethodTreeListing+from pipelex.libraries.method_trees import listing_identity, read_tree_file_as_listed, reporting_host_library_tree_faults+from pipelex.libraries.package_environments import (+ PythonPackageEnvironment,+ current_pipe_func_environment_source,+ resolve_pipe_func_package_environment,+)from pipelex.methods.package_reason import MethodPackageReasonfrom pipelex.plugins.pipe_func_package_offer_registry import PipeFuncPackageRefusal@@ -301,25 +314,21 @@ def _judged_method(*, tree: MethodTree, names_host_paths: bool) -> JudgedMethod:def _read_requirements_text(*, judged: JudgedMethod) -> str:- """The text of a tree's `requirements.txt`.+ """The text of a tree's `requirements.txt`, read against the hash the tree's listing recorded.Raises:- MethodRequirementsFileError: The file cannot be read, or is not UTF-8 text.+ MethodRequirementsFileError: The file is not UTF-8 text.+ MethodTreeChangedError: The file is no longer the one the tree's listing hashed, or can no longer be read."""- location = judged.tree.file_location(REQUIREMENTS_FILE_NAME)source = judged.requirements_source+ data = read_tree_file_as_listed(tree=judged.tree, tree_path=REQUIREMENTS_FILE_NAME)try:- return location.read_bytes().decode("utf-8")+ return data.decode("utf-8")except UnicodeDecodeError as exc:msg = f"{judged.method_name} declares its packages in a {REQUIREMENTS_FILE_NAME} that is not UTF-8 text, so they cannot be read."raise MethodRequirementsFileError(msg, items=[PipelexBundleBlueprintValidationErrorData(source=source, message=msg)]).with_params(**_REQUIREMENTS_INVALID_PARAMS) from exc- except OSError as exc:- msg = f"{judged.method_name} declares its packages in a {REQUIREMENTS_FILE_NAME} that cannot be read: {exc.strerror or type(exc).__name__}."- raise MethodRequirementsFileError(msg, items=[PipelexBundleBlueprintValidationErrorData(source=source, message=msg)]).with_params(- **_REQUIREMENTS_INVALID_PARAMS- ) from excdef _unmet_requirement_params(*, unmet_requirement: UnmetRequirement) -> dict[str, str]:@@ -415,42 +424,70 @@ def _refused_by_deployment_error(return MethodRequirementsNotMetError(msg, items=[item]).with_params(**params)-def _ensure_judged_methods_met(*, judged_methods: Iterable[JudgedMethod], names_host_paths: bool) -> None:+def _ensure_judged_methods_met(*, judged_methods: Iterable[JudgedMethod], names_host_paths: bool, tree_listing: MethodTreeListing | None) -> None:"""Refuse the first of the methods whose `requirements.txt` the standard refuses or its PipeFunc environment does not meet.Each tree is judged once however often it is given, in the order given, against the environment stated for its ownrequirements: in a sandbox-hosted mode, the plugin providing the mode is asked for each tree declaring requirements,- so trees judged together never share an environment stated for another's. Methods declaring none never read it.+ so trees judged together never share an environment stated for another's. Methods declaring none never read it. A+ tree `tree_listing` holds as met against the current execution mode and factory, which an earlier check of the run+ judged, is not judged again, each tree found met is kept there for the run's later checks, and a tree two checks+ judge at once is judged by one of them (`MethodTreeListing.judge_requirements_once`).Raises:- MethodRequirementsFileError: A tree's `requirements.txt` holds a line the standard refuses, or cannot be read.+ MethodRequirementsFileError: A tree's `requirements.txt` holds a line the standard refuses, or is not UTF-8 text.+ MethodTreeChangedError: A tree's `requirements.txt` is no longer the file its tree's listing hashed.MethodRequirementsNotMetError: The environment does not meet a tree's requirements, or the deployment refusesthem with its own reason, no environment of its holding them.PipeFuncPackageSetUnstatedError: PipeFunc steps run out of this process and the deployment states no packageset for them."""judged_trees: set[tuple[str, str, bool]] = set()+ environment_source = current_pipe_func_environment_source() if tree_listing is not None else Nonefor judged in judged_methods:tree_identity = listing_identity(tree=judged.tree)if tree_identity in judged_trees:continuejudged_trees.add(tree_identity)- if not holds_requirements_to_check(tree=judged.tree):+ if tree_listing is None:+ _ensure_judged_method_met(judged=judged, names_host_paths=names_host_paths)continue- parsed = parse_requirements_text(text=_read_requirements_text(judged=judged))- if parsed.refused_lines:- _raise_refused_lines(judged=judged, refused_lines=parsed.refused_lines)- if not parsed.requirements:- continue- environment = resolve_pipe_func_package_environment(requirements=parsed.requirements, names_host_paths=names_host_paths)- if isinstance(environment, PipeFuncPackageRefusal):- raise _refused_by_deployment_error(judged=judged, refusal=environment, requirements=parsed.requirements)- unmet = unmet_requirements(requirements=parsed.requirements, environment=environment)- if unmet:- _raise_unmet(judged=judged, unmet=unmet, environment=environment)+ tree_listing.judge_requirements_once(+ tree=judged.tree,+ judged_against=environment_source,+ judge=functools.partial(_ensure_judged_method_met, judged=judged, names_host_paths=names_host_paths),+ )++def _ensure_judged_method_met(*, judged: JudgedMethod, names_host_paths: bool) -> None:+ """Refuse a method whose `requirements.txt` the standard refuses or its PipeFunc environment does not meet.-def ensure_method_requirements_met(*, trees: Iterable[MethodTree], names_host_paths: bool) -> None:+ A tree with nothing to check, holding no requirements or no Python, meets them.++ Raises:+ MethodRequirementsFileError: The tree's `requirements.txt` holds a line the standard refuses, or is not UTF-8 text.+ MethodTreeChangedError: The tree's `requirements.txt` is no longer the file its listing hashed.+ MethodRequirementsNotMetError: The environment does not meet the tree's requirements, or the deployment refuses+ them with its own reason.+ PipeFuncPackageSetUnstatedError: PipeFunc steps run out of this process and the deployment states no package+ set for them.+ """+ if not holds_requirements_to_check(tree=judged.tree):+ return+ parsed = parse_requirements_text(text=_read_requirements_text(judged=judged))+ if parsed.refused_lines:+ _raise_refused_lines(judged=judged, refused_lines=parsed.refused_lines)+ if not parsed.requirements:+ return+ environment = resolve_pipe_func_package_environment(requirements=parsed.requirements, names_host_paths=names_host_paths)+ if isinstance(environment, PipeFuncPackageRefusal):+ raise _refused_by_deployment_error(judged=judged, refusal=environment, requirements=parsed.requirements)+ unmet = unmet_requirements(requirements=parsed.requirements, environment=environment)+ if unmet:+ _raise_unmet(judged=judged, unmet=unmet, environment=environment)+++def ensure_method_requirements_met(*, trees: Iterable[MethodTree], names_host_paths: bool, tree_listing: MethodTreeListing | None = None) -> None:"""Refuse a method among the trees whose `requirements.txt` the standard refuses or its PipeFunc environment does not meet.Each tree is judged once however often it is given, in the order given, and the first refused ends the check.@@ -459,9 +496,14 @@ def ensure_method_requirements_met(*, trees: Iterable[MethodTree], names_host_patrees: The method trees a load reads, each checked when it lies on disk, declares requirements and holds Python.names_host_paths: Whether a refusal may name the method's root and this interpreter by their paths on this host,as a load's may: its refusal is the caller's own disk's, or the host's fault, withheld from the caller.+ tree_listing: The run's listing of its method trees, which keeps each tree a check of the run found met with the+ execution mode and factory it was judged against: such a tree is not judged again against them, so the+ package-offer factory is asked once per tree in a run, whose verdict does not depend on how a refusal names+ the method. None judges every tree given.Raises:- MethodRequirementsFileError: A tree's `requirements.txt` holds a line the standard refuses, or cannot be read.+ MethodRequirementsFileError: A tree's `requirements.txt` holds a line the standard refuses, or is not UTF-8 text.+ MethodTreeChangedError: A tree's `requirements.txt` is no longer the file its tree's listing hashed.MethodRequirementsNotMetError: The environment does not meet a tree's requirements, or the deployment refusesthem with its own reason.PipeFuncPackageSetUnstatedError: PipeFunc steps run out of this process and the deployment states no package@@ -470,17 +512,20 @@ def ensure_method_requirements_met(*, trees: Iterable[MethodTree], names_host_pa_ensure_judged_methods_met(judged_methods=(_judged_method(tree=tree, names_host_paths=names_host_paths) for tree in trees),names_host_paths=names_host_paths,+ tree_listing=tree_listing,)-def ensure_package_requirements_met(*, tree: MethodTree, package_address: str) -> None:+def ensure_package_requirements_met(*, tree: MethodTree, package_address: str, tree_listing: MethodTreeListing | None = None) -> None:"""Refuse a dependency package whose `requirements.txt` the standard refuses or its PipeFunc environment does not meet.The package is named by its address and its file by `<address>/requirements.txt`, never by the directory it was- installed or fetched into, as its bundles are, since the package is the caller's dependency wherever it loads.+ installed or fetched into, as its bundles are, since the package is the caller's dependency wherever it loads. A+ package `tree_listing` holds as met is not judged again, as a method's tree is not (`ensure_method_requirements_met`).Raises:- MethodRequirementsFileError: The package's `requirements.txt` holds a line the standard refuses, or cannot be read.+ MethodRequirementsFileError: The package's `requirements.txt` holds a line the standard refuses, or is not UTF-8 text.+ MethodTreeChangedError: The package's `requirements.txt` is no longer the file its tree's listing hashed.MethodRequirementsNotMetError: The environment does not meet the package's requirements, or the deploymentrefuses them with its own reason.PipeFuncPackageSetUnstatedError: PipeFunc steps run out of this process and the deployment states no package@@ -491,10 +536,10 @@ def ensure_package_requirements_met(*, tree: MethodTree, package_address: str) -method_name=f"The method package '{package_address}'",requirements_source=f"{package_address}/{REQUIREMENTS_FILE_NAME}",)- _ensure_judged_methods_met(judged_methods=[judged], names_host_paths=False)+ _ensure_judged_methods_met(judged_methods=[judged], names_host_paths=False, tree_listing=tree_listing)-def ensure_served_method_requirements_met(*, library_dirs: Sequence[Path], limits: TreeLimits | None = None) -> None:+def ensure_served_method_requirements_met(*, library_dirs: Sequence[Path], tree_listing: MethodTreeListing | None = None) -> None:"""Check the requirements of a caller's method a server holds in directories of its own, as loading them would, without loading them.For a server that hands the method to a loader elsewhere, such as a validator whose worker receives the method's@@ -504,14 +549,20 @@ def ensure_served_method_requirements_met(*, library_dirs: Sequence[Path], limithost's fault, as a load of a host's library directories reports it, and a refusal names no path on the host, thedirectories' and this interpreter's included.+ The trees are listed through `tree_listing`, which the server hands to the load of the same request, so the method+ loaded is the one checked here and its files are listed and hashed once (`pipelex.libraries.method_tree_listing`),+ and the listing keeps each tree found met, so the load does not judge it again. None lists them for this check alone.+Raises:HostLibraryTreeError: A tree of the directories could not be listed.- MethodRequirementsFileError: A tree's `requirements.txt` holds a line the standard refuses, or cannot be read.+ MethodRequirementsFileError: A tree's `requirements.txt` holds a line the standard refuses, or is not UTF-8 text.+ MethodTreeChangedError: A tree's `requirements.txt` is no longer the file its tree's listing hashed.MethodRequirementsNotMetError: The environment does not meet a tree's requirements, or the deployment refusesthem with its own reason.PipeFuncPackageSetUnstatedError: PipeFunc steps run out of this process and the deployment states no packageset for them."""+ listing = tree_listing if tree_listing is not None else MethodTreeListing()with reporting_host_library_tree_faults():- library_trees = list_library_dir_trees(library_dirs=library_dirs, limits=limits)- ensure_method_requirements_met(trees=[library_tree.tree for library_tree in library_trees], names_host_paths=False)+ library_trees = listing.library_dir_trees(library_dirs=library_dirs)+ ensure_method_requirements_met(trees=[library_tree.tree for library_tree in library_trees], names_host_paths=False, tree_listing=listing)diff --git a/pipelex/libraries/method_tree_listing.py b/pipelex/libraries/method_tree_listing.pynew file mode 100644index 000000000..8c7563a19--- /dev/null+++ b/pipelex/libraries/method_tree_listing.py@@ -0,0 +1,392 @@+"""One run's listing of the method trees it reads: each tree listed once, then handed down to every load of the run.++Listing a method tree walks its directory and hashes every file of it (`pipelex.libraries.method_trees`). A run needs the+tree it reads before it loads it, a CLI to resolve the folder or the bundle file it was handed and to record the tree's+digest, a server to check the requirements of the method it holds before the run starts, so the first lister makes a+`MethodTreeListing`, lists through it, and hands it down to the load. The load takes every tree from it, listing only the+trees nobody listed yet, and so one listing serves the whole run: the digest the run records is the digest of the tree it+loaded.++A listing holds each tree by its root, so a folder handed over as a method, the same folder loaded as a library+directory, and a bundle file named alone inside it are one listing of one tree, whichever is asked first. It reads the+path handed over exactly as the standard's lister does before taking anything from what it holds: a directory, or a+symbolic link naming one, is followed to the directory it names, as the lister follows it, while a `.mthds` file named+alone is never followed, so a symbolic link named alone is refused as the lister refuses it, even when the run already+listed the file it points to.++The files of a tree may change after its listing. Every read of them is checked against the hashes the listing recorded+(`read_tree_file_as_listed`), and a load (`loading`) reads again every file of each tree it takes that the run listed+before any load, once, as it takes it, so a tree edited between its listing and its load is refused before anything of+it is parsed or run, with `MethodTreeChangedError`, in the context of the load that takes it; a tree a load lists itself+is as fresh as that check would be. A listing belongs to one run: it is never kept beyond the run that made it, and a+command that writes the files it lists, such as `fix`, makes a new one after each write.++The listing is handed down by argument, except across the bundle-validator seam, a plugin contract that names no+listing: a host validating through it puts its listing in scope around the call (`scoped_tree_listing`), and the+in-process validation takes it from there (`validate_bundles_in_process`), while a validator dispatching to another+process never reads it.++A listing also keeps the verdicts on the trees whose `requirements.txt` a check of the run found met+(`pipelex.libraries.method_requirements`), each with what it was judged against, the PipeFunc execution mode and the+plugin factory stating its packages, so each tree is judged once per run against one environment, and the package-offer+factory of a sandbox-hosted deployment is asked once for it, even by loads judging it at the same time. A refused tree+is never kept, so a run that goes on after a refusal judges it again.++Its loads may run at the same time in different worker threads of the method-load pool, so each tree is listed, checked+and judged under a lock of its own: a second ask waits for the first and takes its tree, while trees of different paths+are listed side by side.++Every tree is listed under this machine's limits (`[interpreter.method_tree_limits]`), read once for the whole listing,+and a directory a host wrote a tree out as returns the tree as written, as every listing of it does+(`pipelex.libraries.written_trees`).+"""++import threading+from collections.abc import Callable, Generator, Hashable, Iterable+from contextlib import contextmanager+from contextvars import ContextVar+from pathlib import Path++from mthds.method_tree.models import MethodTree, TreeLimits++from pipelex.core.method_trees.tree_bundle import TreeBundle+from pipelex.libraries.method_trees import (+ HandedOverLocation,+ LibraryTree,+ ensure_tree_files_as_listed,+ list_library_dir_tree,+ list_library_dir_trees,+ list_library_trees,+ list_local_method_tree,+ list_package_method_tree,+ listing_identity,+ local_tree_limits,+ locate_handed_over,+ lone_file_member,+ lone_file_tree_path,+)+from pipelex.libraries.written_trees import WRITTEN_METHOD_TREES++TreeIdentity = tuple[str, str, bool]+++def _resolved_or_absolute(*, path: Path) -> Path:+ """A path as a key: resolved, as the lister resolves a directory, or absolute when it cannot be resolved."""+ try:+ return path.resolve()+ except (OSError, RuntimeError):+ return path.absolute()+++class MethodTreeListing:+ """The method trees one run listed, each listed once, with the checks the run made of them.++ Its methods list as the functions of `pipelex.libraries.method_trees` do, and raise as they do, but list each tree+ once: a second ask, under any spelling the standard's lister reads as the same tree, returns the tree the first+ listed. A listing that fails is not held, so asking again lists again and fails again.+ """++ def __init__(self) -> None:+ # Guards the maps below, and is never held while a tree is listed, checked or judged.+ self._lock = threading.Lock()+ # One lock per tree, path or verdict being worked on, so concurrent asks of one wait while the others proceed.+ self._key_locks: dict[Hashable, threading.RLock] = {}+ self._limits: TreeLimits | None = None+ # The tree of each root, as a directory handed over or a library directory lists it, with no entry.+ self._trees_by_root: dict[Path, MethodTree] = {}+ # The tree of each `.mthds` file named alone, by the file's location, its own name never followed.+ self._lone_file_trees: dict[Path, MethodTree] = {}+ # The trees each library directory holds, its own first, by the directory's resolved path.+ self._library_trees: dict[Path, list[MethodTree]] = {}+ # Each dependency package's tree, rooted at the package's own directory, by that directory's resolved path.+ self._package_trees: dict[Path, MethodTree] = {}+ # The trees a load reading this listing took fresh from the disk, by listing them or by reading them again.+ self._trees_read_by_a_load: set[TreeIdentity] = set()+ # What each tree was found to meet its requirements against: the environment sources of its verdicts.+ self._requirements_met: dict[TreeIdentity, list[object]] = {}++ def _key_lock(self, *, key: Hashable) -> threading.RLock:+ with self._lock:+ key_lock = self._key_locks.get(key)+ if key_lock is None:+ key_lock = threading.RLock()+ self._key_locks[key] = key_lock+ return key_lock++ @property+ def limits(self) -> TreeLimits:+ """This machine's limits, read from the configuration once for the whole listing.++ Raises:+ PipelexConfigError: Nothing booted and the configuration files do not validate.+ """+ with self._key_lock(key="limits"):+ if self._limits is None:+ self._limits = local_tree_limits()+ return self._limits++ def _handed_out(self, *, tree: MethodTree, listed_now: bool) -> MethodTree:+ """Hand a tree out: to a load, a tree listed before any load is read again first, once (`loading`).++ `listed_now` says the tree was just read from the disk by the lister, so a load taking it needs no second read. A+ tree a host wrote out is read as written, never again.++ Raises:+ MethodTreeChangedError: A load takes a tree listed before it, and a file of it is not the file its listing hashed.+ """+ if tree.directory is None or not self._is_loading():+ return tree+ if listed_now or WRITTEN_METHOD_TREES.holds_tree(tree=tree):+ # A tree a host wrote out is read as written, from the files the host wrote, and never read again.+ with self._lock:+ self._trees_read_by_a_load.add(listing_identity(tree=tree))+ else:+ self._check_tree(tree=tree)+ return tree++ def _is_loading(self) -> bool:+ return any(listing is self for listing in _LOADING_LISTINGS.get())++ def _held_root_tree(self, *, root: Path) -> MethodTree | None:+ with self._lock:+ return self._trees_by_root.get(root)++ def _hold_root_tree(self, *, tree: MethodTree) -> None:+ """Hold a directory tree under its root, the entry a lone file's listing names dropped, for every later ask of the root."""+ if tree.directory is None or tree.is_one_file:+ return+ root_tree = tree if tree.entry is None else tree.model_copy(update={"entry": None})+ with self._lock:+ self._trees_by_root.setdefault(tree.directory, root_tree)++ def method_tree(self, *, handed_over: Path) -> MethodTree:+ """The method tree of a directory, or of a `.mthds` file named alone, listed once (`list_local_method_tree`).++ Raises:+ EmptyMethodTreeError: The membership rule keeps every file of the directory out of its tree.+ MethodTreeLoadError: The standard refused the tree otherwise, it passed a limit, or it could not be read.+ PipelexConfigError: Nothing booted and the configuration files, read for the limits, do not validate.+ """+ location = locate_handed_over(handed_over=handed_over)+ if location is not None and location.lone_file is not None:+ return self._lone_file_tree(location=location, lone_file=location.lone_file, handed_over=handed_over)+ if location is None or location.root is None:+ # The lister refuses the path before listing anything: it gives the refusal.+ return self._handed_out(tree=list_local_method_tree(handed_over=handed_over, limits=self.limits), listed_now=True)+ with self._key_lock(key=("root", location.root)):+ tree = self._held_root_tree(root=location.root)+ if tree is not None:+ return self._handed_out(tree=tree, listed_now=False)+ tree = list_local_method_tree(handed_over=handed_over, limits=self.limits)+ self._hold_root_tree(tree=tree)+ return self._handed_out(tree=tree, listed_now=True)++ def _library_dir_own_tree(self, *, library_dir: Path, limits: TreeLimits) -> MethodTree | None:+ """The tree a library directory is rooted in, or None when it is empty, taken from the root it shares (`list_library_dir_tree`)."""+ location = locate_handed_over(handed_over=library_dir)+ if location is None or location.directory is None or location.root is None:+ # Not a directory as the lister reads one: it gives what it gives such a path.+ listed = list_library_dir_tree(library_dir=library_dir, limits=limits)+ return self._handed_out(tree=listed, listed_now=True) if listed is not None else None+ with self._key_lock(key=("root", location.root)):+ tree = self._held_root_tree(root=location.root)+ if tree is not None:+ return self._handed_out(tree=tree, listed_now=False)+ tree = list_library_dir_tree(library_dir=library_dir, limits=limits)+ if tree is None:+ return None+ self._hold_root_tree(tree=tree)+ return self._handed_out(tree=tree, listed_now=True)++ def _lone_file_tree(self, *, location: HandedOverLocation, lone_file: Path, handed_over: Path) -> MethodTree:+ with self._key_lock(key=("lone_file", lone_file)):+ with self._lock:+ tree = self._lone_file_trees.get(lone_file)+ listed_now = False+ if tree is None:+ tree = self._lone_file_tree_from_root(location=location, lone_file=lone_file)+ if tree is None:+ tree = list_local_method_tree(handed_over=handed_over, limits=self.limits)+ listed_now = True+ self._hold_root_tree(tree=tree)+ with self._lock:+ self._lone_file_trees.setdefault(lone_file, tree)+ return self._handed_out(tree=tree, listed_now=listed_now)++ def _lone_file_tree_from_root(self, *, location: HandedOverLocation, lone_file: Path) -> MethodTree | None:+ """A lone file's tree from its root's tree listed already, when that tree holds the file, else None for the lister to list it.++ The lister lists a lone file under a `METHODS.toml` as the walk of the root that lists the root itself, naming+ the file as its entry: the walk the root's listing made. A file the walk did not keep, such as one a `.gitignore`+ keeps out, is left to the lister, whose refusal names why.+ """+ if location.root is None:+ return None+ with self._key_lock(key=("root", location.root)):+ root_tree = self._held_root_tree(root=location.root)+ if root_tree is None:+ return None+ entry = lone_file_tree_path(root=location.root, lone_file=lone_file)+ if entry is None or entry not in root_tree.members:+ return None+ return root_tree.model_copy(update={"entry": entry})++ def lone_file_bundle(self, *, bundle_file: Path) -> TreeBundle:+ """A bundle file named alone as the member of its method tree, its tree listed once (`method_tree`).++ Raises:+ MethodTreeLoadError: The tree could not be listed, or the membership rule keeps the file out of it.+ PipelexConfigError: Nothing booted and the configuration files, read for the limits, do not validate.+ """+ return lone_file_member(tree=self.method_tree(handed_over=bundle_file), bundle_file=bundle_file)++ def library_trees(self, *, library_dir: Path) -> list[MethodTree]:+ """The method trees a library directory holds, its own first, listed once (`list_library_trees`).++ The directory's own tree is the tree of its root, which `method_tree` lists too, so one listing serves a directory+ handed over as a method's folder and loaded as a library directory, whichever is asked first.++ Raises:+ MethodTreeLoadError: A tree could not be listed.+ PipelexConfigError: Nothing booted and the configuration files, read for the limits, do not validate.+ """+ key = _resolved_or_absolute(path=library_dir)+ with self._key_lock(key=("library", key)):+ with self._lock:+ trees = self._library_trees.get(key)+ if trees is not None:+ for tree in trees:+ self._handed_out(tree=tree, listed_now=False)+ return trees+ # The directory's own tree is handed out as it is taken (`_library_dir_own_tree`), read again when it was held+ # already, and the trees nested in it are listed now.+ trees = list_library_trees(library_dir=library_dir, limits=self.limits, own_tree_of=self._library_dir_own_tree)+ with self._lock:+ self._library_trees[key] = trees+ for tree in trees:+ self._handed_out(tree=tree, listed_now=True)+ return trees++ def _library_trees_of(self, *, library_dir: Path, limits: TreeLimits) -> list[MethodTree]: # ruff: ignore[unused-method-argument]+ # The listing's own limits, read once, are the ones `list_library_dir_trees` was handed.+ return self.library_trees(library_dir=library_dir)++ def library_dir_trees(self, *, library_dirs: Iterable[Path]) -> list[LibraryTree]:+ """The method trees of the library directories, in the order given, each directory's own first (`list_library_dir_trees`).++ Raises:+ MethodTreeLoadError: A tree could not be listed.+ PipelexConfigError: Nothing booted and the configuration files, read for the limits, do not validate.+ """+ return list_library_dir_trees(library_dirs=library_dirs, limits=self.limits, trees_of=self._library_trees_of)++ def package_tree(self, *, root: Path, package_address: str) -> MethodTree:+ """A dependency package's tree, rooted at the package's own directory, listed once (`list_package_method_tree`).++ Raises:+ MethodTreeLoadError: The standard refused the tree, it passed a limit, or it could not be read, named by the+ package's address.+ PipelexConfigError: Nothing booted and the configuration files, read for the limits, do not validate.+ """+ key = _resolved_or_absolute(path=root)+ with self._key_lock(key=("package", key)):+ with self._lock:+ tree = self._package_trees.get(key)+ if tree is not None:+ return self._handed_out(tree=tree, listed_now=False)+ tree = list_package_method_tree(root=root, package_address=package_address, limits=self.limits)+ with self._lock:+ self._package_trees[key] = tree+ return self._handed_out(tree=tree, listed_now=True)++ def judge_requirements_once(self, *, tree: MethodTree, judged_against: object, judge: Callable[[], None]) -> None:+ """Judge a tree's requirements once per run against one environment: `judge` runs unless a check of the run found them met.++ `judged_against` names what the verdict depends on beyond the tree, compared with `==`: the PipeFunc execution+ mode and the plugin factory stating its packages, so a check and a load judging against different ones never+ share a verdict. Two loads judging the tree at once judge it once, the second taking the first's verdict, or+ judging it again when the first refused it, since a refusal is never kept.++ Raises:+ Whatever `judge` raises: the tree's refusal, which is not kept.+ """+ identity = listing_identity(tree=tree)+ with self._key_lock(key=("requirements", identity)):+ with self._lock:+ if any(environment == judged_against for environment in self._requirements_met.get(identity, ())):+ return+ judge()+ with self._lock:+ self._requirements_met.setdefault(identity, []).append(judged_against)++ def _check_tree(self, *, tree: MethodTree) -> None:+ identity = listing_identity(tree=tree)+ with self._key_lock(key=("check", identity)):+ with self._lock:+ if identity in self._trees_read_by_a_load:+ return+ ensure_tree_files_as_listed(tree=tree)+ with self._lock:+ self._trees_read_by_a_load.add(identity)++ def ensure_unchanged(self, *, tree: MethodTree) -> None:+ """Read a tree a load takes from elsewhere than this listing's methods again, once, as a tree they hand to a load is.++ For a bundle a caller placed in its tree before the load, such as a bundle file a command or a validation read+ alone: the load takes the tree from the bundle, so the check `loading` makes of a tree it takes from the listing+ is made here. A tree a host wrote out, or one read from a map of file entries, is never read again.++ Raises:+ MethodTreeChangedError: A file of the tree is not the file its listing hashed.+ """+ if tree.directory is None or WRITTEN_METHOD_TREES.holds_tree(tree=tree):+ return+ self._check_tree(tree=tree)++ @contextmanager+ def loading(self) -> Generator[None, None, None]:+ """Mark a load reading this listing: each tree it takes that the run listed before any load is read again first.++ A tree the run listed before a load, such as the folder a command resolved or the method a server checked, may+ have changed since: as the load takes it, each of its files is read again and refused unless it is the file the+ listing hashed, so a changed tree is refused before anything of it is parsed, imported or run, and the refusal+ is raised where the load takes the tree, in the context that reports the load's own faults. A tree a load lists+ itself is as fresh as that check, a tree is read again once per run, and a load nested in another reading the+ same listing is that load.+ """+ loading_listings = _LOADING_LISTINGS.get()+ if any(listing is self for listing in loading_listings):+ yield+ return+ token = _LOADING_LISTINGS.set((*loading_listings, self))+ try:+ yield+ finally:+ _LOADING_LISTINGS.reset(token)+++# The listings a load in this context reads, so a tree it lists counts as read and a nested load checks nothing again.+_LOADING_LISTINGS: ContextVar[tuple[MethodTreeListing, ...]] = ContextVar("loading_tree_listings", default=())++# The listing a host put in scope around a call that takes none by argument, such as a bundle validator's.+_SCOPED_TREE_LISTING: ContextVar[MethodTreeListing | None] = ContextVar("scoped_tree_listing", default=None)+++@contextmanager+def scoped_tree_listing(*, tree_listing: MethodTreeListing) -> Generator[None, None, None]:+ """Put a run's listing in scope for the calls made inside the block, and in the worker threads they load in.++ For a host whose listing must cross a seam that takes none by argument, such as the API server validating through a+ bundle validator once it has checked the method's requirements: the in-process validation inside the block loads the+ trees from it. The scope rides the context, so it reaches the method-load pool, which runs each load in a copy of it.+ """+ token = _SCOPED_TREE_LISTING.set(tree_listing)+ try:+ yield+ finally:+ _SCOPED_TREE_LISTING.reset(token)+++def get_scoped_tree_listing() -> MethodTreeListing | None:+ """The listing a host put in scope (`scoped_tree_listing`), or None outside any."""+ return _SCOPED_TREE_LISTING.get()diff --git a/pipelex/libraries/method_trees.py b/pipelex/libraries/method_trees.pyindex 5572b1262..764de681a 100644--- a/pipelex/libraries/method_trees.py+++ b/pipelex/libraries/method_trees.py@@ -20,13 +20,22 @@ stops before hashing any file once a tree passes one, so a directory handed overA directory a host wrote a tree out as, and holds in `WRITTEN_METHOD_TREES`, is never listed: every listing of it returnsthe tree as written, so the membership rule is not applied twice (`pipelex.libraries.written_trees`).++A listing hashes every file of the tree, and a file read from a listed tree afterwards is read against that hash+(`read_tree_file_as_listed`, `ensure_tree_files_as_listed`): a file edited, replaced or removed since the listing is+refused with `MethodTreeChangedError`, so a load never parses, and a run never reads, bytes other than those the tree's+digest names."""+import io+import os+import statfrom collections.abc import Generator, Iterablefrom contextlib import contextmanagerfrom pathlib import Path-from typing import NamedTuple+from typing import NamedTuple, Protocol+from mthds.method_tree.digest import sha256_hexfrom mthds.method_tree.discovery import (BUNDLE_SUFFIX,DEFAULT_DIRECTORY_NAMES,@@ -37,16 +46,18 @@ from mthds.method_tree.discovery import (holds_manifest,holds_venv_marker,kind_of_exact_name,+ on_disk_name,read_directory,)from mthds.method_tree.exceptions import MethodTreeRefusedError, TreeLimitError, TreeSourceErrorfrom mthds.method_tree.gitignore import PatternList-from mthds.method_tree.lister import list_method_tree, list_method_tree_at_root-from mthds.method_tree.models import MethodTree, TreeLimits+from mthds.method_tree.lister import hash_file, list_method_tree, list_method_tree_at_root+from mthds.method_tree.models import MethodTree, TreeFile, TreeLimitsfrom mthds.method_tree.tree_path import nfcfrom mthds.method_tree.vocabulary import ExclusionRule, TreeRefusalCodefrom pipelex import log+from pipelex.base_exceptions import PipelexUnexpectedErrorfrom pipelex.core.method_trees.method_file_store import HeldFilefrom pipelex.core.method_trees.tree_bundle import TreeBundlefrom pipelex.hosted.execution import get_or_load_pipelex_config@@ -55,6 +66,7 @@ from pipelex.libraries.exceptions import (HostLibraryRequirementsError,HostLibraryTreeError,MethodRequirementsError,+ MethodTreeChangedError,MethodTreeLoadError,)from pipelex.libraries.written_trees import WRITTEN_METHOD_TREES@@ -234,22 +246,174 @@ def tree_bundles_of(*, tree: MethodTree) -> list[TreeBundle]:return [TreeBundle(tree=tree, tree_path=tree_path) for tree_path in tree.tree_paths if tree_path.endswith(BUNDLE_SUFFIX)]-def tree_bundle_of_lone_file(*, bundle_file: Path, limits: TreeLimits | None = None) -> TreeBundle:- """A bundle file this host read, named alone, as the member of its method tree.-- Its tree is rooted at the nearest `METHODS.toml` at or above its folder, else it is the one-file tree of the file,- from which a reference to any other file is refused.+def lone_file_member(*, tree: MethodTree, bundle_file: Path) -> TreeBundle:+ """A bundle file named alone as the member of the tree its listing returned, the entry that listing named for it.Raises:- MethodTreeLoadError: The tree could not be listed, or the membership rule keeps the file out of it.+ MethodTreeLoadError: The listing named no entry for the file: the membership rule keeps it out of its tree."""- tree = list_local_method_tree(handed_over=bundle_file, limits=limits)if tree.entry is None:msg = f"Listing the method tree of the bundle file '{bundle_file}' named no entry for it."raise MethodTreeLoadError(msg).with_params(reason=TreeRefusalCode.LONE_FILE_EXCLUDED)return TreeBundle(tree=tree, tree_path=tree.entry)+class HandedOverLocation(NamedTuple):+ """Where a path handed over as a method lies, read as the standard's lister reads it before it lists anything.++ A directory, a symbolic link naming one included, is named by its resolved path, and its tree is rooted at `root`.+ A `.mthds` file named alone is named by its folder resolved and its own name as the folder holds it, never followed,+ so a symbolic link named alone is a location of its own, which its listing refuses; its tree is rooted at `root`, its+ nearest `METHODS.toml`, or is its one-file tree when `root` is None.+ """++ directory: Path | None+ lone_file: Path | None+ root: Path | None+++def locate_handed_over(*, handed_over: Path) -> HandedOverLocation | None:+ """Where a path handed over as a method lies, as `list_method_tree` reads it, or None when that reading fails.++ None is for a path the lister refuses before listing anything, one that does not exist or a file not named `.mthds`+ say: the caller lists it, and the lister gives the refusal.+ """+ try:+ status = os.lstat(handed_over)+ if stat.S_ISDIR(status.st_mode) or (stat.S_ISLNK(status.st_mode) and handed_over.is_dir()):+ directory = handed_over.resolve(strict=True)+ return HandedOverLocation(directory=directory, lone_file=None, root=find_manifest_directory(directory) or directory)+ parent = handed_over.absolute().parent.resolve(strict=True)+ lone_file = parent / on_disk_name(parent, handed_over.name)+ if not lone_file.name.endswith(BUNDLE_SUFFIX):+ return None+ return HandedOverLocation(directory=None, lone_file=lone_file, root=find_manifest_directory(parent))+ except (OSError, RuntimeError, TreeSourceError):+ return None+++def lone_file_tree_path(*, root: Path, lone_file: Path) -> str | None:+ """The tree path of a lone file in the tree rooted at `root`, each name taken as the file system holds it, as the lister takes it.++ None when a name on the way cannot be read, for the caller to list the file and get the lister's own refusal.+ """+ current = root+ held_names: list[str] = []+ try:+ for part in lone_file.relative_to(root).parts:+ raw_name = on_disk_name(current, part)+ held_names.append(nfc(raw_name))+ current /= raw_name+ except (ValueError, TreeSourceError):+ return None+ return "/".join(held_names)+++def _tree_label(*, tree: MethodTree) -> str:+ """How a message names a tree: by its root on this host, except a tree a host wrote out, whose root is the host's."""+ if tree.directory is None or WRITTEN_METHOD_TREES.holds_tree(tree=tree):+ return "the method tree"+ return f"the method tree at '{tree.directory}'"+++def _changed_file_error(*, tree: MethodTree, tree_path: str, finding: str) -> MethodTreeChangedError:+ msg = (+ f"The file '{tree_path}' of {_tree_label(tree=tree)} {finding} since the tree was listed, so it is not the file the "+ "run listed and validated. Run again once the method's files stop changing."+ )+ return MethodTreeChangedError(msg).with_params(path=tree_path)+++def _listed_file(*, tree: MethodTree, tree_path: str) -> TreeFile:+ """The listing's record of the file at `tree_path`.++ Raises:+ MethodTreeChangedError: The listing holds no such file.+ """+ for tree_file in tree.files:+ if tree_file.path == tree_path:+ return tree_file+ raise _changed_file_error(tree=tree, tree_path=tree_path, finding="is not a file of the tree as listed, or was added")+++def read_tree_file_as_listed(*, tree: MethodTree, tree_path: str) -> bytes:+ """The bytes of a file of a tree listed from this host's disk, read now, refused unless they are the bytes its listing hashed.++ Raises:+ MethodTreeChangedError: The file is no longer there or cannot be read, or its bytes are not the ones listed.+ """+ listed = _listed_file(tree=tree, tree_path=tree_path)+ try:+ data = tree.file_location(tree_path).read_bytes()+ except OSError as exc:+ raise _changed_file_error(tree=tree, tree_path=tree_path, finding="was removed or became unreadable") from exc+ if len(data) != listed.size or sha256_hex(data) != listed.sha256:+ raise _changed_file_error(tree=tree, tree_path=tree_path, finding="changed")+ return data+++def read_tree_bundle_as_listed(*, tree_bundle: TreeBundle) -> bytes:+ """The bytes of a bundle file of a listed tree, refused unless they are the bytes its listing hashed (`read_tree_file_as_listed`).++ Raises:+ MethodTreeChangedError: The bundle's file is no longer there or cannot be read, or its bytes are not the ones listed.+ PipelexUnexpectedError: The bundle is the one parsed from a string at a tree's root, which is no file of it.+ """+ if tree_bundle.tree_path is None:+ msg = "A bundle parsed from a string has no file of its method tree to read"+ raise PipelexUnexpectedError(msg)+ return read_tree_file_as_listed(tree=tree_bundle.tree, tree_path=tree_bundle.tree_path)+++def decoded_as_text_file(*, data: bytes) -> str:+ """A file's bytes decoded as UTF-8 text is read from a file, each line ending made a newline (`Path.read_text`).++ Raises:+ UnicodeDecodeError: The bytes are not UTF-8.+ """+ return io.TextIOWrapper(io.BytesIO(data), encoding="utf-8").read()+++def ensure_bundle_text_as_listed(*, tree_bundle: TreeBundle, mthds_content: str) -> None:+ """Refuse the text a caller read from a bundle file of a listed tree unless it is the text of the file the listing hashed.++ A caller that read the bundle's text itself, such as a CLI reading the bundle file it runs, read it at a moment of its+ own, before or after the tree's listing: the text is the bundle's only when it is the listed file's, decoded as the+ caller decoded it, with its line endings kept, or each made a newline as a text read makes them.++ Raises:+ MethodTreeChangedError: The bundle's file is no longer the one the listing hashed, or the text is not its text.+ """+ if tree_bundle.tree_path is None:+ return+ data = read_tree_file_as_listed(tree=tree_bundle.tree, tree_path=tree_bundle.tree_path)+ try:+ listed_text = data.decode("utf-8")+ except UnicodeDecodeError as exc:+ raise _changed_file_error(tree=tree_bundle.tree, tree_path=tree_bundle.tree_path, finding="was read as text other than its own") from exc+ if mthds_content not in {listed_text, decoded_as_text_file(data=data)}:+ raise _changed_file_error(tree=tree_bundle.tree, tree_path=tree_bundle.tree_path, finding="changed")+++def ensure_tree_files_as_listed(*, tree: MethodTree) -> None:+ """Read every file of a tree listed from this host's disk again, refusing the tree unless each is the file its listing hashed.++ A tree read from a map of file entries holds no file on this host, and is never read here.++ Raises:+ MethodTreeChangedError: A file is no longer there or cannot be read, or its bytes are not the ones listed.+ """+ if tree.directory is None:+ return+ for tree_file in tree.files:+ try:+ current = hash_file(tree.file_location(tree_file.path), tree_path=tree_file.path)+ except TreeSourceError as exc:+ raise _changed_file_error(tree=tree, tree_path=tree_file.path, finding="was removed or became unreadable") from exc+ if current.size != tree_file.size or current.sha256 != tree_file.sha256:+ raise _changed_file_error(tree=tree, tree_path=tree_file.path, finding="changed")++def _is_empty_tree(*, refusal_error: MethodTreeRefusedError) -> bool:match refusal_error.refusal.code:case TreeRefusalCode.EMPTY_TREE:@@ -432,7 +596,19 @@ def _library_trees(*, tree: MethodTree, limits: TreeLimits) -> list[MethodTree]:return trees-def list_library_trees(*, library_dir: Path, limits: TreeLimits | None = None) -> list[MethodTree]:+class LibraryDirTreeLister(Protocol):+ """Lists the tree a library directory is rooted in, or None when it is empty, as `list_library_dir_tree` does."""++ def __call__(self, *, library_dir: Path, limits: TreeLimits) -> MethodTree | None: ...+++class LibraryTreesLister(Protocol):+ """Lists the trees a library directory holds, its own first, as `list_library_trees` does."""++ def __call__(self, *, library_dir: Path, limits: TreeLimits) -> list[MethodTree]: ...+++def list_library_trees(*, library_dir: Path, limits: TreeLimits | None = None, own_tree_of: LibraryDirTreeLister | None = None) -> list[MethodTree]:"""The method trees a library directory holds: its own, then those of the packages and repositories nested in it.A directory that does not exist holds none. One whose own tree the membership rule leaves empty, such as a@@ -441,6 +617,8 @@ def list_library_trees(*, library_dir: Path, limits: TreeLimits | None = None) -Args:library_dir: The library directory.limits: The limits to list each tree under, read from the configuration when not given.+ own_tree_of: How the tree the directory is rooted in is listed, such as through a run's listing, which holds it+ for every step of the run; `list_library_dir_tree` when not given.Raises:MethodTreeLoadError: A tree could not be listed.@@ -448,22 +626,57 @@ def list_library_trees(*, library_dir: Path, limits: TreeLimits | None = None) -if not library_dir.exists():log.debug(f"A library directory that does not exist was skipped: {library_dir}")return []+ active_limits = limits or local_tree_limits()+ own_tree = (own_tree_of or list_library_dir_tree)(library_dir=library_dir, limits=active_limits)+ return library_trees_from(library_dir=library_dir, own_tree=own_tree, limits=active_limits)+++def list_library_dir_tree(*, library_dir: Path, limits: TreeLimits | None = None) -> MethodTree | None:+ """The method tree a library directory is rooted in, or None when the membership rule leaves it empty.++ The tree is rooted at the nearest `METHODS.toml` at or above the directory, else at the directory itself, as+ `list_local_method_tree` roots it, so a successful listing of either is the same tree; a directory a host wrote a+ tree out as returns the tree as written. An empty tree is no fault here: a library directory may hold nothing but+ nested methods, which `library_trees_from` finds.++ Args:+ library_dir: The library directory, which exists.+ limits: The limits to list the tree under, read from the configuration when not given.++ Raises:+ MethodTreeLoadError: The standard refused the tree for a reason other than its being empty, it passed a limit,+ or it could not be read.+ """written = WRITTEN_METHOD_TREES.tree_written_at(directory=library_dir)if written is not None:- return [written]- active_limits = limits or local_tree_limits()+ return writtentry:- tree = list_method_tree(library_dir, limits=active_limits)+ return list_method_tree(library_dir, limits=limits or local_tree_limits())except MethodTreeRefusedError as exc:if _is_empty_tree(refusal_error=exc):- return _trees_nested_in_empty_directory(directory=library_dir.resolve(), limits=active_limits)+ return Noneraise MethodTreeLoadError(exc.message) from excexcept TreeLimitError as exc:msg = f"{exc.message} {LIMITS_ADVICE}"raise MethodTreeLoadError(msg) from excexcept TreeSourceError as exc:raise MethodTreeLoadError(exc.message) from exc- return _library_trees(tree=tree, limits=active_limits)+++def library_trees_from(*, library_dir: Path, own_tree: MethodTree | None, limits: TreeLimits) -> list[MethodTree]:+ """The method trees a library directory holds, from the tree it is rooted in, as `list_library_dir_tree` listed it.++ That tree comes first, then the tree of each package and repository nested in it, and theirs, depth first. A tree a+ host wrote out holds no nested tree, since reading it back as written keeps nothing out: so the answer depends on the+ tree alone, never on whether its writer still holds it. A directory whose own tree is empty (`own_tree` None) holds+ the trees nested in it.++ Raises:+ MethodTreeLoadError: A nested tree could not be listed.+ """+ if own_tree is None:+ return _trees_nested_in_empty_directory(directory=library_dir.resolve(), limits=limits)+ return _library_trees(tree=own_tree, limits=limits)class LibraryBundleFile(NamedTuple):@@ -491,7 +704,9 @@ class LibraryTree(NamedTuple):tree: MethodTree-def list_library_dir_trees(*, library_dirs: Iterable[Path], limits: TreeLimits | None = None) -> list[LibraryTree]:+def list_library_dir_trees(+ *, library_dirs: Iterable[Path], limits: TreeLimits | None = None, trees_of: LibraryTreesLister | None = None+) -> list[LibraryTree]:"""The method trees of the library directories, in the order the directories are given, each directory's own first.A tree holding no bundle is listed too, such as one holding only the Python a server received beside a method's@@ -500,6 +715,7 @@ def list_library_dir_trees(*, library_dirs: Iterable[Path], limits: TreeLimits |Args:library_dirs: The library directories.limits: The limits to list each tree under, read from the configuration once when not given.+ trees_of: How a directory's trees are listed, such as through a run's listing; `list_library_trees` when not given.Raises:MethodTreeLoadError: A tree could not be listed.@@ -507,7 +723,7 @@ def list_library_dir_trees(*, library_dirs: Iterable[Path], limits: TreeLimits |active_limits = limits or local_tree_limits()library_trees: list[LibraryTree] = []for library_dir in library_dirs:- for tree in list_library_trees(library_dir=library_dir, limits=active_limits):+ for tree in (trees_of or list_library_trees)(library_dir=library_dir, limits=active_limits):library_trees.append(LibraryTree(library_dir=library_dir, tree=tree))return library_trees@@ -589,7 +805,10 @@ def tree_python_sources(*, tree: MethodTree, library_dir: Path | None) -> dict[sA file outside the library directory, or of a tree that no library directory holds, such as the tree of a bundle filenamed alone, is keyed by its tree path: the key names the file to a caller, never by a path of this host. These arethe sources a sandbox-hosted load scans for structure classes without importing them, so a file that is not UTF-8- text is read with replacement characters.+ text is read with replacement characters. Each is read against the hash its listing recorded.++ Raises:+ MethodTreeChangedError: A Python file is not the one the tree's listing hashed."""if tree.directory is None:return {}@@ -601,7 +820,7 @@ def tree_python_sources(*, tree: MethodTree, library_dir: Path | None) -> dict[skey = tree_pathif spelling is not None and tree_path.startswith(spelling[0]):key = f"{spelling[1]}{tree_path.removeprefix(spelling[0])}"- sources[key] = tree.file_location(tree_path).read_bytes().decode("utf-8", errors="replace")+ sources[key] = read_tree_file_as_listed(tree=tree, tree_path=tree_path).decode("utf-8", errors="replace")return sourcesdiff --git a/pipelex/libraries/package_environments.py b/pipelex/libraries/package_environments.pyindex ce353ee99..6155f861c 100644--- a/pipelex/libraries/package_environments.py+++ b/pipelex/libraries/package_environments.py@@ -19,7 +19,7 @@ import shleximport sysfrom collections.abc import Mapping, Sequencefrom importlib.metadata import PackageNotFoundError, distribution-from typing import Protocol+from typing import NamedTuple, Protocolfrom packaging.markers import default_environmentfrom packaging.utils import canonicalize_name@@ -30,7 +30,12 @@ from pipelex.config import get_config, get_pipe_func_execution_mode, is_pipe_funfrom pipelex.core.method_trees.requirements_file import RequirementLinefrom pipelex.interpreter_hub import get_pipe_func_package_offer_registryfrom pipelex.libraries.exceptions import PipeFuncPackageSetUnstatedError-from pipelex.plugins.pipe_func_package_offer_registry import PipeFuncPackageOffer, PipeFuncPackageOfferRequest, PipeFuncPackageRefusal+from pipelex.plugins.pipe_func_package_offer_registry import (+ PipeFuncPackageOffer,+ PipeFuncPackageOfferFactoryFn,+ PipeFuncPackageOfferRequest,+ PipeFuncPackageRefusal,+)from pipelex.tools.misc.async_utils import is_event_loop_running_on_this_thread@@ -145,6 +150,28 @@ class OfferedPackageEnvironment(PythonPackageEnvironment):)+class PipeFuncEnvironmentSource(NamedTuple):+ """What a requirements verdict is judged against, beside the tree: the PipeFunc execution mode, and the factory stating its packages.++ Two verdicts judged against equal sources are one: a run keeps a tree's met verdict for the checks after it only+ while they judge against the same mode and the same factory, compared as `==` compares them, by identity for a+ function or a plain object.+ """++ execution_mode: str+ offer_factory: PipeFuncPackageOfferFactoryFn | None+++def current_pipe_func_environment_source() -> PipeFuncEnvironmentSource:+ """The source a check judges requirements against now: this interpreter in the `direct` mode, else the mode's registered factory."""+ execution_mode = get_pipe_func_execution_mode()+ if not is_pipe_func_sandbox_hosted():+ return PipeFuncEnvironmentSource(execution_mode=execution_mode, offer_factory=None)+ return PipeFuncEnvironmentSource(+ execution_mode=execution_mode, offer_factory=get_pipe_func_package_offer_registry().get_optional(mode=execution_mode)+ )++def resolve_pipe_func_package_environment(*, requirements: Sequence[RequirementLine], names_host_paths: bool) -> PythonPackageEnvironment | PipeFuncPackageRefusal:diff --git a/pipelex/methods/package_reason.py b/pipelex/methods/package_reason.pyindex a2d7207eb..0483248e3 100644--- a/pipelex/methods/package_reason.py+++ b/pipelex/methods/package_reason.py@@ -42,5 +42,7 @@ class MethodPackageReason(StrEnum):VERSION_INVALID = "version_invalid"# The method's tree cannot be listed: a path it names is missing, one of its files cannot be read, or its manifest does not parse.TREE_UNREADABLE = "tree_unreadable"+ # A file of the method's tree changed, was removed or became unreadable between the tree's listing and a read of it.+ TREE_CHANGED = "tree_changed"# The method's `methods.lock` is not UTF-8 text, does not parse, or records an entry the lock format refuses.LOCK_INVALID = "lock_invalid"diff --git a/pipelex/mthds_parsing/parser.py b/pipelex/mthds_parsing/parser.pyindex 1adfa87fa..5fb78595f 100644--- a/pipelex/mthds_parsing/parser.py+++ b/pipelex/mthds_parsing/parser.py@@ -11,7 +11,7 @@ from pipelex.mthds_parsing.exceptions import BundleElaboratorError, MthdsParserEfrom pipelex.mthds_parsing.pipelex_bundle_blueprint import PipelexBundleBlueprintfrom pipelex.mthds_parsing.validation_error_categorizer import PIPELEX_BUNDLE_BLUEPRINT_SOURCE_FIELD, categorize_blueprint_validation_errorfrom pipelex.tools.misc.exceptions import TomlError-from pipelex.tools.misc.toml_utils import load_toml_from_content, load_toml_from_path+from pipelex.tools.misc.toml_utils import load_toml_from_bytes, load_toml_from_content, load_toml_from_pathfrom pipelex.tools.typing.pydantic_utils import format_pydantic_validation_error@@ -63,6 +63,7 @@ class MthdsParser(BaseModel):cls,bundle_path: Path | None = None,*,+ bundle_data: bytes | None = None,mthds_content: str | None = None,mthds_source: str | None = None,file_origin: BundleFileOrigin = BundleFileOrigin.METHOD_TREE,@@ -79,7 +80,9 @@ class MthdsParser(BaseModel):categorizer reads off that dict — mirroring the real file path the diskpath already records, and overriding any ``source`` the bundle textitself declared. With ``bundle_path``, it names the file in place of its path on- this host, as a host names a file of a tree it wrote out for a caller by its tree path.+ this host, as a host names a file of a tree it wrote out for a caller by its tree path. ``bundle_data`` is the+ file's bytes when the caller read them already, such as a loader that read them against the hash its method+ tree's listing recorded: they are parsed in place of a read of ``bundle_path``, which still names the file.``file_origin`` says where the bundle's steps find the files they name, and so the rule a filereference, such as a ``PipeDocGen`` step's ``template_file``, is held to when the blueprint validates.@@ -93,7 +96,11 @@ class MthdsParser(BaseModel):validation_context: BundleValidationContexttry:if bundle_path is not None:- blueprint_dict = load_toml_from_path(path=str(bundle_path))+ blueprint_dict = (+ load_toml_from_bytes(data=bundle_data, source=bundle_path)+ if bundle_data is not None+ else load_toml_from_path(path=str(bundle_path))+ )blueprint_dict[PIPELEX_BUNDLE_BLUEPRINT_SOURCE_FIELD] = mthds_source if mthds_source is not None else str(bundle_path)validation_context = BundleValidationContext(file_origin=BundleFileOrigin.METHOD_TREE)elif mthds_content is not None:diff --git a/pipelex/pipeline/execution_seams.py b/pipelex/pipeline/execution_seams.pyindex 02bfd200c..3b20be214 100644--- a/pipelex/pipeline/execution_seams.py+++ b/pipelex/pipeline/execution_seams.py@@ -44,6 +44,8 @@ from pipelex.interpreter_hub import (from pipelex.kernel.memory_ops import shape_inputsfrom pipelex.libraries.bundle_resources import check_submitted_filesfrom pipelex.libraries.method_load_threads import run_method_load+from pipelex.libraries.method_tree_listing import MethodTreeListing+from pipelex.libraries.method_trees import ensure_bundle_text_as_listedfrom pipelex.libraries.pipe.pipe_suggestions import declared_pipe_refsfrom pipelex.mthds_parsing.parser import MthdsParserfrom pipelex.pipe_machinery.pipe_abstract import PipeAbstract@@ -67,6 +69,7 @@ from pipelex.system.storage_scope import validate_read_scope, validate_storage_sfrom pipelex.tools.misc.file_utils import reject_bare_str_or_pathif TYPE_CHECKING:+ from pipelex.core.method_trees.tree_bundle import TreeBundlefrom pipelex.mthds_parsing.pipelex_bundle_blueprint import PipelexBundleBlueprintfrom pipelex.system.trace_context import TraceContext@@ -91,6 +94,7 @@ def acquire_library(library_dirs_are_callers: bool = False,methods_dirs: list[Path] | None = None,submitted_files: Mapping[str, bytes] | None = None,+ tree_listing: MethodTreeListing | None = None,) -> tuple[str, str | None]:"""Set the current library, open it, and load dirs + blueprints into it.@@ -138,6 +142,12 @@ def acquire_library(contents load, as files of the tree the contents sit at the root of, and the run's crate names that tree by itsdigest. A file none of the contents' steps names is refused with a ``BundleResourceNotNamedError``.+ ``tree_listing`` is the run's listing of the method trees it reads, made by a caller that listed them first, such as+ a CLI that resolved the folder or the bundle file it runs, or a server that checked the requirements of the method it+ holds: the directories and the bundle files load from the trees it holds, listing through it only what nobody listed+ yet, so the run lists each tree once and the tree whose digest it records is the tree it loaded. ``None`` makes one+ listing for this load, which the directories and the contents share.+Returns the ``library_id`` and the bundle's qualified ``main_pipe`` ref(``domain.pipe_code``) when an ``mthds_contents`` bundle declares one, else``None``. Pipe resolution is the caller's responsibility.@@ -154,6 +164,7 @@ def acquire_library(library_dirs_are_callers=library_dirs_are_callers,methods_dirs=methods_dirs,submitted_files=submitted_files,+ tree_listing=tree_listing,)return acquired_id, qualified_main_pipe@@ -167,6 +178,7 @@ async def acquire_library_async(library_dirs_are_callers: bool = False,methods_dirs: list[Path] | None = None,submitted_files: Mapping[str, bytes] | None = None,+ tree_listing: MethodTreeListing | None = None,) -> tuple[str, str | None]:""":func:`acquire_library` for asynchronous code: the same library, loaded in a worker thread, off the event loop.@@ -192,6 +204,7 @@ async def acquire_library_async(library_dirs_are_callers=library_dirs_are_callers,methods_dirs=methods_dirs,submitted_files=submitted_files,+ tree_listing=tree_listing,)return acquired_id, qualified_main_pipe@@ -241,6 +254,7 @@ def _load_acquired_library(library_dirs_are_callers: bool,methods_dirs: list[Path] | None,submitted_files: Mapping[str, bytes] | None,+ tree_listing: MethodTreeListing | None,) -> str | None:"""Load the directories and the contents into the library :func:`acquire_library` opened and made current.@@ -248,6 +262,8 @@ def _load_acquired_library(:func:`acquire_library_async`. Returns the bundle's qualified ``main_pipe``, or ``None``."""library_manager = get_library_manager()+ # One listing for the whole load: the directories' trees and a bundle file's own tree are each listed once.+ listing = tree_listing if tree_listing is not None else MethodTreeListing()effective_dirs, source_label = resolve_library_dirs(library_dirs)if effective_dirs:effective_dirs_desc = ", ".join(str(dir_path) for dir_path in effective_dirs)@@ -258,7 +274,7 @@ def _load_acquired_library(translate_to_validate_bundle_error() if library_dirs_are_callers else reporting_host_library_faults())with library_dirs_translation:- library_manager.load_libraries(library_id=library_id, library_dirs=effective_dirs, methods_dirs=methods_dirs)+ library_manager.load_libraries(library_id=library_id, library_dirs=effective_dirs, methods_dirs=methods_dirs, tree_listing=listing)else:log.verbose(f"No library directories to load ({source_label})")@@ -268,7 +284,7 @@ def _load_acquired_library(host_library_withholding: AbstractContextManager[None] = (nullcontext() if library_dirs_are_callers else withholding_host_library_files(library_dirs=effective_dirs, caller_contents=mthds_contents))- with host_library_withholding, translate_to_validate_bundle_error():+ with host_library_withholding, translate_to_validate_bundle_error(), listing.loading():# The file on this host each content was read from, where the caller names one that exists: that bundle# is the file's, whose steps find the files they name in its method tree. Every other content belongs to# no tree, and its steps read only the files submitted with it, so its blueprint validates under that rule.@@ -296,16 +312,29 @@ def _load_acquired_library(if bundle_uris:current_library = library_manager.get_library(library_id=library_id)blueprints_to_load = []- for blueprint, uri, resolved_uri, host_bundle_file in zip(all_blueprints, bundle_uris, resolved_uris, host_bundle_files, strict=True):+ for blueprint, content, uri, resolved_uri, host_bundle_file in zip(+ all_blueprints, mthds_contents, bundle_uris, resolved_uris, host_bundle_files, strict=True+ ):+ # A content read from a file on this host was read before this load, at a moment of its reader's:+ # it is the bundle the run loads only when it is the text of the file the run's listing hashed, the+ # file a library directory's load read when that load read it.if resolved_uri in current_library.loaded_mthds_paths:+ loaded_tree_bundle = _loaded_tree_bundle(library_id=library_id, bundle_file=resolved_uri)+ if host_bundle_file is not None and loaded_tree_bundle is not None:+ ensure_bundle_text_as_listed(tree_bundle=loaded_tree_bundle, mthds_content=content)log.verbose(f"Bundle '{uri}' was skipped, already loaded from library directories")else:if host_bundle_file is not None:+ tree_bundle = listing.lone_file_bundle(bundle_file=host_bundle_file)+ ensure_bundle_text_as_listed(tree_bundle=tree_bundle, mthds_content=content)blueprint.mark_read_from_host_file(bundle_file=host_bundle_file)+ blueprint.place_in_tree(tree_bundle=tree_bundle)blueprints_to_load.append(blueprint)if blueprints_to_load:- library_manager.load_from_blueprints(library_id=library_id, blueprints=blueprints_to_load, methods_dirs=methods_dirs)+ library_manager.load_from_blueprints(+ library_id=library_id, blueprints=blueprints_to_load, methods_dirs=methods_dirs, tree_listing=listing+ )# Qualify main_pipe with domain to avoid ambiguity when multiple domains define pipes with# the same code — via the one shared selection rule (first declaring main_pipe, else first).@@ -313,6 +342,20 @@ def _load_acquired_library(return qualified_main_pipe+def _loaded_tree_bundle(*, library_id: str, bundle_file: Path) -> "TreeBundle | None":+ """The bundle a load of the library read from `bundle_file`, resolved, as a member of its tree, or None when none did."""+ for blueprint in get_library_manager().get_accumulated_blueprints(library_id):+ tree_bundle = blueprint.tree_bundle+ if tree_bundle is None or tree_bundle.tree_path is None:+ continue+ try:+ if tree_bundle.location.resolve() == bundle_file:+ return tree_bundle+ except (OSError, RuntimeError):+ continue+ return None++def load_libraries_and_activate(library_dirs: Sequence[str | Path] | None = None) -> str:"""Open a fresh library, set it current, resolve + load ``library_dirs``, and leave it loaded.diff --git a/pipelex/pipeline/fixes/fix_loop.py b/pipelex/pipeline/fixes/fix_loop.pyindex 81d26f931..76630c34c 100644--- a/pipelex/pipeline/fixes/fix_loop.py+++ b/pipelex/pipeline/fixes/fix_loop.py@@ -25,12 +25,17 @@ A method's folder is fixed the same way (``fix_method_folder``): the loop validamethod tree, as ``validate_bundle(method_dir=...)`` judges it, with no entry file. The tree isrooted at the nearest ``METHODS.toml`` at or above the folder, so its root is in the write scope,not only the folder named, and so are the ``library_dirs`` passed beside it.++Each round lists the method's trees once (``pipelex.libraries.method_tree_listing``): the snapshot+and the validation of the round read the same listing, so the files a fix is bound to are those the+validation loaded. A round that writes a file ends its listing, and the next round lists anew. The+first round's listing is the caller's, when it listed the trees to resolve what it fixes."""import os-from collections.abc import Awaitable, Callable, Iterable, Sequence+from collections.abc import Awaitable, Iterable, Sequencefrom pathlib import Path-from typing import Any, NamedTuple, Self, cast+from typing import Any, NamedTuple, Protocol, Self, castimport tomlkitfrom pydantic import BaseModel, ConfigDict, Field, model_validator@@ -47,8 +52,8 @@ from pipelex.fix_ops.file_transaction import (read_file_snapshot,)from pipelex.interpreter_hub import resolve_library_dirs-from pipelex.libraries.library_utils import get_pipelex_mthds_files_from_dirs-from pipelex.libraries.method_trees import method_tree_root+from pipelex.libraries.method_tree_listing import MethodTreeListing+from pipelex.libraries.method_trees import library_bundle_files_of, method_tree_rootfrom pipelex.pipe_machinery.pipe_blueprint import SIGNATURE_ONLY_KEYSfrom pipelex.pipeline.exceptions import ValidateBundleErrorfrom pipelex.pipeline.fixes.applicability import is_safe_fix_for_load_scope, is_target_in_write_scope@@ -173,21 +178,22 @@ def _absolute_source_path(path: Path) -> Path:return Path(os.path.abspath(path))-def _snapshot_loaded_sources(*, entry_source_path: Path | None, effective_dirs: Sequence[Path]) -> dict[Path, FileSnapshot]:+def _snapshot_loaded_sources(+ *, entry_source_path: Path | None, effective_dirs: Sequence[Path], tree_listing: MethodTreeListing+) -> dict[Path, FileSnapshot]:"""Snapshot every source identity validation is about to load.Keys are lexical absolute source names, while each snapshot is pinned to the destinationresolved at the same time. The resolved destination is also an alias when unambiguous, whichaccommodates injected managers that already report canonical source paths. A method's folder- has no entry file: its bundles are those of its directory, among ``effective_dirs``.+ has no entry file: its bundles are those of its directory, among ``effective_dirs``. The trees+ are listed through ``tree_listing``, the listing the validation that follows loads them from."""source_paths: set[Path] = set()if entry_source_path is not None:source_paths.add(_absolute_source_path(entry_source_path))- source_paths.update(- _absolute_source_path(mthds_path)- for mthds_path in get_pipelex_mthds_files_from_dirs(dirs={_absolute_source_path(directory) for directory in effective_dirs})- )+ library_trees = tree_listing.library_dir_trees(library_dirs=sorted({_absolute_source_path(directory) for directory in effective_dirs}))+ source_paths.update(_absolute_source_path(bundle_file.path) for bundle_file in library_bundle_files_of(library_trees=library_trees))snapshots_by_source: dict[Path, FileSnapshot] = {}for source_path in sorted(source_paths, key=str):target_path = source_path.resolve()@@ -447,6 +453,7 @@ async def fix_bundle_file(max_iterations: int | None = None,select_codes: Sequence[str] | None = None,ignore_codes: Sequence[str] | None = None,+ tree_listing: MethodTreeListing | None = None,) -> FixBundleResult:"""Fix a bundle in place until valid, out of fixes, no progress, or max_iterations.@@ -459,13 +466,17 @@ async def fix_bundle_file(reports ``pending_signatures`` and ``is_runnable``. ``max_iterations=None`` resolves to the``fix_loop_max_attempts`` builder config. ``select_codes`` / ``ignore_codes`` filter which fixrules may apply (see ``_safe_fixes``); the CLI validates the codes before calling.+ ``tree_listing`` is the listing the caller listed the bundle's trees through, which the first+ round reads; None lists them in that round."""entry_source_path = _absolute_source_path(mthds_file_path)effective_dirs, _ = resolve_library_dirs(library_dirs)write_scope_dirs = library_dirs if writable_library_dirs is None else writable_library_dirs- async def validate_the_bundle() -> ValidateBundleResult:- return await validate_bundle(mthds_file_path=mthds_file_path, library_dirs=library_dirs, allow_signatures=allow_signatures)+ async def validate_the_bundle(*, round_listing: MethodTreeListing) -> ValidateBundleResult:+ return await validate_bundle(+ mthds_file_path=mthds_file_path, library_dirs=library_dirs, allow_signatures=allow_signatures, tree_listing=round_listing+ )return await _fix_to_fixed_point(validate=validate_the_bundle,@@ -475,6 +486,7 @@ async def fix_bundle_file(max_iterations=max_iterations,select_codes=select_codes,ignore_codes=ignore_codes,+ tree_listing=tree_listing,)@@ -487,6 +499,7 @@ async def fix_method_folder(max_iterations: int | None = None,select_codes: Sequence[str] | None = None,ignore_codes: Sequence[str] | None = None,+ tree_listing: MethodTreeListing | None = None,) -> FixBundleResult:"""Fix a method's folder in place, judged whole, until valid, out of fixes, no progress, or max_iterations.@@ -500,6 +513,8 @@ async def fix_method_folder(``writable_library_dirs`` when given, as the ``--diff`` preview's sandbox gives them, having mirrored that root.The snapshot taken before each validation lists the same directories the validation loads, the folder's listingbeing its whole tree's, so every file a fix can target was snapshotted under the name the load gives it.+ ``tree_listing`` is the listing the caller listed the folder's tree through, which the first round reads; None+ lists it in that round."""beside_dirs = dirs_beside_method(method_dir=method_dir, library_dirs=library_dirs)loaded_dirs = [method_dir, *beside_dirs]@@ -514,12 +529,13 @@ async def fix_method_folder(else:write_scope_dirs = writable_library_dirs- async def validate_the_method() -> ValidateBundleResult:+ async def validate_the_method(*, round_listing: MethodTreeListing) -> ValidateBundleResult:return await validate_bundle(method_dir=method_dir,library_dirs=library_dirs,library_dirs_are_callers=True,allow_signatures=allow_signatures,+ tree_listing=round_listing,)return await _fix_to_fixed_point(@@ -530,20 +546,32 @@ async def fix_method_folder(max_iterations=max_iterations,select_codes=select_codes,ignore_codes=ignore_codes,+ tree_listing=tree_listing,)+class _RoundValidation(Protocol):+ """What the fix loop validates in a round: the bundle file or the method's folder, its trees taken from the round's listing."""++ def __call__(self, *, round_listing: MethodTreeListing) -> Awaitable[ValidateBundleResult]: ...++async def _fix_to_fixed_point(*,- validate: Callable[[], Awaitable[ValidateBundleResult]],+ validate: _RoundValidation,entry_source_path: Path | None,effective_dirs: Sequence[Path],writable_dirs: list[Path],max_iterations: int | None,select_codes: Sequence[str] | None,ignore_codes: Sequence[str] | None,+ tree_listing: MethodTreeListing | None,) -> FixBundleResult:- """The fix loop over what ``validate`` judges: a bundle file and its entry, or a method's folder with none."""+ """The fix loop over what ``validate`` judges: a bundle file and its entry, or a method's folder with none.++ Each round's snapshot and validation read one listing of the trees: the caller's ``tree_listing`` in the first round,+ and a new one in each round after a write, since the files it listed changed.+ """if max_iterations is None:max_iterations = get_config().interpreter.builder.fix_loop_max_attemptsentry_path = entry_source_path.resolve() if entry_source_path is not None else None@@ -557,11 +585,14 @@ async def _fix_to_fixed_point(last_error_items: list[ValidationErrorItem] = []last_error_codes: list[ValidationItemCode] = []+ round_listing = tree_listing if tree_listing is not None else MethodTreeListing()for _ in range(max_iterations):# The snapshot lists the method's trees: one that cannot be listed is the invalid verdict, with nothing to fix.try:with translate_to_validate_bundle_error():- snapshots_by_source = _snapshot_loaded_sources(entry_source_path=entry_source_path, effective_dirs=effective_dirs)+ snapshots_by_source = _snapshot_loaded_sources(+ entry_source_path=entry_source_path, effective_dirs=effective_dirs, tree_listing=round_listing+ )except ValidateBundleError as exc:return FixBundleResult(is_valid=False,@@ -572,7 +603,7 @@ async def _fix_to_fixed_point(remaining_error_codes=exc.item_codes(),)try:- validation_result = await validate()+ validation_result = await validate(round_listing=round_listing)pending_signatures = _pending_signatures_from_validation_result(validation_result)return FixBundleResult(is_valid=True,@@ -693,10 +724,12 @@ async def _fix_to_fixed_point(written_paths.add(target_path)files_written.append(str(target_path))apply_rounds += 1+ # The round wrote files its listing hashed, so the next validation lists the trees anew.+ round_listing = MethodTreeListing()# max_iterations apply rounds done — the final verdict comes from one last validation.try:- validation_result = await validate()+ validation_result = await validate(round_listing=round_listing)pending_signatures = _pending_signatures_from_validation_result(validation_result)return FixBundleResult(is_valid=True,diff --git a/pipelex/pipeline/pipeline_run_setup.py b/pipelex/pipeline/pipeline_run_setup.pyindex a717ca8e1..e1858aaa8 100644--- a/pipelex/pipeline/pipeline_run_setup.py+++ b/pipelex/pipeline/pipeline_run_setup.py@@ -20,6 +20,7 @@ from pipelex.interpreter_hub import (get_selectable_own_pipe_refs,set_current_library,)+from pipelex.libraries.method_tree_listing import MethodTreeListingfrom pipelex.pipe_run.pipe_job import PipeJobfrom pipelex.pipe_run.pipe_run_params import (FORCE_DRY_RUN_MODE_ENV_KEY,@@ -82,6 +83,7 @@ async def pipeline_run_setup(library_dirs_are_callers: bool = False,methods_dirs: list[Path] | None = None,submitted_files: Mapping[str, bytes] | None = None,+ tree_listing: MethodTreeListing | None = None,) -> tuple[PipeJob, str, str]:"""Set up a pipeline for execution.@@ -196,6 +198,12 @@ async def pipeline_run_setup(files of the tree the contents sit at the root of, and the job's crate names that tree by its digest,carrying none of their bytes: the step that uses one reads it through the method file store. See:func:`acquire_library`.+ tree_listing:+ The run's listing of the method trees it reads, made by a caller that listed them before the run, such as a CLI+ that resolved the folder or the bundle file it runs, or a server that checked the requirements of the method it+ holds. The run loads each tree from it rather than listing it again, so the tree whose digest the run records,+ ``RunMetadata.method_tree_digest``, is the tree it loaded. ``None`` lists the trees for this run. See+ :func:`acquire_library`.Returns:-------@@ -338,6 +346,7 @@ async def pipeline_run_setup(library_dirs_are_callers=library_dirs_are_callers,methods_dirs=methods_dirs,submitted_files=submitted_files,+ tree_listing=tree_listing,)library_acquired = Truediff --git a/pipelex/pipeline/runner.py b/pipelex/pipeline/runner.pyindex 192950930..34f5ce5f8 100644--- a/pipelex/pipeline/runner.py+++ b/pipelex/pipeline/runner.py@@ -52,6 +52,7 @@ if TYPE_CHECKING:from pipelex.core.memory.working_memory import WorkingMemoryfrom pipelex.core.pipes.pipe_output import PipeOutput+ from pipelex.libraries.method_tree_listing import MethodTreeListingfrom pipelex.pipe_run.delivery_assignment import DeliveryAssignmentfrom pipelex.pipe_run.pipe_job import PipeJobfrom pipelex.pipe_run.pipe_run_protocol import PipeRunProtocol@@ -266,6 +267,7 @@ class PipelexMTHDSProtocol(MTHDSProtocol["PipeOutput"]):extra: dict[str, Any] | None = None,delivery_assignment: DeliveryAssignment | None = None,request_id: str | None = None,+ tree_listing: MethodTreeListing | None = None,) -> PipelexRunResultExecute:"""Execute a pipeline and wait for its completion.@@ -309,6 +311,16 @@ class PipelexMTHDSProtocol(MTHDSProtocol["PipeOutput"]):extension. It must be one to 128 ASCII letters, digits, ``_``or ``-`` (``validate_request_id``): any other value is the host'sbug, refused with a ``RunParameterError`` before the run is set up.+ tree_listing:+ The listing of the method trees this run reads, when the caller+ listed them before the run, such as a CLI that resolved the+ folder or the bundle file it runs, or a server that checked the+ requirements of the method it holds: the run loads each tree from+ it rather than listing it again, so one listing serves the whole+ run and the tree whose digest the run records is the tree it+ loaded. A listing serves one run: pass a new one to each+ ``execute``, since the files may change between two runs.+ ``None`` lists the trees for this run.Returns:-------@@ -367,6 +379,7 @@ class PipelexMTHDSProtocol(MTHDSProtocol["PipeOutput"]):methods_dirs=self.methods_dirs,submitted_files=self.submitted_files,request_id=request_id,+ tree_listing=tree_listing,)effective_pipe_run = self._pipe_run or get_pipe_run()with method_source_scope(source=self._method_source(mthds_contents=mthds_contents)):diff --git a/pipelex/pipeline/validate_bundle.py b/pipelex/pipeline/validate_bundle.pyindex 625af3b9f..a53856ade 100644--- a/pipelex/pipeline/validate_bundle.py+++ b/pipelex/pipeline/validate_bundle.py@@ -25,9 +25,9 @@ from pipelex.interpreter_hub import (set_current_library,)from pipelex.libraries.bundle_resources import check_submitted_files-from pipelex.libraries.library_utils import get_pipelex_mthds_files_from_dirsfrom pipelex.libraries.method_load_threads import run_method_load-from pipelex.libraries.method_trees import list_local_method_tree+from pipelex.libraries.method_tree_listing import MethodTreeListing+from pipelex.libraries.method_trees import LibraryBundleFile, library_bundle_files_of, read_tree_bundle_as_listed, read_tree_file_as_listedfrom pipelex.libraries.pipe.exceptions import EntryPipeNotFoundErrorfrom pipelex.libraries.pipe.pipe_suggestions import close_pipe_refs, declared_pipe_refsfrom pipelex.mthds_parsing.parser import MthdsParser@@ -142,6 +142,7 @@ async def validate_bundle(submitted_files: Mapping[str, bytes] | None = None,method_dir: Path | None = None,methods_dirs: Sequence[Path] | None = None,+ tree_listing: MethodTreeListing | None = None,) -> ValidateBundleResult:"""Load one bundle, or one method's tree, into a fresh library and dry-run its pipes.@@ -180,6 +181,10 @@ async def validate_bundle(run does, among the packages shipped in ``methods_dirs``, then the methods installed on this host, then by a fetchon a miss when fetch-on-miss is on. The library is opened and made current here, in the caller's context, and eachload reads that binding from the worker thread's copy of it; the dry runs stay on the loop.++ ``tree_listing`` is the listing the caller already listed the method's trees through, such as the folder a command+ resolved before validating it (``pipelex.libraries.method_tree_listing``): every load of the validation takes its+ trees from it, so each tree is listed once. None lists them for this validation."""provided_params = sum([@@ -225,6 +230,7 @@ async def validate_bundle(msg = "submitted_files accompany mthds_contents: a bundle file reads the files of its method tree"raise PipelexUnexpectedError(msg)+ listing = tree_listing if tree_listing is not None else MethodTreeListing()library_manager = get_library_manager()library_id, library = library_manager.open_library()success = False@@ -242,6 +248,7 @@ async def validate_bundle(dry_run_pipe_codes=dry_run_pipe_codes,caller_identity=caller_identity,collect_dry_run_warnings=collect_dry_run_warnings,+ tree_listing=listing,)success = Truereturn result@@ -271,7 +278,7 @@ async def validate_bundle(if effective_dirs:log.verbose(f"Loading libraries from {len(effective_dirs)} directory(ies) ({source_label}) for validation")with host_library_tree_faults:- await run_method_load(library_manager.load_libraries, library_id=library_id, library_dirs=effective_dirs)+ await run_method_load(library_manager.load_libraries, library_id=library_id, library_dirs=effective_dirs, tree_listing=listing)else:log.verbose(f"No library directories to load ({source_label})")if mthds_contents is not None:@@ -282,6 +289,7 @@ async def validate_bundle(mthds_sources=mthds_sources,submitted_files=submitted_files,library_dirs_are_callers=library_dirs_are_callers,+ tree_listing=listing,)dry_run_results = await BundleValidator().validate_pipes(pipes=_pipes_to_dry_run(loaded_pipes, dry_run_pipe_codes=dry_run_pipe_codes),@@ -303,7 +311,9 @@ async def validate_bundle(else:assert mthds_file_path is not None- blueprint, loaded_pipes = await run_method_load(_load_bundle_file, library_id=library_id, mthds_file_path=mthds_file_path)+ blueprint, loaded_pipes = await run_method_load(+ _load_bundle_file, library_id=library_id, mthds_file_path=mthds_file_path, tree_listing=listing+ )loaded_blueprints = [blueprint]dry_run_results = await BundleValidator().validate_pipes(@@ -353,7 +363,7 @@ def _ensure_tree_python_parses(*, tree: MethodTree) -> None:if not tree_path.endswith(PYTHON_SUFFIX):continuetry:- source = tree.file_location(tree_path).read_bytes().decode("utf-8")+ source = read_tree_file_as_listed(tree=tree, tree_path=tree_path).decode("utf-8")except UnicodeDecodeError:items.append(_tree_python_item(tree_path=tree_path, finding="not UTF-8 text", message=f"The Python file '{tree_path}' is not UTF-8 text.")@@ -427,6 +437,7 @@ def _load_contents(mthds_sources: Sequence[str | None] | None,submitted_files: Mapping[str, bytes] | None,library_dirs_are_callers: bool,+ tree_listing: MethodTreeListing,) -> tuple[list[PipelexBundleBlueprint], list[PipeAbstract]]:"""Parse submitted contents and load them, with the files submitted beside them, into the library.@@ -448,11 +459,13 @@ def _load_contents(if not library_dirs_are_callers:# A refusal the caller reads offers the pipes of the bundles it sent, never the host's.library_manager.get_library(library_id=library_id).pipe_library.set_caller_pipe_refs(pipe_refs=declared_pipe_refs(blueprints=blueprints))- loaded_pipes = library_manager.load_from_blueprints(library_id=library_id, blueprints=blueprints)+ loaded_pipes = library_manager.load_from_blueprints(library_id=library_id, blueprints=blueprints, tree_listing=tree_listing)return blueprints, loaded_pipes-def _load_bundle_file(*, library_id: str, mthds_file_path: Path) -> tuple[PipelexBundleBlueprint, list[PipeAbstract]]:+def _load_bundle_file(+ *, library_id: str, mthds_file_path: Path, tree_listing: MethodTreeListing+) -> tuple[PipelexBundleBlueprint, list[PipeAbstract]]:"""Read and parse a bundle file, and load it unless its library directories already did.Blocking work, run in the method-load pool by ``validate_bundle``.@@ -462,10 +475,19 @@ def _load_bundle_file(*, library_id: str, mthds_file_path: Path) -> tuple[Pipele"""library_manager = get_library_manager()library = library_manager.get_library(library_id=library_id)- blueprint = MthdsParser.make_pipelex_bundle_blueprint(bundle_path=mthds_file_path)- if mthds_file_path.resolve() not in library.loaded_mthds_paths:- # File not yet loaded - load it from the blueprint- return blueprint, library_manager.load_from_blueprints(library_id=library_id, blueprints=[blueprint])+ with tree_listing.loading():+ # A bundle file this machine will not let the command read is the file's refusal (`OSError`), given before its+ # tree is listed, whose listing would refuse the tree instead.+ with mthds_file_path.open("rb"):+ pass+ # The bundle is parsed from the bytes its tree's listing hashed, the tree the load reads it in.+ tree_bundle = tree_listing.lone_file_bundle(bundle_file=mthds_file_path)+ bundle_data = read_tree_bundle_as_listed(tree_bundle=tree_bundle)+ blueprint = MthdsParser.make_pipelex_bundle_blueprint(bundle_path=mthds_file_path, bundle_data=bundle_data)+ blueprint.place_in_tree(tree_bundle=tree_bundle)+ if mthds_file_path.resolve() not in library.loaded_mthds_paths:+ # File not yet loaded - load it from the blueprint+ return blueprint, library_manager.load_from_blueprints(library_id=library_id, blueprints=[blueprint], tree_listing=tree_listing)# File already loaded - get existing pipes from library by their refs. A bundle's `[pipe.<code>]` keys are bare and# belong to the bundle's own domain, so they are qualified here rather than searched for: the library is keyed by# qualified ref.@@ -479,23 +501,28 @@ def _load_method_dir(methods_dirs: Sequence[Path] | None,library_dirs: Sequence[Path] | None,library_id: str,+ tree_listing: MethodTreeListing,) -> tuple[MethodTree, list[PipeAbstract]]:"""List the method's tree, check its Python parses, and load it with the caller's directories beside it.Blocking work, run in a worker thread by ``_validate_method_dir``. The tree's packages called by address resolve asa run resolves them: the packages shipped in ``methods_dirs`` first, then the methods installed on this host and a- fetch on a miss, so the verdict is the one the run of the same method meets.+ fetch on a miss, so the verdict is the one the run of the same method meets. The tree is listed through+ ``tree_listing``, which the load takes it from, so it is listed once.Returns:The method's tree, and the pipes the load built, those of the directories beside it included."""- tree = list_local_method_tree(handed_over=method_dir)- _ensure_tree_python_parses(tree=tree)- loaded_pipes = get_library_manager().load_libraries(- library_id=library_id,- library_dirs=[method_dir, *dirs_beside_method(method_dir=method_dir, library_dirs=library_dirs)],- methods_dirs=list(methods_dirs) if methods_dirs is not None else None,- )+ # One load from the listing to the pipes: a tree listed before it is read again once, as the load takes it.+ with tree_listing.loading():+ tree = tree_listing.method_tree(handed_over=method_dir)+ _ensure_tree_python_parses(tree=tree)+ loaded_pipes = get_library_manager().load_libraries(+ library_id=library_id,+ library_dirs=[method_dir, *dirs_beside_method(method_dir=method_dir, library_dirs=library_dirs)],+ methods_dirs=list(methods_dirs) if methods_dirs is not None else None,+ tree_listing=tree_listing,+ )return tree, loaded_pipes@@ -509,6 +536,7 @@ async def _validate_method_dir(dry_run_pipe_codes: list[str] | None,caller_identity: CallerIdentity | None,collect_dry_run_warnings: bool,+ tree_listing: MethodTreeListing,) -> ValidateBundleResult:"""Load the tree of a caller's method into the open library and dry-run its pipes (see ``validate_bundle``).@@ -527,6 +555,7 @@ async def _validate_method_dir(methods_dirs=methods_dirs,library_dirs=library_dirs,library_id=library_id,+ tree_listing=tree_listing,)blueprints = library_manager.get_accumulated_blueprints(library_id)# Only the tree's own bundles are the method. The directories beside it, and the packages nested in it that the@@ -555,22 +584,47 @@ async def _validate_method_dir()-def _load_directory(*, directory: Path, mthds_files: list[Path], library_id: str) -> tuple[list[PipelexBundleBlueprint], list[PipeAbstract]]:+def _list_directory_bundle_files(*, directory: Path, tree_listing: MethodTreeListing) -> list[LibraryBundleFile]:+ """The bundle files a load of the directory reads, listed through the listing the load takes its trees from.++ Blocking work, run in the method-load pool.++ Raises:+ MethodTreeLoadError: A tree could not be listed.+ """+ # Listed for the load that follows, which parses and loads them from this listing.+ with tree_listing.loading():+ library_trees = tree_listing.library_dir_trees(library_dirs=[directory])+ return library_bundle_files_of(library_trees=library_trees)+++def _load_directory(+ *, directory: Path, mthds_files: list[LibraryBundleFile], library_id: str, tree_listing: MethodTreeListing+) -> tuple[list[PipelexBundleBlueprint], list[PipeAbstract]]:"""Parse a directory's bundles and load the directory: blocking work, run in the method-load pool.+ Each bundle is parsed from the bytes its tree's listing hashed, as the load reads it.+Returns:The blueprint of each bundle of the directory, and the pipes the load built."""- blueprints = [MthdsParser.make_pipelex_bundle_blueprint(bundle_path=mthds_file) for mthds_file in mthds_files]- loaded_pipes = get_library_manager().load_libraries(library_id=library_id, library_dirs=[Path(directory)])+ blueprints = [+ MthdsParser.make_pipelex_bundle_blueprint(+ bundle_path=mthds_file.path, bundle_data=read_tree_bundle_as_listed(tree_bundle=mthds_file.tree_bundle)+ )+ for mthds_file in mthds_files+ ]+ loaded_pipes = get_library_manager().load_libraries(library_id=library_id, library_dirs=[Path(directory)], tree_listing=tree_listing)return blueprints, loaded_pipesasync def validate_bundles_from_directory(directory: Path, *, allow_signatures: bool = False) -> ValidateBundleResult:# A directory whose tree cannot be listed is the invalid verdict, as a bundle of it that does not load is, and it is- # given before any library opens. The listing, like the parsing and the load below, runs off the event loop.+ # given before any library opens. The listing, like the parsing and the load below, runs off the event loop, and the+ # load takes the trees from it, so each is listed once.+ tree_listing = MethodTreeListing()with translate_to_validate_bundle_error():- mthds_files = await run_method_load(get_pipelex_mthds_files_from_dirs, dirs={directory})+ mthds_files = await run_method_load(_list_directory_bundle_files, directory=directory, tree_listing=tree_listing)library_manager = get_library_manager()library_id, library = library_manager.open_library()@@ -579,7 +633,9 @@ async def validate_bundles_from_directory(directory: Path, *, allow_signatures:try:set_current_library(library_id=library_id)with translate_to_validate_bundle_error():- all_blueprints, loaded_pipes = await run_method_load(_load_directory, directory=directory, mthds_files=mthds_files, library_id=library_id)+ all_blueprints, loaded_pipes = await run_method_load(+ _load_directory, directory=directory, mthds_files=mthds_files, library_id=library_id, tree_listing=tree_listing+ )dry_run_results = await BundleValidator().validate_pipes(pipes=loaded_pipes, library_id=library_id, allow_signatures=allow_signatures)result = ValidateBundleResult(blueprints=all_blueprints,diff --git a/pipelex/pipeline/validate_in_process.py b/pipelex/pipeline/validate_in_process.pyindex cf003dcce..c50858c4a 100644--- a/pipelex/pipeline/validate_in_process.py+++ b/pipelex/pipeline/validate_in_process.py@@ -27,7 +27,7 @@ from pipelex import logfrom pipelex.base_exceptions import PipelexError, ValidationErrorItemfrom pipelex.interpreter_hub import clear_current_library, get_current_library_id_or_none, get_library_manager, set_current_libraryfrom pipelex.libraries.method_load_threads import run_method_load-from pipelex.libraries.method_trees import list_local_method_tree+from pipelex.libraries.method_tree_listing import MethodTreeListing, get_scoped_tree_listingfrom pipelex.mthds_parsing.pipelex_bundle_blueprint import PipelexBundleBlueprintfrom pipelex.pipe_run.dry_run_in_process import best_effort_graph_specfrom pipelex.pipe_run.dry_run_warning_collector import collecting_dry_run_warnings@@ -50,19 +50,22 @@ if TYPE_CHECKING:from pipelex.graph.graphspec import GraphSpec-def _graph_target_ref(*, graph_pipe_code: str | None, method_dir: Path | None, blueprints: list[PipelexBundleBlueprint]) -> str | None:+def _graph_target_ref(+ *, graph_pipe_code: str | None, method_dir: Path | None, blueprints: list[PipelexBundleBlueprint], tree_listing: MethodTreeListing+) -> str | None:"""The pipe the graph arm draws: the explicit target, else a method tree's entry pipe, else the batch's `main_pipe`.A method tree starts at its entry pipe by the MTHDS protocol's rule (`select_method_entry`), so its graph is that- pipe's, and a tree with no entry pipe, which a run naming no pipe refuses, gets no graph. Listing and reading the tree- is blocking work, so `validate_bundles_in_process` runs this in a worker thread.+ pipe's, and a tree with no entry pipe, which a run naming no pipe refuses, gets no graph. The tree is the one the+ validation loaded, taken from its listing; reading it is blocking work, so `validate_bundles_in_process` runs this+ in a worker thread."""if graph_pipe_code is not None:return graph_pipe_codeif method_dir is None:return select_primary_blueprint(blueprints).main_pipe_reftry:- return select_method_entry(tree=list_local_method_tree(handed_over=method_dir)).pipe_ref+ return select_method_entry(tree=tree_listing.method_tree(handed_over=method_dir)).pipe_refexcept MethodEntryError:return None@@ -80,6 +83,7 @@ async def validate_bundles_in_process(submitted_files: Mapping[str, bytes] | None = None,method_dir: Path | None = None,methods_dirs: Sequence[Path] | None = None,+ tree_listing: MethodTreeListing | None = None,) -> PipelexValidationReport:"""Parse, validate, and dry-run MTHDS bundles in-process; assemble the canonical report.@@ -119,6 +123,10 @@ async def validate_bundles_in_process(method_dir: The directory of the caller's method tree, validated whole in place of `mthds_contents`, such asthe tree a host wrote out for a request (see `validate_bundle`).methods_dirs: Directories laid out like `.mthds/methods/` holding the packages shipped with `method_dir`.+ tree_listing: The listing the caller already listed the method's trees through: the validation and its graph+ arm take every tree from it, so each is listed once (`pipelex.libraries.method_tree_listing`). None takes+ the listing a host put in scope (`scoped_tree_listing`), such as the API server around its validator once+ it has checked the method's requirements, else lists the trees for this validation.Returns:PipelexValidationReport with the structural artifacts of a valid bundle.@@ -138,6 +146,7 @@ async def validate_bundles_in_process(submitted_files=submitted_files,method_dir=method_dir,methods_dirs=methods_dirs,+ tree_listing=tree_listing or get_scoped_tree_listing() or MethodTreeListing(),validation_report=validation_report,)@@ -154,6 +163,7 @@ async def _validate_bundles_in_scope(submitted_files: Mapping[str, bytes] | None,method_dir: Path | None,methods_dirs: Sequence[Path] | None,+ tree_listing: MethodTreeListing,validation_report: MethodValidationReport,) -> PipelexValidationReport:"""The body of :func:`validate_bundles_in_process`, run inside the caller's scope and timed by `validation_report`."""@@ -184,6 +194,7 @@ async def _validate_bundles_in_scope(submitted_files=submitted_files,method_dir=method_dir,methods_dirs=methods_dirs,+ tree_listing=tree_listing,)# Capture the validation library id ONCE, right after validate_bundle leaves it# current — the graph arm and the finally must target the SAME library even if@@ -210,7 +221,7 @@ async def _validate_bundles_in_scope(validation_report.count_warnings(count=len(warnings))# A method tree's entry pipe is read from its files on this host's disk: blocking work, done in a worker thread.graph_target_ref = await run_method_load(- _graph_target_ref, graph_pipe_code=graph_pipe_code, method_dir=method_dir, blueprints=result.blueprints+ _graph_target_ref, graph_pipe_code=graph_pipe_code, method_dir=method_dir, blueprints=result.blueprints, tree_listing=tree_listing)# The graph arm dry-runs its target once more, and finds again what the sweep found, which `warnings`# already carries: it is collected and dropped here rather than logged as if nothing reported it.diff --git a/pipelex/plugins/pipe_func_package_offer_registry.py b/pipelex/plugins/pipe_func_package_offer_registry.pyindex 786651f60..0c966cf3d 100644--- a/pipelex/plugins/pipe_func_package_offer_registry.py+++ b/pipelex/plugins/pipe_func_package_offer_registry.py@@ -89,8 +89,11 @@ class PipeFuncPackageOfferRequest(NamedTuple):# plugin's reason when no environment of the mode can hold them, or ``None`` when the deployment cannot state its# packages at all. The requirements arrive with their markers unevaluated: the factory evaluates them against the marker# environment of the place it would offer, and a refusal rests only on those whose markers hold there, never on one the-# place does not need (``requirements_needed_in``). Called synchronously, once for each method tree declaring-# requirements each time a check judges it, never at registration.+# place does not need (``requirements_needed_in``). Called synchronously, never at registration, once per run for each+# method tree or dependency package declaring requirements, against the run's execution mode and this factory: the run's+# listing keeps the tree's verdict when the offer meets it, so a later check of the run, a load judging the tree at the+# same moment included, asks nothing more, while a check against another mode or factory, a refused tree, and every+# new run ask again (`MethodTreeListing.judge_requirements_once`).## Its threading contract: a factory is never called on a thread running an event loop. Every method load pipelex reaches# from asynchronous code, such as a validation or a run on the API server or in a worker's activity, runs in the@@ -99,11 +102,11 @@ class PipeFuncPackageOfferRequest(NamedTuple):# load itself from asynchronous code moves it off the loop the same way. Pipelex enforces it: a factory about to be# called on a thread running an event loop is not called, and the load fails with `PipelexUnexpectedError`. A factory# may therefore block for as long as its work takes, and must not use an event loop of its caller's: none runs on its-# thread, and the one its caller's code runs on, if any, belongs to another thread. Caching its answers-# by the requirements is still the plugin's job: a factory whose answer takes time to compute, such as one resolving-# the requirements with a resolver, bounds the work with a timeout of its own and caches by the requirements' texts, so-# only a process's first check of a requirement set pays for it. A fault of its own, such as a resolver it cannot-# reach, is raised as an error of the plugin's, never answered as a refusal.+# thread, and the one its caller's code runs on, if any, belongs to another thread. Caching its answers across runs is+# the plugin's job, since pipelex keeps a verdict for one run alone: a factory whose answer takes time to compute, such+# as one resolving the requirements with a resolver, bounds the work with a timeout of its own and keeps its answers by+# the requirements' texts, so only a process's first check of a requirement set pays for it. A fault of its own, such+# as a resolver it cannot reach, is raised as an error of the plugin's, never answered as a refusal.PipeFuncPackageOfferFactoryFn: TypeAlias = Callable[[PipeFuncPackageOfferRequest], PipeFuncPackageOffer | PipeFuncPackageRefusal | None]diff --git a/tests/data/errors/error_identity.txt b/tests/data/errors/error_identity.txtindex a8d601568..15cbfb822 100644--- a/tests/data/errors/error_identity.txt+++ b/tests/data/errors/error_identity.txt@@ -216,6 +216,7 @@ MethodRequirementsError | abstractMethodRequirementsFileError | method.package_invalidMethodRequirementsNotMetError | method.dependency_missingMethodStructuresRefusedError | pipelex.deployment.python_structures_refused+MethodTreeChangedError | method.package_invalidMethodTreeLoadError | method.package_invalidMigrationError | abstractMigrationGoldenError | runtime.unexpecteddiff --git a/tests/integration/pipelex/cli/test_agent_bundle_file_reads.py b/tests/integration/pipelex/cli/test_agent_bundle_file_reads.pyindex 48c6ba50a..67b3a9d4e 100644--- a/tests/integration/pipelex/cli/test_agent_bundle_file_reads.py+++ b/tests/integration/pipelex/cli/test_agent_bundle_file_reads.py@@ -146,7 +146,7 @@ class TestAgentRunBundleListsTheTreeFirst:self, mocker: MockerFixture, tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None:"""`run bundle` lists a bundle file's tree before reading it, so a file past the limits is refused, never read whole."""- mocker.patch("pipelex.libraries.method_trees.local_tree_limits", return_value=TreeLimits(max_file_bytes=16))+ mocker.patch("pipelex.libraries.method_tree_listing.local_tree_limits", return_value=TreeLimits(max_file_bytes=16))bundle_file = tmp_path / "run.mthds"bundle_file.write_text(_BUNDLE, encoding="utf-8")diff --git a/tests/integration/pipelex/cli/test_save_csv_empty_result_row_model.py b/tests/integration/pipelex/cli/test_save_csv_empty_result_row_model.pyindex 170cbc18d..2d0ec1624 100644--- a/tests/integration/pipelex/cli/test_save_csv_empty_result_row_model.py+++ b/tests/integration/pipelex/cli/test_save_csv_empty_result_row_model.py@@ -54,6 +54,7 @@ class TestSaveCsvEmptyResultRowModel:bundle_path=None,mthds_content=_FLAT_ROW_MTHDS,library_dir=None,+ tree_listing=None,)assert flat_field_names(row_model) == ["name", "country"], "The header a zero-row CSV would carry"diff --git a/tests/integration/pipelex/libraries/test_method_loads_off_the_event_loop.py b/tests/integration/pipelex/libraries/test_method_loads_off_the_event_loop.pyindex 8e5c5dfcb..2fd71e206 100644--- a/tests/integration/pipelex/libraries/test_method_loads_off_the_event_loop.py+++ b/tests/integration/pipelex/libraries/test_method_loads_off_the_event_loop.py@@ -5,7 +5,7 @@ from pathlib import Pathimport pytestfrom mthds.method_tree.digest import digest_of_contents-from mthds.method_tree.models import MethodTree+from mthds.method_tree.models import MethodTree, TreeLimitsfrom pytest_mock import MockerFixturefrom pipelex.base_exceptions import PipelexUnexpectedError@@ -111,28 +111,31 @@ class TestMethodLoadsOffTheEventLoop:async def test_a_validation_of_a_method_tree_asks_the_package_offer_off_the_event_loop(self, held_tree: Path, recording_offer: _RecordingOffer) -> None:+ """The validation asks the factory off the loop, once for the tree, however many steps of its load judge it."""result = await validate_bundle(method_dir=held_tree)assert [pipe.code for pipe in result.pipes] == ["echo"]+ assert len(recording_offer.thread_names) == 1, recording_offer.thread_namesassert _all_in_the_method_load_pool(recording_offer.thread_names), recording_offer.thread_names- async def test_the_in_process_validator_reads_the_tree_entry_for_its_graph_off_the_event_loop(+ async def test_the_in_process_validator_lists_the_tree_once_off_the_event_loop_and_draws_its_graph_from_it(self, mocker: MockerFixture, held_tree: Path, recording_offer: _RecordingOffer) -> None:- """Beside the load, the validator lists the tree once more to find the entry pipe its graph draws, off the loop too."""+ """The validation lists the tree off the loop, and the entry pipe its graph draws is read from that same listing."""listing_threads: list[str] = []- def list_after_noting_the_thread(*, handed_over: Path) -> MethodTree:+ def list_after_noting_the_thread(*, handed_over: Path, limits: TreeLimits | None = None) -> MethodTree:listing_threads.append(threading.current_thread().name)- return list_local_method_tree(handed_over=handed_over)+ return list_local_method_tree(handed_over=handed_over, limits=limits)- mocker.patch("pipelex.pipeline.validate_in_process.list_local_method_tree", side_effect=list_after_noting_the_thread)+ mocker.patch("pipelex.libraries.method_tree_listing.list_local_method_tree", side_effect=list_after_noting_the_thread)report = await validate_bundles_in_process(method_dir=held_tree)assert report.graph_spec is not Noneassert len(listing_threads) == 1assert _all_in_the_method_load_pool(listing_threads), listing_threads+ assert len(recording_offer.thread_names) == 1, recording_offer.thread_namesassert _all_in_the_method_load_pool(recording_offer.thread_names), recording_offer.thread_namesasync def test_an_asynchronous_acquisition_loads_off_the_event_loop_and_leaves_the_library_current(@@ -146,6 +149,7 @@ class TestMethodLoadsOffTheEventLoop:assert library_id == "off-the-loop-acquisition"assert qualified_main_pipe is Noneassert get_current_library_id_or_none() == library_id+ assert len(recording_offer.thread_names) == 1, recording_offer.thread_namesassert _all_in_the_method_load_pool(recording_offer.thread_names), recording_offer.thread_namesfinally:library_manager.teardown(library_id=library_id)diff --git a/tests/integration/pipelex/libraries/test_method_requirements_load.py b/tests/integration/pipelex/libraries/test_method_requirements_load.pyindex 1f651c13a..4518b2e3d 100644--- a/tests/integration/pipelex/libraries/test_method_requirements_load.py+++ b/tests/integration/pipelex/libraries/test_method_requirements_load.py@@ -239,9 +239,9 @@ class TestMethodRequirementsLoad:library_manager.teardown(library_id=library_id)assert sorted(pipe.domain_code for pipe in pipes) == ["requirements_excel", "requirements_load"]- # A load judges a tree at each of its checks, so the factory is asked again for the same requirements, which is- # why a plugin caches by them; each request carries one tree's requirements, never another's.- assert {tuple(requirement_texts) for requirement_texts in asked} == {("pandas>=3", "xlsxwriter"), ("polars>=1",)}+ # A load judges each tree once, through the listing its steps share, so the factory is asked once per tree, and+ # each request carries one tree's requirements, never another's.+ assert sorted(tuple(requirement_texts) for requirement_texts in asked) == [("pandas>=3", "xlsxwriter"), ("polars>=1",)]assert not sentinel.exists()@pytest.mark.usefixtures("sandbox_hosted_mode")diff --git a/tests/integration/pipelex/libraries/test_one_listing_per_run.py b/tests/integration/pipelex/libraries/test_one_listing_per_run.pynew file mode 100644index 000000000..a06bc3cf4--- /dev/null+++ b/tests/integration/pipelex/libraries/test_one_listing_per_run.py@@ -0,0 +1,227 @@+from collections.abc import Generator+from pathlib import Path+from typing import Any++import pytest+from mthds.method_tree.lister import list_method_tree+from mthds.method_tree.models import MethodTree+from pytest_mock import MockerFixture++from pipelex.cli.bundle_target_resolution import list_bundle_file_tree, list_method_folder, resolve_method_folder_run+from pipelex.interpreter_hub import clear_current_library, get_current_library_id_or_none, get_library_manager, set_current_library+from pipelex.libraries.exceptions import MethodTreeChangedError+from pipelex.libraries.method_tree_listing import MethodTreeListing+from pipelex.pipeline.exceptions import ValidateBundleError+from pipelex.pipeline.execution_seams import acquire_library_async+from pipelex.pipeline.fixes.fix_loop import fix_method_folder+from pipelex.pipeline.validate_bundle import validate_bundle+from pipelex.pipeline.validate_in_process import validate_bundles_in_process++_ECHO_MTHDS = """\+domain = "one_listing"+main_pipe = "echo"++[pipe.echo]+type = "PipeCompose"+description = "Echo the data"+inputs = { data = "Text" }+output = "Text"+template = "$data"+"""++_MANIFEST = '[package]\naddress = "github.com/pipelex-tests/one-listing"\nversion = "1.0.0"\ndescription = "A listed method"\n'+++class _ListedPaths:+ """The paths the standard's lister was asked to list a method tree for, as each was handed to it."""++ def __init__(self) -> None:+ self.paths: list[Path] = []++ def under(self, *, directory: Path) -> list[Path]:+ """The paths listed that lie in a directory, the method's own, leaving out the library directories around it."""+ root = directory.resolve()+ return [path for path in self.paths if path.resolve().is_relative_to(root)]++[email protected](name="listed_paths")+def listed_paths_fixture(mocker: MockerFixture) -> _ListedPaths:+ """Note each path a method tree is listed for, then list it as the standard does."""+ listed = _ListedPaths()++ def list_after_noting_the_path(path: Path, **kwargs: Any) -> MethodTree:+ listed.paths.append(path)+ return list_method_tree(path, **kwargs)++ mocker.patch("pipelex.libraries.method_trees.list_method_tree", side_effect=list_after_noting_the_path)+ return listed++[email protected](name="method_dir")+def method_dir_fixture(tmp_path: Path) -> Path:+ """A method's folder holding one bundle, whose `main_pipe` is its entry."""+ method_dir = tmp_path / "method"+ method_dir.mkdir()+ (method_dir / "main.mthds").write_text(_ECHO_MTHDS, encoding="utf-8")+ return method_dir++[email protected](name="rooted_method")+def rooted_method_fixture(tmp_path: Path) -> Path:+ """A method rooted at its `METHODS.toml`, whose bundle file holds the template its steps would read, returned as the bundle file."""+ method_dir = tmp_path / "rooted"+ (method_dir / "templates").mkdir(parents=True)+ (method_dir / "METHODS.toml").write_text(_MANIFEST, encoding="utf-8")+ (method_dir / "templates" / "letter.md").write_text("Dear $data,\n", encoding="utf-8")+ bundle_file = method_dir / "main.mthds"+ bundle_file.write_text(_ECHO_MTHDS, encoding="utf-8")+ return bundle_file++[email protected](name="outer_library")+def outer_library_fixture() -> Generator[None, None, None]:+ """Tear down the library a test leaves current, and make the library that was current before it current again."""+ outer_library_id = get_current_library_id_or_none()+ yield+ library_id = get_current_library_id_or_none()+ if library_id is not None and library_id != outer_library_id:+ get_library_manager().teardown(library_id=library_id)+ if outer_library_id is not None:+ set_current_library(library_id=outer_library_id)+ else:+ clear_current_library()++[email protected]("outer_library")[email protected](loop_scope="class")+class TestOneListingPerRun:+ async def test_a_folder_run_loads_the_tree_its_command_listed_without_listing_it_again(+ self, listed_paths: _ListedPaths, method_dir: Path+ ) -> None:+ """`run bundle <folder>` resolves the folder, then loads it as a library directory and its entry bundle by its file."""+ tree_listing = MethodTreeListing()+ folder_run = resolve_method_folder_run(directory=method_dir, pipe_override=None, tree_listing=tree_listing)+ entry_file = folder_run.entry_bundle_file+ assert entry_file is not None++ library_id, _ = await acquire_library_async(+ "one-listing-folder-run",+ library_dirs=[str(method_dir)],+ mthds_contents=[entry_file.read_text(encoding="utf-8")],+ bundle_uris=[str(entry_file)],+ tree_listing=tree_listing,+ )++ assert listed_paths.under(directory=method_dir) == [method_dir]+ loaded_trees = [+ blueprint.tree_bundle.tree for blueprint in get_library_manager().get_accumulated_blueprints(library_id) if blueprint.tree_bundle+ ]+ assert loaded_trees, "the run loaded no bundle of the method"+ # The tree whose digest the run records is the very tree it loaded.+ assert all(loaded_tree is folder_run.tree for loaded_tree in loaded_trees)++ async def test_a_bundle_file_named_alone_is_placed_in_the_tree_its_command_listed(self, listed_paths: _ListedPaths, method_dir: Path) -> None:+ bundle_file = method_dir / "main.mthds"+ tree_listing = MethodTreeListing()+ method_tree = list_bundle_file_tree(bundle_file=bundle_file, tree_listing=tree_listing)++ library_id, _ = await acquire_library_async(+ "one-listing-file-run",+ mthds_contents=[bundle_file.read_text(encoding="utf-8")],+ bundle_uris=[str(bundle_file)],+ tree_listing=tree_listing,+ )++ assert listed_paths.under(directory=method_dir) == [bundle_file]+ (blueprint,) = get_library_manager().get_accumulated_blueprints(library_id)+ assert blueprint.tree_bundle is not None+ assert blueprint.tree_bundle.tree is method_tree++ async def test_a_load_handed_no_listing_lists_each_tree_once(self, listed_paths: _ListedPaths, method_dir: Path) -> None:+ """A load that is the first to read the method lists it itself, once, for every step of the load."""+ await acquire_library_async(+ "one-listing-own-load",+ library_dirs=[str(method_dir)],+ mthds_contents=[_ECHO_MTHDS],+ bundle_uris=[str(method_dir / "main.mthds")],+ )++ assert listed_paths.under(directory=method_dir) == [method_dir]++ async def test_validating_a_folder_its_command_listed_lists_it_no_more(self, listed_paths: _ListedPaths, method_dir: Path) -> None:+ """`validate bundle <folder>` lists the folder to resolve it, and the validation loads the tree from that listing."""+ tree_listing = MethodTreeListing()+ list_method_folder(directory=method_dir, tree_listing=tree_listing)++ result = await validate_bundle(method_dir=method_dir, library_dirs=[method_dir], library_dirs_are_callers=True, tree_listing=tree_listing)++ assert [pipe.code for pipe in result.pipes] == ["echo"]+ assert listed_paths.under(directory=method_dir) == [method_dir]++ async def test_the_in_process_validator_lists_a_method_once_for_its_load_and_its_graph(+ self, listed_paths: _ListedPaths, method_dir: Path+ ) -> None:+ report = await validate_bundles_in_process(method_dir=method_dir)++ assert report.graph_spec is not None+ assert listed_paths.under(directory=method_dir) == [method_dir]++ async def test_a_fix_of_a_valid_folder_its_command_listed_lists_it_no_more(self, listed_paths: _ListedPaths, method_dir: Path) -> None:+ """The fix loop's snapshot and validation read the listing the command resolved the folder through."""+ tree_listing = MethodTreeListing()+ list_method_folder(directory=method_dir, tree_listing=tree_listing)++ result = await fix_method_folder(method_dir=method_dir, tree_listing=tree_listing)++ assert result.is_valid is True+ assert listed_paths.under(directory=method_dir) == [method_dir]++ async def test_a_bundle_edited_after_its_command_listed_it_is_refused_before_the_run_loads_it(self, method_dir: Path) -> None:+ """The run's digest names the tree the command listed: a bundle edited since is refused, never loaded in its place."""+ tree_listing = MethodTreeListing()+ folder_run = resolve_method_folder_run(directory=method_dir, pipe_override=None, tree_listing=tree_listing)+ entry_file = folder_run.entry_bundle_file+ assert entry_file is not None+ content_read = entry_file.read_text(encoding="utf-8")+ entry_file.write_text(_ECHO_MTHDS.replace("Echo the data", "Echo the edited data"), encoding="utf-8")++ with pytest.raises(ValidateBundleError) as exc_info:+ await acquire_library_async(+ "one-listing-edited-bundle",+ library_dirs=[str(method_dir)],+ library_dirs_are_callers=True,+ mthds_contents=[content_read],+ bundle_uris=[str(entry_file)],+ tree_listing=tree_listing,+ )++ assert isinstance(exc_info.value.__cause__, MethodTreeChangedError)+ assert "'main.mthds'" in exc_info.value.message++ async def test_a_template_edited_after_its_command_listed_the_bundle_is_refused_before_the_run_loads_it(self, rooted_method: Path) -> None:+ """A file the bundle's steps read is part of the listed tree: edited after the listing, the run refuses the tree."""+ tree_listing = MethodTreeListing()+ list_bundle_file_tree(bundle_file=rooted_method, tree_listing=tree_listing)+ (rooted_method.parent / "templates" / "letter.md").write_text("Dear $data, edited\n", encoding="utf-8")++ with pytest.raises(ValidateBundleError) as exc_info:+ await acquire_library_async(+ "one-listing-edited-template",+ mthds_contents=[rooted_method.read_text(encoding="utf-8")],+ bundle_uris=[str(rooted_method)],+ tree_listing=tree_listing,+ )++ assert isinstance(exc_info.value.__cause__, MethodTreeChangedError)+ assert "'templates/letter.md'" in exc_info.value.message++ async def test_validating_a_bundle_whose_template_changed_since_its_listing_refuses_it(self, rooted_method: Path) -> None:+ tree_listing = MethodTreeListing()+ list_bundle_file_tree(bundle_file=rooted_method, tree_listing=tree_listing)+ (rooted_method.parent / "templates" / "letter.md").write_text("Dear $data, edited\n", encoding="utf-8")++ with pytest.raises(ValidateBundleError) as exc_info:+ await validate_bundle(mthds_file_path=rooted_method, library_dirs=[], tree_listing=tree_listing)++ assert isinstance(exc_info.value.__cause__, MethodTreeChangedError)+ assert "'templates/letter.md'" in exc_info.value.messagediff --git a/tests/unit/pipelex/cli/agent_cli/commands/test_bundle_path_resolver.py b/tests/unit/pipelex/cli/agent_cli/commands/test_bundle_path_resolver.pyindex cd17da096..2ce881142 100644--- a/tests/unit/pipelex/cli/agent_cli/commands/test_bundle_path_resolver.py+++ b/tests/unit/pipelex/cli/agent_cli/commands/test_bundle_path_resolver.py@@ -83,7 +83,7 @@ class TestResolveBundleTarget:self, mocker: MockerFixture, tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None:"""Listing the tree reads its limits from the configuration: its refusal is the failed boot's problem, no verdict."""- mocker.patch("pipelex.libraries.method_trees.local_tree_limits", side_effect=PipelexConfigError("the configuration is out of date"))+ mocker.patch("pipelex.libraries.method_tree_listing.local_tree_limits", side_effect=PipelexConfigError("the configuration is out of date"))write_method_folder(directory=tmp_path, shape=MethodFolderShape.ONE_DECLARED_MAIN_PIPE)with pytest.raises(typer.Exit) as exc_info:diff --git a/tests/unit/pipelex/cli/agent_cli/test_folder_entry_commands.py b/tests/unit/pipelex/cli/agent_cli/test_folder_entry_commands.pyindex 62e5e3bcd..a0c2e05ab 100644--- a/tests/unit/pipelex/cli/agent_cli/test_folder_entry_commands.py+++ b/tests/unit/pipelex/cli/agent_cli/test_folder_entry_commands.py@@ -15,6 +15,7 @@ from pipelex.cli.agent_cli.commands.inputs.bundle_cmd import inputs_bundle_cmdfrom pipelex.cli.agent_cli.commands.run.bundle_cmd import run_bundle_cmdfrom pipelex.cli.agent_cli.commands.validate.bundle_cmd import validate_bundle_cmdfrom pipelex.cli.bundle_target_resolution import resolve_method_folder_run+from pipelex.libraries.method_tree_listing import MethodTreeListingfrom pipelex.pipeline.exceptions import MethodEntryErrorfrom pipelex.pipeline.fixes.fix_loop import FixBundleResultfrom tests.helpers.method_folders import NAMED_PIPE, MethodFolderShape, write_method_folder@@ -37,7 +38,7 @@ _VALID_RESULT: dict[str, Any] = {"success": True, "is_valid": True, "is_runnabledef _run_refusal(*, directory: Path) -> str:"""The message `run bundle` refuses the folder with: no single pipe to start at is the run's own refusal, no verdict."""with pytest.raises(MethodEntryError) as exc_info:- resolve_method_folder_run(directory=directory, pipe_override=None)+ resolve_method_folder_run(directory=directory, pipe_override=None, tree_listing=MethodTreeListing())return exc_info.value.message@@ -231,7 +232,7 @@ class TestAgentFolderEntryCommands:`validate` and `fix` exit 2 on it, no verdict, as the human `pipelex validate bundle` and `fix bundle` do;`inputs` and `run` exit 1, as a failed boot does."""- mocker.patch("pipelex.libraries.method_trees.local_tree_limits", side_effect=PipelexConfigError("the configuration is out of date"))+ mocker.patch("pipelex.libraries.method_tree_listing.local_tree_limits", side_effect=PipelexConfigError("the configuration is out of date"))write_method_folder(directory=tmp_path, shape=MethodFolderShape.ONE_DECLARED_MAIN_PIPE)with pytest.raises(typer.Exit) as exc_info:diff --git a/tests/unit/pipelex/cli/commands/test_bundle_path_resolver.py b/tests/unit/pipelex/cli/commands/test_bundle_path_resolver.pyindex ad4392930..cf64de9f1 100644--- a/tests/unit/pipelex/cli/commands/test_bundle_path_resolver.py+++ b/tests/unit/pipelex/cli/commands/test_bundle_path_resolver.py@@ -17,6 +17,7 @@ from pipelex.base_exceptions import PipelexConfigErrorfrom pipelex.cli.bundle_target_resolution import resolve_method_folder_runfrom pipelex.cli.commands.bundle_path_resolver import resolve_build_bundle_target, resolve_bundle_targetfrom pipelex.cli.judged_target import JudgedBundleTarget+from pipelex.libraries.method_tree_listing import MethodTreeListingfrom pipelex.pipeline.exceptions import MethodEntryError, ValidateBundleErrorfrom tests.helpers.method_folders import NAMED_PIPE, MethodFolderShape, write_method_folder@@ -26,7 +27,8 @@ if TYPE_CHECKING:from pytest_mock import MockerFixture_HINT = " To fix a bundle, pass a .mthds file or directory: pipelex fix bundle <path>"-_METHOD_TREES_MODULE = "pipelex.libraries.method_trees"+# The module a command's listing reads this machine's tree limits through.+_TREE_LISTING_MODULE = "pipelex.libraries.method_tree_listing"def _run_refusal(*, directory: Path) -> str:@@ -37,7 +39,7 @@ def _run_refusal(*, directory: Path) -> str:sentence and whose cause is that same `MethodEntryError`, the one a `build` command reports as it is."""with pytest.raises((MethodEntryError, ValidateBundleError)) as exc_info:- resolve_method_folder_run(directory=directory, pipe_override=None)+ resolve_method_folder_run(directory=directory, pipe_override=None, tree_listing=MethodTreeListing())refusal = exc_info.valueif isinstance(refusal, ValidateBundleError):assert isinstance(refusal.__cause__, MethodEntryError)@@ -52,7 +54,9 @@ class TestHumanBundlePathResolver:target = resolve_bundle_target(str(bundle_file), library_dir=None, command="fix", not_a_bundle_hint=_HINT)- assert target == JudgedBundleTarget(bundle_path=bundle_file, method_dir=None, method_tree=None, library_dirs=None)+ assert target == JudgedBundleTarget(+ bundle_path=bundle_file, method_dir=None, method_tree=None, library_dirs=None, tree_listing=target.tree_listing+ )assert target.judged_path == bundle_filedef test_tilde_in_path_is_expanded(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:@@ -136,7 +140,7 @@ class TestHumanBundlePathResolver:self, mocker: MockerFixture, tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None:"""Listing the tree reads its limits from the configuration: a refusal there is no traceback."""- mocker.patch(f"{_METHOD_TREES_MODULE}.local_tree_limits", side_effect=PipelexConfigError("the configuration is out of date"))+ mocker.patch(f"{_TREE_LISTING_MODULE}.local_tree_limits", side_effect=PipelexConfigError("the configuration is out of date"))write_method_folder(directory=tmp_path, shape=MethodFolderShape.ONE_DECLARED_MAIN_PIPE)with pytest.raises(typer.Exit) as exc_info:@@ -148,7 +152,7 @@ class TestHumanBundlePathResolver:def test_a_build_command_reports_a_configuration_that_does_not_validate_before_the_boot(self, mocker: MockerFixture, tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None:- mocker.patch(f"{_METHOD_TREES_MODULE}.local_tree_limits", side_effect=PipelexConfigError("the configuration is out of date"))+ mocker.patch(f"{_TREE_LISTING_MODULE}.local_tree_limits", side_effect=PipelexConfigError("the configuration is out of date"))write_method_folder(directory=tmp_path, shape=MethodFolderShape.ONE_DECLARED_MAIN_PIPE)with pytest.raises(typer.Exit) as exc_info:diff --git a/tests/unit/pipelex/cli/commands/test_folder_entry_commands.py b/tests/unit/pipelex/cli/commands/test_folder_entry_commands.pyindex 7ba193e02..10d0832dd 100644--- a/tests/unit/pipelex/cli/commands/test_folder_entry_commands.py+++ b/tests/unit/pipelex/cli/commands/test_folder_entry_commands.py@@ -15,6 +15,7 @@ from pipelex.cli.commands.build.output.bundle_cmd import build_output_bundle_cmdfrom pipelex.cli.commands.fix.bundle_cmd import fix_bundle_cmdfrom pipelex.cli.commands.run.bundle_cmd import run_bundle_cmdfrom pipelex.cli.commands.validate.bundle_cmd import validate_bundle_cmd+from pipelex.libraries.method_tree_listing import MethodTreeListingfrom pipelex.pipeline.exceptions import MethodEntryErrorfrom tests.helpers.method_folders import NAMED_PIPE, MethodFolderShape, write_method_folder@@ -31,7 +32,7 @@ _BUILD_COMMANDS = ["build_inputs", "build_output"]def _run_refusal(*, directory: Path) -> str:"""The message `run bundle` refuses the folder with: no single pipe to start at is the run's own refusal, no verdict."""with pytest.raises(MethodEntryError) as exc_info:- resolve_method_folder_run(directory=directory, pipe_override=None)+ resolve_method_folder_run(directory=directory, pipe_override=None, tree_listing=MethodTreeListing())return exc_info.value.message@@ -179,7 +180,7 @@ class TestHumanFolderEntryCommands:`validate` and `fix` exit 2 on it, no verdict, as the agent CLI's `validate bundle` and `fix bundle` do; `run`and `build` exit 1, as their other failures do."""- mocker.patch("pipelex.libraries.method_trees.local_tree_limits", side_effect=PipelexConfigError("the configuration is out of date"))+ mocker.patch("pipelex.libraries.method_tree_listing.local_tree_limits", side_effect=PipelexConfigError("the configuration is out of date"))write_method_folder(directory=tmp_path, shape=MethodFolderShape.ONE_DECLARED_MAIN_PIPE)with pytest.raises(typer.Exit) as exc_info:diff --git a/tests/unit/pipelex/cli/test_judged_target.py b/tests/unit/pipelex/cli/test_judged_target.pyindex 9de36948a..0a82ebe7f 100644--- a/tests/unit/pipelex/cli/test_judged_target.py+++ b/tests/unit/pipelex/cli/test_judged_target.py@@ -5,6 +5,7 @@ from pytest_mock import MockerFixturefrom pipelex.base_exceptions import PipelexUnexpectedErrorfrom pipelex.cli.judged_target import JudgedBundleTarget, fix_judged_target+from pipelex.libraries.method_tree_listing import MethodTreeListingfrom pipelex.pipeline.fixes.fix_loop import FixBundleResult_MODULE = "pipelex.cli.judged_target"@@ -22,12 +23,14 @@ class TestJudgedTarget:ids=["bundle_file", "method_folder"],)async def test_the_judged_path_is_the_folder_else_the_file(self, bundle_path: Path | None, method_dir: Path | None, expected: Path) -> None:- target = JudgedBundleTarget(bundle_path=bundle_path, method_dir=method_dir, method_tree=None, library_dirs=None)+ target = JudgedBundleTarget(+ bundle_path=bundle_path, method_dir=method_dir, method_tree=None, library_dirs=None, tree_listing=MethodTreeListing()+ )assert target.judged_path == expectedasync def test_a_target_naming_neither_is_unexpected(self) -> None:- target = JudgedBundleTarget(bundle_path=None, method_dir=None, method_tree=None, library_dirs=None)+ target = JudgedBundleTarget(bundle_path=None, method_dir=None, method_tree=None, library_dirs=None, tree_listing=MethodTreeListing())with pytest.raises(PipelexUnexpectedError, match="neither a bundle file nor a method's folder"):_ = target.judged_path@@ -37,7 +40,9 @@ class TestJudgedTarget:file_fix = mocker.patch(f"{_MODULE}.fix_bundle_file", new=mocker.AsyncMock(return_value=_RESULT))method_dir = Path("my_method")library_dirs = [method_dir, Path("libs")]- target = JudgedBundleTarget(bundle_path=None, method_dir=method_dir, method_tree=None, library_dirs=library_dirs)+ target = JudgedBundleTarget(+ bundle_path=None, method_dir=method_dir, method_tree=None, library_dirs=library_dirs, tree_listing=MethodTreeListing()+ )result = await fix_judged_target(target=target, writable_library_dirs=None, allow_signatures=True, max_iterations=4, select_codes=None, ignore_codes=("strip-namespace",)@@ -52,6 +57,7 @@ class TestJudgedTarget:max_iterations=4,select_codes=None,ignore_codes=("strip-namespace",),+ tree_listing=target.tree_listing,)file_fix.assert_not_called()@@ -60,7 +66,7 @@ class TestJudgedTarget:file_fix = mocker.patch(f"{_MODULE}.fix_bundle_file", new=mocker.AsyncMock(return_value=_RESULT))bundle_path = Path("my_bundle.mthds")writable = [Path("copy")]- target = JudgedBundleTarget(bundle_path=bundle_path, method_dir=None, method_tree=None, library_dirs=None)+ target = JudgedBundleTarget(bundle_path=bundle_path, method_dir=None, method_tree=None, library_dirs=None, tree_listing=MethodTreeListing())await fix_judged_target(target=target, writable_library_dirs=writable, allow_signatures=False, max_iterations=None, select_codes=None, ignore_codes=None@@ -74,5 +80,6 @@ class TestJudgedTarget:max_iterations=None,select_codes=None,ignore_codes=None,+ tree_listing=target.tree_listing,)folder_fix.assert_not_called()diff --git a/tests/unit/pipelex/cli/test_run_core_execution.py b/tests/unit/pipelex/cli/test_run_core_execution.pyindex c35f8000a..dc2756308 100644--- a/tests/unit/pipelex/cli/test_run_core_execution.py+++ b/tests/unit/pipelex/cli/test_run_core_execution.py@@ -473,7 +473,13 @@ class TestRunCoreExecution:csv_codec_mock.assert_called_once()assert csv_codec_mock.call_args.kwargs["row_model"] is TextContent# The reload sees exactly the load inputs the run was handed, and is torn down again.- assert reload.acquire.call_args.kwargs == {"library_id": "", "library_dirs": ["libs"], "mthds_contents": None, "bundle_uris": None}+ assert reload.acquire.call_args.kwargs == {+ "library_id": "",+ "library_dirs": ["libs"],+ "mthds_contents": None,+ "bundle_uris": None,+ "tree_listing": None,+ }self._assert_reload_library_torn_down(reload)@pytest.mark.usefixtures("config_mock", "console")diff --git a/tests/unit/pipelex/fix_ops/test_fix_file_transaction.py b/tests/unit/pipelex/fix_ops/test_fix_file_transaction.pyindex c37bcbe9a..8d98a2eb5 100644--- a/tests/unit/pipelex/fix_ops/test_fix_file_transaction.py+++ b/tests/unit/pipelex/fix_ops/test_fix_file_transaction.py@@ -13,6 +13,7 @@ from pipelex.fix_ops.file_transaction import (commit_file_updates,read_file_snapshot,)+from pipelex.libraries.method_tree_listing import MethodTreeListingfrom pipelex.pipeline.fixes.fix_loop import (_bind_fixes_to_snapshots, # pyright: ignore[reportPrivateUsage]_snapshot_loaded_sources, # pyright: ignore[reportPrivateUsage]@@ -28,7 +29,7 @@ class TestFixFileTransaction:second_target.write_text("second\n", encoding="utf-8")source_link = tmp_path / "library.mthds"source_link.symlink_to(first_target)- snapshots = _snapshot_loaded_sources(entry_source_path=source_link, effective_dirs=[])+ snapshots = _snapshot_loaded_sources(entry_source_path=source_link, effective_dirs=[], tree_listing=MethodTreeListing())fix = SuggestedFix(fix_code="test", description="test", safety=FixSafety.SAFE, source=str(source_link), ops=[])source_link.unlink()source_link.symlink_to(second_target)diff --git a/tests/unit/pipelex/libraries/test_method_requirements_check.py b/tests/unit/pipelex/libraries/test_method_requirements_check.pyindex 56b2fcd94..cb6430e3f 100644--- a/tests/unit/pipelex/libraries/test_method_requirements_check.py+++ b/tests/unit/pipelex/libraries/test_method_requirements_check.py@@ -24,6 +24,7 @@ from pipelex.libraries.method_requirements import (ensure_served_method_requirements_met,requirements_needed_in,)+from pipelex.libraries.method_tree_listing import MethodTreeListingfrom pipelex.libraries.method_trees import reporting_host_library_tree_faultsfrom pipelex.libraries.package_environments import OfferedPackageEnvironmentfrom pipelex.pipeline.exceptions import ValidateBundleError@@ -229,6 +230,40 @@ class TestMethodRequirementsCheck:assert "pipelex-env-pandas" not in messageassert [(item.source, item.line) for item in exc_info.value.blueprint_validation_errors] == [("requirements.txt", 1)]+ def test_a_run_s_listing_asks_the_factory_once_for_a_tree_found_met_and_again_for_a_refused_one(+ self, tmp_path: Path, sandbox_offering: Callable[..., PipeFuncConfig]+ ) -> None:+ """The listing keeps the trees a check of the run found met, however the check names them, and never a refused one."""+ offers_by_requirements: dict[tuple[str, ...], PipeFuncPackageOffer] = {+ ("polars>=1",): MethodRequirementsTestData.environment_offer(snapshot_name="pipelex-env-polars", distributions={"polars": "1.2.0"}),+ ("pandas>=4",): MethodRequirementsTestData.environment_offer(snapshot_name="pipelex-env-stale", distributions={"pandas": "3.0.1"}),+ }+ asked: list[tuple[str, ...]] = []++ def offer_for(request: PipeFuncPackageOfferRequest) -> PipeFuncPackageOffer | PipeFuncPackageRefusal | None:+ requirement_texts = tuple(line.text for line in request.requirements)+ asked.append(requirement_texts)+ return offers_by_requirements[requirement_texts]++ sandbox_offering(offers={"daytona": offer_for})+ polars_tree = list_method_tree(_method(directory=tmp_path / "polars", requirements="polars>=1\n"))+ stale_tree = list_method_tree(_method(directory=tmp_path / "stale", requirements="pandas>=4\n"))+ tree_listing = MethodTreeListing()++ # A server's check, then the load of the same request, which names host paths, then a later step of that load.+ ensure_method_requirements_met(trees=[polars_tree], names_host_paths=False, tree_listing=tree_listing)+ ensure_method_requirements_met(trees=[polars_tree], names_host_paths=True, tree_listing=tree_listing)+ ensure_method_requirements_met(trees=[polars_tree], names_host_paths=True, tree_listing=tree_listing)+ assert asked == [("polars>=1",)]++ for _ in range(2):+ with pytest.raises(MethodRequirementsNotMetError):+ ensure_method_requirements_met(trees=[polars_tree, stale_tree], names_host_paths=True, tree_listing=tree_listing)+ assert asked == [("polars>=1",), ("pandas>=4",), ("pandas>=4",)]++ ensure_method_requirements_met(trees=[polars_tree], names_host_paths=True, tree_listing=MethodTreeListing())+ assert asked == [("polars>=1",), ("pandas>=4",), ("pandas>=4",), ("polars>=1",)]+def test_a_factory_refusing_only_what_its_sandbox_needs_lets_a_false_marker_requirement_load(self, tmp_path: Path, sandbox_offering: Callable[..., PipeFuncConfig]) -> None:diff --git a/tests/unit/pipelex/libraries/test_method_tree_listing.py b/tests/unit/pipelex/libraries/test_method_tree_listing.pynew file mode 100644index 000000000..468050a98--- /dev/null+++ b/tests/unit/pipelex/libraries/test_method_tree_listing.py@@ -0,0 +1,293 @@+import threading+from concurrent.futures import ThreadPoolExecutor+from pathlib import Path++import pytest+from mthds.method_tree.models import MethodTree, TreeLimits+from mthds.package.dependency_resolver import ResolvedDependency+from mthds.package.manifest.schema import MethodsManifest+from mthds.protocol.problem.registry import CoreCode+from pytest_mock import MockerFixture++from pipelex.errors.problem import ProblemView+from pipelex.interpreter_hub import get_library_manager, scoped_current_library+from pipelex.libraries import method_requirements, method_tree_listing+from pipelex.libraries.exceptions import MethodTreeChangedError, MethodTreeLoadError+from pipelex.libraries.library_manager import LibraryManager+from pipelex.libraries.method_tree_listing import MethodTreeListing+from pipelex.libraries.method_trees import ensure_bundle_text_as_listed, list_local_method_tree, listing_identity, read_tree_file_as_listed+from pipelex.methods.package_reason import MethodPackageReason++_MANIFEST = '[package]\naddress = "github.com/pipelex-tests/listing"\nversion = "1.0.0"\ndescription = "A listed package"\n'+_BUNDLE = "domain = 'listing_tests'\n"+_PACKAGE_ADDRESS = "github.com/pipelex-tests/listing-dependency"+# How long a thread waits for another before the test gives up on it, and how long a judgement holds the tree while a+# second ask arrives: a second judgement arriving in that time is a second call the listing should have prevented.+_WAIT_SECONDS = 5.0+_HOLD_SECONDS = 0.3+++def _touch(*, directory: Path, tree_path: str, content: str = _BUNDLE) -> Path:+ file_path = directory / tree_path+ file_path.parent.mkdir(parents=True, exist_ok=True)+ file_path.write_text(content, encoding="utf-8")+ return file_path+++def _refusal_params(*, error: MethodTreeLoadError) -> dict[str, object]:+ problem = error.to_problem(view=ProblemView.CALLER)+ return problem.params.as_wire() if problem.params is not None else {}+++class TestMethodTreeListing:+ def test_a_bundle_file_named_alone_is_rooted_at_the_manifest_above_it_else_alone(self, tmp_path: Path) -> None:+ """Named alone, a bundle reads the files of the method a METHODS.toml above it roots, and with none only itself."""+ lone = _touch(directory=tmp_path, tree_path="loose/lone.mthds")+ _touch(directory=tmp_path, tree_path="loose/notes.md")+ _touch(directory=tmp_path, tree_path="method/METHODS.toml", content=_MANIFEST)+ rooted = _touch(directory=tmp_path, tree_path="method/bundles/main.mthds")+ tree_listing = MethodTreeListing()++ lone_bundle = tree_listing.lone_file_bundle(bundle_file=lone)+ rooted_bundle = tree_listing.lone_file_bundle(bundle_file=rooted)++ assert lone_bundle.tree.is_one_file+ assert lone_bundle.tree.tree_paths == ["lone.mthds"]+ assert not rooted_bundle.tree.is_one_file+ assert rooted_bundle.tree_path == "bundles/main.mthds"+ assert rooted_bundle.tree.tree_paths == ["METHODS.toml", "bundles/main.mthds"]++ @pytest.mark.parametrize("with_manifest", [True, False])+ @pytest.mark.parametrize("real_file_first", [True, False])+ def test_a_symbolic_link_named_alone_is_refused_whichever_the_run_listed_first(+ self, tmp_path: Path, with_manifest: bool, real_file_first: bool+ ) -> None:+ """The link is refused as the lister refuses it, even once the run listed the file it points to, and the file still lists."""+ if with_manifest:+ _touch(directory=tmp_path, tree_path="method/METHODS.toml", content=_MANIFEST)+ real_file = _touch(directory=tmp_path, tree_path="method/main.mthds")+ link = tmp_path / "elsewhere" / "link.mthds"+ link.parent.mkdir()+ link.symlink_to(real_file)+ with pytest.raises(MethodTreeLoadError) as lister_refusal:+ list_local_method_tree(handed_over=link)+ tree_listing = MethodTreeListing()++ if real_file_first:+ tree_listing.lone_file_bundle(bundle_file=real_file)+ with pytest.raises(MethodTreeLoadError) as listing_refusal:+ tree_listing.method_tree(handed_over=link)+ real_bundle = tree_listing.lone_file_bundle(bundle_file=real_file)++ assert listing_refusal.value.message == lister_refusal.value.message+ assert _refusal_params(error=listing_refusal.value) == _refusal_params(error=lister_refusal.value)+ assert real_bundle.tree_path == "main.mthds"++ @pytest.mark.parametrize("real_directory_first", [True, False])+ def test_a_symbolic_link_naming_a_directory_is_followed_as_the_lister_follows_it(+ self, mocker: MockerFixture, tmp_path: Path, real_directory_first: bool+ ) -> None:+ """A directory and a link naming it are one tree, listed once, whichever the run asks for first."""+ real_directory = tmp_path / "method"+ _touch(directory=real_directory, tree_path="main.mthds")+ link = tmp_path / "linked-method"+ link.symlink_to(real_directory, target_is_directory=True)+ expected = list_local_method_tree(handed_over=link)+ lister = mocker.spy(method_tree_listing, "list_local_method_tree")+ tree_listing = MethodTreeListing()++ first, second = (real_directory, link) if real_directory_first else (link, real_directory)+ first_tree = tree_listing.method_tree(handed_over=first)+ second_tree = tree_listing.method_tree(handed_over=second)++ assert second_tree is first_tree+ assert lister.call_count == 1+ assert listing_identity(tree=first_tree) == listing_identity(tree=expected)++ @pytest.mark.parametrize("bundle_file_first", [True, False])+ def test_a_bundle_file_named_alone_inside_a_library_directory_shares_the_directory_s_listing(+ self, mocker: MockerFixture, tmp_path: Path, bundle_file_first: bool+ ) -> None:+ """`run bundle dir/bundles/main.mthds -L dir` lists the tree rooted at `dir` once, for the file and for the directory."""+ library_dir = tmp_path / "library"+ _touch(directory=library_dir, tree_path="METHODS.toml", content=_MANIFEST)+ bundle_file = _touch(directory=library_dir, tree_path="bundles/main.mthds")+ expected = list_local_method_tree(handed_over=bundle_file)+ lone_file_lister = mocker.spy(method_tree_listing, "list_local_method_tree")+ library_dir_lister = mocker.spy(method_tree_listing, "list_library_dir_tree")+ tree_listing = MethodTreeListing()++ if bundle_file_first:+ tree_bundle = tree_listing.lone_file_bundle(bundle_file=bundle_file)+ library_trees = tree_listing.library_trees(library_dir=library_dir)+ else:+ library_trees = tree_listing.library_trees(library_dir=library_dir)+ tree_bundle = tree_listing.lone_file_bundle(bundle_file=bundle_file)++ assert lone_file_lister.call_count + library_dir_lister.call_count == 1+ assert tree_bundle.tree_path == "bundles/main.mthds"+ assert tree_bundle.tree.entry == expected.entry+ assert listing_identity(tree=tree_bundle.tree) == listing_identity(tree=expected)+ assert [listing_identity(tree=tree) for tree in library_trees] == [listing_identity(tree=expected)]++ def test_concurrent_asks_of_one_tree_list_it_once_while_other_trees_list_beside_them(self, mocker: MockerFixture, tmp_path: Path) -> None:+ """Two asks of one tree wait for one listing, and a third tree lists at the same moment: the barrier needs both listings at once."""+ shared_directory = tmp_path / "shared"+ other_directory = tmp_path / "other"+ _touch(directory=shared_directory, tree_path="main.mthds")+ _touch(directory=other_directory, tree_path="main.mthds")+ list_for_real = method_tree_listing.list_local_method_tree+ both_listing = threading.Barrier(2, timeout=_WAIT_SECONDS)+ listed: list[Path] = []++ def list_beside_another(*, handed_over: Path, limits: TreeLimits | None = None) -> MethodTree:+ listed.append(handed_over)+ both_listing.wait()+ return list_for_real(handed_over=handed_over, limits=limits)++ mocker.patch.object(method_tree_listing, "list_local_method_tree", side_effect=list_beside_another)+ tree_listing = MethodTreeListing()+ assert tree_listing.limits is not None++ with ThreadPoolExecutor(max_workers=3) as pool:+ shared_asks = [pool.submit(tree_listing.method_tree, handed_over=shared_directory) for _ in range(2)]+ other_ask = pool.submit(tree_listing.method_tree, handed_over=other_directory)+ shared_trees = [ask.result(timeout=_WAIT_SECONDS * 2) for ask in shared_asks]+ other_tree = other_ask.result(timeout=_WAIT_SECONDS * 2)++ assert shared_trees[0] is shared_trees[1]+ assert other_tree.directory != shared_trees[0].directory+ assert sorted(listed) == sorted([shared_directory, other_directory])++ def test_concurrent_judgements_of_one_tree_judge_it_once_per_environment(self, tmp_path: Path) -> None:+ """A second judgement waits for the first and takes its verdict; another environment, or a refusal, judges again."""+ _touch(directory=tmp_path, tree_path="main.mthds")+ tree_listing = MethodTreeListing()+ tree = tree_listing.method_tree(handed_over=tmp_path)+ first_judging = threading.Event()+ second_judging = threading.Event()+ judgements: list[str] = []++ def judge() -> None:+ judgements.append("judged")+ if len(judgements) == 1:+ first_judging.set()+ second_judging.wait(timeout=_HOLD_SECONDS)+ else:+ second_judging.set()++ with ThreadPoolExecutor(max_workers=2) as pool:+ first = pool.submit(tree_listing.judge_requirements_once, tree=tree, judged_against="daytona", judge=judge)+ assert first_judging.wait(timeout=_WAIT_SECONDS)+ second = pool.submit(tree_listing.judge_requirements_once, tree=tree, judged_against="daytona", judge=judge)+ first.result(timeout=_WAIT_SECONDS)+ second.result(timeout=_WAIT_SECONDS)+ assert judgements == ["judged"]++ tree_listing.judge_requirements_once(tree=tree, judged_against="direct", judge=judge)+ tree_listing.judge_requirements_once(tree=tree, judged_against="daytona", judge=judge)+ assert judgements == ["judged", "judged"]++ def refuse() -> None:+ judgements.append("refused")+ msg = "refused"+ raise ValueError(msg)++ for _ in range(2):+ with pytest.raises(ValueError, match="refused"):+ tree_listing.judge_requirements_once(tree=tree, judged_against="elsewhere", judge=refuse)+ assert judgements == ["judged", "judged", "refused", "refused"]++ def test_a_tree_edited_after_its_listing_is_refused_as_a_load_takes_it(self, tmp_path: Path) -> None:+ """A template edited between the command's listing and the load is refused with `tree_changed`, naming the file."""+ _touch(directory=tmp_path, tree_path="main.mthds")+ template = _touch(directory=tmp_path, tree_path="templates/letter.md", content="Dear $name,\n")+ tree_listing = MethodTreeListing()+ tree_listing.method_tree(handed_over=tmp_path)++ template.write_text("Dear $name, edited\n", encoding="utf-8")+ tree_listing.method_tree(handed_over=tmp_path)+ with tree_listing.loading(), pytest.raises(MethodTreeChangedError) as exc_info:+ tree_listing.method_tree(handed_over=tmp_path)++ problem = exc_info.value.to_problem(view=ProblemView.CALLER)+ assert problem.code == CoreCode.METHOD_PACKAGE_INVALID+ assert _refusal_params(error=exc_info.value) == {"reason": MethodPackageReason.TREE_CHANGED, "path": "templates/letter.md"}+ assert "'templates/letter.md'" in exc_info.value.message++ def test_a_load_reads_a_tree_listed_before_it_again_once_and_a_tree_it_lists_never(self, mocker: MockerFixture, tmp_path: Path) -> None:+ listed_before = tmp_path / "listed-before"+ listed_by_the_load = tmp_path / "listed-by-the-load"+ _touch(directory=listed_before, tree_path="main.mthds")+ _touch(directory=listed_by_the_load, tree_path="main.mthds")+ tree_listing = MethodTreeListing()+ tree_listing.method_tree(handed_over=listed_before)+ read_again = mocker.spy(method_tree_listing, "ensure_tree_files_as_listed")++ with tree_listing.loading():+ tree_listing.method_tree(handed_over=listed_before)+ tree_listing.method_tree(handed_over=listed_by_the_load)+ with tree_listing.loading():+ tree_listing.method_tree(handed_over=listed_before)+ tree_listing.method_tree(handed_over=listed_by_the_load)+ with tree_listing.loading():+ tree_listing.method_tree(handed_over=listed_before)++ assert [call.kwargs["tree"].directory for call in read_again.call_args_list] == [listed_before.resolve()]++ def test_a_bundle_read_after_its_listing_is_read_as_listed(self, tmp_path: Path) -> None:+ """A reader's text of a listed bundle is the bundle's when it is the listed file's, its line endings kept or made newlines."""+ bundle_file = tmp_path / "main.mthds"+ bundle_file.write_bytes(b"domain = 'listing_tests'\r\n")+ tree_listing = MethodTreeListing()+ tree_bundle = tree_listing.lone_file_bundle(bundle_file=bundle_file)++ assert read_tree_file_as_listed(tree=tree_bundle.tree, tree_path="main.mthds") == b"domain = 'listing_tests'\r\n"+ ensure_bundle_text_as_listed(tree_bundle=tree_bundle, mthds_content="domain = 'listing_tests'\n")+ ensure_bundle_text_as_listed(tree_bundle=tree_bundle, mthds_content="domain = 'listing_tests'\r\n")+ with pytest.raises(MethodTreeChangedError):+ ensure_bundle_text_as_listed(tree_bundle=tree_bundle, mthds_content="domain = 'another'\n")++ bundle_file.write_bytes(b"domain = 'edited'\n")+ with pytest.raises(MethodTreeChangedError, match="changed since the tree was listed"):+ read_tree_file_as_listed(tree=tree_bundle.tree, tree_path="main.mthds")+ bundle_file.unlink()+ with pytest.raises(MethodTreeChangedError, match="was removed or became unreadable"):+ read_tree_file_as_listed(tree=tree_bundle.tree, tree_path="main.mthds")++ def test_a_dependency_package_is_listed_and_judged_once_per_run(self, mocker: MockerFixture, tmp_path: Path) -> None:+ """Two loads of one run reading one package list its tree once and judge its requirements once."""+ package_root = tmp_path / "package"+ package_bundle = _touch(directory=package_root, tree_path="notes.mthds", content="domain = 'listing_dependency'\n")+ _touch(directory=package_root, tree_path="helpers.py", content="VALUE = 1\n")+ _touch(directory=package_root, tree_path="requirements.txt", content="pydantic>=2\n")+ resolved_dep = ResolvedDependency(+ alias="listing_dependency",+ address=_PACKAGE_ADDRESS,+ manifest=MethodsManifest(address=_PACKAGE_ADDRESS, version="1.0.0", description="A listed dependency"),+ package_root=package_root,+ mthds_files=[package_bundle],+ exported_pipe_codes=None,+ )+ package_lister = mocker.spy(method_tree_listing, "list_package_method_tree")+ environment_resolver = mocker.spy(method_requirements, "resolve_pipe_func_package_environment")+ library_manager = get_library_manager()+ assert isinstance(library_manager, LibraryManager)+ tree_listing = MethodTreeListing()++ for _ in range(2):+ library_id, library = library_manager.open_library()+ try:+ with scoped_current_library(library_id=library_id):+ library_manager._load_single_dependency( # ruff: ignore[private-member-access] # pyright: ignore[reportPrivateUsage]+ library=library,+ library_id=library_id,+ resolved_dep=resolved_dep,+ package_address=_PACKAGE_ADDRESS,+ tree_listing=tree_listing,+ )+ finally:+ library_manager.teardown(library_id=library_id)++ assert package_lister.call_count == 1+ assert environment_resolver.call_count == 1diff --git a/tests/unit/pipelex/libraries/test_method_trees.py b/tests/unit/pipelex/libraries/test_method_trees.pyindex 705303819..9d0ee614d 100644--- a/tests/unit/pipelex/libraries/test_method_trees.py+++ b/tests/unit/pipelex/libraries/test_method_trees.py@@ -18,7 +18,6 @@ from pipelex.libraries.method_trees import (list_package_method_tree,reporting_host_library_tree_faults,submitted_tree_files,- tree_bundle_of_lone_file,tree_python_sources,)from pipelex.methods.package_reason import MethodPackageReason@@ -110,22 +109,6 @@ class TestMethodTrees:assert caller.params is not Noneassert caller.params.as_wire() == {"reason": MethodPackageReason.TOO_MANY_FILES, "limit": 1}- def test_a_bundle_file_named_alone_is_rooted_at_the_manifest_above_it_else_alone(self, tmp_path: Path) -> None:- """Named alone, a bundle reads the files of the method a METHODS.toml above it roots, and with none only itself."""- lone = _touch(directory=tmp_path, tree_path="loose/lone.mthds")- _touch(directory=tmp_path, tree_path="loose/notes.md")- _touch(directory=tmp_path, tree_path="method/METHODS.toml", content=_MANIFEST)- rooted = _touch(directory=tmp_path, tree_path="method/bundles/main.mthds")-- lone_bundle = tree_bundle_of_lone_file(bundle_file=lone)- rooted_bundle = tree_bundle_of_lone_file(bundle_file=rooted)-- assert lone_bundle.tree.is_one_file- assert lone_bundle.tree.tree_paths == ["lone.mthds"]- assert not rooted_bundle.tree.is_one_file- assert rooted_bundle.tree_path == "bundles/main.mthds"- assert rooted_bundle.tree.tree_paths == ["METHODS.toml", "bundles/main.mthds"]-def test_a_library_directory_holding_only_nested_methods_loads_them_all(self, tmp_path: Path) -> None:"""Its own tree is empty, yet the packages and repositories nested in it load, and those nested in an empty one of them."""_touch(directory=tmp_path, tree_path=".hidden-notes.md", content="kept out")diff --git a/tests/unit/pipelex/pipeline/fixes/test_fix_method_folder.py b/tests/unit/pipelex/pipeline/fixes/test_fix_method_folder.pyindex 02cc5cc83..d05626000 100644--- a/tests/unit/pipelex/pipeline/fixes/test_fix_method_folder.py+++ b/tests/unit/pipelex/pipeline/fixes/test_fix_method_folder.py@@ -5,6 +5,7 @@ import tomlkitfrom pytest_mock import MockerFixturefrom pipelex.core.exceptions import PipesAndConceptValidationErrorData+from pipelex.libraries.method_tree_listing import MethodTreeListingfrom pipelex.pipeline.exceptions import ValidateBundleErrorfrom pipelex.pipeline.fixes import fix_loopfrom pipelex.pipeline.fixes.fix_loop import fix_method_folder@@ -87,12 +88,18 @@ class TestFixMethodFolder:assert summary_path.read_text(encoding="utf-8") == _SUMMARY_MTHDSassert validate_mock.await_count == 2for call in validate_mock.await_args_list:- assert call.kwargs == {+ assert {key: value for key, value in call.kwargs.items() if key != "tree_listing"} == {"method_dir": method_dir,"library_dirs": [method_dir],"library_dirs_are_callers": True,"allow_signatures": False,}+ # Each validation reads the listing its round's snapshot was taken from, and the round that wrote the fix+ # ended its listing, so the validation after it lists the trees anew.+ first_listing, second_listing = [call.kwargs["tree_listing"] for call in validate_mock.await_args_list]+ assert isinstance(first_listing, MethodTreeListing)+ assert isinstance(second_listing, MethodTreeListing)+ assert second_listing is not first_listingasync def test_a_sourceless_fix_is_dropped_since_a_folder_has_no_entry_file(self, tmp_path: Path, mocker: MockerFixture) -> None:"""With no entry file, a fix that names no file has nowhere to land: nothing is written."""diff --git a/tests/unit/pipelex/pipeline/test_runner_validate_plumbing.py b/tests/unit/pipelex/pipeline/test_runner_validate_plumbing.pyindex 919f1db66..4a205a610 100644--- a/tests/unit/pipelex/pipeline/test_runner_validate_plumbing.py+++ b/tests/unit/pipelex/pipeline/test_runner_validate_plumbing.py@@ -99,6 +99,7 @@ class TestRunnerValidatePlumbing:submitted_files=submitted_files,method_dir=None,methods_dirs=None,+ tree_listing=mocker.ANY,)async def test_no_library_dirs_passes_none_and_strict_default(self, mocker: MockerFixture) -> None:@@ -118,6 +119,7 @@ class TestRunnerValidatePlumbing:submitted_files=None,method_dir=None,methods_dirs=None,+ tree_listing=mocker.ANY,)async def test_validation_runs_with_the_protocols_caller_in_scope(self, mocker: MockerFixture) -> None: