Build AI methods with your coding agent, then run them anywhere: as an MCP for chatbots, as a webapp for people, or via API for your software.
feature/Runtime-reads-trees · L-261015-2a5c46 (#1432)
Louis Choquel12f7d128306a
diff --git a/.drift/acks/cli-docs.toml b/.drift/acks/cli-docs.tomlindex 070ef6036..2f63444cf 100644--- a/.drift/acks/cli-docs.toml+++ b/.drift/acks/cli-docs.toml@@ -1,8 +1,8 @@contract = "cli-docs"-digest = "sha256:051eca7283ff3508d8f7cdea3df9d9b2fea5ff3b7ad9ccd8d965084ec764c8d0"-reviewed_by = "Louis Choquel"-reviewed_at = "2026-10-12T01:17:29Z"-rationale = "clean-pass: both validate cores now ask the sweep to collect the dry-run warnings explicitly (collect_dry_run_warnings=True), which keeps what every validate surface carries exactly as docs/tools/cli/validate.md, docs/tools/cli/agent-cli.md and the agent CLI guide already describe; no CLI doc change needed."+digest = "sha256:36a85341cb88dc54c1e7898ed2acca893097e68c77386b0200b9dce7ca74a397"+reviewed_by = "Claude Opus 5.5"+reviewed_at = "2026-10-16T00:08:00Z"+rationale = "real-catch: docs/tools/cli/run.md now says a hosted folder run sends its main pipe as domain.pipe_code and that output folders are labelled by the bare pipe code on hosted runs too; docs/tools/cli/agent-cli.md and pipelex/cli/agent_cli/CLAUDE.md say pipelex-agent run writes into results/<pipe_code>_output_NN/ with a .gitignore beside its bundle, else under mthds-wip/, with the override cases."[trigger_files]"pipelex/cli/__init__.py" = "blob:8b137891791fe96927ad78e64b0aad7bded08bdc"@@ -36,11 +36,11 @@ rationale = "clean-pass: both validate cores now ask the sweep to collect the dr"pipelex/cli/agent_cli/commands/plxt_passthrough.py" = "blob:289adc65f49a185ca9f81bb73f87b4c68facb62b""pipelex/cli/agent_cli/commands/run/__init__.py" = "blob:e69de29bb2d1d6434b8b29ae775ad8c2e48c5391""pipelex/cli/agent_cli/commands/run/_output_helpers.py" = "blob:40ed1700e0d5e86d7696eadd7f9887baeb8059f6"-"pipelex/cli/agent_cli/commands/run/_run_core.py" = "blob:6049690867e53e8efcf92d0e44bd4311dfe9f7f8"+"pipelex/cli/agent_cli/commands/run/_run_core.py" = "blob:d71aee873856cfa72eb0ff18eb82c79e348f124a""pipelex/cli/agent_cli/commands/run/_run_hosted.py" = "blob:cea011e61feb082a7b9d82d7693b4fd3246737e5""pipelex/cli/agent_cli/commands/run/app.py" = "blob:7546d9b31ea2dc22b1ddae538edfbc8e7f7d08df"-"pipelex/cli/agent_cli/commands/run/bundle_cmd.py" = "blob:86988b3506ebc057df6a55d8862105ba6ceb693b"-"pipelex/cli/agent_cli/commands/run/method_cmd.py" = "blob:a83bb4df195d696e72a9c51704ee6c681a6d50a1"+"pipelex/cli/agent_cli/commands/run/bundle_cmd.py" = "blob:593854e8d3e39f5fe31d6b0b0c71b136933f06a7"+"pipelex/cli/agent_cli/commands/run/method_cmd.py" = "blob:776d1cf4bfbdce645f3cbfcd28be46bfb97174b0""pipelex/cli/agent_cli/commands/run/pipe_cmd.py" = "blob:f421c75ee696c18f85a7f63805455ef23637302f""pipelex/cli/agent_cli/commands/run/stdin_resolver.py" = "blob:c7c7d7495508f1c4e5a46eba336a4ab1694ebcf7""pipelex/cli/agent_cli/commands/validate/__init__.py" = "blob:e69de29bb2d1d6434b8b29ae775ad8c2e48c5391"@@ -105,17 +105,19 @@ rationale = "clean-pass: both validate cores now ask the sweep to collect the dr"pipelex/cli/commands/plugins_cmd.py" = "blob:9657b713c3650ff4a929e92ae68af453d896393e""pipelex/cli/commands/resolve_cmd.py" = "blob:88237df0f048bbf5a02e585baef932bfc355e1ee""pipelex/cli/commands/run/__init__.py" = "blob:e69de29bb2d1d6434b8b29ae775ad8c2e48c5391"-"pipelex/cli/commands/run/_hosted_sources.py" = "blob:e7ce03fcff7d474f013d6de0882b9c468274a461"+"pipelex/cli/commands/run/_hosted_sources.py" = "blob:d2e7efc8e0f9bb5f061d30afc586165c47d2dac8""pipelex/cli/commands/run/_inputs_file_loader.py" = "blob:957e6eb00531d168b3b1e179a517717d069cb7fc""pipelex/cli/commands/run/_inputs_path_resolver.py" = "blob:c8ec89d98718e18af7aaaded44c9f545be60adab""pipelex/cli/commands/run/_main_stuff_file.py" = "blob:ca806e55085aa2e5748492751521296ce1f42960"-"pipelex/cli/commands/run/_run_core.py" = "blob:f812f4b766a2ce3d348aa031783ebcd37892fd0e"-"pipelex/cli/commands/run/_run_hosted.py" = "blob:dd5801877a636f71187e9b23c2fab18d4d25ba32"+"pipelex/cli/commands/run/_method_folder.py" = "blob:11bd4f37a1a4fdee324c7aa5623ddab85cbebf81"+"pipelex/cli/commands/run/_run_core.py" = "blob:5b46ee698300deca87491d0491177fd4fe891220"+"pipelex/cli/commands/run/_run_hosted.py" = "blob:261439461abc1532d16393f33baf32f770207814"+"pipelex/cli/commands/run/_run_output_folder.py" = "blob:5834889a5e35e6457acf24761b408bc6a7bbfd6c""pipelex/cli/commands/run/app.py" = "blob:e312b159625e0a72bc9a18d3bfd2c73e7ac59f24"-"pipelex/cli/commands/run/bundle_cmd.py" = "blob:aba274dc8d6d0ac08baa3f21afbc06b2fdbec356"+"pipelex/cli/commands/run/bundle_cmd.py" = "blob:801027bff8303114da7b1c2d43c8e3715b0be3c2""pipelex/cli/commands/run/exceptions.py" = "blob:e9552226e597922b6ac0757f7b52eb880a2e1472""pipelex/cli/commands/run/method_cmd.py" = "blob:924bca6e242d8490e32eeb9ed2329db3d1b58b8d"-"pipelex/cli/commands/run/pipe_cmd.py" = "blob:f58885186d59142146eed340451b21832526af47"+"pipelex/cli/commands/run/pipe_cmd.py" = "blob:eafdc4bcf3f3f84042ecf9644e0f17fc355e0e17""pipelex/cli/commands/show_cmd.py" = "blob:4c87c70058ea3b15827d03a370f3345569a2b705""pipelex/cli/commands/update_cmd.py" = "blob:b2db6170ddd9c67f17d88096ba1f1796a897557d""pipelex/cli/commands/usage_stats_cmd.py" = "blob:b3723c64f911d1b69e1e1aa126610a132a16e3da"diff --git a/.drift/acks/config-docs.toml b/.drift/acks/config-docs.tomlindex 8953fc986..286bef0e7 100644--- a/.drift/acks/config-docs.toml+++ b/.drift/acks/config-docs.toml@@ -1,8 +1,8 @@contract = "config-docs"-digest = "sha256:2af32cbadd596a5408a68ca05db6073d687342949b12c224c77ae66318097426"-reviewed_by = "Louis Choquel"-reviewed_at = "2026-10-12T21:51:11Z"-rationale = "clean-pass: DynamoDBTracingConfig gains retention_days, an int of at least 1 with its default of 30 in pipelex/pipelex.toml and in the kit and project copies. The change is additive and its bound is an annotation the fingerprint records, so the regenerated head golden accounts for it and no ledger entry is needed. docs/configuration has no tracing page to go stale; the setting is documented with the tracing backends in docs/under-the-hood/execution-graph-tracing.md."+digest = "sha256:1de0bee628b09fb93a93f501aaaca3d5871fa21372874986b66cdc64d9da5a1a"+reviewed_by = "Claude Opus 5.5"+reviewed_at = "2026-10-15T23:13:43Z"+rationale = "real-catch: InterpreterConfig gains method_tree_limits ([interpreter.method_tree_limits], max_files and max_total_bytes required with their defaults in pipelex/pipelex.toml and the kit and project copies, max_file_bytes optional). docs/configuration/index.md now names the tree limits among the interpreter sections and library-config.md documents which files a library directory loads, the limits, and that [interpreter.scan] excluded_dirs now only scopes the Python scan; the section is documented in full in docs/building-methods/method-trees.md. The change is additive with its bounds in annotations, so the regenerated head goldens account for it and no ledger entry is needed."[trigger_files]"pipelex/cogt/config_cogt.py" = "blob:293ef22ae123266e6a1886d4ffb07f455e5be9e3"@@ -10,14 +10,14 @@ rationale = "clean-pass: DynamoDBTracingConfig gains retention_days, an int of a"pipelex/graph/graph_config.py" = "blob:dc0939a6d2730c0a10b88aa1a6e0b2dd583462a6""pipelex/language/mthds_config.py" = "blob:353009ad8c59dc5266bdd9dfa42b1ad18b67dca7""pipelex/methods/methods_config.py" = "blob:76ca9a0e64f8c10ef6a355aafd0d79d36b09f06a"-"pipelex/pipelex.toml" = "blob:ec186eff52f9a612964ec661cd56bf5b198a698b"+"pipelex/pipelex.toml" = "blob:bf914ead3600e40f228577afef8a196de0ce771f""pipelex/system/configuration/__init__.py" = "blob:8b137891791fe96927ad78e64b0aad7bded08bdc""pipelex/system/configuration/config_check.py" = "blob:2d5de29aba3d2ce205d3d82b4623f37b53108332""pipelex/system/configuration/config_loader.py" = "blob:8703fc3e5a82d7c6a3d148ea195c25044c2f0fe0""pipelex/system/configuration/config_model.py" = "blob:b7d141d559c0302f0d8c56859ae0a8edeb9bfdf0""pipelex/system/configuration/config_root.py" = "blob:c7d43851270912d17413cfc6a511247d3c7cc160""pipelex/system/configuration/config_surface.py" = "blob:49424a8103a5611f5b2885135dd699273bde57c8"-"pipelex/system/configuration/configs.py" = "blob:fb032dd59182f0c188201567026dbab0cc8f8774"+"pipelex/system/configuration/configs.py" = "blob:74175f5e5aac2d50af6f319d3706db0d90dd5eab""pipelex/system/configuration/pipe_func_config.py" = "blob:dc8890e914fbcdb611147e42eaa0cd87b32288a5""pipelex/system/telemetry/telemetry_config.py" = "blob:068a785d97a2d52ee43e82e723cd944b8b6601bc""pipelex/tools/aws/aws_config.py" = "blob:0a7c9dbf276e61355b1fa79b9db8caf252450a39"diff --git a/.pipelex/pipelex.toml b/.pipelex/pipelex.tomlindex 665b27ceb..c230ffc40 100644--- a/.pipelex/pipelex.toml+++ b/.pipelex/pipelex.toml@@ -229,6 +229,17 @@ excluded_dirs = ["results",]+####################################################################################################+# Method Tree Limits Config+####################################################################################################++[interpreter.method_tree_limits]+# How large a method tree this machine lists when it reads a method: a method's folder, a .mthds file named alone, a+# library directory or a dependency package. Listing stops before reading any file once a tree passes a limit.+# Raise them here for a method that needs more.+max_files = 10000+max_total_bytes = 1073741824+##################################################################################################### Builder Config####################################################################################################diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 210d2e59f..20fb84092 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -2,8 +2,23 @@## [v0.86.1] - 2026-10-16+### Added++- **`[interpreter.method_tree_limits]`**: the limits every method tree is listed under on this machine, `max_files` (10,000) and `max_total_bytes` (1 GiB) by default with `max_file_bytes` unset, so a folder handed over by mistake is refused before any file is hashed, with the advice to run the method's own folder or to raise the limits; these are not the hosted API's limits, which a deployment sets.+- **A run records its method tree**: `pipelex run bundle` prints the digest of the method tree it read with its recap and writes `method_tree.json` (the digest, the tree's root, its entry and each file's path, hash and size) in its output folder, and `pipelex-agent run bundle` adds `method_tree_digest` to its output file and to its `--with-memory` output.+- **`MethodTreeLoadError`, `MethodEntryError`, `CrateIntegrityError` and `HostLibraryTreeError`**: the `error_type` of a method tree that cannot be listed (a symbolic link, a tree past its limits), of a method folder with no single pipe to start at, of a library crate naming a method tree it does not carry, and of a server's own library directory whose tree cannot be listed beside the bundles a caller submitted, a configuration fault whose message, naming the server's paths, never reaches the caller.+### Changed+- **Library directories load through their method trees (Breaking)**: a library directory's `.mthds` files are the bundles of its method tree and of the trees of the nested packages and repositories under it, each loaded as a method of its own, so a hidden, gitignored or virtual-environment `.mthds` file is no longer loaded, a symbolic link in the tree refuses the load, and `[interpreter.scan] excluded_dirs` now applies only to the scan for Python classes and functions; a hosted run sends the same bundles. A directory holding nothing but nested methods loads them, a nested package or repository holding no `.mthds` file is skipped before it is listed, and a dependency package whose tree cannot be listed is refused naming the package by its address, never the directory it was installed into.+- **`-L <subdir>` inside a package loads the whole package (Breaking)**: a library directory is listed as its method tree, which is rooted at the nearest `METHODS.toml` at or above it, so naming a subfolder of a package with `-L`, or as a library directory of a runner, loads every bundle of the package.+- **A `.mthds` file named alone belongs to its method tree (Breaking)**: a bundle file handed to `run bundle`, `validate bundle` or a runner is the member of the tree rooted at the nearest `METHODS.toml` at or above its folder, and with none its method is that one file, so a step naming any other file, such as a `template_file`, is refused with the advice to run the method's folder or to add a `METHODS.toml`.+- **`template_file` resolves in the bundle's method tree (Breaking)**: a PipeDocGen `template_file` read from disk is looked up among the files of its bundle's method tree, where a `..` climbing to another folder of the tree is now allowed, and a file the tree does not hold (above its root, gitignored, hidden or in a nested package) is refused, the message naming each file by its path in the tree; a URL is refused when the bundle parses, and a bundle loaded from a string still reads only the files sent with it.+- **`run bundle <folder>` starts at the method's main pipe (Breaking)**: with no `--pipe`, `pipelex run bundle` and `pipelex-agent run bundle` on a folder, locally as on the hosted API, start at its `METHODS.toml`'s `main_pipe`, or with no manifest at the one `main_pipe` its bundles declare, and refuse none or several, naming each bundle by its path in the tree; a `bundle.mthds` or a single `.mthds` file in the folder is no longer looked for. A hosted run names that pipe as `domain.pipe_code`, so it starts where the local run would when two domains of the method define the same code.+- **Run output folders hold a `.gitignore`**: every folder `pipelex run` creates for a run's outputs holds a `.gitignore` ignoring everything in it, so a run started inside a method's folder leaves the method's tree, and its digest, unchanged; the folder is named after the pipe's own code even when the pipe was named as `domain.pipe_code`, on a hosted run too.+- **`pipelex-agent run` writes into an output folder (Breaking)**: its output file (`live_run.json` or `dry_run.json`) and its graph files go into a new numbered folder, `results/<pipe_code>_output_<NN>/` beside the bundle it ran, else under `mthds-wip/`, holding the same `.gitignore`, instead of beside the bundle itself, so two runs of a method read the same tree, with the same `method_tree_digest`.+- **`LibraryCrate` carries method trees (Breaking)**: `host_bundle_files` gives way to the transport-only `method_trees`, each tree's listing as JSON keyed by the tree, and `tree_bundles`, each pipe's bundle's place in its tree, so a crate loaded again, in a Temporal workflow included, resolves a step's file in the tree the host listed.+- **`mthds` and `pipelex-sdk` versions, MTHDS protocol 1.0.0 and standard 7.0.0 (Breaking)**: the pins move to `mthds==0.22.0`, the release that lists method trees, carries protocol 1.0.0 and implements MTHDS standard 7.0.0, and `pipelex-sdk==0.45.0`, the release built on it, so the runner and `GET /v1/version` report `protocol_version` `1.0.0`, a crate pipelex normalizes is stamped `mthds_version` `7.0.0`, and a `METHODS.toml` declaring `mthds_version = "^5.0.0"` now warns that the engine does not satisfy it. A dependency package fetched before is fetched again into the release's new cache layout, a package's `METHODS.toml` is found only at its root and manifest discovery stops at any `.git` entry; everyone downstream inherits those exact versions.- **A scoped event log captures usage events**: a usage event emitted while `runtime_hub.scoped_event_log` is set now goes into the scoped event log and nowhere else, ahead of an event log registered with `ReportingManager.set_event_log` and of the per-process runner fallback, including inside an isolated sub-execution such as a Temporal activity, where it always went to the fallback. Code running a leaf inside an activity can therefore scope a fresh `InMemoryEventLog` around it and return the leaf's usage events with its result, which is how a backend with no shared tracing store, such as Mistral Workflows, collects a run's usage; with no scoped event log set, usage events are routed as before.## [v0.86.0] - 2026-10-13diff --git a/docs/api-server/openapi/pipelex-api.openapi.yaml b/docs/api-server/openapi/pipelex-api.openapi.yamlindex e1d9acb8e..0d14273e6 100644--- a/docs/api-server/openapi/pipelex-api.openapi.yaml+++ b/docs/api-server/openapi/pipelex-api.openapi.yaml@@ -2,8 +2,8 @@openapi: 3.1.0info:title: Pipelex API- summary: The source-available Pipelex runner — implements MTHDS Protocol v0.7.0.- description: 'This server implements the [MTHDS Protocol](https://mthds.ai) v0.7.0 (`POST /execute`, `POST /start`, `POST+ summary: The source-available Pipelex runner — implements MTHDS Protocol v1.0.0.+ description: 'This server implements the [MTHDS Protocol](https://mthds.ai) v1.0.0 (`POST /execute`, `POST /start`, `POST/validate`, `GET /models`, `GET /version` — marked `x-mthds-protocol: true`) plus the Pipelex API extensions: resolveand codegen (`/resolve`, `/codegen`), the pipe I/O route (`/pipe-io`), the model reference check (`/models/check`), andeditor tooling (`/lint`, `/format`). Contract layering: MTHDS Protocol ⊂ Pipelex API (this server) ⊂ Pipelex hosted API.diff --git a/docs/building-methods/method-trees.md b/docs/building-methods/method-trees.mdnew file mode 100644index 000000000..71cbc9711--- /dev/null+++ b/docs/building-methods/method-trees.md@@ -0,0 +1,60 @@+---+title: "Method Trees"+description: "How Pipelex reads a method from your disk: the method tree a folder or a bundle file roots, the files it keeps out, where a run starts, and what a run records."+---++# Method Trees++A method is not one `.mthds` file but a folder of files: its bundles, the template files its steps print with, its Python modules, its `METHODS.toml`. The MTHDS standard calls the set of files a method is made of its **method tree**, and Pipelex reads a method from your disk only through that tree, locally as on the hosted API. A file the tree does not hold is invisible to the method's steps, however close to the bundle it lies.++## Where a tree is rooted++What you hand Pipelex decides the tree it reads:++- **A folder**, as in `pipelex run bundle invoicing/` or `-L invoicing/`, is the method rooted at the nearest `METHODS.toml` at or above it, or at the folder itself when there is none.+- **A `.mthds` file named alone**, as in `pipelex run bundle invoicing/invoice.mthds` or `pipelex validate bundle invoicing/invoice.mthds`, is the member of the tree rooted at the nearest `METHODS.toml` at or above its folder. With no `METHODS.toml` there, its method is **that one file**: a step of the bundle naming any other file, such as a `template_file`, is refused with a message saying to run the method's folder instead, or to add a `METHODS.toml` to it. Sending a folder is always your explicit act, so a stray `.mthds` file never carries its neighbours anywhere.+- **A dependency package** is the tree rooted at the package's own directory.++## What a tree holds++A tree holds the regular files under its root, except those its membership rule keeps out:++- every file or folder whose name starts with a dot, such as `.git/`, `.venv/` or `.env`;+- `__pycache__/` and `node_modules/` folders, and any folder holding a Python virtual environment;+- a nested repository (a folder holding `.git`) and a nested package (a folder holding its own `METHODS.toml`), each a tree of its own;+- every file a `.gitignore` of the tree ignores, matched as git matches it, and every file the manifest's `[package] exclude` patterns name.++A symbolic link anywhere in the tree refuses the whole tree: a tree holds regular files only, so replace the link with the file it names, or keep it out with a `.gitignore` pattern.++The tree's `.gitignore` files are what keeps a run's outputs out of the method: every folder `pipelex run` creates for its outputs holds a `.gitignore` ignoring everything in it, so a run started from the method's own folder leaves its tree, and its digest, as they were.++## File references++A file a bundle names, such as a [`PipeDocGen`](pipes/pipe-operators/PipeDocGen.md) step's `template_file`, is a path relative to the bundle's file, written with forward slashes, and it is resolved in the bundle's tree: joined to the bundle's own place in the tree, a `..` may climb out of the bundle's folder to a file elsewhere in the tree, and the result must be a file of the tree. So a bundle in `bundles/` can name `../templates/invoice.docx`, while a path climbing above the tree's root, an absolute path, a file a `.gitignore` keeps out and a file of a nested package are all refused, each refusal naming the files by their paths in the tree. Pipelex looks the path up among the tree's files and never opens a path a bundle wrote.++## Library directories++A library directory, such as a `-L` directory or one of `PIPELEXPATH`, is a place methods are found rather than one method. Pipelex loads its tree, then the tree of each nested package and nested repository under it, each as a method of its own whose steps read only its own files, so every method in the directory loads. A `.mthds` file kept out by the membership rule, a hidden one, a gitignored one or one inside a virtual environment, is not loaded.++## Where a folder's run starts++`pipelex run bundle <folder>` with no `--pipe` starts where the MTHDS protocol says a run naming no pipe starts, since the bundles of a folder have no order:++1. With a `METHODS.toml` at the tree's root, the run starts at its `main_pipe`, in the bundle of a domain exporting it that defines it. A manifest declaring no `main_pipe` is refused.+2. With no `METHODS.toml`, the run starts at the one `main_pipe` the tree's bundles declare. None, or several, is refused, the message naming each bundle by its path in the tree.++`--pipe` names the pipe to start at, as `pipe_code` or `domain.pipe_code`, and needs neither. A `.mthds` file named alone starts at its own `main_pipe`.++## Limits and the tree digest++Listing a tree reads and hashes every file in it. A local load lists each tree under this machine's own limits, set under `[interpreter.method_tree_limits]`: `max_files` (10,000 by default) and `max_total_bytes` (1 GiB by default), with `max_file_bytes`, the size of one file, unchecked unless you set it. The lister counts before it hashes anything, so a folder handed over by mistake, such as a home directory, is refused in moments, and the refusal says to run the method's own folder or to raise the limits in your project's `.pipelex/pipelex.toml`:++```toml+[interpreter.method_tree_limits]+max_files = 50000+max_total_bytes = 4294967296+```++These limits are not the hosted API's. A deployment sets its own, which a local run cannot know, and applying them locally would refuse a method rooted in a cluttered folder that loads today; a sender warned of the hosted limits before it sends is the hosted run's concern, not a local load's.++Every listing computes the tree's digest, the SHA-256 of its canonical listing of paths, hashes and sizes, which names the exact files of a method: two folders with the same files have the same digest. A local run records the digest of the tree it ran. `pipelex run bundle` prints it with the run's recap and writes `method_tree.json` into the run's output folder, holding the digest, the tree's root on this machine, its entry and each file's path, hash and size. `pipelex-agent run bundle` adds `method_tree_digest` to the run's output file and, with `--with-memory`, to its output.diff --git a/docs/building-methods/packages.md b/docs/building-methods/packages.mdindex efc90474a..edb74f95c 100644--- a/docs/building-methods/packages.md+++ b/docs/building-methods/packages.md@@ -170,7 +170,7 @@ your-project/└── requirements.txt```-The `METHODS.toml` file sits at the project root. Pipelex discovers it by walking up from bundle locations until it finds the manifest.+The `METHODS.toml` file sits at the project root. Pipelex discovers it by walking up from bundle locations until it finds the manifest, stopping at a repository's root, and the folder holding it roots the package's [method tree](method-trees.md): the files of the package its steps can read.## Current Public CLI Workflowdiff --git a/docs/building-methods/pipes/pipe-operators/PipeDocGen.md b/docs/building-methods/pipes/pipe-operators/PipeDocGen.mdindex 5494b8cc4..cc36e3bab 100644--- a/docs/building-methods/pipes/pipe-operators/PipeDocGen.md+++ b/docs/building-methods/pipes/pipe-operators/PipeDocGen.md@@ -81,7 +81,7 @@ To print the same invoice on the plugin's `pipelex-pdf`, the step names it: `mod| `format` | string | `pdf`, `xlsx`, `docx` or `pptx`. It sets the file's type and suffix. | Yes || `model` | string | The document engine that prints the file, a `doc_gen` model, alias or preset, such as `reportlab-pdf` or, with the plugin, `pipelex-pdf`. It defaults to the model deck's engine for the format and source. | No || `template` | string | An inline HTML and Jinja2 template, for `pdf` only. A `pdf` with a template is printed by the plugin. | No |-| `template_file` | string | A template file relative to the bundle's file, inside its bundle and written with forward slashes: `.html` for `pdf`, `.xlsx`, `.docx` or `.pptx`. An empty or absolute path, a `..` segment, a backslash and a colon are refused, the rule the files sent with a bundle are keyed by, so any template file can be sent with its bundle. Read beside the bundle when it is loaded from a directory, and from the files sent with it when it is loaded from a string. It excludes `template`. `pptx` requires one. | No |+| `template_file` | string | A template file relative to the bundle's file, written with forward slashes: `.html` for `pdf`, `.xlsx`, `.docx` or `.pptx`. An empty or absolute path, a URL, a backslash and a colon are refused. When the bundle is read from a folder, the file is the one its [method tree](../../method-trees.md#file-references) holds at that path, a `..` climbing to another folder of the tree included, and a file the tree keeps out, such as a gitignored one, is refused; a bundle file run alone with no `METHODS.toml` at or above it reads no other file. When the bundle is loaded from a string, the file is one of the files sent with it, where a `..` segment names none. It excludes `template`. `pptx` requires one. | No || `filename` | string | A Jinja expression over the inputs for the file's name, without its suffix, such as `invoice-{{ invoice.number }}`. It defaults to the pipe's code. The rendered name is made into one every storage reader serves, the hosted API's included, spelled in ASCII with hyphens for the rest: see [The file's name](#the-files-name). | No |### The file's name@@ -145,7 +145,7 @@ When a bundle is sent as content rather than read from a directory, as it is thr}```-- **The key is the path the step names**, relative to the bundle and written with forward slashes: a step with `template_file = "templates/invoice.docx"` reads the file sent as `templates/invoice.docx`, and `./templates/invoice.docx` names the same file. Every bundle of the request is read as sitting in one directory. An absolute path, a `..` segment, a backslash and a colon are refused, as they are in a `template_file`.+- **The key is the path the step names**, relative to the bundle and written with forward slashes: a step with `template_file = "templates/invoice.docx"` reads the file sent as `templates/invoice.docx`, and `./templates/invoice.docx` names the same file. Every bundle of the request is read as sitting in one directory. An absolute path, a `..` segment, a backslash and a colon are refused.- **The value is the file's bytes** in standard base64.- **Every file sent must be one a step names.** A file no step reads is refused, so a misspelled key fails loudly rather than leaving its step without a template.- **The files are checked like files on disk.** An `.html` template file is checked against the inputs when the method loads, and the engine's template check runs on an office template file at validation and in the dry run.diff --git a/docs/configuration/config-technical/library-config.md b/docs/configuration/config-technical/library-config.mdindex 5558835ef..9081250ca 100644--- a/docs/configuration/config-technical/library-config.md+++ b/docs/configuration/config-technical/library-config.md@@ -59,7 +59,7 @@ This is the recommended approach for newcomers and simple projects. Pipelex readWhen you initialize Pipelex with `Pipelex.make()`, the system:-1. **Scans your project directory** for all `.mthds` files+1. **Lists the method trees of your library directories** and loads the `.mthds` files they hold2. **Discovers Python structure classes** that inherit from `StructuredContent`3. **Loads pipeline definitions** including domains, concepts, and pipes4. **Registers custom functions** decorated with `@pipe_func()`@@ -273,23 +273,17 @@ pipe_output = response.pipe_output5. **Include structure class directories**: Remember that `library_dirs` must contain both `.mthds` files AND Python files defining `StructuredContent` classes.-## Excluded Directories+## Which Files Are Loaded-To improve performance and avoid loading unnecessary files, Pipelex automatically excludes common directories from discovery:+A library directory's `.mthds` files are the bundles of its [method trees](../../building-methods/method-trees.md): the directory's own tree, and the tree of each nested package (a folder holding its own `METHODS.toml`) and nested repository (a folder holding `.git`) under it, each loaded as a method of its own whose steps read only its own files. A file the membership rule keeps out is never loaded:-- `.venv`, `venv`, `env`, `virtualenv`, `.virtualenv` - Virtual environments-- `.git` - Git repository data-- `__pycache__` - Python bytecode cache-- `.pytest_cache` - Pytest cache-- `.mypy_cache` - Mypy type checker cache-- `.ruff_cache` - Ruff linter cache-- `node_modules` - Node.js dependencies-- `.env` - Environment files-- `results` - Common output directory+- a hidden file or folder, such as `.venv/`, `.git/` or `.pytest_cache/`;+- a `__pycache__/` or `node_modules/` folder, or a folder holding a Python virtual environment, such as `venv/`;+- a file a `.gitignore` of the tree ignores, such as a run's output folder, which holds a `.gitignore` ignoring everything in it.-Files in these directories will not be scanned, even if they contain `.mthds` files or structure classes.+A library directory whose own tree is empty, one holding nothing but nested packages or repositories, loads the methods nested in it all the same. A nested package or repository that holds no `.mthds` file is skipped before its tree is listed, so a checkout with nothing to load, such as a submodule holding a symbolic link, never refuses the load; one that holds a bundle follows every rule below. A symbolic link inside a library directory's tree refuses the load, naming the link. A directory with more files than `[interpreter.method_tree_limits]` allows is refused before any file is read, with the advice to point at the method's own folder or to raise the limits. The limits are read once per load. When a server loads its own library directories beside the bundles a caller submits, a tree of those directories that cannot be listed is a `HostLibraryTreeError`, a fault of the server's configuration whose message, naming its paths, never reaches the caller; a dependency package's tree that cannot be listed is refused naming the package by its address, never by the directory it was installed into.-The list is the `excluded_dirs` array under `[interpreter.scan]` in `pipelex.toml`, each entry naming a directory; a value that is not an array, such as a single string, fails the boot.+The structure classes and `@pipe_func()` functions of a library directory are discovered by scanning its Python files, which skips the directories listed in the `excluded_dirs` array under `[interpreter.scan]` in `pipelex.toml` (virtual environments, `.git`, caches, `node_modules`, `results` and the like); a value that is not an array, such as a single string, fails the boot. That list no longer decides which `.mthds` files load.## Project Organizationdiff --git a/docs/configuration/index.md b/docs/configuration/index.mdindex c3c1b52f4..ed23fb1e5 100644--- a/docs/configuration/index.md+++ b/docs/configuration/index.md@@ -70,7 +70,7 @@ The configuration is organized into these main sections. The first three mirror1. `[runtime]` - process-scoped infrastructure: storage, secrets, logging, cloud credentials, outbound network posture, reporting, tracing, observation, and the plugin denylist2. `[inference]` - the model-calling seam: the model deck, LLM, image generation, extraction, the default templating style, and dry-run mocks-3. `[interpreter]` - library-scoped method machinery: MTHDS parsing, pipe runs, pipe functions, pipeline execution, source scanning, and the builder+3. `[interpreter]` - library-scoped method machinery: MTHDS parsing, the limits a method tree is listed under, pipe runs, pipe functions, pipeline execution, source scanning, and the builder4. `[run]` - where `pipelex run` and `pipelex-agent run` execute a method by default: on this machine or on the hosted Pipelex API, see [Run Configuration](config-practical/run-config.md)5. `[kit]` - settings for the `pipelex-dev` kit tooling6. `[usage_stats]` - `enabled = false` turns off Pipelex's anonymous usage statistics for everyone working in the project; it is read from the project's own `.pipelex/pipelex.toml` and from the configuration Pipelex is packaged with, never from `~/.pipelex/`, and `true` has no effect, see [Usage Statistics](../setup/usage-statistics.md)diff --git a/docs/errors/authoring-and-language.md b/docs/errors/authoring-and-language.mdindex 6e8b047ed..e0a092b8c 100644--- a/docs/errors/authoring-and-language.md+++ b/docs/errors/authoring-and-language.md@@ -116,13 +116,16 @@ own page. Classes are grouped by subsystem.- [`ConceptLibraryError`](concept-library-error.md) — Concept library- [`ConceptLoadingError`](concept-loading-error.md) — Concept loading+- [`CrateIntegrityError`](crate-integrity-error.md) — Crate integrity error- [`CrateNormalizationError`](crate-normalization-error.md) — Crate normalization error- [`DomainLibraryError`](domain-library-error.md) — Domain library- [`DomainLoadingError`](domain-loading-error.md) — Domain loading- [`EntryPipeAmbiguousError`](entry-pipe-ambiguous-error.md) — Entry pipe ambiguous- [`EntryPipeNotFoundError`](entry-pipe-not-found-error.md) — Entry pipe not found+- [`HostLibraryTreeError`](host-library-tree-error.md) — Host library tree- [`LibraryError`](library-error.md) — Library error- [`LibraryLoadingError`](library-loading-error.md) — Library loading+- [`MethodTreeLoadError`](method-tree-load-error.md) — Method tree load- [`PipeLibraryError`](pipe-library-error.md) — Pipe library- [`PipeLoadingError`](pipe-loading-error.md) — Pipe loading- [`PipeNotFoundError`](pipe-not-found-error.md) — Pipe not founddiff --git a/docs/errors/crate-integrity-error.md b/docs/errors/crate-integrity-error.mdnew file mode 100644index 000000000..b39b0cf9d--- /dev/null+++ b/docs/errors/crate-integrity-error.md@@ -0,0 +1,21 @@+---+title: "Crate integrity error"+description: "Reference for the `CrateIntegrityError` Pipelex error class."+---++<!-- pipelex:generated -->++# Crate integrity error++Raised when a library crate names a method tree it does not carry: the host that assembled it built it wrong.++| Field | Value |+|---|---|+| `error_type` | `CrateIntegrityError` |+| `title` | Crate integrity error |+| `type_uri` | `https://docs.pipelex.com/latest/errors/crate-integrity-error/` |+| `error_domain` | _(inherited from parent)_ |+| Defined in | `pipelex.libraries.exceptions` |+| Parent class | [`LibraryError`](library-error.md) |++[Back to Error Reference](index.md)diff --git a/docs/errors/execution-and-runtime.md b/docs/errors/execution-and-runtime.mdindex 3e713272d..309045385 100644--- a/docs/errors/execution-and-runtime.md+++ b/docs/errors/execution-and-runtime.md@@ -25,6 +25,7 @@ own page. Classes are grouped by subsystem.## Pipeline execution+- [`MethodEntryError`](method-entry-error.md) — Method entry- [`PipeExecutionError`](pipe-execution-error.md) — Pipe execution- [`PipeIOContractError`](pipe-io-contract-error.md) — Pipe IO contract- [`PipeStackOverflowError`](pipe-stack-overflow-error.md) — Pipe stack overflowdiff --git a/docs/errors/host-library-tree-error.md b/docs/errors/host-library-tree-error.mdnew file mode 100644index 000000000..9d6e90de0--- /dev/null+++ b/docs/errors/host-library-tree-error.md@@ -0,0 +1,22 @@+---+title: "Host library tree"+description: "Reference for the `HostLibraryTreeError` Pipelex error class."+---++<!-- pipelex:generated -->++# Host library tree++Raised when a library directory of this host, one a load reads beside a caller's bundles, cannot be read as a method tree.++| Field | Value |+|---|---|+| `error_type` | `HostLibraryTreeError` |+| `title` | Host library tree |+| `type_uri` | `https://docs.pipelex.com/latest/errors/host-library-tree-error/` |+| `error_domain` | `config` |+| Defined in | `pipelex.libraries.exceptions` |+| Parent class | [`PipelexError`](pipelex-error.md) |+| `retryable` | `false` |++[Back to Error Reference](index.md)diff --git a/docs/errors/method-entry-error.md b/docs/errors/method-entry-error.mdnew file mode 100644index 000000000..4d9bd7f85--- /dev/null+++ b/docs/errors/method-entry-error.md@@ -0,0 +1,23 @@+---+title: "Method entry"+description: "Reference for the `MethodEntryError` Pipelex error class."+---++<!-- pipelex:generated -->++# Method entry++Raised when a method run with no pipe named has no single pipe to start at.++| Field | Value |+|---|---|+| `error_type` | `MethodEntryError` |+| `title` | Method entry |+| `type_uri` | `https://docs.pipelex.com/latest/errors/method-entry-error/` |+| `error_domain` | `input` |+| Defined in | `pipelex.pipeline.exceptions` |+| Parent class | [`PipelexError`](pipelex-error.md) |+| `retryable` | `false` |+| `user_action` | `change_input` — Name the pipe to run, or declare the method's one main_pipe: in its METHODS.toml, or in one of its bundles |++[Back to Error Reference](index.md)diff --git a/docs/errors/method-tree-load-error.md b/docs/errors/method-tree-load-error.mdnew file mode 100644index 000000000..67a92f664--- /dev/null+++ b/docs/errors/method-tree-load-error.md@@ -0,0 +1,22 @@+---+title: "Method tree load"+description: "Reference for the `MethodTreeLoadError` Pipelex error class."+---++<!-- pipelex:generated -->++# Method tree load++Raised when a method tree a load reads from this host's disk cannot be listed.++| Field | Value |+|---|---|+| `error_type` | `MethodTreeLoadError` |+| `title` | Method tree load |+| `type_uri` | `https://docs.pipelex.com/latest/errors/method-tree-load-error/` |+| `error_domain` | `input` |+| Defined in | `pipelex.libraries.exceptions` |+| Parent class | [`LibraryLoadingError`](library-loading-error.md) |+| `retryable` | `false` |++[Back to Error Reference](index.md)diff --git a/docs/tools/cli/agent-cli.md b/docs/tools/cli/agent-cli.mdindex a2d09d7b9..25fcf0e76 100644--- a/docs/tools/cli/agent-cli.md+++ b/docs/tools/cli/agent-cli.md@@ -51,13 +51,15 @@ pipelex-agent run method <NAME> [OPTIONS]For `bundle` and `method`, use `--pipe` to target a specific pipe.+`run bundle` reads its method through its [method tree](../../building-methods/method-trees.md), as `pipelex run bundle` does. A folder is the method rooted at the nearest `METHODS.toml` at or above it, else at the folder itself, and with no `--pipe` the run starts at the manifest's `main_pipe`, or, with no manifest, at the one `main_pipe` its bundles declare; none or several is refused as a `ValidateBundleError` naming each bundle by its path in the tree. A bundle file named alone starts at its own `main_pipe`, and with no `METHODS.toml` at or above its folder its method is that one file, so a step naming a file beside it, such as a `template_file`, is refused with the advice to run the folder or to add a `METHODS.toml`. The run's output file (`live_run.json` or `dry_run.json`, in the run's output folder) carries `method_tree_digest`, the digest of the tree the run read, and so does the output under `--with-memory`.+#### Running on the hosted API-With `--runner hosted` (or `--hosted`, or `[run] execution = "hosted"`), the run executes on the hosted Pipelex API and nothing local is booted: no provider key or inference configuration is needed on this machine, only a Pipelex API key in `PIPELEX_API_KEY`, saved in `~/.pipelex/.env` by `pipelex login` or exported in the shell. A `.env` file Pipelex loads wins over the shell, and the working directory's wins over `~/.pipelex/.env`, so an exported key is read only when neither file sets the variable. What the run sends is what a local run would load, and only the contents of its `.mthds` files: a PipeFunc's Python, structure classes, `METHODS.toml` and the packages a dependency alias names stay on this machine, so a local bundle relying on them runs with `--runner local`. `run bundle` sends the bundle, then every `.mthds` file of its library directories. `run pipe` sends its library directories' files (the installed method exporting the pipe and `-L`, else `PIPELEXPATH`) and is refused when it has none. `run method` sends an installed or local method's files, but a published address (`github.com/owner/repo[/name][@tag]`) goes as a `method_ref` and a stored method's catalog id (`mt_…`) as a `method_id`, both resolved by the hosted API, so nothing is fetched or read locally and `-L` is refused with them. A local file named at a document or image input is uploaded first (a relative path in an inputs file resolves against that file's directory, and a `file://` URI is read as its path), and the run's inputs carry its storage URI, while a `.csv` path at a structured-list input is not uploaded, so give its rows as a JSON list; an explicit `null`, such as an optional image left out, reaches the run as `null`. The method's signature is read only when some value of the inputs names an existing file on this machine, so inputs naming none go straight to the run, a method calling another one by its address included. Such a method cannot have a local file uploaded yet: give the file as an `https://` URL. `run bundle` on a bundle file with no `-L` sends `PIPELEXPATH`'s files too, as a local run loads them, and `run method` takes a method installed under a name spelled like a catalog id (`mt_reports`) as that method. `--dry-run` and `--mock-inputs` are refused on a hosted run, while `--graph` / `--no-graph` and `--costs` / `--no-costs` are accepted and have no effect there, since a hosted run writes nothing to disk and emits no cost events on this machine (`pipelex run --hosted` refuses `--costs`). Its requests name their client in the `User-Agent` as `pipelex-cli/<version>`, as `pipelex run --hosted` does, so the hosted API counts them as CLI runs.+With `--runner hosted` (or `--hosted`, or `[run] execution = "hosted"`), the run executes on the hosted Pipelex API and nothing local is booted: no provider key or inference configuration is needed on this machine, only a Pipelex API key in `PIPELEX_API_KEY`, saved in `~/.pipelex/.env` by `pipelex login` or exported in the shell. A `.env` file Pipelex loads wins over the shell, and the working directory's wins over `~/.pipelex/.env`, so an exported key is read only when neither file sets the variable. What the run sends is what a local run would load, and only the contents of its `.mthds` files: a PipeFunc's Python, structure classes, `METHODS.toml` and the packages a dependency alias names stay on this machine, so a local bundle relying on them runs with `--runner local`. `run bundle` sends the bundle, then every `.mthds` file of its library directories' method trees, so a bundle the membership rule keeps out is never sent. `run pipe` sends its library directories' files (the installed method exporting the pipe and `-L`, else `PIPELEXPATH`) and is refused when it has none. `run method` sends an installed or local method's files, but a published address (`github.com/owner/repo[/name][@tag]`) goes as a `method_ref` and a stored method's catalog id (`mt_…`) as a `method_id`, both resolved by the hosted API, so nothing is fetched or read locally and `-L` is refused with them. A local file named at a document or image input is uploaded first (a relative path in an inputs file resolves against that file's directory, and a `file://` URI is read as its path), and the run's inputs carry its storage URI, while a `.csv` path at a structured-list input is not uploaded, so give its rows as a JSON list; an explicit `null`, such as an optional image left out, reaches the run as `null`. The method's signature is read only when some value of the inputs names an existing file on this machine, so inputs naming none go straight to the run, a method calling another one by its address included. Such a method cannot have a local file uploaded yet: give the file as an `https://` URL. `run bundle` on a bundle file with no `-L` sends `PIPELEXPATH`'s files too, as a local run loads them, and `run method` takes a method installed under a name spelled like a catalog id (`mt_reports`) as that method. `--dry-run` and `--mock-inputs` are refused on a hosted run, while `--graph` / `--no-graph` and `--costs` / `--no-costs` are accepted and have no effect there, since a hosted run writes nothing to disk and emits no cost events on this machine (`pipelex run --hosted` refuses `--costs`). Its requests name their client in the `User-Agent` as `pipelex-cli/<version>`, as `pipelex run --hosted` does, so the hosted API counts them as CLI runs.The success envelope has the shape of a local run's. Compact, it is the main output's content. With `--with-memory`, `main_stuff.json` is that content (the hosted API renders no Markdown, so the markdown output shows the JSON), `working_memory` is the whole working memory, `pipeline_run_id` names the run on the hosted API, and `uploads` lists the files uploaded for it. A hosted run writes nothing to disk.-A run refuses an invalid bundle while loading it, before any pipe runs, with exit code `1` and an error envelope whose `error_type` is `ValidateBundleError` and whose `validation_errors` array holds the items `validate` gives for that bundle, each with its pipe, its field and its suggested fix when it has one. A refusal found while loading a library directory you pass with `-L` (or the bundle's own directory) is reported the same way, its items naming their file in `source`; a bundle file run on its own is loaded from its text, so the items of a refusal found while loading it carry no `source`, while a bundle that does not parse when `run bundle` looks for its `main_pipe` (no `--pipe`) names its file. In markdown, the items render as the same grouped prose `validate` prints.+A run refuses an invalid bundle while loading it, before any pipe runs, with exit code `1` and an error envelope whose `error_type` is `ValidateBundleError` and whose `validation_errors` array holds the items `validate` gives for that bundle, each with its pipe, its field and its suggested fix when it has one. A refusal found while loading a library directory you pass with `-L` (or the method's folder, for `run bundle <dir>`) is reported the same way, its items naming their file in `source`; a bundle file run on its own is loaded from its text, so the items of a refusal found while loading it carry no `source`, while a bundle that does not parse when `run bundle` looks for its `main_pipe` (no `--pipe`) names its file. In markdown, the items render as the same grouped prose `validate` prints.!!! note "Stdin inputs stay JSON"The agent CLI can also read inputs from stdin (a flat dict or a `working_memory` envelope). Stdin inputs are **JSON-only** — the extension-based TOML discrimination applies to `--inputs` file paths only. Like the main CLI, `run bundle <dir>` auto-detects `inputs.json` / `inputs.toml` when `--inputs` is omitted, erroring if both exist. In a `working_memory` envelope each stuff must name its concept as the ref string; anything else there — the full concept object an older runtime dumped, for instance — is refused under `"error_type": "StdinEnvelopeShapeError"`, which is the label for an envelope that parsed as JSON but does not hold the shape the contract asks for, as distinct from the `JSONDecodeError` that means the JSON itself did not parse.@@ -226,4 +228,4 @@ For the structured commands, exactly one such object is written to stderr per faGraph visualization is available through the `validate bundle` subcommand with the `--graph` flag, and through `run` subcommands where it is enabled by default. Use `--no-graph` on `run` to disable it.-A `run` with the graph on writes, beside its bundle, the ReactFlow viewer (`dry_run.html` or `live_run.html`) and the graph itself: `graphspec.json` on a dry run, `live_run_graph.json` on a live run. Beside the graph, under their canonical names whatever the graph file is called, sit the three files that describe its data: `pipe_io_contracts.json`, `input_form.json` and `output_form.json`, the same artifacts `validate` reports under those names, keyed by namespaced `pipe_ref`. A graph viewer resolves them from the graph file's directory, not from its name. With `--with-memory`, the JSON envelope's `graph_files` object names every one of these paths (`graph_html`, `graph_spec` on a live run, `pipe_io_contracts`, `input_form`, `output_form`).+A `run` writes its outputs into a new numbered folder, `results/<pipe_code>_output_<NN>/` beside its bundle, or under `mthds-wip/` when it ran no bundle file; a folder run that named its pipe writes under the folder's `results/`, and a fetched method under `results/` in the working directory. The folder holds a `.gitignore` ignoring everything in it, so no output ever joins the method tree and the next run reads the same tree, with the same `method_tree_digest`. With the graph on, that folder holds the ReactFlow viewer (`dry_run.html` or `live_run.html`) and the graph itself: `graphspec.json` on a dry run, `live_run_graph.json` on a live run. Beside the graph, under their canonical names whatever the graph file is called, sit the three files that describe its data: `pipe_io_contracts.json`, `input_form.json` and `output_form.json`, the same artifacts `validate` reports under those names, keyed by namespaced `pipe_ref`. A graph viewer resolves them from the graph file's directory, not from its name. With `--with-memory`, the JSON envelope's `graph_files` object names every one of these paths (`graph_html`, `graph_spec` on a live run, `pipe_io_contracts`, `input_form`, `output_form`).diff --git a/docs/tools/cli/run.md b/docs/tools/cli/run.mdindex 6cecdb2ab..1c414bedc 100644--- a/docs/tools/cli/run.md+++ b/docs/tools/cli/run.md@@ -80,15 +80,17 @@ pipelex run pipe my_pipe -L ./pipelines --hosted --inputs data.jsonpipelex run bundle <PATH> [OPTIONS]```-Runs a pipeline from a bundle file (`.mthds`) or a pipeline directory. When a directory is given, the bundle file is auto-detected inside it — and so is an inputs file (`inputs.json` or `inputs.toml`) when `--inputs` is not passed (see [Input File Formats](#input-file-formats)).+Runs a method from a bundle file (`.mthds`) or a method's folder, read through its [method tree](../../building-methods/method-trees.md). A folder is the method rooted at the nearest `METHODS.toml` at or above it, else at the folder itself, and with no `--pipe` the run starts at the manifest's `main_pipe`, or, with no manifest, at the one `main_pipe` its bundles declare; none or several is refused, naming each bundle by its path in the tree. An inputs file (`inputs.json` or `inputs.toml`) in the folder is auto-detected when `--inputs` is not passed (see [Input File Formats](#input-file-formats)). A bundle file named alone starts at its own `main_pipe`, and with no `METHODS.toml` at or above its folder its method is that one file, so a step naming a file beside it, such as a `template_file`, is refused: run the folder instead, or add a `METHODS.toml`.++The run prints the digest of the tree it read with its recap, and records the tree as `method_tree.json` in its output folder (see [The results directory](#the-results-directory)).**Arguments:**-- `PATH` - Path to a `.mthds` bundle file or a directory containing one+- `PATH` - Path to a `.mthds` bundle file or a method's folder**Options:**-- `--pipe PIPE_CODE` - Run a specific pipe from the bundle (defaults to the bundle's main pipe)+- `--pipe PIPE_CODE` - Run a specific pipe, as `pipe_code` or `domain.pipe_code` (defaults to the method's main pipe)- `--inputs`, `-i` - Path to a JSON or TOML file containing inputs (discriminated by file extension), or inline JSON starting with `{` — see [Input File Formats](#input-file-formats)- `--output-dir`, `-o` - Directory to save outputs- `--save-main-stuff` / `--no-save-main-stuff` - Whether to save the main output@@ -202,7 +204,7 @@ Every request a hosted run sends names its client in its `User-Agent` as `pipele**What a hosted run sends** is what a local run would load:-- `run bundle` sends the bundle, then every `.mthds` file of its library directories (the bundle's own directory for `run bundle <dir>`, then `-L`; for a bundle file with no `-L`, `PIPELEXPATH`), each once. Without `--pipe`, the bundle's `main_pipe` is read from the bundle here, as a local run reads it.+- `run bundle` sends the bundle, then every `.mthds` file of its library directories' method trees (the method's folder for `run bundle <dir>`, then `-L`; for a bundle file with no `-L`, `PIPELEXPATH`), each once, so a bundle the membership rule keeps out is never sent. Without `--pipe`, the pipe is the one a local run would start at: the bundle's `main_pipe`, or for a folder the method's, selected here as a local run selects it and sent as `domain.pipe_code`, so the hosted API starts at that pipe even when another domain sent with it defines the same code.- `run pipe` sends every `.mthds` file of its library directories: the installed method exporting the pipe and `-L`, else `PIPELEXPATH`. With none of them, the run is refused and asks for `-L`.- `run method` sends an installed or local method's files. A published address goes as a reference and a catalog id (`mt_…`) as an id, both resolved by the hosted API: nothing is fetched or read locally, `-L` is refused with them, and a relative `--inputs` path resolves against the working directory rather than the method's. A method installed under a name spelled like a catalog id, such as `mt_reports`, is that installed method, not an id.@@ -283,7 +285,7 @@ The output JSON contains the complete working memory after pipeline execution, i### The results directory-Each run that saves anything gets its own numbered directory under `--output-dir` (`results/<pipe_code>_output_01/`, then `_02`, and so on). With `--save-working-memory` it holds `working_memory.json`, with `--save-main-stuff` the `main_stuff.*` renders, and with `--graph` the graph outputs that `graphs_inclusion` enables: `graphspec.json`, the Mermaid code and viewer, and the ReactFlow viewer. When the main output is a Document or an Image, `--save-main-stuff` also copies the file itself into the directory under its own name, so a PDF that a [`PipeDocGen`](../../building-methods/pipes/pipe-operators/PipeDocGen.md) step produced lands as `invoice-INV-2026-0142.pdf` and opens with a double-click, and the run prints its path. It never replaces a file the run writes in the directory: a file whose name is taken, such as a document passed through from an earlier run as `main_stuff.json`, is saved as `main_stuff-1.json`. A dry run produces no file, so it copies nothing. In `working_memory.json` each stuff names its concept by ref — `{"stuff_code": …, "stuff_name": …, "concept": "<domain>.<Code>", "content": …}` — and carries no concept definition; see [Working Memory](../../building-methods/pipes/working-memory.md#in-memory-and-on-the-wire).+Each run that saves anything gets its own numbered directory under `--output-dir` (`results/<pipe_code>_output_01/`, then `_02`, and so on), named after the pipe's own code even when it was named as `domain.pipe_code` or through a dependency's alias, on a hosted run too. The directory holds a `.gitignore` ignoring everything in it, so a run started from inside a method's folder never adds its outputs to the method's tree. A local run of a bundle or a folder also writes `method_tree.json` there, the record of the [method tree](../../building-methods/method-trees.md#limits-and-the-tree-digest) it read: its digest, its root on this machine, its entry and each file's path, hash and size. With `--save-working-memory` it holds `working_memory.json`, with `--save-main-stuff` the `main_stuff.*` renders, and with `--graph` the graph outputs that `graphs_inclusion` enables: `graphspec.json`, the Mermaid code and viewer, and the ReactFlow viewer. When the main output is a Document or an Image, `--save-main-stuff` also copies the file itself into the directory under its own name, so a PDF that a [`PipeDocGen`](../../building-methods/pipes/pipe-operators/PipeDocGen.md) step produced lands as `invoice-INV-2026-0142.pdf` and opens with a double-click, and the run prints its path. It never replaces a file the run writes in the directory: a file whose name is taken, such as a document passed through from an earlier run as `main_stuff.json`, is saved as `main_stuff-1.json`. A dry run produces no file, so it copies nothing. In `working_memory.json` each stuff names its concept by ref — `{"stuff_code": …, "stuff_name": …, "concept": "<domain>.<Code>", "content": …}` — and carries no concept definition; see [Working Memory](../../building-methods/pipes/working-memory.md#in-memory-and-on-the-wire).Beside `graphspec.json`, and gated by the same `graphspec_json` flag, the run writes the three files that describe the data the graph carries:@@ -299,7 +301,7 @@ A run whose main output resolves as a recorded absence (an optional `?` output t## An Invalid Bundle-A run loads its bundle before any pipe runs, and refuses an invalid one there, before spending anything. The refusal is reported the way [`pipelex validate`](validate.md) reports the same bundle: the grouped invalid-bundle panel, with one located item per refusal (its pipe, its field, its message and its suggested fix when it has one), and exit code `1`. A bundle file run on its own is loaded from its text, so the items of a refusal found while loading it name no file, and the panel names the bundle instead; a bundle that does not parse when `run bundle` looks for its `main_pipe` (no `--pipe`) names its file on the item too. That holds whether the parse or the load refuses the bundle, and for a refusal found while loading a library directory you pass with `-L` (or the bundle's own directory, for `run bundle <dir>`). When the items carry automatic fixes, the panel names the `pipelex fix bundle` command that applies them.+A run loads its bundle before any pipe runs, and refuses an invalid one there, before spending anything. The refusal is reported the way [`pipelex validate`](validate.md) reports the same bundle: the grouped invalid-bundle panel, with one located item per refusal (its pipe, its field, its message and its suggested fix when it has one), and exit code `1`. A bundle file run on its own is loaded from its text, so the items of a refusal found while loading it name no file, and the panel names the bundle instead; a bundle that does not parse when `run bundle` looks for its `main_pipe` (no `--pipe`) names its file on the item too. That holds whether the parse or the load refuses the bundle, and for a refusal found while loading a library directory you pass with `-L` (or the method's folder, for `run bundle <dir>`). When the items carry automatic fixes, the panel names the `pipelex fix bundle` command that applies them.## Related Documentationdiff --git a/mkdocs.yml b/mkdocs.ymlindex e53f77809..bb474e731 100644--- a/mkdocs.yml+++ b/mkdocs.yml@@ -262,6 +262,7 @@ plugins:- building-methods/pipes/index.md: "Design and Run Methods Overview"- building-methods/libraries.md: "Libraries"- building-methods/packages.md: "Packages"+ - building-methods/method-trees.md: "Method Trees"- building-methods/pipes/executing-pipelines.md: "Executing Pipelines"- building-methods/pipes/running-on-the-hosted-api.md: "Running on the Hosted API from Python"- building-methods/pipes/run-modes-and-backends.md: "Run Modes & Backends"diff --git a/pipelex/cli/agent_cli/CLAUDE.md b/pipelex/cli/agent_cli/CLAUDE.mdindex ac42fb36e..5a7ef16b7 100644--- a/pipelex/cli/agent_cli/CLAUDE.md+++ b/pipelex/cli/agent_cli/CLAUDE.md@@ -17,7 +17,7 @@ Only the error format is backed by a module-level `ContextVar` in `agent_output.Markdown structure per command:-- `run`: `# Pipeline run complete`, a `## Result` section (the rendered `main_stuff` markdown with `--with-memory`, otherwise the concept JSON in a fenced block), and output / graph file paths. With the graph on, the `graph_files` object of the `--with-memory` envelope names the viewer (`graph_html`), the graph on a live run (`graph_spec`, the graphspec renamed `live_run_graph.json`) and the graph's three companions written beside it under their canonical names (`pipe_io_contracts`, `input_form`, `output_form` — the validate artifacts describing the graph's data, keyed by `pipe_ref`); the agent CLI settles its graph inclusion (graphspec on, viewer on) before the run rather than at render time, since the companions are built inside the run's library window under that same flag; a viewer resolves them from the graph file's directory, so the rename never moves them. Cost is machine-first: on the local runner, when costs are on (`--costs`, the default) and the run did reportable work (any tokens or any cost), a best-effort `cost_report` object (`{total_cost, by_model: [...]}`, real USD) rides the JSON `--with-memory` envelope — there is **no** Rich cost table on stderr (unlike the human `pipelex run` CLI). The markdown renderer ignores the `cost_report` key, so it is JSON-only. A free/zero-price model still reports its token usage (with `total_cost` 0). The `cost_report` is **absent** when the run did no reportable work (dry runs), when `--no-costs` is passed, on a hosted run (which assembles no local usage), or if cost-summary aggregation fails (caught and skipped so it never fails an otherwise-successful run) — so consumers must treat it as optional.+- `run`: `# Pipeline run complete`, a `## Result` section (the rendered `main_stuff` markdown with `--with-memory`, otherwise the concept JSON in a fenced block), and output / graph file paths. With the graph on, the `graph_files` object of the `--with-memory` envelope names the viewer (`graph_html`), the graph on a live run (`graph_spec`, the graphspec renamed `live_run_graph.json`) and the graph's three companions written beside it under their canonical names (`pipe_io_contracts`, `input_form`, `output_form` — the validate artifacts describing the graph's data, keyed by `pipe_ref`); the agent CLI settles its graph inclusion (graphspec on, viewer on) before the run rather than at render time, since the companions are built inside the run's library window under that same flag; a viewer resolves them from the graph file's directory, so the rename never moves them. Cost is machine-first: on the local runner, when costs are on (`--costs`, the default) and the run did reportable work (any tokens or any cost), a best-effort `cost_report` object (`{total_cost, by_model: [...]}`, real USD) rides the JSON `--with-memory` envelope — there is **no** Rich cost table on stderr (unlike the human `pipelex run` CLI). The markdown renderer ignores the `cost_report` key, so it is JSON-only. A free/zero-price model still reports its token usage (with `total_cost` 0). The `cost_report` is **absent** when the run did no reportable work (dry runs), when `--no-costs` is passed, on a hosted run (which assembles no local usage), or if cost-summary aggregation fails (caught and skipped so it never fails an otherwise-successful run) — so consumers must treat it as optional. A local `run bundle` also carries `method_tree_digest`, the digest of the method tree it read, in its output file and its `--with-memory` envelope.- `validate`: `# Validation passed`, the bundle path when relevant, and a list of validated pipes with their status. Every entry in `validated_pipes` (markdown and JSON) identifies its pipe by the namespaced `pipe_ref` (`domain.code`) — the same unambiguous identity across `validate all`, `validate bundle`, and `validate pipe`, so the same pipe is never reported under two identifiers. `validate bundle`, `validate method`, and `validate all` carry `pending_signatures` — the library-wide list of pipes still declared as `PipeSignature` (unimplemented forward declarations), namespaced by `pipe_ref` — rendered as a "Pending signatures" section in markdown. They also carry a derived `is_runnable` boolean (`true` ⇔ `pending_signatures` is empty), and the markdown adds an explicit plain-English runnability verdict on those surfaces: a complete bundle/library states it is runnable, while one with pending signatures states it is NOT yet runnable immediately above the verbatim "Pending signatures" heading. These whole-bundle/whole-library surfaces are **strict by default**: they exit non-zero on `not is_runnable` unless `--allow-signatures` is passed (which both tolerates the placeholders for the exit code and mock-runs the signature pipes in the sweep). Single-pipe surfaces make no library-wide runnability claim and never gate the exit code on it: bare `validate pipe <code>` omits the key and emits no verdict, while `validate bundle`/`validate method` with `--pipe` still surface the library-wide `pending_signatures` for information but do not gate on it (the requested slice can be fully implemented even when unrelated placeholders remain elsewhere). The whole-bundle and whole-library surfaces also carry a `warnings` array — the advisory lints, then what the sweep's dry runs found without failing (a `PipeDocGen` template check's `template_check_warning`s), rendered as a "Warnings" section in markdown — assembled by `pipelex/pipeline/advisory_warnings.py`, the ONE composition point every advisory-bearing channel calls (this CLI, the bare CLI and the protocol report). Bare `validate pipe <code>` carries the dry-run findings alone (`pipelex/pipeline/dry_run_warnings.py`), since one pipe has no flow to lint but what its dry run found is about it; `validate bundle` / `validate method` with `--pipe` carry every family, because they validate the whole bundle and only narrow the dry run, so the array stays bundle-wide there. Add a new advisory family there, never at a call site: the sites used to assemble their own and disagreed about which families they carried.- `fix`: `# Fix applied - bundle is valid` after one or more fixes, `# Bundle already valid` when the loop had nothing to apply, or `# Bundle valid but not runnable` when pending `PipeSignature` placeholders remain. The JSON success envelope carries `is_valid`, `bundle_path`, `iterations`, `fixes_applied`, `files_written`, `remaining_errors`, and, on successful validation, `pending_signatures` plus `is_runnable`. Whole-bundle fix runs are strict by default: they emit the success envelope but exit 1 on `not is_runnable` unless `--allow-signatures` is passed. A still-invalid verdict exits 1 via `FixBundleError` on stderr with the same structured result fields plus any `bail_reason`.- `migrate`: `# Configuration migration`, the mode (applied / dry run) and verdict, the directories walked, a files-walked/changed/written count and, when a former release left something, its file count, then one `## <path>` section per file the former-release cleanup touches, listing its changes and its backup, a `## Pipelex still cannot start` section when the cleanup's check, run on every run, found something, and one per file that is not clean listing the entries applied, the entries blocked with their reason and guidance, and any unexplained paths. Paths, operation kinds, ledger-supplied values and the names of retired backends and profiles only — no value read from the user's file is ever rendered, in either format.@@ -102,9 +102,9 @@ commands/- **The `migration` field on a configuration error**: A `PipelexConfigError` carries one when a scan of this machine's configuration directories found something — and only then, so its presence is the test for whether the migration history has anything to say. Whether a command *repairs* it is the separate `would_write` field: true means the configuration is *old*, and the loop is `pipelex-agent migrate --dry-run --format json`, show the user, then `--yes`; false means the block is a diagnosis its `plans` carry and the command would rewrite nothing, so branching on presence alone sends an agent to a run with nothing to do. Its `plans` are the same shape that command emits under its own `plans` key. `error_domain` stays `config`: a domain the hook specification does not know routes to BLOCK, which would stop an agent instead of telling it what to run. See `docs/migration-ledger.md` → "Reporting a stale configuration on a validation error".- **Init**: All commands that need Pipelex use `make_pipelex_for_agent_cli(library_dirs)`. It catches init errors and routes them through `agent_error()`.- **Async core**: Run and validate are async — commands use `asyncio.run()`.-- **File convention**: Generated outputs go to `mthds-wip/` with incremental naming (`pipeline_01/`, `pipeline_02/`). Exception: `run method` on a fetched target (a method address or GitHub URL) anchors its run outputs in `results/` under the caller's CWD via `run_pipeline_core`'s `output_dir_override` — the fetched clone is an ephemeral temp dir deleted at process exit, so bundle-adjacent outputs would be lost.+- **File convention**: Generated outputs go to `mthds-wip/` with incremental naming (`pipeline_01/`, `pipeline_02/`). A `run` writes into a new numbered folder, `results/<pipe_code>_output_<NN>/`, made by `make_run_output_folder` (`pipelex/cli/commands/run/_run_output_folder.py`) beside the bundle it ran, else under `mthds-wip/`; the folder holds a `.gitignore` of `*`, so a run's outputs never join the method tree and two runs read the same tree, with the same `method_tree_digest`. `run_pipeline_core`'s `output_dir_override` replaces the directory the folder is made under: `run method` on a fetched target (a method address or GitHub URL) passes `results/` under the caller's CWD, since the fetched clone is an ephemeral temp dir deleted at process exit, and `run bundle <folder> --pipe` passes the folder's `results/`, having no entry bundle to write beside.- **Validate refusals are verdicts**: the `validate` commands run their library load inside `translate_to_validate_bundle_error` (`pipelex/pipeline/validate_bundle_translation.py`), so every input-domained refusal raised while loading a bundle, the unknown model (`PipeOperatorModelChoiceError`, an `unknown_model` item) included, reaches the command as a `ValidateBundleError` and leaves with the invalid-verdict envelope and exit 1. No `validate` command keeps an arm of its own for a model-choice error; exit 2 stays for a failure of the tool or its environment (config, runtime, security).-- **Run refusals are verdicts too**: a run loads its bundle inside the same translation (`acquire_library`), so an invalid bundle leaves `run pipe` / `run bundle` / `run method` as a `ValidateBundleError` with exit 1, through the generic `agent_error` arm, and `_assemble_error_payload` copies the report's `validation_errors` into the envelope for any cause that carries them. `run_pipeline_core` sets `library_dirs_are_callers=True`, so a refusal in a `-L` directory (or the bundle's own directory) is the same verdict; `run bundle`'s own parse of a bundle without `--pipe` goes through the translation too. So no `run` command keeps an arm of its own for a model-choice error either: an unknown model reaches it as the verdict's `unknown_model` item.+- **Run refusals are verdicts too**: a run loads its bundle inside the same translation (`acquire_library`), so an invalid bundle leaves `run pipe` / `run bundle` / `run method` as a `ValidateBundleError` with exit 1, through the generic `agent_error` arm, and `_assemble_error_payload` copies the report's `validation_errors` into the envelope for any cause that carries them. `run_pipeline_core` sets `library_dirs_are_callers=True`, so a refusal in a `-L` directory (or the bundle's own directory) is the same verdict; `run bundle`'s own parse of a bundle without `--pipe` goes through the translation too, and so does its reading of the method tree: a folder resolves through `pipelex/cli/commands/run/_method_folder.py` (`resolve_method_folder_run`, the protocol's entry rule in `pipelex/pipeline/method_entry.py`) and a lone file through `list_bundle_file_tree`, so a folder with no single `main_pipe`, a tree past its limits or a symbolic link in the tree is a `ValidateBundleError` too. So no `run` command keeps an arm of its own for a model-choice error either: an unknown model reaches it as the verdict's `unknown_model` item.- **Hosted runs**: every `run` subcommand takes `--runner local|hosted`, `--hosted/--local` and `--base-url`; `resolve_agent_run_execution()` (`run/_run_hosted.py`) settles them through `pipelex.hosted.execution.resolve_run_execution` (the flag, then `[run] execution`, then local), refusing a `--runner` that contradicts the flag pair and a `--base-url` on a local run; a `PipelexConfigError` there carries `AGENT_INIT_FAILURE_HINT`, the hint a failed boot gives (`agent_cli_factory.py`). A hosted run boots nothing: it builds a `HostedRunRequest` from what `pipelex/cli/commands/run/_hosted_sources.py` resolves (the bundle and its library's `.mthds` files, a pipe's library, or a method's files; an address or a catalog id as `method_ref` / `method_id`, never fetched) and `run_hosted_for_agent()` runs it through `pipelex.hosted.hosted_run.run_hosted`, which reads the signature and uploads local input files (only when some input string names an existing local file, so a method with an address-based dependency runs with any other inputs), then drives the lifecycle `start_and_wait` drives step by step (the `/v1/version` handshake, `start`, `wait_for_result`, and the blocking `execute` on a server with no run store), so a failure while following an acknowledged run is a `HostedRunPollingError` naming the run, and a transport failure after the run request was sent is a `HostedRunOutcomeUnknownError` rather than an `ApiUnreachableError`. `run_hosted`'s `on_started` callback records the acknowledged run's id, so Ctrl-C (`KeyboardInterrupt`, or the `CancelledError` of a cancelled task) leaves as a `HostedRunInterruptedError` envelope carrying `pipeline_run_id`, with exit code 130. `run bundle` on a file with no `-L` sends `PIPELEXPATH`'s files (`hosted_library_dirs`), and an installed method named like a catalog id is that method. An exception no hosted arm names leaves through the command-boundary catch-all as the envelope. Its success envelope is `build_run_output()`'s, with the main output's content as `json`, an empty `markdown`, the working memory dump and `pipeline_run_id`; nothing is written to disk. `--dry-run` / `--mock-inputs` are refused on it, while `--graph` and `--costs` are accepted and ignored. There is no global `--runner` any more, and no `ctx.obj`.- **Hosted refusals**: `agent_error_hosted()` routes pipelex-sdk's `ApiResponseError` (every `PipelexAPIClient` route raises it on a non-2xx answer) to `agent_error_api_response()`. `api_response_error_payload()` builds the envelope from the problem document alone: `error_type` is the runner's class (`ValidateBundleError` at load, as a local run reports it, or the root fault of a failed run, which a local run reports as `cause_type` under `PipelineExecutionError`; this path has no `cause_type`), `message` is the error's own `API POST /v1/start failed (<status>): <reason>` without the next step it appends, `hint` is `user_action.detail`, and `error_domain` / `error_category` / `retryable` / `validation_errors` / `pipe_code` / `pipe_stack` / `request_id` are the runner's, plus `error_code` (the problem's `code`), `retry_after_seconds` (the `Retry-After` header) and `http_status`. It deliberately bypasses `agent_error()`'s lookups: the dict hints keyed on a class name local commands (`doctor`, `check-model`) that cannot fix a remote refusal, and the domain is only the runner's to say, so one with no `error_domain` carries none. Both `error_domain` and `retryable` come from `pipelex.hosted.error_rendering.sent_verdict()`, the members the problem document carried, never from the verdict pipelex-sdk decides for every `ApiResponseError`, which fills an unsent member from its own status table and reads every 5xx, a 503 included, as retryable. A refusal with no `user_action` — the hosted plane's own answers in front of the runner: the gateway's 403 for a bad key, the rate limiter's 429, the unreachable runner's 503 — gets its status's next step from `pipelex.hosted.error_rendering` (`HOSTED_REFUSAL_NEXT_STEPS_BY_STATUS`, the table the human CLI reads too; 401 and 403 point at `PIPELEX_API_KEY` and `pipelex login`, 404 at the method and the origin), else `HOSTED_REFUSAL_CALLER_NEXT_STEP` for any other 4xx and `HOSTED_SERVER_FAULT_NEXT_STEP` for the rest. Only a 429 that did not say otherwise is inferred `retryable`: the platform can answer a 503 after the run completed, when it failed to record the result, so its hint warns that a retry can repeat a paid run. Every other failure, the SDK's and pipelex's own `HostedRunError` subclasses alike, goes through `agent_error()` with what `describe_hosted_error()` reads off it, the view the human CLI prints too: the class, message, next step, domain and `retryable`, the `pipeline_run_id` of any failure met once the run was acknowledged (`RunFailedError`, `RunTimeoutError`, `MissingMainStuffError`, `HostedRunPollingError`, `HostedRunInterruptedError`), a `retryable` passed whenever the hosted API gave a verdict, `false` included, so `RETRYABLE_ERROR_TYPES` keyed on the runner's class never overrides it, and the `validation_errors` of a failed run or of a `HostedMethodInvalidError` (pipe-io's verdict on a method that does not load, whose items name the faulty file in `source`); a `RunFailedError` adds `run_status` and its report's category, model and provider. The markdown renderer keeps a runner item this pipelex's `ValidationErrorItem` cannot type (a newer `error_type`, an unknown field) as JSON under Details instead of failing.- **Method-reference failures**: the `run`/`validate`/`inputs` method commands call `resolve_method_target(..., raise_ref_errors=True)` and shape the typed `MethodRefError` subclasses into `agent_error(...)` themselves (validate uses `exit_code=2`, its no-verdict convention) — the human CLI's red-text rendering stays in `resolve_method_target`'s default arm.diff --git a/pipelex/cli/agent_cli/commands/run/_run_core.py b/pipelex/cli/agent_cli/commands/run/_run_core.pyindex 604969086..d71aee873 100644--- a/pipelex/cli/agent_cli/commands/run/_run_core.py+++ b/pipelex/cli/agent_cli/commands/run/_run_core.py@@ -10,6 +10,7 @@ from typing import TYPE_CHECKING, Anyfrom pipelex import logfrom pipelex.base_exceptions import PipelexErrorfrom pipelex.cli.agent_cli.commands.run._output_helpers import build_run_output+from pipelex.cli.commands.run._run_output_folder import RUN_RESULTS_DIR_NAME, make_run_output_folder, run_output_labelfrom pipelex.cogt.usage.cost_registry import CostRegistryfrom pipelex.config import get_configfrom pipelex.core.memory.absence import AbsenceRecord@@ -23,6 +24,8 @@ from pipelex.tools.log.error_fields import error_fieldsfrom pipelex.tools.misc.json_utils import clean_json_dumpsif TYPE_CHECKING:+ from mthds.method_tree.models import MethodTree+from pipelex.system.configuration.configs import PipelineExecutionConfig@@ -52,6 +55,7 @@ async def run_pipeline_core(with_memory: bool = False,inputs_base_dir: Path | None = None,output_dir_override: Path | None = None,+ method_tree: MethodTree | None = None,) -> dict[str, Any]:"""Core logic for running a pipeline and returning JSON-serializable output.@@ -69,9 +73,12 @@ async def run_pipeline_core(return compact concept JSON only (False, default).inputs_base_dir: Directory bare relative file paths in ``inputs`` resolve against (SmartInputs D3) — the inputs file's parent when file-loaded, else ``None``.- output_dir_override: Directory to write run outputs (output JSON, graph files) into,- overriding the bundle-adjacent default. The caller passes one when the bundle lives- in a directory that will not survive the process — a fetched method's ephemeral clone.+ output_dir_override: Directory the run's output folder (output JSON, graph files) is created under,+ overriding the default, `results/` beside the bundle. The caller passes one when the bundle lives+ in a directory that will not survive the process — a fetched method's ephemeral clone — or when a+ folder run named its pipe and has no entry bundle to write beside.+ method_tree: The method tree the run reads, when the command listed one: its digest is recorded as+ ``method_tree_digest`` beside the run's other side effects, on disk and under ``with_memory``.Returns:Dictionary with execution results suitable for JSON serialization.@@ -141,22 +148,27 @@ async def run_pipeline_core(compact_result=compact_result,)- # Determine output directory: an explicit override wins (a fetched method's bundle lives in an- # ephemeral clone deleted at process exit, so the caller re-anchors outputs somewhere durable);- # otherwise next to the bundle, or mthds-wip/ fallback- output_dir: Path+ # The run writes into a new numbered folder, `<pipe_code>_output_NN/`, under the directory its outputs go to: an+ # explicit override wins (a fetched method's bundle lives in an ephemeral clone deleted at process exit, so the+ # caller re-anchors outputs somewhere durable); otherwise `results/` beside the bundle, or under mthds-wip/. The+ # folder holds a `.gitignore` ignoring everything in it, as `pipelex run`'s does, so a run beside its bundle never+ # adds a file to the method tree it ran, and the next run reads the same tree, with the same digest.+ outputs_dir: Pathif output_dir_override is not None:- output_dir = output_dir_override+ outputs_dir = output_dir_overrideelif bundle_uris and not bundle_uris[0].startswith(("http://", "https://")):- output_dir = Path(bundle_uris[0]).parent+ outputs_dir = Path(bundle_uris[0]).parent / RUN_RESULTS_DIR_NAMEelse:- output_dir = Path("mthds-wip")- output_dir.mkdir(parents=True, exist_ok=True)+ outputs_dir = Path("mthds-wip") / RUN_RESULTS_DIR_NAME+ output_dir = make_run_output_folder(output_dir=outputs_dir, label=run_output_label(pipe_ref=pipe_code))# Side-effect metadata (file paths) is tracked separately so it never# leaks into the compact stdout output. It is written to the on-disk# JSON file and, when with_memory is True, included in the returned dict.side_effects: dict[str, Any] = {}+ # The digest of the method tree the run read names the exact files of the method it ran.+ if method_tree is not None:+ side_effects["method_tree_digest"] = method_tree.digest# Cost report: best-effort structured `cost_report` object in the JSON output (machine-first; no Rich# table on stderr like the human CLI). build_cost_summary returns None for runs that did no reportablediff --git a/pipelex/cli/agent_cli/commands/run/bundle_cmd.py b/pipelex/cli/agent_cli/commands/run/bundle_cmd.pyindex 86988b350..593854e8d 100644--- a/pipelex/cli/agent_cli/commands/run/bundle_cmd.py+++ b/pipelex/cli/agent_cli/commands/run/bundle_cmd.py@@ -5,7 +5,7 @@ from __future__ import annotationsimport asyncioimport functoolsfrom pathlib import Path-from typing import Annotated, Any+from typing import TYPE_CHECKING, Annotated, Anyimport typer@@ -29,17 +29,21 @@ from pipelex.cli.agent_cli.commands.run._run_hosted import ()from pipelex.cli.agent_cli.commands.run.stdin_resolver import parse_cli_inputsfrom pipelex.cli.commands.run._hosted_sources import collect_mthds_files, hosted_library_dirs+from pipelex.cli.commands.run._method_folder import list_bundle_file_tree, resolve_method_folder_run+from pipelex.cli.commands.run._run_output_folder import RUN_RESULTS_DIR_NAMEfrom pipelex.hosted.exceptions import HostedRunSourceErrorfrom pipelex.hosted.hosted_run import HostedRunRequestfrom pipelex.hosted.run_config import RunExecution # ruff: ignore[typing-only-first-party-import] - typer reads the --runner annotation at runtime-from pipelex.mthds_parsing.helpers import MTHDS_EXTENSION, is_pipelex_file+from pipelex.mthds_parsing.helpers import is_pipelex_filefrom pipelex.mthds_parsing.parser import MthdsParserfrom pipelex.pipe_operators.exceptions import PipeOperatorModelAvailabilityErrorfrom pipelex.pipelex import Pipelex-from pipelex.pipeline.default_file_names import DEFAULT_BUNDLE_FILE_NAMEfrom pipelex.pipeline.exceptions import PipelineExecutionError, ValidateBundleErrorfrom pipelex.pipeline.validate_bundle_translation import translate_to_validate_bundle_error+if TYPE_CHECKING:+ from mthds.method_tree.models import MethodTree+def run_bundle_cmd(path: Annotated[@@ -123,23 +127,27 @@ def run_bundle_cmd(auto_inputs_dir: Path | None = Nonetarget_path = Path(path)+ # The method tree the run reads, whose digest the run records: the folder's, or the bundle file's own.+ method_tree: MethodTree | None = None+ # Where a folder run that named its pipe writes its output folders: `results/` in the folder, which has no entry+ # bundle to write beside.+ output_dir_override: Path | None = None+ # The pipe a hosted run names: `--pipe`, else the main pipe the local run starts at, qualified by its domain, so the+ # hosted API starts at the same pipe when two domains of the files it is sent define the same code.+ hosted_pipe_code: str | None = pipeif target_path.is_dir():- # Directory mode: auto-detect bundle, inputs, and library dir- bundle_file = target_path / DEFAULT_BUNDLE_FILE_NAME- if bundle_file.is_file():- bundle_path = str(bundle_file)+ # Directory mode: the method rooted at or above the folder, started where the protocol's rule says+ try:+ folder_run = resolve_method_folder_run(directory=target_path, pipe_override=pipe)+ except ValidateBundleError as exc:+ agent_error(exc.message, error_type="ValidateBundleError", cause=exc)+ method_tree = folder_run.tree+ pipe_code = folder_run.pipe_code+ if folder_run.entry is not None and folder_run.entry_bundle_file is not None:+ bundle_path = str(folder_run.entry_bundle_file)+ hosted_pipe_code = folder_run.entry.pipe_refelse:- mthds_files = list(target_path.glob(f"*{MTHDS_EXTENSION}"))- if len(mthds_files) == 0:- agent_error(f"No .mthds bundle file found in directory '{path}'", error_type="FileNotFoundError")- if len(mthds_files) > 1:- mthds_names = ", ".join(mthds_file.name for mthds_file in mthds_files)- agent_error(- f"Multiple .mthds files found in '{path}' ({mthds_names}) and no '{DEFAULT_BUNDLE_FILE_NAME}'. "- f"Pass the .mthds file directly instead.",- error_type="ArgumentError",- )- bundle_path = str(mthds_files[0])+ output_dir_override = target_path / RUN_RESULTS_DIR_NAME# Hand the directory to parse_cli_inputs as the lowest-priority inputs source: it probes# for inputs.json / inputs.toml (and applies the ambiguity rule) only after --inputs and@@ -155,6 +163,10 @@ def run_bundle_cmd(elif is_pipelex_file(target_path):bundle_path = path+ try:+ method_tree = list_bundle_file_tree(bundle_file=target_path)+ except ValidateBundleError as exc:+ agent_error(exc.message, error_type="ValidateBundleError", cause=exc)else:agent_error(f"'{path}' is not a .mthds file or directory. Use 'run pipe <code>' for pipe codes, or 'run bundle <path>' for .mthds files/directories.",@@ -167,6 +179,7 @@ def run_bundle_cmd(try:mthds_content = Path(bundle_path).read_text(encoding="utf-8")if not pipe_code:+ # A bundle file named alone starts at its own main_pipe.# A bundle that does not parse is the invalid verdict, with the items `validate` gives,# as it is when the run refuses it while loading.with translate_to_validate_bundle_error():@@ -178,6 +191,7 @@ def run_bundle_cmd(error_type="BundleError",)pipe_code = main_pipe_code+ hosted_pipe_code = main_pipe_codeexcept FileNotFoundError as exc:agent_error(f"Bundle file not found: {bundle_path}", error_type="FileNotFoundError", cause=exc)except (OSError, UnicodeDecodeError) as exc:@@ -194,12 +208,14 @@ def run_bundle_cmd(refuse_local_only_flags(dry_run=dry_run, mock_inputs=mock_inputs)# The bundle goes first, then the rest of the library a local run would load with it: -L, else PIPELEXPATH.try:- bundle_files = collect_mthds_files(primary=Path(bundle_path), library_dirs=hosted_library_dirs(library_dirs=library_dir))+ bundle_files = collect_mthds_files(+ primary=Path(bundle_path) if bundle_path else None, library_dirs=hosted_library_dirs(library_dirs=library_dir)+ )except HostedRunSourceError as exc:agent_error(exc.message, error_type=type(exc).__name__, cause=exc)request = HostedRunRequest(mthds_files=bundle_files,- pipe_code=pipe_code,+ pipe_code=hosted_pipe_code,inputs=pipeline_inputs,inputs_base_dir=parsed_inputs.inputs_base_dir,)@@ -222,6 +238,8 @@ def run_bundle_cmd(costs=costs,with_memory=with_memory,inputs_base_dir=parsed_inputs.inputs_base_dir,+ output_dir_override=output_dir_override,+ method_tree=method_tree,))agent_success_formatted(diff --git a/pipelex/cli/agent_cli/commands/run/method_cmd.py b/pipelex/cli/agent_cli/commands/run/method_cmd.pyindex a83bb4df1..776d1cf4b 100644--- a/pipelex/cli/agent_cli/commands/run/method_cmd.py+++ b/pipelex/cli/agent_cli/commands/run/method_cmd.py@@ -30,6 +30,7 @@ from pipelex.cli.agent_cli.commands.run._run_hosted import (from pipelex.cli.agent_cli.commands.run.stdin_resolver import parse_cli_inputsfrom pipelex.cli.commands.run._hosted_sources import resolve_hosted_method_targetfrom pipelex.cli.commands.run._inputs_file_loader import resolve_inputs_arg_against_dir+from pipelex.cli.commands.run._run_output_folder import RUN_RESULTS_DIR_NAMEfrom pipelex.cli.method_resolver import method_output_base_dir, resolve_method_targetfrom pipelex.hosted.exceptions import HostedRunSourceErrorfrom pipelex.hosted.hosted_run import HostedRunRequest@@ -163,10 +164,10 @@ def run_method_cmd(agent_error(str(exc), error_type=type(exc).__name__, cause=exc)# A fetched method's package directory is an ephemeral clone deleted at process exit —- # anchor run outputs (output JSON, graph files) in a durable location instead.+ # anchor run output folders (output JSON, graph files) in a durable location instead.output_dir_override: Path | None = Noneif method.provenance is not None:- output_dir_override = method_output_base_dir(method=method) / "results"+ output_dir_override = method_output_base_dir(method=method) / RUN_RESULTS_DIR_NAMEbundle_path: str | None = Nonemthds_content: str | None = Nonediff --git a/pipelex/cli/commands/run/_hosted_sources.py b/pipelex/cli/commands/run/_hosted_sources.pyindex e7ce03fcf..d2e7efc8e 100644--- a/pipelex/cli/commands/run/_hosted_sources.py+++ b/pipelex/cli/commands/run/_hosted_sources.py@@ -22,14 +22,15 @@ from typing import TYPE_CHECKINGfrom pipelex_sdk.crate_models import MthdsFileItemfrom pydantic import BaseModel, ConfigDict+from pipelex.cli.commands.run._run_output_folder import run_output_labelfrom pipelex.cli.installed_methods import DuplicateMethodNameError, MethodNotFoundError, find_method_by_namefrom pipelex.cli.method_resolver import method_output_base_dir, resolve_method_targetfrom pipelex.hosted.exceptions import HostedRunSourceError-from pipelex.hosted.execution import get_or_load_pipelex_config+from pipelex.libraries.exceptions import MethodTreeLoadError+from pipelex.libraries.method_trees import list_library_bundle_filesfrom pipelex.methods.method_ref import looks_like_method_ref, parse_method_ref-from pipelex.mthds_parsing.helpers import MTHDS_EXTENSION, is_pipelex_file+from pipelex.mthds_parsing.helpers import is_pipelex_filefrom pipelex.system.environment import PIPELEXPATH_ENV_KEY, get_pipelexpath_dirs-from pipelex.tools.misc.file_utils import find_files_in_dirif TYPE_CHECKING:from collections.abc import Sequence@@ -71,15 +72,15 @@ def hosted_library_dirs(*, library_dirs: Sequence[str] | None) -> Sequence[str |def collect_mthds_files(*, primary: Path | None, library_dirs: Sequence[str | Path] | None) -> list[MthdsFileItem]:"""Read the `.mthds` files a local run would load: `primary` first, then each library directory's, each file once.- A library directory is scanned recursively, skipping the configured `[interpreter.scan] excluded_dirs`, as a- local load scans it; a library entry naming a `.mthds` file is taken as it is; a missing one is skipped, as a- local load skips it. Each file carries its path as its label: the hosted API names it in the validation errors- of the method's signature, read before the run, while the run route takes the bare contents.+ A library directory's files are the bundles of its method tree and of the trees nested in it, as a local load+ reads them (`pipelex.libraries.method_trees`), so a file the membership rule keeps out, a `.gitignore`d or hidden+ one, is never sent; a library entry naming a `.mthds` file is taken as it is; a missing one is skipped, as a local+ load skips it. Each file carries its path as its label: the hosted API names it in the validation errors of the+ method's signature, read before the run, while the run route takes the bare contents.Raises:- HostedRunSourceError: If a file cannot be read.+ HostedRunSourceError: If a tree cannot be listed or a file cannot be read."""- excluded_dirs = list(get_or_load_pipelex_config().interpreter.scan.excluded_dirs)ordered: list[Path] = []if primary is not None:ordered.append(primary)@@ -88,8 +89,12 @@ def collect_mthds_files(*, primary: Path | None, library_dirs: Sequence[str | Paif library_path.is_file() and is_pipelex_file(library_path):ordered.append(library_path)elif library_path.is_dir():- found = find_files_in_dir(dir_path=library_path, pattern=f"*{MTHDS_EXTENSION}", excluded_dirs=excluded_dirs)- ordered.extend(path for path in found if is_pipelex_file(path))+ try:+ library_bundle_files = list_library_bundle_files(library_dirs=[library_path])+ except MethodTreeLoadError as exc:+ msg = f"Could not list the method tree of '{library_path}' to send its bundles to the hosted API: {exc.message}"+ raise HostedRunSourceError(msg) from exc+ ordered.extend(bundle_file.path for bundle_file in library_bundle_files)seen: set[Path] = set()files: list[MthdsFileItem] = []@@ -178,7 +183,11 @@ def resolve_hosted_method_target(*, name: str, pipe_override: str | None, librar"Install it and run it by name, or run its bundle with 'run bundle <path> -L <dir>'.")raise HostedRunSourceError(msg)- label = pipe_override or (parse_method_ref(remote_ref).address.rsplit("/", 1)[-1] if remote_ref else name)+ label: str+ if pipe_override:+ label = run_output_label(pipe_ref=pipe_override)+ else:+ label = parse_method_ref(remote_ref).address.rsplit("/", 1)[-1] if remote_ref else namereturn HostedMethodTarget(method_ref=remote_ref,method_id=remote_id,@@ -197,7 +206,7 @@ def resolve_hosted_method_target(*, name: str, pipe_override: str | None, librarreturn HostedMethodTarget(mthds_files=mthds_files,pipe_code=pipe_code,- label=pipe_code,+ label=run_output_label(pipe_ref=pipe_code),output_base_dir=method_output_base_dir(method=method),inputs_anchor_dir=Path(method_library_dirs[0]),)diff --git a/pipelex/cli/commands/run/_method_folder.py b/pipelex/cli/commands/run/_method_folder.pynew file mode 100644index 000000000..11bd4f37a--- /dev/null+++ b/pipelex/cli/commands/run/_method_folder.py@@ -0,0 +1,62 @@+"""What `run bundle` runs when it is handed a method's folder or a bundle file: its method tree, and where it starts.++Both CLIs read their target here, so `pipelex run bundle` and `pipelex-agent run bundle` run the same pipe of the same+tree for the same arguments, locally as on the hosted API:++- A folder is the method rooted at the nearest `METHODS.toml` at or above it, else at the folder itself. With no+ `--pipe`, the run starts where the MTHDS protocol says a run naming no pipe starts (`pipelex.pipeline.method_entry`):+ the manifest's `main_pipe`, else the one `main_pipe` its bundles declare; none or several is refused, the bundles+ named by their tree paths.+- A `.mthds` file named alone is the member of its own file's tree: rooted at the nearest `METHODS.toml` at or above+ its folder, else the one-file tree of that file, from which a reference to any other file is refused. It starts at+ its own `main_pipe`, as it always has.++The tree is listed under this machine's limits (`[interpreter.method_tree_limits]`), and its digest is what a run+records as the method it ran.+"""++from pathlib import Path+from typing import NamedTuple++from mthds.method_tree.models import MethodTree++from pipelex.libraries.method_trees import list_local_method_tree+from pipelex.pipeline.method_entry import MethodEntry, select_method_entry+from pipelex.pipeline.validate_bundle_translation import translate_to_validate_bundle_error+++class MethodFolderRun(NamedTuple):+ """A method's folder to run: its tree, the pipe the run starts at, and the bundle defining it when it was selected."""++ tree: MethodTree+ #: `--pipe` as given, else the selected entry's `domain.pipe_code`.+ pipe_code: str+ #: The pipe the protocol's rule selected, or None when `--pipe` named one.+ entry: MethodEntry | None+ #: The file on this host of the bundle defining the selected entry, or None when `--pipe` named the pipe.+ entry_bundle_file: Path | None+++def resolve_method_folder_run(*, directory: Path, pipe_override: str | None) -> MethodFolderRun:+ """List a method folder's tree and settle the pipe its run starts at.++ Raises:+ ValidateBundleError: The tree is refused or past this machine's limits, a bundle does not parse, or the method+ has no single pipe to start at, each as the invalid verdict `validate` gives.+ """+ with translate_to_validate_bundle_error():+ tree = list_local_method_tree(handed_over=directory)+ if pipe_override:+ return MethodFolderRun(tree=tree, pipe_code=pipe_override, entry=None, entry_bundle_file=None)+ entry = select_method_entry(tree=tree)+ return MethodFolderRun(tree=tree, pipe_code=entry.pipe_ref, entry=entry, entry_bundle_file=tree.file_location(entry.tree_path))+++def list_bundle_file_tree(*, bundle_file: Path) -> MethodTree:+ """The method tree of a `.mthds` file named alone, as the run loads it.++ Raises:+ ValidateBundleError: The tree is refused or past this machine's limits, as the invalid verdict `validate` gives.+ """+ with translate_to_validate_bundle_error():+ return list_local_method_tree(handed_over=bundle_file)diff --git a/pipelex/cli/commands/run/_run_core.py b/pipelex/cli/commands/run/_run_core.pyindex f812f4b76..5b46ee698 100644--- a/pipelex/cli/commands/run/_run_core.py+++ b/pipelex/cli/commands/run/_run_core.py@@ -15,6 +15,7 @@ from pipelex.cli.cli_factory import make_pipelex_for_clifrom pipelex.cli.commands.run._inputs_file_loader import load_inputs_dict_from_pathfrom pipelex.cli.commands.run._inputs_path_resolver import resolve_inputs_pathsfrom pipelex.cli.commands.run._main_stuff_file import save_main_stuff_file+from pipelex.cli.commands.run._run_output_folder import make_run_output_folder, run_output_label, write_method_tree_recordfrom pipelex.cli.deck_notice import warn_if_deck_stalefrom pipelex.cli.error_handlers import (ErrorContext,@@ -49,13 +50,14 @@ from pipelex.system.telemetry.events import EventPropertyfrom pipelex.system.usage_stats.usage_stats_contract import UsageStatsMethodSourcefrom pipelex.system.usage_stats.usage_stats_runs import method_source_scopefrom pipelex.tools.misc.exceptions import JsonTypeError, TomlError-from pipelex.tools.misc.file_utils import get_incremental_directory_pathfrom pipelex.tools.misc.json_utils import save_as_json_to_pathfrom pipelex.tools.misc.package_utils import get_package_versionfrom pipelex.tools.tabular.csv_codec import assert_supported_table_suffix, csv_from_list_content, flat_field_namesfrom pipelex.tools.tabular.exceptions import CsvErrorif TYPE_CHECKING:+ from mthds.method_tree.models import MethodTree+from pipelex.core.concepts.concept import Conceptfrom pipelex.core.stuffs.stuff_content import StuffContent@@ -193,10 +195,12 @@ async def _execute_run(costs: bool | None = None,dynamic_output_concept_ref: str | None = None,save_csv: str | None = None,+ method_tree: MethodTree | None = None,) -> None:"""Core async execution logic for running a pipe.- Shared between the ``method`` and ``pipe`` subcommands.+ Shared between the ``method``, ``pipe`` and ``bundle`` subcommands. ``method_tree`` is the method tree the run+ reads, when the command listed one, recorded as ``method_tree.json`` in the run's output folder."""# Fail fast on an unusable --save-csv target BEFORE running the pipeline, so a bad path (empty,# or an unsupported suffix like .xlsx) doesn't waste a full inference run. The output-shape and@@ -312,8 +316,13 @@ async def _execute_run(needs_output_path = (graph_spec is not None) or save_main_stuff or save_working_memoryif needs_output_path:- output_path = get_incremental_directory_path(base_path=Path(output_dir), base_name=f"{pipe_code}_output")- output_path.mkdir(parents=True, exist_ok=True)+ # Named after the pipe's own code: a qualified `domain.pipe_code` keeps its domain out of the folder's name.+ output_path = make_run_output_folder(output_dir=Path(output_dir), label=run_output_label(pipe_ref=pipe_code))++ # The method tree the run read, recorded beside its outputs: its digest names the exact files of the method.+ method_tree_record_path: Path | None = None+ if output_path is not None and method_tree is not None:+ method_tree_record_path = write_method_tree_record(output_path=output_path, method_tree=method_tree)# Save graph outputs if requestedsaved_graphs: list[str] = []@@ -480,6 +489,10 @@ async def _execute_run(console.print(f" [green]✓[/green] {_WORKING_MEMORY_FILENAME}")else:console.print(f" [green]✓[/green] working_memory: {escape(working_memory_output_path)}")+ if method_tree_record_path:+ console.print(f" [green]✓[/green] {method_tree_record_path.name}")+ if method_tree is not None:+ console.print(f" Method tree: {escape(method_tree.digest)}, {len(method_tree.files)} files")# CSV output is written to a literal cwd-relative path (CQ1), independent of --output-dir.if save_csv is not None:console.print(f" [green]✓[/green] CSV saved to [bold magenta]{escape(save_csv)}[/bold magenta]")@@ -506,6 +519,7 @@ def execute_run(orchestrator: str | None = None,dynamic_output_concept_ref: str | None = None,save_csv: str | None = None,+ method_tree: MethodTree | None = None,) -> None:"""Synchronous entry point that wraps the async execution with Pipelex setup/teardown.@@ -549,6 +563,7 @@ def execute_run(costs=costs,dynamic_output_concept_ref=dynamic_output_concept_ref,save_csv=save_csv,+ method_tree=method_tree,))diff --git a/pipelex/cli/commands/run/_run_hosted.py b/pipelex/cli/commands/run/_run_hosted.pyindex dd5801877..261439461 100644--- a/pipelex/cli/commands/run/_run_hosted.py+++ b/pipelex/cli/commands/run/_run_hosted.py@@ -19,6 +19,7 @@ from rich.markup import escapefrom pipelex.base_exceptions import PipelexErrorfrom pipelex.cli.commands.run._run_core import load_run_inputs+from pipelex.cli.commands.run._run_output_folder import make_run_output_folderfrom pipelex.cli.error_handlers import handle_validate_bundle_error, print_traceback_if_requestedfrom pipelex.hosted.client_factory import make_hosted_clientfrom pipelex.hosted.error_rendering import describe_hosted_error@@ -30,7 +31,6 @@ from pipelex.mthds_parsing.parser import MthdsParserfrom pipelex.pipeline.exceptions import ValidateBundleErrorfrom pipelex.pipeline.validate_bundle_translation import translate_to_validate_bundle_errorfrom pipelex.runtime_hub import get_console-from pipelex.tools.misc.file_utils import get_incremental_directory_pathfrom pipelex.tools.misc.json_utils import save_as_json_to_pathfrom pipelex.tools.misc.pretty import pretty_print, pretty_print_md@@ -266,8 +266,7 @@ def execute_hosted_run(output_path: Path | None = Nonesaved: list[str] = []if save_main_stuff or save_working_memory or (graph is not False and results.graph_spec is not None):- output_path = get_incremental_directory_path(base_path=Path(output_dir), base_name=f"{output_label}_output")- output_path.mkdir(parents=True, exist_ok=True)+ output_path = make_run_output_folder(output_dir=Path(output_dir), label=output_label)if output_path is not None and save_main_stuff:save_as_json_to_path(object_to_save=results.main_stuff, path=output_path / "main_stuff.json")diff --git a/pipelex/cli/commands/run/_run_output_folder.py b/pipelex/cli/commands/run/_run_output_folder.pynew file mode 100644index 000000000..5834889a5--- /dev/null+++ b/pipelex/cli/commands/run/_run_output_folder.py@@ -0,0 +1,60 @@+"""The folder a `pipelex run` writes its outputs to, kept out of every method tree, and the record of the tree it ran.++A run writes its outputs to a new numbered folder, `<label>_output_<NN>`, under `--output-dir` for `pipelex run`, and+under `results/` beside the bundle it ran for `pipelex-agent run`. That folder often+lies inside a method's folder, the working directory being the method's, so the run writes a `.gitignore` holding `*`+into it: the membership rule honours the tree's `.gitignore` files as git does (`mthds/docs/spec/method-tree.md`), so+none of a run's outputs, nor the `.gitignore` itself, ever joins the tree, and a second run of the method reads the+same tree, with the same digest, as the first.++A local run also records there the method tree it read, as `method_tree.json`: its digest, which names the exact files+the run read, and the files themselves, by tree path, hash and size.+"""++import json+from pathlib import Path++from mthds.method_tree.models import MethodTree++from pipelex.tools.misc.file_utils import get_incremental_directory_path++RUN_OUTPUT_GITIGNORE_NAME = ".gitignore"+RUN_OUTPUT_GITIGNORE_CONTENT = "# Written by pipelex run: a run's outputs are never part of a method.\n*\n"+METHOD_TREE_RECORD_NAME = "method_tree.json"+#: The directory `pipelex-agent run` creates its output folders under, beside the bundle it ran.+RUN_RESULTS_DIR_NAME = "results"+++def run_output_label(*, pipe_ref: str) -> str:+ """The label a run's output folder takes from the pipe it ran: the pipe's own code, without its domain or alias.++ A pipe named as `domain.pipe_code`, or as `alias->domain.pipe_code` for a dependency's, labels its folder by+ `pipe_code`, as a pipe named by its bare code does.+ """+ return pipe_ref.rsplit("->", 1)[-1].rsplit(".", 1)[-1]+++def make_run_output_folder(*, output_dir: Path, label: str) -> Path:+ """Create the next numbered output folder for a run under `output_dir`, holding a `.gitignore` that ignores it all."""+ output_path = get_incremental_directory_path(output_dir, base_name=f"{label}_output")+ output_path.mkdir(parents=True, exist_ok=True)+ (output_path / RUN_OUTPUT_GITIGNORE_NAME).write_text(RUN_OUTPUT_GITIGNORE_CONTENT, encoding="utf-8")+ return output_path+++def method_tree_record(*, method_tree: MethodTree) -> dict[str, object]:+ """The record of the method tree a run read: its digest, its root on this host, its entry, and its files."""+ return {+ "digest": method_tree.digest,+ "root": str(method_tree.directory) if method_tree.directory is not None else None,+ "is_one_file": method_tree.is_one_file,+ "entry": method_tree.entry,+ "files": [{"path": tree_file.path, "sha256": tree_file.sha256, "size": tree_file.size} for tree_file in method_tree.files],+ }+++def write_method_tree_record(*, output_path: Path, method_tree: MethodTree) -> Path:+ """Write `method_tree.json` into a run's output folder, and return its path."""+ record_path = output_path / METHOD_TREE_RECORD_NAME+ record_path.write_text(json.dumps(method_tree_record(method_tree=method_tree), indent=2, ensure_ascii=False) + "\n", encoding="utf-8")+ return record_pathdiff --git a/pipelex/cli/commands/run/bundle_cmd.py b/pipelex/cli/commands/run/bundle_cmd.pyindex aba274dc8..801027bff 100644--- a/pipelex/cli/commands/run/bundle_cmd.py+++ b/pipelex/cli/commands/run/bundle_cmd.py@@ -1,18 +1,25 @@from __future__ import annotationsfrom pathlib import Path-from typing import Annotated+from typing import TYPE_CHECKING, Annotatedimport typer+from pipelex.base_exceptions import PipelexUnexpectedErrorfrom pipelex.cli.commands.run._hosted_sources import collect_mthds_files, hosted_library_dirsfrom pipelex.cli.commands.run._inputs_file_loader import find_default_inputs_file+from pipelex.cli.commands.run._method_folder import list_bundle_file_tree, resolve_method_folder_runfrom pipelex.cli.commands.run._run_core import COMMAND, execute_run, validate_run_flag_combinationfrom pipelex.cli.commands.run._run_hosted import bundle_main_pipe_code, execute_hosted_run, refuse_local_only_flags, resolve_cli_run_execution+from pipelex.cli.commands.run._run_output_folder import run_output_labelfrom pipelex.cli.commands.run.exceptions import AmbiguousInputsFilesError+from pipelex.cli.error_handlers import handle_validate_bundle_errorfrom pipelex.hosted.exceptions import HostedRunSourceError-from pipelex.mthds_parsing.helpers import MTHDS_EXTENSION, is_pipelex_file-from pipelex.pipeline.default_file_names import DEFAULT_BUNDLE_FILE_NAME+from pipelex.mthds_parsing.helpers import is_pipelex_file+from pipelex.pipeline.exceptions import ValidateBundleError++if TYPE_CHECKING:+ from mthds.method_tree.models import MethodTreedef run_bundle_cmd(@@ -140,31 +147,23 @@ def run_bundle_cmd(bundle_path: str | None = Nonetarget_path = Path(path)+ # The method tree the run reads, whose digest the run records: the folder's, or the bundle file's own.+ method_tree: MethodTree+ # The pipe a hosted run names: `--pipe`, else the main pipe the local run starts at, qualified by its domain, so the+ # hosted API starts at the same pipe when two domains of the files it is sent define the same code.+ hosted_pipe_code: str | None = pipeif target_path.is_dir():- # Directory mode: auto-detect bundle and inputs- bundle_file = target_path / DEFAULT_BUNDLE_FILE_NAME- if bundle_file.is_file():- bundle_path = str(bundle_file)- else:- mthds_files = list(target_path.glob(f"*{MTHDS_EXTENSION}"))- if len(mthds_files) == 0:- typer.secho(- f"Failed to run: no .mthds bundle file found in directory '{path}'",- fg=typer.colors.RED,- err=True,- )- raise typer.Exit(1)- if len(mthds_files) > 1:- mthds_names = ", ".join(mthds_file.name for mthds_file in mthds_files)- typer.secho(- f"Failed to run: multiple .mthds files found in '{path}' ({mthds_names}) "- f"and no '{DEFAULT_BUNDLE_FILE_NAME}'. "- f"Pass the .mthds file directly, e.g.: pipelex run bundle {target_path / mthds_files[0].name}",- fg=typer.colors.RED,- err=True,- )- raise typer.Exit(1)- bundle_path = str(mthds_files[0])+ # Directory mode: the method rooted at or above the folder, started where the protocol's rule says+ try:+ folder_run = resolve_method_folder_run(directory=target_path, pipe_override=pipe)+ except ValidateBundleError as exc:+ handle_validate_bundle_error(exc, library_dirs=[Path(one_dir) for one_dir in library_dir] if library_dir else None)+ method_tree = folder_run.tree+ pipe_code = folder_run.pipe_code+ if folder_run.entry is not None and folder_run.entry_bundle_file is not None:+ bundle_path = str(folder_run.entry_bundle_file)+ hosted_pipe_code = folder_run.entry.pipe_ref+ typer.echo(f"Starting at the method's main pipe {folder_run.entry.pipe_ref}, in {folder_run.entry.tree_path}")# Auto-detect inputs (inputs.json or inputs.toml) if --inputs not explicitly providedif not inputs:@@ -184,10 +183,14 @@ def run_bundle_cmd(elif target_dir_str not in library_dir:library_dir = [target_dir_str, *library_dir]- typer.echo(f"Auto-detected bundle: {bundle_path}")-elif is_pipelex_file(target_path):bundle_path = path+ try:+ method_tree = list_bundle_file_tree(bundle_file=target_path)+ except ValidateBundleError as exc:+ handle_validate_bundle_error(+ exc, bundle_path=target_path, library_dirs=[Path(one_dir) for one_dir in library_dir] if library_dir else None+ )else:typer.secho(f"Failed to run: '{path}' is not a .mthds file or directory.\n"@@ -209,10 +212,17 @@ def run_bundle_cmd(costs=costs,graph_full_data=graph_full_data,)- hosted_pipe_code = pipe_code or bundle_main_pipe_code(bundle_path=bundle_path, library_dirs=library_dir)- # The bundle goes first, then the rest of the library a local run would load with it: -L, else PIPELEXPATH.+ if hosted_pipe_code is None:+ if bundle_path is None:+ msg = "A hosted run of a method's folder names its pipe, or the bundle defining its main pipe."+ raise PipelexUnexpectedError(msg)+ hosted_pipe_code = bundle_main_pipe_code(bundle_path=bundle_path, library_dirs=library_dir)+ # The entry bundle goes first, then the rest of the library a local run would load with it: the method's folder+ # and -L, else PIPELEXPATH.try:- bundle_files = collect_mthds_files(primary=Path(bundle_path), library_dirs=hosted_library_dirs(library_dirs=library_dir))+ bundle_files = collect_mthds_files(+ primary=Path(bundle_path) if bundle_path else None, library_dirs=hosted_library_dirs(library_dirs=library_dir)+ )except HostedRunSourceError as exc:typer.secho(f"Failed to run: {exc.message}", fg=typer.colors.RED, err=True)raise typer.Exit(1) from exc@@ -222,7 +232,7 @@ def run_bundle_cmd(inputs=inputs,dynamic_output_concept_ref=dynamic_output_concept_ref,base_url=base_url,- output_label=hosted_pipe_code,+ output_label=run_output_label(pipe_ref=hosted_pipe_code),output_dir=output_dir,save_working_memory=save_working_memory,working_memory_path=working_memory_path,@@ -252,4 +262,5 @@ def run_bundle_cmd(orchestrator=orchestrator,dynamic_output_concept_ref=dynamic_output_concept_ref,save_csv=save_csv,+ method_tree=method_tree,)diff --git a/pipelex/cli/commands/run/pipe_cmd.py b/pipelex/cli/commands/run/pipe_cmd.pyindex f58885186..eafdc4bcf 100644--- a/pipelex/cli/commands/run/pipe_cmd.py+++ b/pipelex/cli/commands/run/pipe_cmd.py@@ -8,6 +8,7 @@ import typerfrom pipelex.cli.commands.run._hosted_sources import hosted_pipe_library_filesfrom pipelex.cli.commands.run._run_core import COMMAND, execute_run, validate_run_flag_combinationfrom pipelex.cli.commands.run._run_hosted import execute_hosted_run, refuse_local_only_flags, resolve_cli_run_execution+from pipelex.cli.commands.run._run_output_folder import run_output_labelfrom pipelex.cli.method_resolver import resolve_pipe_from_exportsfrom pipelex.hosted.exceptions import HostedRunSourceErrorfrom pipelex.mthds_parsing.helpers import MTHDS_EXTENSION, is_pipelex_file@@ -179,7 +180,7 @@ def run_pipe_cmd(inputs=inputs,dynamic_output_concept_ref=dynamic_output_concept_ref,base_url=base_url,- output_label=pipe_code,+ output_label=run_output_label(pipe_ref=pipe_code),output_dir=output_dir,save_working_memory=save_working_memory,working_memory_path=working_memory_path,diff --git a/pipelex/core/method_trees/__init__.py b/pipelex/core/method_trees/__init__.pynew file mode 100644index 000000000..e69de29bbdiff --git a/pipelex/core/method_trees/building_bundle.py b/pipelex/core/method_trees/building_bundle.pynew file mode 100644index 000000000..2351cc4bb--- /dev/null+++ b/pipelex/core/method_trees/building_bundle.py@@ -0,0 +1,31 @@+"""The method tree member a pipe is being built from, for the factories that resolve a file reference while building.++The library manager builds every pipe of a bundle it read from disk inside `building_pipe_of`, naming the bundle's+member of its method tree, and every pipe of a bundle parsed from a content string with none. A factory that resolves+a file the step names, such as `PipeDocGen`'s `template_file`, reads it here: the tree is the only place it looks, so+neither a `source` a caller labelled a bundle with nor one the bundle's own text wrote can point a read at this host's+disk.+"""++from collections.abc import Generator+from contextlib import contextmanager+from contextvars import ContextVar++from pipelex.core.method_trees.tree_bundle import TreeBundle++_BUILDING_TREE_BUNDLE: ContextVar[TreeBundle | None] = ContextVar("building_tree_bundle", default=None)+++@contextmanager+def building_pipe_of(*, tree_bundle: TreeBundle | None) -> Generator[None, None, None]:+ """Build the pipes inside this block as pipes of `tree_bundle`, or of a bundle parsed from a string when None."""+ token = _BUILDING_TREE_BUNDLE.set(tree_bundle)+ try:+ yield+ finally:+ _BUILDING_TREE_BUNDLE.reset(token)+++def get_building_tree_bundle() -> TreeBundle | None:+ """The member of its method tree the pipe being built was read from, or None for a bundle parsed from a string."""+ return _BUILDING_TREE_BUNDLE.get()diff --git a/pipelex/core/method_trees/file_references.py b/pipelex/core/method_trees/file_references.pynew file mode 100644index 000000000..50bb3abb3--- /dev/null+++ b/pipelex/core/method_trees/file_references.py@@ -0,0 +1,93 @@+"""Resolving a file a bundle field names, such as a `PipeDocGen` step's `template_file`, in the bundle's method tree.++The standard's resolution (`mthds/docs/spec/method-tree.md`, "File References") is `mthds-python`'s: the path is+joined to the bundle's own tree path, a `..` may climb as long as it stays inside the tree, and the result must be a+file of the tree, which is a lookup in the set of tree paths the listing produced. So a file the membership rule keeps+out of the tree, a dotfile, a `.gitignore`d file or a file of a nested package, is invisible to the step, and nothing+outside the tree is ever opened.+"""++from mthds.method_tree.exceptions import MethodTreeRefusedError+from mthds.method_tree.references import resolve_reference+from mthds.method_tree.vocabulary import ReferenceContext, TreeRefusalCode++from pipelex.base_exceptions import PipelexUnexpectedError+from pipelex.core.method_trees.tree_bundle import TreeBundle++ONE_FILE_TREE_ADVICE = "Run the method's folder instead, or add a METHODS.toml to it."+++def file_reference_form_refusal(*, reference: str) -> str | None:+ """Why a path written in a bundle field cannot name a file of any method tree, or None when its form can.++ The checks a blueprint can make on its own, before any tree is at hand: the standard's form check in path+ context. Whether the path names a file is decided when the bundle loads, in its tree. A `..` segment passes here:+ it may climb to a file elsewhere in the tree. The reason never repeats the path, which a host library's step would+ otherwise put in front of a caller.+ """+ if not reference.strip():+ return "it is empty"+ if "://" in reference:+ return "it is a URL; write a path relative to the bundle's file"+ if reference.startswith("/"):+ return "it is absolute; write it relative to the bundle's file"+ if "\\" in reference:+ return "it holds a backslash; write the path with forward slashes"+ # No tree path holds a colon, so a reference holding one, such as a drive letter or a scheme, names no file.+ if ":" in reference:+ return "it holds a colon, which no file of a method tree can"+ return None+++def resolve_bundle_file_reference(*, reference: str, tree_bundle: TreeBundle) -> str:+ """The tree path of the file a bundle field names, resolved against the bundle's tree path in its method tree.++ Raises:+ MethodTreeRefusedError: When the reference is refused; `refusal.code` says at which step of the resolution,+ and `reference_refusal_reason` words it for the bundle's author.+ """+ resolved = resolve_reference(reference, holder=tree_bundle.tree_path, context=ReferenceContext.PATH, tree=tree_bundle.tree)+ if resolved is None:+ msg = f"A reference in path context is always a file reference, yet '{reference}' was resolved as none."+ raise PipelexUnexpectedError(msg)+ return resolved+++def reference_refusal_reason(*, refusal_error: MethodTreeRefusedError, tree_bundle: TreeBundle) -> str:+ """Why a file reference was refused, as a sentence naming files by their tree paths only, never by a host path.++ The refusal of a reference from a one-file tree, the tree of a `.mthds` file named alone with no `METHODS.toml`+ at or above it, says how to give the bundle the files beside it. Every other refusal is `mthds-python`'s own+ wording, which names the reference and the bundle by tree path.+ """+ match refusal_error.refusal.code:+ case TreeRefusalCode.REFERENCE_IN_ONE_FILE_TREE:+ references = ", ".join(f"'{name}'" for name in refusal_error.refusal.names)+ return (+ f"The file reference {references} in '{tree_bundle.tree_path}' names a file beside the bundle, which its method does not hold: "+ f"'{tree_bundle.tree_path}' was run alone, with no METHODS.toml at or above its folder, so its method is that one file. "+ f"{ONE_FILE_TREE_ADVICE}"+ )+ case (+ TreeRefusalCode.REFERENCE_INVALID+ | TreeRefusalCode.REFERENCE_LEAVES_TREE+ | TreeRefusalCode.REFERENCE_NOT_A_TREE_PATH+ | TreeRefusalCode.REFERENCE_NOT_IN_TREE+ | TreeRefusalCode.SYMBOLIC_LINK+ | TreeRefusalCode.SPECIAL_FILE+ | TreeRefusalCode.INVALID_TREE_PATH+ | TreeRefusalCode.NORMALIZATION_COLLISION+ | TreeRefusalCode.PATH_COLLISION+ | TreeRefusalCode.EMPTY_TREE+ | TreeRefusalCode.LONE_FILE_EXCLUDED+ | TreeRefusalCode.GITIGNORE_NOT_UTF8+ | TreeRefusalCode.DUPLICATE_KEY+ | TreeRefusalCode.EXCLUDED_NAME+ | TreeRefusalCode.MALFORMED_ENTRY+ | TreeRefusalCode.MISSING_BLOB+ | TreeRefusalCode.FUNCTION_NAME_WITHOUT_MODULE+ | TreeRefusalCode.FUNCTION_NAME_INVALID_SEGMENT+ | TreeRefusalCode.MODULE_NOT_IN_TREE+ | TreeRefusalCode.MODULE_AMBIGUOUS+ ):+ return refusal_error.messagediff --git a/pipelex/core/method_trees/tree_bundle.py b/pipelex/core/method_trees/tree_bundle.pynew file mode 100644index 000000000..fa25cfae7--- /dev/null+++ b/pipelex/core/method_trees/tree_bundle.py@@ -0,0 +1,50 @@+"""A bundle as a member of its method tree: the tree it was listed in, and its tree path there.++A bundle read from this host's disk is always read as a member of a method tree (`mthds/docs/spec/method-tree.md`),+listed by `mthds-python`'s lister: a library directory's tree, the tree of a method's folder, the tree of a `.mthds`+file named alone, or a dependency package's tree. A file the bundle's steps name, such as a `PipeDocGen` step's+`template_file`, is then resolved in that tree, by a lookup in its set of tree paths, and never by opening a path the+bundle wrote. A bundle parsed from a content string belongs to no tree here.+"""++from pathlib import Path++from mthds.method_tree.models import MethodTree+from pydantic import BaseModel, ConfigDict+++def tree_key_of(*, tree: MethodTree) -> str:+ """The key a tree is held under where several travel together, such as in a library crate.++ Two listings of one directory with the same files share a key, and a one-file tree never shares one with the+ directory tree of its folder, since only the latter lets a reference reach another file.+ """+ kind = "one-file" if tree.is_one_file else "directory"+ return f"{kind}:{tree.directory}:{tree.digest}"+++class TreeBundle(BaseModel):+ """A bundle file of a method tree: the tree, listed from this host's disk, and the bundle's tree path in it."""++ model_config = ConfigDict(frozen=True)++ tree: MethodTree+ tree_path: str++ @property+ def tree_key(self) -> str:+ return tree_key_of(tree=self.tree)++ @property+ def location(self) -> Path:+ """Where the bundle's file lies on this host."""+ return self.tree.file_location(self.tree_path)+++class TreeBundleRef(BaseModel):+ """A bundle file named by the key of its tree and its tree path, as a library crate carries it beside its trees."""++ model_config = ConfigDict(frozen=True, extra="forbid")++ tree_key: str+ tree_path: strdiff --git a/pipelex/kit/configs/pipelex.toml b/pipelex/kit/configs/pipelex.tomlindex 665b27ceb..c230ffc40 100644--- a/pipelex/kit/configs/pipelex.toml+++ b/pipelex/kit/configs/pipelex.toml@@ -229,6 +229,17 @@ excluded_dirs = ["results",]+####################################################################################################+# Method Tree Limits Config+####################################################################################################++[interpreter.method_tree_limits]+# How large a method tree this machine lists when it reads a method: a method's folder, a .mthds file named alone, a+# library directory or a dependency package. Listing stops before reading any file once a tree passes a limit.+# Raise them here for a method that needs more.+max_files = 10000+max_total_bytes = 1073741824+##################################################################################################### Builder Config####################################################################################################diff --git a/pipelex/libraries/bundle_resources.py b/pipelex/libraries/bundle_resources.pyindex 8672f4a57..b82dd00f1 100644--- a/pipelex/libraries/bundle_resources.py+++ b/pipelex/libraries/bundle_resources.py@@ -31,9 +31,10 @@ _FALLBACK_STORED_STEM = "file"def bundle_resource_reads(*, pipe_blueprints: Iterable[PipeBlueprintUnion]) -> dict[str, BundleResourceReadStage]:"""The bundle files these pipes read, keyed by canonical path, and when each is first read.- A path no file inside a bundle can have, such as one with a `..` segment, is left out: no submitted file can- answer it, and a step's blueprint refuses to name one. A file read at load by one step and at run by another- counts as read at load, which is the stricter way to carry it.+ A path no file submitted with a bundle can have, such as one with a `..` segment, is left out: no submitted file+ can answer it, and the step that names it is refused when it is built, as naming a file not supplied with its+ bundle. A file read at load by one step and at run by another counts as read at load, which is the stricter way to+ carry it."""reads: dict[str, BundleResourceReadStage] = {}for pipe_blueprint in pipe_blueprints:diff --git a/pipelex/libraries/exceptions.py b/pipelex/libraries/exceptions.pyindex e0fca00fb..238780658 100644--- a/pipelex/libraries/exceptions.py+++ b/pipelex/libraries/exceptions.py@@ -1,4 +1,4 @@-from pipelex.base_exceptions import PipelexError+from pipelex.base_exceptions import ErrorDomain, PipelexErrorfrom pipelex.core.exceptions import PipelexBundleBlueprintValidationErrorData, PipesAndConceptValidationErrorData@@ -12,6 +12,12 @@ class CrateNormalizationError(LibraryError):_declared_title = "Crate normalization error"+class CrateIntegrityError(LibraryError):+ """Raised when a library crate names a method tree it does not carry: the host that assembled it built it wrong."""++ _declared_title = "Crate integrity error"++class LibraryLoadingError(LibraryError):"""Error raised when loading library components fails.@@ -85,3 +91,31 @@ class PipeLoadingError(LibraryLoadingError):self.source = sourceself.original_error = original_errorsuper().__init__(message)+++class MethodTreeLoadError(LibraryLoadingError):+ """Raised when a method tree a load reads from this host's disk cannot be listed.++ The tree of a method's folder, of a library directory, of a `.mthds` file named alone or of a dependency package:+ the standard refused it (a symbolic link, a name that is not a tree path, a lone file the membership rule keeps+ out), it passed a limit this machine lists trees under, or a directory or a file of it could not be read. The+ tree is the caller's own method, so the message, which names the tree's root and the files by their tree paths,+ is theirs to read, and the same load is refused again until the tree changes.+ """++ error_domain = ErrorDomain.INPUT+ retryable = False+ _authors_caller_facing_message = True+++class HostLibraryTreeError(PipelexError):+ """Raised when a library directory of this host, one a load reads beside a caller's bundles, cannot be read as a method tree.++ Such a directory is the host's own (installed libraries, the defaults, `PIPELEXPATH` on a server), so a tree of it+ that the standard refuses, that passes a limit or that cannot be read is a fault of the host's configuration, not+ of the caller's method: it is not an `INPUT` error, and its message, which names paths on the host, is not+ caller-facing. It is deliberately no `LibraryError`, which a validation would report as the caller's invalid bundle.+ """++ error_domain = ErrorDomain.CONFIG+ retryable = Falsediff --git a/pipelex/libraries/library_crate.py b/pipelex/libraries/library_crate.pyindex 765d1d5b2..faf85a89a 100644--- a/pipelex/libraries/library_crate.py+++ b/pipelex/libraries/library_crate.py@@ -1,11 +1,14 @@import hashlibimport json+from mthds.method_tree.models import MethodTreefrom pydantic import BaseModel, ConfigDict, Fieldfrom pipelex.core.bundle_resources.bundle_resource import BundleResourcesfrom pipelex.core.concepts.concept_blueprint import ConceptBlueprintfrom pipelex.core.domains.domain_blueprint import DomainBlueprint+from pipelex.core.method_trees.tree_bundle import TreeBundle, TreeBundleRef+from pipelex.libraries.exceptions import CrateIntegrityErrorfrom pipelex.mthds_parsing.pipelex_bundle_blueprint import ElaborationMetadata, PipeBlueprintUnion@@ -61,17 +64,45 @@ class LibraryCrate(BaseModel):`python_sources`, it is transport-only: EXCLUDED from the fingerprint and from the normalized crate."""- host_bundle_files: dict[str, str] = Field(default_factory=dict)- """pipe_ref -> the bundle file the pipe was read from, on the host that assembled the crate, for a pipe whose bundle- that host read from a file (a library directory, a bundle file).+ method_trees: dict[str, str] = Field(default_factory=dict)+ """tree key (`tree_key_of`) -> a method tree the host that assembled the crate listed its bundles' files in, as+ the JSON of its listing, a `MethodTree`, so the crate stays plain text wherever it travels, a Temporal payload+ included.- A pipe reads a file beside its bundle, such as a PipeDocGen `template_file`, only through this map, never through- `source_map`: a source is a label, which a caller sends with content strings and a bundle's own text can write, so- a pipe loaded from content strings has no entry and reads only the files submitted with it (`bundle_resources`).- Transport-only, like `python_sources`: EXCLUDED from the fingerprint and from the normalized crate, so a host path- never reaches a caller through `/resolve`.+ A pipe reads a file its bundle names, such as a PipeDocGen `template_file`, only by resolving it in its bundle's+ tree (`tree_bundles`), never through `source_map`: a source is a label, which a caller sends with content strings+ and a bundle's own text can write, so a pipe loaded from content strings has no tree and reads only the files+ submitted with it (`bundle_resources`). Transport-only, like `python_sources`: EXCLUDED from the fingerprint and+ from the normalized crate, so a host path never reaches a caller through `/resolve`."""+ tree_bundles: dict[str, TreeBundleRef] = Field(default_factory=dict)+ """pipe_ref -> the member of `method_trees` the pipe's bundle was read as, for a pipe whose bundle the host that+ assembled the crate read from its disk. Transport-only, like `method_trees`."""++ def tree_bundles_by_pipe_ref(self) -> dict[str, TreeBundle]:+ """Each pipe whose bundle was read from a host's disk, with the member of its method tree the bundle was read as.++ Each tree is read back once, however many pipes its bundles define. A pipe of a bundle parsed from a string has+ no entry.++ Raises:+ CrateIntegrityError: A pipe names a tree the crate does not carry.+ """+ trees: dict[str, MethodTree] = {}+ tree_bundles: dict[str, TreeBundle] = {}+ for pipe_ref, tree_bundle_ref in self.tree_bundles.items():+ tree = trees.get(tree_bundle_ref.tree_key)+ if tree is None:+ tree_json = self.method_trees.get(tree_bundle_ref.tree_key)+ if tree_json is None:+ msg = f"The crate names the method tree '{tree_bundle_ref.tree_key}' for pipe '{pipe_ref}' but carries no such tree."+ raise CrateIntegrityError(msg)+ tree = MethodTree.model_validate_json(tree_json)+ trees[tree_bundle_ref.tree_key] = tree+ tree_bundles[pipe_ref] = TreeBundle(tree=tree, tree_path=tree_bundle_ref.tree_path)+ return tree_bundles+fingerprint: str = """""SHA-256 hex digest of the serialized concepts + pipes content (source is intentionally excluded)."""diff --git a/pipelex/libraries/library_crate_factory.py b/pipelex/libraries/library_crate_factory.pyindex 424a0dd5b..6a70b1311 100644--- a/pipelex/libraries/library_crate_factory.py+++ b/pipelex/libraries/library_crate_factory.py@@ -3,6 +3,7 @@ from typing import TYPE_CHECKING, NamedTuplefrom pipelex.core.bundle_resources.bundle_resource import BundleResourcesfrom pipelex.core.concepts.concept_factory import ConceptFactoryfrom pipelex.core.domains.domain_blueprint import DomainBlueprint+from pipelex.core.method_trees.tree_bundle import TreeBundleReffrom pipelex.libraries.collision_messages import duplicate_ref_msgfrom pipelex.libraries.concept.exceptions import ConceptLibraryErrorfrom pipelex.libraries.contract_match import contracts_match@@ -13,6 +14,8 @@ from pipelex.mthds_parsing.pipelex_bundle_blueprint import ElaborationMetadata,from pipelex.pipe_machinery.pipe_factory import PipeFactoryif TYPE_CHECKING:+ from mthds.method_tree.models import MethodTree+from pipelex.core.concepts.concept_blueprint import ConceptBlueprint@@ -64,13 +67,20 @@ class LibraryCrateFactory:pipes: dict[str, PipeBlueprintUnion] = {}domains: dict[str, DomainBlueprint] = {}source_map: dict[str, str] = {}- host_bundle_files: dict[str, str] = {}+ method_trees: dict[str, MethodTree] = {}+ tree_bundles: dict[str, TreeBundleRef] = {}elaboration_metadata: dict[str, ElaborationMetadata] = {}for blueprint in blueprints:domain_code = blueprint.domainsource = blueprint.source- host_bundle_file = str(blueprint.host_bundle_file) if blueprint.host_bundle_file is not None else None+ # The member of its method tree the bundle was read as, which a step reading a file the bundle names resolves+ # it in; None for a bundle parsed from a string.+ tree_bundle_ref: TreeBundleRef | None = None+ if blueprint.tree_bundle is not None:+ tree_key = blueprint.tree_bundle.tree_key+ method_trees.setdefault(tree_key, blueprint.tree_bundle.tree)+ tree_bundle_ref = TreeBundleRef(tree_key=tree_key, tree_path=blueprint.tree_bundle.tree_path)# Domain metadata: order-independent, omission-quiet merge (see merge_domain_metadata_field).if domain_code not in domains:@@ -149,11 +159,11 @@ class LibraryCrateFactory:else:source_map.pop(pipe_ref, None)if winner.blueprint is pipe_blueprint:- # The bundle file a step reads beside follows the winner too.- if host_bundle_file is not None:- host_bundle_files[pipe_ref] = host_bundle_file+ # The tree a step resolves the files it names in follows the winner too.+ if tree_bundle_ref is not None:+ tree_bundles[pipe_ref] = tree_bundle_refelse:- host_bundle_files.pop(pipe_ref, None)+ tree_bundles.pop(pipe_ref, None)cls._track_elaboration(elaboration_metadata=elaboration_metadata, pipe_ref=pipe_ref, elaboration=blueprint.get_elaboration_for(pipe_code))@@ -161,8 +171,8 @@ class LibraryCrateFactory:pipes[pipe_ref] = pipe_blueprintif source:source_map[pipe_ref] = source- if host_bundle_file is not None:- host_bundle_files[pipe_ref] = host_bundle_file+ if tree_bundle_ref is not None:+ tree_bundles[pipe_ref] = tree_bundle_refcls._track_elaboration(elaboration_metadata=elaboration_metadata, pipe_ref=pipe_ref, elaboration=blueprint.get_elaboration_for(pipe_code))@@ -177,7 +187,9 @@ class LibraryCrateFactory:pipes=pipes,domains=domains,source_map=source_map,- host_bundle_files=host_bundle_files,+ # Only the trees a winning declaration was read from: a tree whose every pipe lost a collision travels nowhere.+ method_trees={tree_key: method_trees[tree_key].model_dump_json() for tree_key in sorted({ref.tree_key for ref in tree_bundles.values()})},+ tree_bundles=tree_bundles,elaboration_metadata=elaboration_metadata,python_sources=python_sources or {},bundle_resources=dict(bundle_resources) if bundle_resources else {},diff --git a/pipelex/libraries/library_manager.py b/pipelex/libraries/library_manager.pyindex 4acd25668..73177292b 100644--- a/pipelex/libraries/library_manager.py+++ b/pipelex/libraries/library_manager.py@@ -7,6 +7,7 @@ from typing import TYPE_CHECKING, Any, ForwardRef, Literal, get_args, get_originfrom kajson.class_registry import ClassRegistryfrom kajson.kajson_manager import KajsonManager+from mthds.method_tree.models import TreeLimitsfrom mthds.package.dependency_resolver import ResolvedDependency, determine_exported_pipes, resolve_all_dependenciesfrom mthds.package.discovery import find_package_manifestfrom mthds.package.exceptions import DependencyResolveError, ManifestError@@ -26,6 +27,7 @@ from pipelex.core.concepts.native.concept_native import NativeConceptCodefrom pipelex.core.domains.domain_blueprint import DomainBlueprintfrom pipelex.core.domains.domain_factory import DomainFactoryfrom pipelex.core.exceptions import PipelexBundleBlueprintValidationErrorData, PipesAndConceptValidationErrorData+from pipelex.core.method_trees.building_bundle import building_pipe_offrom pipelex.core.pipes.exceptions import PipeLoadRefusalError, PipeOperatorModelChoiceErrorfrom pipelex.core.qualified_ref import QualifiedReffrom pipelex.core.stuffs.structured_content import StructuredContent@@ -44,8 +46,13 @@ from pipelex.libraries.library_crate import LibraryCratefrom pipelex.libraries.library_crate_factory import LibraryCrateFactoryfrom pipelex.libraries.library_factory import LibraryFactoryfrom pipelex.libraries.library_manager_abstract import LibraryManagerAbstract-from pipelex.libraries.library_utils import (- get_pipelex_mthds_files_from_dirs,+from pipelex.libraries.method_trees import (+ LibraryBundleFile,+ list_library_bundle_files,+ list_package_method_tree,+ local_tree_limits,+ tree_bundle_of_lone_file,+ tree_bundles_of,)from pipelex.libraries.pipe.exceptions import PipeLibraryErrorfrom pipelex.libraries.visibility_utils import check_visibility_for_blueprints, make_visibility_checker@@ -77,6 +84,7 @@ if TYPE_CHECKING:from pipelex.core.concepts.concept import Conceptfrom pipelex.core.domains.domain import Domain+ from pipelex.core.method_trees.tree_bundle import TreeBundleMTHDS_METHODS_DIRNAME = ".mthds/methods"@@ -118,21 +126,6 @@ def find_methods_dirs_from_blueprints(blueprints: list[PipelexBundleBlueprint])return result-def _dependency_bundle_source(*, package_address: str, package_root: Path, mthds_path: Path) -> str:- """Name a dependency's bundle by the package's address and the bundle's path inside the package.-- The name is the bundle's ``source``, which rides every item a refusal inside the package gives, the- files a duplicate declaration names included, and a verdict is caller-facing: where the host installed- the package is the host's own path, which a hosted caller must never read. A bundle found outside the- package root, which discovery does not produce, is named by its file name alone rather than by its path.- """- for candidate_root in (package_root, package_root.resolve()):- for candidate_path in (mthds_path, mthds_path.resolve()):- if candidate_path.is_relative_to(candidate_root):- return f"{package_address}/{candidate_path.relative_to(candidate_root).as_posix()}"- return f"{package_address}/{mthds_path.name}"--def _parse_dependency_bundle(*, mthds_path: Path, bundle_source: str, package_address: str) -> PipelexBundleBlueprint:"""Parse one bundle of a dependency package under its caller-facing source, refusing the load when it does not parse.@@ -585,26 +578,28 @@ class LibraryManager(LibraryManagerAbstract):library_dirs = []all_dirs: list[Path] = []- all_mthds_paths: list[Path] = []all_dirs.extend(library_dirs)- all_mthds_paths.extend(get_pipelex_mthds_files_from_dirs(set(library_dirs)))-- if library_file_paths:- all_mthds_paths.extend(library_file_paths)-- # Combine and deduplicate- seen_absolute_paths: set[str] = set()- valid_mthds_paths: list[Path] = []- for mthds_path in all_mthds_paths:- try:- absolute_path = str(mthds_path.resolve())- except (OSError, RuntimeError):- # For paths that can't be resolved (e.g., in zipped packages), use string representation- absolute_path = str(mthds_path)+ # Every bundle is read as a member of its method tree: each library directory's tree and the trees nested+ # in it, then the tree of each file named alone. A file the membership rule keeps out is never read.+ # This machine's limits are read once for the whole load.+ tree_limits = local_tree_limits()+ all_bundle_files: list[LibraryBundleFile] = list_library_bundle_files(library_dirs=library_dirs, limits=tree_limits)+ for library_file_path in library_file_paths or []:+ all_bundle_files.append(+ LibraryBundleFile(+ tree_bundle=tree_bundle_of_lone_file(bundle_file=library_file_path, limits=tree_limits),+ path=library_file_path,+ )+ )- if absolute_path not in seen_absolute_paths:- valid_mthds_paths.append(mthds_path)- seen_absolute_paths.add(absolute_path)+ # Each file once, as the member of the first tree that holds it.+ seen_locations: set[Path] = set()+ bundle_files: list[LibraryBundleFile] = []+ for bundle_file in all_bundle_files:+ location = bundle_file.tree_bundle.location+ if location not in seen_locations:+ bundle_files.append(bundle_file)+ seen_locations.add(location)# Import modules and register in global registries# Import from user directories@@ -658,9 +653,9 @@ class LibraryManager(LibraryManagerAbstract):# Load MTHDS files into the specific librarylog.verbose("Loading the MTHDS files into the library",- fields={"library_id": library_id, "mthds_paths": [str(mthds_path) for mthds_path in valid_mthds_paths]},+ fields={"library_id": library_id, "mthds_paths": [str(bundle_file.path) for bundle_file in bundle_files]},)- return self._load_mthds_files_into_library(library_id=library_id, valid_mthds_paths=valid_mthds_paths)+ return self._load_mthds_files_into_library(library_id=library_id, bundle_files=bundle_files)@overridedef is_crate_loaded(self, *, library_id: str, fingerprint: str) -> bool:@@ -795,6 +790,9 @@ class LibraryManager(LibraryManagerAbstract):# signature change across PipeFactory and belongs with the concept-side work.qualified_pipes = qualify_crate(crate).pipes+ # The member of its method tree each pipe's bundle was read as, each tree read back once+ tree_bundles_by_pipe_ref = crate.tree_bundles_by_pipe_ref()+# Load pipes with domain-filtered concept codesall_pipes: list[PipeAbstract] = []for pipe_ref, pipe_blueprint in qualified_pipes.items():@@ -808,13 +806,16 @@ class LibraryManager(LibraryManagerAbstract):concept_codes_for_domain = domain_concept_codes.get(domain_code, [])source = crate.source_map.get(pipe_ref)- # A pipe that reads a file beside its bundle (a PipeDocGen `template_file`) resolves it against the- # bundle file the host that assembled the crate read it from, and against nothing else: the pipe's- # `source` is overwritten with that file, or cleared, so neither a caller's label (`source_map`) nor a- # `source` the bundle's text wrote on the pipe can point the read at a directory on this host.- pipe_blueprint = pipe_blueprint.model_copy(update={"source": crate.host_bundle_files.get(pipe_ref)})- with _locating_pipe_build_refusals(- pipe_code=pipe_code, domain_code=domain_code, source=source, elaboration=crate.elaboration_metadata.get(pipe_ref)+ # A pipe that reads a file its bundle names (a PipeDocGen `template_file`) resolves it in the method tree+ # the host that assembled the crate listed its bundle in, and in nothing else: the pipe's `source` is+ # cleared, so neither a caller's label (`source_map`) nor a `source` the bundle's text wrote on the pipe+ # can point the read at a directory on this host. A bundle parsed from a string has no tree.+ pipe_blueprint = pipe_blueprint.model_copy(update={"source": None})+ with (+ building_pipe_of(tree_bundle=tree_bundles_by_pipe_ref.get(pipe_ref)),+ _locating_pipe_build_refusals(+ pipe_code=pipe_code, domain_code=domain_code, source=source, elaboration=crate.elaboration_metadata.get(pipe_ref)+ ),):pipe = PipeFactory[PipeAbstract].make_from_blueprint(domain_code=domain_code,@@ -861,6 +862,18 @@ class LibraryManager(LibraryManagerAbstract):# (concept resolution, class-registry access) resolve through the ambient current# library, so a load into a non-current library must not read the caller's binding.with scoped_current_library(library_id=library_id):+ # A bundle read from a file on this host that no loader placed in a method tree, such as a bundle file a CLI+ # names alone, is read as a member of its own file's tree: the steps of every bundle read from this host's+ # disk read only the files of their tree.+ unplaced_host_blueprints = [+ blueprint for blueprint in blueprints if blueprint.tree_bundle is None and blueprint.host_bundle_file is not None+ ]+ if unplaced_host_blueprints:+ tree_limits = local_tree_limits()+ for blueprint in unplaced_host_blueprints:+ if blueprint.host_bundle_file is not None:+ blueprint.place_in_tree(tree_bundle=tree_bundle_of_lone_file(bundle_file=blueprint.host_bundle_file, limits=tree_limits))+# Accumulate blueprints for later crate construction via get_crate()self._blueprints.setdefault(library_id, []).extend(blueprints)self._crate_cache.pop(library_id, None)@@ -1050,23 +1063,26 @@ class LibraryManager(LibraryManagerAbstract):# Private helper methods############################################################- def _load_mthds_files_into_library(self, *, library_id: str, valid_mthds_paths: list[Path]) -> list[PipeAbstract]:+ def _load_mthds_files_into_library(self, *, library_id: str, bundle_files: list[LibraryBundleFile]) -> list[PipeAbstract]:"""Load MTHDS files into a specific library.This method:- 1. Parses blueprints from MTHDS files+ 1. Parses blueprints from MTHDS files, each placed in the method tree it was listed in2. Finds and loads dependency packages (if manifest has dependencies with local paths)3. Loads blueprints into the specified libraryArgs:library_id: The ID of the library to load into- valid_mthds_paths: List of MTHDS file paths to load+ bundle_files: The bundle files to load, each the member of its method tree"""+ valid_mthds_paths = [bundle_file.path for bundle_file in bundle_files]blueprints: list[PipelexBundleBlueprint] = []- for mthds_file_path in valid_mthds_paths:+ for bundle_file in bundle_files:+ mthds_file_path = bundle_file.pathtry:blueprint = MthdsParser.make_pipelex_bundle_blueprint(bundle_path=mthds_file_path)blueprint.source = str(mthds_file_path)+ blueprint.place_in_tree(tree_bundle=bundle_file.tree_bundle)except FileNotFoundError as file_not_found_error:msg = f"Could not find MTHDS bundle at '{mthds_file_path}'"raise LibraryLoadingError(msg) from file_not_found_error@@ -1282,11 +1298,14 @@ class LibraryManager(LibraryManagerAbstract):library = self.get_library(library_id=library_id)+ # This machine's limits are read once for every package of the manifest.+ tree_limits = local_tree_limits()for resolved_dep in resolved_deps:self._load_single_dependency(library=library,resolved_dep=resolved_dep,package_address=resolved_dep.address,+ tree_limits=tree_limits,)# Wire concept resolver after all deps are loaded so cross-package@@ -1299,6 +1318,7 @@ class LibraryManager(LibraryManagerAbstract):*,resolved_dep: ResolvedDependency,package_address: str,+ tree_limits: TreeLimits | None = None,) -> None:"""Load a single resolved dependency into an isolated child library.@@ -1306,9 +1326,10 @@ class LibraryManager(LibraryManagerAbstract):into it, registers it in library.dependency_libraries, and adds aliasedentries to the main library for backward-compatible cross-package lookups.- Each bundle's ``source`` is ``<package_address>/<path inside the package>``, never the file's- path on the host (see ``_dependency_bundle_source``), so a refusal inside the dependency names it- that way on every surface.+ The bundles are the ``.mthds`` files of the package's method tree, rooted at its own directory, so a file+ the membership rule keeps out of the package is neither loaded nor read by one of its steps. Each bundle's+ ``source`` is ``<package_address>/<tree path>``, never the file's path on the host, so a refusal inside the+ dependency names it that way on every surface.A bundle of the dependency that does not parse refuses the load (``_parse_dependency_bundle``): apackage shipped with a request is the caller's own content, and a fetched package's author is owed@@ -1318,20 +1339,26 @@ class LibraryManager(LibraryManagerAbstract):library: The main library to load intoresolved_dep: The resolved dependency infopackage_address: The dependency's address as a reference names it, without any ``@<tag>``+ tree_limits: The limits the package's tree is listed under, read from the configuration when not givenRaises:MthdsParserError: A bundle of the dependency does not parse, its items named by the bundle's source."""alias = resolved_dep.alias- # Parse dependency blueprints+ # Parse dependency blueprints: the bundles of the package's method tree, rooted at its own directory, so a file+ # the membership rule keeps out of the package is neither loaded nor read by one of its steps.dep_blueprints: list[PipelexBundleBlueprint] = []- # Where each bundle is on this host, by its caller-facing source, for the pipes that read a file beside it.- bundle_files_by_source: dict[str, Path] = {}- for mthds_path in resolved_dep.mthds_files:- bundle_source = _dependency_bundle_source(package_address=package_address, package_root=resolved_dep.package_root, mthds_path=mthds_path)- blueprint = _parse_dependency_bundle(mthds_path=mthds_path, bundle_source=bundle_source, package_address=package_address)- bundle_files_by_source[bundle_source] = mthds_path+ # Each bundle as the member of the package's tree, by its caller-facing source, for the pipes reading a file it names.+ tree_bundles_by_source: dict[str, TreeBundle] = {}+ # Listed under the package's address: a refusal never names the directory it was installed or fetched into.+ package_tree = list_package_method_tree(root=resolved_dep.package_root, package_address=package_address, limits=tree_limits)+ for tree_bundle in tree_bundles_of(tree=package_tree):+ # The package's address and the bundle's tree path: never the host's path to the package.+ bundle_source = f"{package_address}/{tree_bundle.tree_path}"+ blueprint = _parse_dependency_bundle(mthds_path=tree_bundle.location, bundle_source=bundle_source, package_address=package_address)+ blueprint.place_in_tree(tree_bundle=tree_bundle)+ tree_bundles_by_source[bundle_source] = tree_bundledep_blueprints.append(blueprint)if not dep_blueprints:@@ -1433,20 +1460,23 @@ class LibraryManager(LibraryManagerAbstract):if pipe_ref not in loaded_pipe_refs:continuedependency_source = crate.source_map.get(pipe_ref)- # A pipe that reads a file beside its bundle (a PipeDocGen `template_file`) finds it from the- # bundle's file on this host, which its blueprint carries into the factory and no further. The- # package's address stays the source every refusal names, since the host's path is not the caller's.- bundle_file = bundle_files_by_source.get(dependency_source) if dependency_source is not None else None- # Overwritten either way, so a `source` the package's own text wrote on the pipe never names a directory.- pipe_blueprint = pipe_blueprint.model_copy(update={"source": str(bundle_file) if bundle_file is not None else None})+ # A pipe that reads a file its bundle names (a PipeDocGen `template_file`) resolves it in the package's+ # tree, from the bundle's tree path there. The package's address stays the source every refusal names,+ # since the host's path is not the caller's.+ dependency_tree_bundle = tree_bundles_by_source.get(dependency_source) if dependency_source is not None else None+ # Cleared, so a `source` the package's own text wrote on the pipe never names a directory.+ pipe_blueprint = pipe_blueprint.model_copy(update={"source": None})try:# The same location the main load path attaches, so a dependency pipe's refusal# names the dependency's own file.- with _locating_pipe_build_refusals(- pipe_code=pipe_code,- domain_code=domain_code,- source=dependency_source,- elaboration=crate.elaboration_metadata.get(pipe_ref),+ with (+ building_pipe_of(tree_bundle=dependency_tree_bundle),+ _locating_pipe_build_refusals(+ pipe_code=pipe_code,+ domain_code=domain_code,+ source=dependency_source,+ elaboration=crate.elaboration_metadata.get(pipe_ref),+ ),):pipe = PipeFactory[PipeAbstract].make_from_blueprint(domain_code=domain_code,diff --git a/pipelex/libraries/library_utils.py b/pipelex/libraries/library_utils.pyindex 15a370edb..07c6d9e70 100644--- a/pipelex/libraries/library_utils.py+++ b/pipelex/libraries/library_utils.py@@ -4,10 +4,10 @@ from pathlib import Pathfrom pipelex import logfrom pipelex.config import get_config-from pipelex.mthds_parsing.helpers import MTHDS_EXTENSION, is_pipelex_file+from pipelex.libraries.method_trees import list_library_bundle_files+from pipelex.mthds_parsing.helpers import is_pipelex_filefrom pipelex.system.telemetry.otel_constants import OTelLogAttrfrom pipelex.tools.log.error_fields import error_fields-from pipelex.tools.misc.file_utils import find_files_in_dirdef get_pipelex_mthds_files_from_package() -> list[Path]:@@ -65,25 +65,13 @@ def get_pipelex_package_dir_for_imports() -> Path | None:def get_pipelex_mthds_files_from_dirs(dirs: set[Path]) -> list[Path]:- """Get all valid Pipelex MTHDS files from the given directories."""- all_mthds_paths: list[Path] = []+ """The bundle files a load of these library directories reads, each spelled from its directory as given.- for dir_path in dirs:- if not dir_path.exists():- log.debug(f"A directory that does not exist was skipped: {dir_path}")- continue+ The files are the `.mthds` members of each directory's method tree and of the trees nested in it+ (`pipelex.libraries.method_trees`), so a file the membership rule keeps out, such as one in a run's output folder,+ is never listed. A directory that does not exist holds none.- # Find all .mthds files in the directory, excluding problematic directories- mthds_files = find_files_in_dir(- dir_path=dir_path,- pattern=f"*{MTHDS_EXTENSION}",- excluded_dirs=list(get_config().interpreter.scan.excluded_dirs),- )-- # Filter to only include valid Pipelex files- for mthds_file in mthds_files:- if is_pipelex_file(mthds_file):- all_mthds_paths.append(mthds_file)- else:- log.debug(f"An MTHDS file that is not a Pipelex file was skipped: {mthds_file}")- return all_mthds_paths+ Raises:+ MethodTreeLoadError: A tree could not be listed.+ """+ return [bundle_file.path for bundle_file in list_library_bundle_files(library_dirs=sorted(dirs))]diff --git a/pipelex/libraries/method_trees.py b/pipelex/libraries/method_trees.pynew file mode 100644index 000000000..8d39177e8--- /dev/null+++ b/pipelex/libraries/method_trees.py@@ -0,0 +1,437 @@+"""Listing the method trees a local load reads its bundles from, under this machine's limits.++Every bundle this host reads from its own disk is read as a member of a method tree, listed by `mthds-python`+(`mthds/docs/spec/method-tree.md`): the tree a directory is rooted in, at the nearest `METHODS.toml` at or above it,+else at the directory itself; the tree of a `.mthds` file named alone, rooted at the nearest `METHODS.toml` at or+above its folder, else the one-file tree of that file; and a dependency package's tree, rooted at the package's own+directory. The membership rule decides what a tree holds, so a file it keeps out, a dotfile, a `.gitignore`d file, a+virtual environment, is never loaded and never read by a step.++A library directory, such as a `-L` directory or one of `PIPELEXPATH`, is not a method but a place methods are+found: its own tree is loaded, then each nested package and nested repository the listing kept out of it, each as a+tree of its own, so every method under it loads as it did before trees, and each reads only its own files.++A package or repository nested in a library directory that holds no `.mthds` file is skipped before it is listed, so+a nested checkout with nothing to load, a submodule say, never fails the load for a file it holds. A library directory+whose own tree is empty, one holding nothing but nested methods, still has its nested methods found and loaded.++Each tree is listed under `[interpreter.method_tree_limits]`, this machine's own limits, read once per load: the lister+stops before hashing any file once a tree passes one, so a directory handed over by mistake is refused in moments.+"""++from collections.abc import Generator, Iterable+from contextlib import contextmanager+from pathlib import Path+from typing import NamedTuple++from mthds.method_tree.discovery import (+ BUNDLE_SUFFIX,+ DEFAULT_DIRECTORY_NAMES,+ GITIGNORE_NAME,+ DirectoryEntry,+ find_manifest_directory,+ holds_git_entry,+ holds_manifest,+ holds_venv_marker,+ kind_of_exact_name,+ read_directory,+)+from mthds.method_tree.exceptions import MethodTreeRefusedError, TreeLimitError, TreeSourceError+from mthds.method_tree.gitignore import PatternList+from mthds.method_tree.lister import list_method_tree, list_method_tree_at_root+from mthds.method_tree.models import MethodTree, TreeLimits+from mthds.method_tree.tree_path import nfc+from mthds.method_tree.vocabulary import ExclusionRule, TreeRefusalCode++from pipelex import log+from pipelex.core.method_trees.tree_bundle import TreeBundle+from pipelex.hosted.execution import get_or_load_pipelex_config+from pipelex.libraries.exceptions import HostLibraryTreeError, MethodTreeLoadError++LIMITS_ADVICE = (+ "Hand over the method's own folder rather than a directory above it, or raise the limits of "+ "interpreter.method_tree_limits in your project's .pipelex/pipelex.toml."+)+++def local_tree_limits() -> TreeLimits:+ """The limits this machine lists a method tree under, from `[interpreter.method_tree_limits]`.++ Read from the booted configuration, or from the configuration files when nothing booted, since a CLI lists the+ tree it runs before it boots, and a hosted run never boots.++ Raises:+ PipelexConfigError: If nothing booted and the configuration files do not validate.+ """+ limits_config = get_or_load_pipelex_config().interpreter.method_tree_limits+ return TreeLimits(+ max_files=limits_config.max_files,+ max_total_bytes=limits_config.max_total_bytes,+ max_file_bytes=limits_config.max_file_bytes,+ )+++def list_local_method_tree(*, handed_over: Path, limits: TreeLimits | None = None) -> MethodTree:+ """List the method tree of a directory, or of a `.mthds` file named alone, under this machine's limits.++ Args:+ handed_over: The directory or the `.mthds` file handed over.+ limits: The limits to list under, read from the configuration when not given.++ Raises:+ MethodTreeLoadError: The standard refused the tree, it passed a limit, or it could not be read.+ """+ try:+ return list_method_tree(handed_over, limits=limits or local_tree_limits())+ except TreeLimitError as exc:+ msg = f"{exc.message} {LIMITS_ADVICE}"+ raise MethodTreeLoadError(msg) from exc+ except (MethodTreeRefusedError, TreeSourceError) as exc:+ raise MethodTreeLoadError(exc.message) from exc+++def _with_root_named(*, message: str, root: Path, label: str) -> str:+ """`message` with every spelling of `root` it holds replaced by `label`, the longest spelling first."""+ spellings = {str(root), str(root.absolute())}+ try:+ spellings.add(str(root.resolve()))+ except (OSError, RuntimeError):+ pass+ for spelling in sorted(spellings, key=len, reverse=True):+ message = message.replace(spelling, label)+ return message+++def list_package_method_tree(*, root: Path, package_address: str, limits: TreeLimits | None = None) -> MethodTree:+ """List a dependency package's tree, rooted at the package's own directory, under this machine's limits.++ A refusal names the package by its address and its files by their tree paths, never by the directory the package+ was installed or fetched into, which is a path on this host: the package is the caller's dependency, so the refusal+ is theirs to read, and nothing in it is theirs to change on this host, so it gives no advice about its limits.++ Raises:+ MethodTreeLoadError: The standard refused the tree, it passed a limit, or it could not be read.+ """+ try:+ return list_method_tree_at_root(root, limits=limits or local_tree_limits())+ except (MethodTreeRefusedError, TreeLimitError, TreeSourceError) as exc:+ msg = _with_root_named(message=exc.message, root=root, label=package_address)+ raise MethodTreeLoadError(msg) from exc+++@contextmanager+def reporting_host_library_tree_faults() -> Generator[None, None, None]:+ """Report a method tree of this host's own library directories that cannot be listed as the host's fault.++ Around a load of library directories that are not the caller's, such as the ones a server loads beside a request's+ bundles, a `MethodTreeLoadError` leaves as a `HostLibraryTreeError`, which is a configuration fault and whose+ message, naming paths on the host, never reaches the caller.++ Raises:+ HostLibraryTreeError: A tree of the directories could not be listed.+ """+ try:+ yield+ except MethodTreeLoadError as exc:+ msg = f"A library directory of this host cannot be read as a method tree: {exc.message}"+ raise HostLibraryTreeError(msg) from exc+++def tree_bundles_of(*, tree: MethodTree) -> list[TreeBundle]:+ """The `.mthds` files of a tree, in tree path order, each as a member of the tree."""+ return [TreeBundle(tree=tree, tree_path=tree_path) for tree_path in tree.tree_paths if tree_path.endswith(BUNDLE_SUFFIX)]+++def tree_bundle_of_lone_file(*, bundle_file: Path, limits: TreeLimits | None = None) -> TreeBundle:+ """A bundle file this host read, named alone, as the member of its method tree.++ Its tree is rooted at the nearest `METHODS.toml` at or above its folder, else it is the one-file tree of the file,+ from which a reference to any other file is refused.++ Raises:+ MethodTreeLoadError: The tree could not be listed, or the membership rule keeps the file out of it.+ """+ tree = list_local_method_tree(handed_over=bundle_file, limits=limits)+ if tree.entry is None:+ msg = f"Listing the method tree of the bundle file '{bundle_file}' named no entry for it."+ raise MethodTreeLoadError(msg)+ return TreeBundle(tree=tree, tree_path=tree.entry)+++def _is_empty_tree(*, refusal_error: MethodTreeRefusedError) -> bool:+ match refusal_error.refusal.code:+ case TreeRefusalCode.EMPTY_TREE:+ return True+ case (+ TreeRefusalCode.SYMBOLIC_LINK+ | TreeRefusalCode.SPECIAL_FILE+ | TreeRefusalCode.INVALID_TREE_PATH+ | TreeRefusalCode.NORMALIZATION_COLLISION+ | TreeRefusalCode.PATH_COLLISION+ | TreeRefusalCode.LONE_FILE_EXCLUDED+ | TreeRefusalCode.GITIGNORE_NOT_UTF8+ | TreeRefusalCode.DUPLICATE_KEY+ | TreeRefusalCode.EXCLUDED_NAME+ | TreeRefusalCode.MALFORMED_ENTRY+ | TreeRefusalCode.MISSING_BLOB+ | TreeRefusalCode.REFERENCE_INVALID+ | TreeRefusalCode.REFERENCE_LEAVES_TREE+ | TreeRefusalCode.REFERENCE_NOT_A_TREE_PATH+ | TreeRefusalCode.REFERENCE_NOT_IN_TREE+ | TreeRefusalCode.REFERENCE_IN_ONE_FILE_TREE+ | TreeRefusalCode.FUNCTION_NAME_WITHOUT_MODULE+ | TreeRefusalCode.FUNCTION_NAME_INVALID_SEGMENT+ | TreeRefusalCode.MODULE_NOT_IN_TREE+ | TreeRefusalCode.MODULE_AMBIGUOUS+ ):+ return False+++def _holds_a_tree_of_its_own(*, rule: ExclusionRule) -> bool:+ """Whether a directory a library directory's listing kept out by `rule` is a tree of its own, loaded beside it."""+ match rule:+ case ExclusionRule.NESTED_PACKAGE | ExclusionRule.NESTED_REPOSITORY:+ return True+ case (+ ExclusionRule.DOT_SEGMENT+ | ExclusionRule.PYCACHE_OR_NODE_MODULES+ | ExclusionRule.VIRTUAL_ENVIRONMENT+ | ExclusionRule.GITIGNORE+ | ExclusionRule.PACKAGE_EXCLUDE+ ):+ return False+++def _holds_a_bundle_name(*, directory: Path) -> bool:+ """Whether any file under `directory` is named like a bundle, judged by names alone, before any tree is listed.++ A cheap scan: it skips what the membership rule keeps out by name (a dot segment, `__pycache__` and+ `node_modules`, a virtual environment) and reads no `.gitignore`, so it may find a bundle a listing then keeps out,+ never miss one a listing keeps. A directory it cannot read holds nothing it can find.+ """+ pending = [directory]+ while pending:+ current = pending.pop()+ try:+ entries = read_directory(current)+ except OSError:+ continue+ if holds_venv_marker(entries):+ continue+ for entry in entries:+ if entry.raw_name.startswith("."):+ continue+ if entry.kind.is_directory:+ if entry.raw_name not in DEFAULT_DIRECTORY_NAMES:+ pending.append(current / entry.raw_name)+ elif entry.raw_name.endswith(BUNDLE_SUFFIX):+ return True+ return False+++def _with_gitignore(+ *, location: Path, tree_path: str, entries: list[DirectoryEntry], pattern_lists: tuple[PatternList, ...]+) -> tuple[PatternList, ...]:+ """The pattern lists that apply to a directory's entries: its parents', and its own `.gitignore` file's when it holds one."""+ kind = kind_of_exact_name(entries, GITIGNORE_NAME)+ if kind is None or not kind.is_regular_file:+ return pattern_lists+ try:+ pattern_list = PatternList.make_from_bytes((location / GITIGNORE_NAME).read_bytes(), base=tree_path)+ except (OSError, UnicodeDecodeError):+ # The listing that found this tree empty read the file; one that changed since keeps nothing out.+ return pattern_lists+ return (*pattern_lists, pattern_list)+++def _kept_out_by_patterns(*, tree_path: str, pattern_lists: tuple[PatternList, ...]) -> bool:+ """Whether the `.gitignore` patterns keep a directory out, as git decides: the deepest file with a matching pattern."""+ for pattern_list in reversed(pattern_lists):+ matched = pattern_list.last_match(tree_path, is_dir=True)+ if matched is not None:+ return not matched.negative+ return False+++def _nested_tree_roots(*, directory: Path) -> list[Path]:+ """The roots of the packages and repositories nested in a directory whose own tree the membership rule leaves empty.++ The lister refuses an empty tree without saying what it kept out, so this walks the directory as the lister does,+ judging each directory by its name, then by its own entries, then by the `.gitignore` patterns, and descends into+ every directory it keeps: a nested package or repository is a root, and what the rule keeps out is never entered.+ The tree is rooted at the directory itself, since a tree rooted at a `METHODS.toml` holds that file and is never empty.+ """+ roots: list[Path] = []+ pending: list[tuple[Path, str, tuple[PatternList, ...]]] = [(directory, "", ())]+ while pending:+ location, tree_path, parent_pattern_lists = pending.pop()+ try:+ entries = read_directory(location)+ except OSError:+ continue+ pattern_lists = _with_gitignore(location=location, tree_path=tree_path, entries=entries, pattern_lists=parent_pattern_lists)+ children: list[tuple[Path, str, tuple[PatternList, ...]]] = []+ for entry in sorted(entries, key=lambda directory_entry: directory_entry.raw_name):+ name = nfc(entry.raw_name)+ if not entry.kind.is_directory or name.startswith(".") or name in DEFAULT_DIRECTORY_NAMES:+ continue+ child_location = location / entry.raw_name+ child_tree_path = f"{tree_path}/{name}" if tree_path else name+ try:+ child_entries = read_directory(child_location)+ except OSError:+ continue+ if holds_venv_marker(child_entries):+ continue+ if holds_git_entry(child_entries) or holds_manifest(child_entries):+ roots.append(child_location)+ continue+ if _kept_out_by_patterns(tree_path=child_tree_path, pattern_lists=pattern_lists):+ continue+ children.append((child_location, child_tree_path, pattern_lists))+ pending.extend(reversed(children))+ return roots+++def _trees_nested_in_empty_directory(*, directory: Path, limits: TreeLimits) -> list[MethodTree]:+ """The trees of the packages and repositories nested in a directory whose own tree is empty, and theirs."""+ trees: list[MethodTree] = []+ for nested_root in _nested_tree_roots(directory=directory):+ trees.extend(_nested_library_trees(root=nested_root, limits=limits))+ return trees+++def _nested_library_trees(*, root: Path, limits: TreeLimits) -> list[MethodTree]:+ """The tree of a package or repository nested in a library directory, then the trees nested in it, depth first.++ One that holds no file named like a bundle is skipped before it is listed, so a nested checkout with nothing to+ load never refuses the load for a file it holds; one that does follows every rule of the standard.++ Raises:+ MethodTreeLoadError: A tree could not be listed.+ """+ if not _holds_a_bundle_name(directory=root):+ log.debug(f"A nested directory holding no bundle was skipped: {root}")+ return []+ try:+ tree = list_method_tree_at_root(root, limits=limits)+ except MethodTreeRefusedError as exc:+ if _is_empty_tree(refusal_error=exc):+ return _trees_nested_in_empty_directory(directory=root, limits=limits)+ raise MethodTreeLoadError(exc.message) from exc+ except TreeLimitError as exc:+ msg = f"{exc.message} {LIMITS_ADVICE}"+ raise MethodTreeLoadError(msg) from exc+ except TreeSourceError as exc:+ raise MethodTreeLoadError(exc.message) from exc+ return _library_trees(tree=tree, limits=limits)+++def _library_trees(*, tree: MethodTree, limits: TreeLimits) -> list[MethodTree]:+ """`tree`, then the tree of each nested package and nested repository it kept out, and theirs, depth first."""+ trees = [tree]+ if tree.directory is None:+ return trees+ for tree_path, exclusion in tree.excluded_directories.items():+ if not _holds_a_tree_of_its_own(rule=exclusion.rule):+ continue+ on_disk = tree.raw_paths.get(tree_path, tree_path)+ trees.extend(_nested_library_trees(root=tree.directory.joinpath(*on_disk.split("/")), limits=limits))+ return trees+++def list_library_trees(*, library_dir: Path, limits: TreeLimits | None = None) -> list[MethodTree]:+ """The method trees a library directory holds: its own, then those of the packages and repositories nested in it.++ A directory that does not exist holds none. One whose own tree the membership rule leaves empty, such as a+ directory holding nothing but nested methods, holds the trees nested in it.++ Args:+ library_dir: The library directory.+ limits: The limits to list each tree under, read from the configuration when not given.++ Raises:+ MethodTreeLoadError: A tree could not be listed.+ """+ if not library_dir.exists():+ log.debug(f"A library directory that does not exist was skipped: {library_dir}")+ return []+ active_limits = limits or local_tree_limits()+ try:+ tree = list_method_tree(library_dir, limits=active_limits)+ except MethodTreeRefusedError as exc:+ if _is_empty_tree(refusal_error=exc):+ return _trees_nested_in_empty_directory(directory=library_dir.resolve(), limits=active_limits)+ raise MethodTreeLoadError(exc.message) from exc+ except TreeLimitError as exc:+ msg = f"{exc.message} {LIMITS_ADVICE}"+ raise MethodTreeLoadError(msg) from exc+ except TreeSourceError as exc:+ raise MethodTreeLoadError(exc.message) from exc+ return _library_trees(tree=tree, limits=active_limits)+++class LibraryBundleFile(NamedTuple):+ """A bundle file of a library directory: the member of its method tree, and its path spelled from the directory as given."""++ tree_bundle: TreeBundle+ path: Path+++def _spelled_from(*, library_dir: Path, location: Path) -> Path:+ """`location` spelled from `library_dir` as the caller gave it, so a relative library directory names relative files.++ A file outside the directory, in a tree rooted above it at a `METHODS.toml`, keeps its absolute path.+ """+ try:+ return library_dir / location.relative_to(library_dir.resolve())+ except ValueError:+ return location+++def list_library_bundle_files(*, library_dirs: Iterable[Path], limits: TreeLimits | None = None) -> list[LibraryBundleFile]:+ """The bundle files of the library directories, each as a member of its method tree, each file once.++ The directories are read in the order given, and a file two of their trees hold, such as a directory and one of+ its subdirectories both given, is the member of the first tree that holds it.++ Args:+ library_dirs: The library directories.+ limits: The limits to list each tree under, read from the configuration once when not given.++ Raises:+ MethodTreeLoadError: A tree could not be listed.+ """+ active_limits = limits or local_tree_limits()+ seen_locations: set[Path] = set()+ bundle_files: list[LibraryBundleFile] = []+ for library_dir in library_dirs:+ for tree in list_library_trees(library_dir=library_dir, limits=active_limits):+ for tree_bundle in tree_bundles_of(tree=tree):+ location = tree_bundle.location+ if location in seen_locations:+ continue+ seen_locations.add(location)+ bundle_files.append(LibraryBundleFile(tree_bundle=tree_bundle, path=_spelled_from(library_dir=library_dir, location=location)))+ return bundle_files+++def bundle_files_under(*, library_dirs: Iterable[Path]) -> list[Path]:+ """Every `.mthds` file under the library directories and the trees they are rooted in, whatever the membership rule.++ Never raises: it is for a caller that must name every file a load of the directories could have named even when+ listing their trees fails, such as one withholding a host's files from a verdict, so it returns a superset of the+ files a load names, spelled from each directory as given.+ """+ bundle_files: list[Path] = []+ for library_dir in library_dirs:+ try:+ tree_root = find_manifest_directory(library_dir.resolve()) or library_dir+ except (OSError, RuntimeError, TreeSourceError):+ tree_root = library_dir+ for search_root in dict.fromkeys((library_dir, tree_root)):+ try:+ bundle_files.extend(sorted(search_root.rglob(f"*{BUNDLE_SUFFIX}")))+ except OSError:+ continue+ return list(dict.fromkeys(bundle_files))diff --git a/pipelex/migration/goldens/pipelex-config/[email protected] b/pipelex/migration/goldens/pipelex-config/[email protected]index ec186eff5..bf914ead3 100644--- a/pipelex/migration/goldens/pipelex-config/[email protected]+++ b/pipelex/migration/goldens/pipelex-config/[email protected]@@ -475,6 +475,19 @@ excluded_dirs = ["results",]+####################################################################################################+# Method tree limits — how large a method tree a local load lists+####################################################################################################++[interpreter.method_tree_limits]+# A local run, validation or load reads every method from its method tree: a method's folder, a .mthds file named+# alone, a library directory, a dependency package. Listing a tree stops before reading any file once the tree+# passes one of these limits, so a directory handed over by mistake, such as a home directory, is refused in moments+# rather than read in full. They are this machine's own, not the hosted plane's: raise them in your project's+# .pipelex/pipelex.toml for a method that needs more. max_file_bytes, the size of one file, is not checked unless set.+max_files = 10000+max_total_bytes = 1073741824+[interpreter.builder]fix_loop_max_attempts = 10default_output_dir = "."diff --git a/pipelex/migration/goldens/pipelex-config/[email protected] b/pipelex/migration/goldens/pipelex-config/[email protected]index b0b973792..3cba268e1 100644--- a/pipelex/migration/goldens/pipelex-config/[email protected]+++ b/pipelex/migration/goldens/pipelex-config/[email protected]@@ -527,6 +527,37 @@"default": 10,"open_node": false},+ "interpreter.method_tree_limits": {+ "value_type": "table",+ "required": true,+ "open_node": false+ },+ "interpreter.method_tree_limits.max_file_bytes": {+ "value_type": "int",+ "required": false,+ "open_node": false,+ "constraints": {+ "ge": 0+ }+ },+ "interpreter.method_tree_limits.max_files": {+ "value_type": "int",+ "required": false,+ "default": 10000,+ "open_node": false,+ "constraints": {+ "ge": 1+ }+ },+ "interpreter.method_tree_limits.max_total_bytes": {+ "value_type": "int",+ "required": false,+ "default": 1073741824,+ "open_node": false,+ "constraints": {+ "ge": 0+ }+ },"interpreter.methods": {"value_type": "table","required": true,diff --git a/pipelex/mthds_parsing/pipelex_bundle_blueprint.py b/pipelex/mthds_parsing/pipelex_bundle_blueprint.pyindex 6ff6068b0..d138c8752 100644--- a/pipelex/mthds_parsing/pipelex_bundle_blueprint.py+++ b/pipelex/mthds_parsing/pipelex_bundle_blueprint.py@@ -11,6 +11,7 @@ from pipelex.core.concepts.native.concept_native import NativeConceptCodefrom pipelex.core.concepts.validation import is_concept_code_validfrom pipelex.core.domains.exceptions import DomainCodeErrorfrom pipelex.core.domains.validation import validate_domain_code+from pipelex.core.method_trees.tree_bundle import TreeBundlefrom pipelex.core.pipes.variable_multiplicity import parse_concept_with_multiplicityfrom pipelex.mthds_parsing.exceptions import InvalidPipeCodeSyntaxError, NativeConceptRedeclarationErrorfrom pipelex.pipe_controllers.batch.pipe_batch_blueprint import PipeBatchBlueprint@@ -127,19 +128,34 @@ class PipelexBundleBlueprint(BaseModel):# The file on this host the bundle was read from, set by the parser when it reads one and by a loader that reads a# bundle's file itself. Never serialized, and never taken from `source`: `source` is a label, which a caller sends# with content strings (`mthds_sources`) and the bundle's own text can write, so it names no directory to read.- # This is what lets a step read a file beside its bundle, such as a PipeDocGen `template_file`, and lets an- # address-based reference find a `.mthds/methods/` directory above it; a bundle parsed from a string has none.+ # It lets an address-based reference find a `.mthds/methods/` directory above the bundle, and it is what the+ # library manager lists the bundle's method tree from when no loader placed the bundle in one._host_bundle_file: Path | None = PrivateAttr(default=None)+ # The bundle as a member of its method tree, set by the loader that listed the tree. A step reads a file the bundle+ # names, such as a PipeDocGen `template_file`, only by resolving it in this tree; a bundle parsed from a string has+ # none, and reads only the files submitted with it.+ _tree_bundle: TreeBundle | None = PrivateAttr(default=None)@propertydef host_bundle_file(self) -> Path | None:"""The file on this host the bundle was read from, or None for a bundle parsed from a string."""+ if self._tree_bundle is not None:+ return self._tree_bundle.locationreturn self._host_bundle_file+ @property+ def tree_bundle(self) -> TreeBundle | None:+ """The bundle as a member of its method tree, or None for a bundle parsed from a string or not yet placed in one."""+ return self._tree_bundle+def mark_read_from_host_file(self, *, bundle_file: Path) -> None:- """Record that this host read the bundle from `bundle_file`, which its steps may read beside."""+ """Record that this host read the bundle from `bundle_file`, whose method tree the load lists."""self._host_bundle_file = bundle_file+ def place_in_tree(self, *, tree_bundle: TreeBundle) -> None:+ """Record the bundle as `tree_bundle`, the member of the method tree a loader listed it in."""+ self._tree_bundle = tree_bundle+@field_validator("domain", mode="before")@classmethoddef validate_domain_syntax(cls, domain_code: str) -> str:diff --git a/pipelex/pipe_machinery/pipe_blueprint.py b/pipelex/pipe_machinery/pipe_blueprint.pyindex a36ca14dc..814ce857a 100644--- a/pipelex/pipe_machinery/pipe_blueprint.py+++ b/pipelex/pipe_machinery/pipe_blueprint.py@@ -229,9 +229,9 @@ class PipeBlueprint(ABC, BaseModel):model_config = ConfigDict(extra="forbid")source: str | None = None- """Not an authoring field. The library manager overwrites it, just before the pipe is built, with the bundle file this- host read the pipe's bundle from, or None, so whatever a bundle's text writes here is never read (see- `LibraryCrate.host_bundle_files`)."""+ """Not an authoring field. The library manager clears it just before the pipe is built, so whatever a bundle's text+ writes here is never read: a step finds a file its bundle names in the bundle's method tree, which the library+ manager hands the factory (see `LibraryCrate.method_trees` and `building_pipe_of`)."""pipe_category: Any = Field(exclude=True) # Technical field for Union discrimination, not user-facingtype: Any # TODO: Find a better way to handle this.description: strdiff --git a/pipelex/pipe_operators/doc_gen/pipe_doc_gen_blueprint.py b/pipelex/pipe_operators/doc_gen/pipe_doc_gen_blueprint.pyindex 71fcf66e0..de6e5d1ee 100644--- a/pipelex/pipe_operators/doc_gen/pipe_doc_gen_blueprint.py+++ b/pipelex/pipe_operators/doc_gen/pipe_doc_gen_blueprint.py@@ -7,7 +7,8 @@ from pipelex.cogt.doc_gen.doc_gen_format import DocGenFormatfrom pipelex.cogt.doc_gen.doc_gen_setting import DocGenModelChoicefrom pipelex.cogt.templating.exceptions import TemplateSigilSyntaxErrorfrom pipelex.cogt.templating.template_preprocessor import preprocess_template-from pipelex.core.bundle_resources.bundle_resource import BundleResourceReadStage, resource_path_refusal+from pipelex.core.bundle_resources.bundle_resource import BundleResourceReadStage+from pipelex.core.method_trees.file_references import file_reference_form_refusalfrom pipelex.core.pipes.variable_multiplicity import parse_concept_with_multiplicityfrom pipelex.pipe_machinery.pipe_blueprint import PipeBlueprintfrom pipelex.tools.jinja2.exceptions import Jinja2TemplateSyntaxError@@ -67,12 +68,13 @@ class PipeDocGenBlueprint(PipeBlueprint):@model_validator(mode="after")def validate_template_sources(self) -> Self:- # The rule the files submitted with a bundle are keyed by, so a template file can be submitted whatever loads the- # bundle. The message never repeats the path: a host library's step would put the host's own path in front of a caller.+ # The form of a file reference, the one check a blueprint can make before its method tree is at hand: whether the+ # path names a file, a `..` climbing elsewhere in the tree included, is decided when the bundle loads, in its tree.+ # The message never repeats the path: a host library's step would put the host's own path in front of a caller.if self.template_file is not None:- refusal = resource_path_refusal(path=self.template_file)+ refusal = file_reference_form_refusal(reference=self.template_file)if refusal is not None:- msg = f"PipeDocGen's 'template_file' is a path relative to the bundle's file, inside its bundle, and this one cannot be: {refusal}."+ msg = f"PipeDocGen's 'template_file' is a path relative to the bundle's file, in its method, and this one cannot be: {refusal}."raise ValueError(msg)if self.template is not None and self.template_file is not None:msg = "PipeDocGen cannot have both 'template' and 'template_file': use one or the other."diff --git a/pipelex/pipe_operators/doc_gen/pipe_doc_gen_factory.py b/pipelex/pipe_operators/doc_gen/pipe_doc_gen_factory.pyindex 3e1877ab8..d3108e087 100644--- a/pipelex/pipe_operators/doc_gen/pipe_doc_gen_factory.py+++ b/pipelex/pipe_operators/doc_gen/pipe_doc_gen_factory.py@@ -2,10 +2,14 @@ from collections.abc import Callablefrom pathlib import Pathfrom typing import Any+from mthds.method_tree.exceptions import MethodTreeRefusedErrorfrom typing_extensions import overridefrom pipelex.core.bundle_resources.bundle_resource import BundleResourcefrom pipelex.core.concepts.native.concept_native import NativeConceptCode+from pipelex.core.method_trees.building_bundle import get_building_tree_bundle+from pipelex.core.method_trees.file_references import reference_refusal_reason, resolve_bundle_file_reference+from pipelex.core.method_trees.tree_bundle import TreeBundlefrom pipelex.core.pipes.inputs.input_stuff_specs import InputStuffSpecsfrom pipelex.core.pipes.stuff_spec.stuff_spec import StuffSpecfrom pipelex.interpreter_hub import get_concept_library, get_native_concept@@ -17,22 +21,6 @@ from pipelex.pipe_operators.doc_gen.template_file_resource import bundle_resourcfrom pipelex.tools.jinja2.template_category import TemplateCategory-def bundle_dir_of_source(source: str | None) -> Path | None:- """The directory of the bundle file a pipe was loaded from, or None for a bundle loaded from a string.-- The library manager sets a pipe blueprint's `source` to the bundle file this host read the bundle from just before- the pipe is built, and clears it otherwise (`LibraryCrate.host_bundle_files`), so the label a caller sends with- content strings, or a `source` the bundle's own text writes, never reaches here: a content-string load has no- directory, and nothing beside it is read from this host's disk.- """- if source is None or "://" in source:- return None- source_path = Path(source)- if not source_path.is_file():- return None- return source_path.resolve().parent--class PipeDocGenFactory(PipeFactoryProtocol[PipeDocGenBlueprint, PipeDocGen]):@classmethod@override@@ -59,16 +47,17 @@ class PipeDocGenFactory(PipeFactoryProtocol[PipeDocGenBlueprint, PipeDocGen]):template = blueprint.templatetemplate_path: str | None = Noneif blueprint.template_file is not None:- bundle_dir = bundle_dir_of_source(blueprint.source)+ tree_bundle = get_building_tree_bundle()read_template_bytes: Callable[[], bytes]- if bundle_dir is None:+ if tree_bundle is None:# A bundle loaded from content strings reads only the files submitted with it, never this host's disk. An# office template is read from them when the step prints, so the pipe keeps only its name, and the run# carries the file in its crate.read_template_bytes = cls._submitted_template_file(pipe_code=pipe_code, template_file=blueprint.template_file).inline_byteselse:- # A bundle this host read from a file finds its template file beside it, and never among submitted files.- resolved_path = cls._resolve_template_file(pipe_code=pipe_code, template_file=blueprint.template_file, bundle_dir=bundle_dir)+ # A bundle this host read from its disk finds its template file in its method tree, and never among+ # submitted files.+ resolved_path = cls._resolve_template_file(pipe_code=pipe_code, template_file=blueprint.template_file, tree_bundle=tree_bundle)read_template_bytes = resolved_path.read_bytesif not blueprint.format.is_template_html:template_path = str(resolved_path)@@ -92,20 +81,22 @@ class PipeDocGenFactory(PipeFactoryProtocol[PipeDocGenBlueprint, PipeDocGen]):)@classmethod- def _resolve_template_file(cls, *, pipe_code: str, template_file: str, bundle_dir: Path) -> Path:- """The template file's path, beside the bundle and inside its directory, checked to exist.+ def _resolve_template_file(cls, *, pipe_code: str, template_file: str, tree_bundle: TreeBundle) -> Path:+ """Where the template file lies on this host, resolved in the bundle's method tree.- The messages name the template file as the method does and never the host's path to it: the bundle a- refusal is located in already says which one, and a dependency's install path is not the caller's to read.+ The file is the tree's file the reference names, resolved against the bundle's tree path, `..` included as long+ as it stays in the tree, by a lookup in the tree's set of tree paths: a file the membership rule keeps out of the+ tree, or any file outside it, is refused, and its path is never opened. The messages name the template file as+ the method does and files by their tree paths, never the host's path to them: the bundle a refusal is located in+ already says which one, and a dependency's install path is not the caller's to read."""- resolved_path = (bundle_dir / template_file).resolve()- if not resolved_path.is_relative_to(bundle_dir):- msg = f"PipeDocGen '{pipe_code}' names the template file '{template_file}', which is outside its bundle's directory."- raise PipeDocGenFactoryError(msg)- if not resolved_path.is_file():- msg = f"PipeDocGen '{pipe_code}' names the template file '{template_file}', which does not exist beside its bundle."- raise PipeDocGenFactoryError(msg)- return resolved_path+ try:+ resolved_tree_path = resolve_bundle_file_reference(reference=template_file, tree_bundle=tree_bundle)+ except MethodTreeRefusedError as exc:+ reason = reference_refusal_reason(refusal_error=exc, tree_bundle=tree_bundle)+ msg = f"PipeDocGen '{pipe_code}' names the template file '{template_file}', which its method tree cannot give it. {reason}"+ raise PipeDocGenFactoryError(msg) from exc+ return tree_bundle.tree.file_location(resolved_tree_path)@classmethoddef _submitted_template_file(cls, *, pipe_code: str, template_file: str) -> BundleResource:@@ -115,8 +106,8 @@ class PipeDocGenFactory(PipeFactoryProtocol[PipeDocGenBlueprint, PipeDocGen]):msg = (f"PipeDocGen '{pipe_code}' names the template file '{template_file}', which was not supplied with its ""bundle. A bundle loaded from content strings has no directory to find the file in, so the file must be "- f"submitted with it, keyed by the same path, '{template_file}'. A bundle loaded from a directory "- "finds it beside its file."+ f"submitted with it, keyed by the same path, '{template_file}'. A bundle loaded from a folder "+ "finds it in its method tree.")raise PipeDocGenFactoryError(msg)return resourcediff --git a/pipelex/pipelex.toml b/pipelex/pipelex.tomlindex ec186eff5..bf914ead3 100644--- a/pipelex/pipelex.toml+++ b/pipelex/pipelex.toml@@ -475,6 +475,19 @@ excluded_dirs = ["results",]+####################################################################################################+# Method tree limits — how large a method tree a local load lists+####################################################################################################++[interpreter.method_tree_limits]+# A local run, validation or load reads every method from its method tree: a method's folder, a .mthds file named+# alone, a library directory, a dependency package. Listing a tree stops before reading any file once the tree+# passes one of these limits, so a directory handed over by mistake, such as a home directory, is refused in moments+# rather than read in full. They are this machine's own, not the hosted plane's: raise them in your project's+# .pipelex/pipelex.toml for a method that needs more. max_file_bytes, the size of one file, is not checked unless set.+max_files = 10000+max_total_bytes = 1073741824+[interpreter.builder]fix_loop_max_attempts = 10default_output_dir = "."diff --git a/pipelex/pipeline/exceptions.py b/pipelex/pipeline/exceptions.pyindex e6fb8ab02..5add07780 100644--- a/pipelex/pipeline/exceptions.py+++ b/pipelex/pipeline/exceptions.py@@ -472,3 +472,21 @@ class PipelineInputUrlMissingError(PipelineInputContentError):"""_authors_caller_facing_message = True+++class MethodEntryError(PipelexError):+ """Raised when a method run with no pipe named has no single pipe to start at.++ The rule is the MTHDS protocol's for a run that names no `pipe_code`: the manifest's `main_pipe` when the method's+ tree holds a `METHODS.toml` at its root, else the one `main_pipe` its bundles declare. A manifest declaring none or+ a pipe no bundle defines, and a method with no manifest whose bundles declare none or several, are refused, with+ the bundles named by their tree paths. The method is the caller's own, so the message is theirs to read.+ """++ error_domain = ErrorDomain.INPUT+ retryable = False+ user_action = UserAction(+ kind=UserActionKind.CHANGE_INPUT,+ detail="Name the pipe to run, or declare the method's one main_pipe: in its METHODS.toml, or in one of its bundles",+ )+ _authors_caller_facing_message = Truediff --git a/pipelex/pipeline/execution_seams.py b/pipelex/pipeline/execution_seams.pyindex 664fe06a1..6b70cebcd 100644--- a/pipelex/pipeline/execution_seams.py+++ b/pipelex/pipeline/execution_seams.py@@ -42,6 +42,7 @@ from pipelex.interpreter_hub import ()from pipelex.kernel.memory_ops import shape_inputsfrom pipelex.libraries.bundle_resources import check_bundle_resources+from pipelex.libraries.method_trees import reporting_host_library_tree_faultsfrom pipelex.mthds_parsing.parser import MthdsParserfrom pipelex.pipe_machinery.pipe_abstract import PipeAbstractfrom pipelex.pipe_run.pipe_job import PipeJob@@ -139,8 +140,9 @@ def acquire_library(if effective_dirs:effective_dirs_desc = ", ".join(str(dir_path) for dir_path in effective_dirs)log.verbose(f"Loading libraries from {len(effective_dirs)} directory(ies) ({source_label}): {effective_dirs_desc}")+ # The caller's directories refuse as their invalid bundle; a host's report a tree it cannot list as its own fault.library_dirs_translation: AbstractContextManager[None] = (- translate_to_validate_bundle_error() if library_dirs_are_callers else nullcontext()+ translate_to_validate_bundle_error() if library_dirs_are_callers else reporting_host_library_tree_faults())with library_dirs_translation:library_manager.load_libraries(library_id=library_id, library_dirs=effective_dirs)diff --git a/pipelex/pipeline/fixes/fix_loop.py b/pipelex/pipeline/fixes/fix_loop.pyindex 6d0520726..51a8951c2 100644--- a/pipelex/pipeline/fixes/fix_loop.py+++ b/pipelex/pipeline/fixes/fix_loop.py@@ -23,7 +23,7 @@ Two scoping rules bound what gets touched:"""import os-from collections.abc import Sequence+from collections.abc import Iterable, Sequencefrom pathlib import Pathfrom typing import Any, NamedTuple, cast@@ -47,6 +47,7 @@ from pipelex.pipe_machinery.pipe_blueprint import SIGNATURE_ONLY_KEYSfrom pipelex.pipeline.exceptions import ValidateBundleErrorfrom pipelex.pipeline.fixes.applicability import is_safe_fix_for_load_scope, is_target_in_write_scopefrom pipelex.pipeline.validate_bundle import validate_bundle+from pipelex.pipeline.validate_bundle_translation import translate_to_validate_bundle_errorfrom pipelex.suggested_fix import DeleteKeyOp, DeleteTableOp, EnsureTableOp, FixOp, MoveKeyOp, RemapValueOp, RenameTableKeyOp, SetKeyOp, SuggestedFixfrom pipelex.tools.misc.exceptions import TomlErrorfrom pipelex.tools.misc.toml_utils import load_toml_from_path@@ -241,9 +242,12 @@ class _FilePipeCodes(NamedTuple):codes: set[str]-def _pipe_codes_by_file(*, entry_path: Path, effective_dirs: Sequence[Path]) -> dict[Path, _FilePipeCodes]:+def _pipe_codes_by_file(*, loaded_paths: Iterable[Path]) -> dict[Path, _FilePipeCodes]:"""Concrete pipe declaration keys (with the declaring domain) of every loaded bundle file.+ ``loaded_paths`` are the files this round's snapshot took, the entry included, so the method's trees are not+ listed again between the validation and the fixes.+The domain is recorded because the collision scope is **per-domain**: in-body refs resolvewithin their own domain and the library keys pipes by ``domain.code``, so a same-nameddeclaration in another domain is not a collision at all — only two files of the same domain@@ -255,8 +259,7 @@ def _pipe_codes_by_file(*, entry_path: Path, effective_dirs: Sequence[Path]) ->file that fails to parse contributes nothing here — its own problems surface throughvalidation, not this scan."""- all_paths: set[Path] = {entry_path}- all_paths.update(mthds_path.resolve() for mthds_path in get_pipelex_mthds_files_from_dirs(dirs=set(effective_dirs)))+ all_paths: set[Path] = {loaded_path.resolve() for loaded_path in loaded_paths}codes_by_file: dict[Path, _FilePipeCodes] = {}for mthds_path in all_paths:try:@@ -412,7 +415,18 @@ async def fix_bundle_file(last_error_items: list[ValidationErrorItem] = []for _ in range(max_iterations):- snapshots_by_source = _snapshot_loaded_sources(entry_source_path=entry_source_path, effective_dirs=effective_dirs)+ # The snapshot lists the method's trees: one that cannot be listed is the invalid verdict, with nothing to fix.+ try:+ with translate_to_validate_bundle_error():+ snapshots_by_source = _snapshot_loaded_sources(entry_source_path=entry_source_path, effective_dirs=effective_dirs)+ except ValidateBundleError as exc:+ return FixBundleResult(+ is_valid=False,+ iterations=apply_rounds,+ fixes_applied=fixes_applied,+ files_written=files_written,+ remaining_errors=_validation_error_items(exc),+ )try:validation_result = await validate_bundle(mthds_file_path=mthds_file_path,@@ -463,7 +477,7 @@ async def fix_bundle_file()if not is_single_file:- codes_by_file = _pipe_codes_by_file(entry_path=entry_path, effective_dirs=effective_dirs)+ codes_by_file = _pipe_codes_by_file(loaded_paths=snapshots_by_source)in_scope_fixes, colliding_names = _split_cross_file_collisions(in_scope_fixes, codes_by_file=codes_by_file)if not in_scope_fixes:colliding = ", ".join(f"'{name}'" for name in sorted(set(colliding_names)))diff --git a/pipelex/pipeline/method_entry.py b/pipelex/pipeline/method_entry.pynew file mode 100644index 000000000..a3b61ca4e--- /dev/null+++ b/pipelex/pipeline/method_entry.py@@ -0,0 +1,164 @@+"""Which pipe a method starts at when a run of its folder names none.++The rule is the MTHDS protocol's for a request that names no `pipe_code` (`mthds/docs/spec/protocol.md`): a method+tree has no order, so no bundle is "the first" any more. The run starts at the manifest's `main_pipe` when the tree+holds a `METHODS.toml` at its root, or else, in a method with no manifest, at the one `main_pipe` its bundles declare.+A manifest declaring no `main_pipe`, or a method with no manifest whose bundles declare none or several, is refused,+with the bundles named by their tree paths.++`select_entry` applies the rule to bundles already parsed, wherever their tree was read from; `select_method_entry`+reads a tree listed from this host's disk.+"""++from typing import NamedTuple++from mthds.method_tree.discovery import MANIFEST_NAME+from mthds.method_tree.models import MethodTree+from mthds.package.discovery import read_package_manifest_at+from mthds.package.exceptions import ManifestError+from mthds.package.manifest.schema import MethodsManifest++from pipelex.base_exceptions import PipelexUnexpectedError+from pipelex.libraries.method_trees import tree_bundles_of+from pipelex.mthds_parsing.parser import MthdsParser+from pipelex.mthds_parsing.pipelex_bundle_blueprint import PipelexBundleBlueprint+from pipelex.pipe_machinery.pipe_factory import PipeFactory+from pipelex.pipeline.exceptions import MethodEntryError+++class EntryCandidate(NamedTuple):+ """A bundle of the method, named by its tree path, as its text declares it."""++ tree_path: str+ blueprint: PipelexBundleBlueprint+++class MethodEntry(NamedTuple):+ """The pipe a run of a method starts at, and the tree path of the bundle that defines it."""++ tree_path: str+ pipe_code: str+ pipe_ref: str+++def _listed(*, names: list[str]) -> str:+ return ", ".join(f"'{name}'" for name in names)+++def _entry_of(*, candidate: EntryCandidate, pipe_code: str) -> MethodEntry:+ return MethodEntry(+ tree_path=candidate.tree_path,+ pipe_code=pipe_code,+ pipe_ref=PipeFactory.make_pipe_ref_with_domain(domain_code=candidate.blueprint.domain, pipe_code=pipe_code),+ )+++def select_entry(*, candidates: list[EntryCandidate], manifest: MethodsManifest | None, method_label: str) -> MethodEntry:+ """The pipe a run of a method starts at when it names none, by the MTHDS protocol's rule.++ Args:+ candidates: The method's bundles, in tree path order.+ manifest: The `METHODS.toml` at the tree's root, or None when the tree holds none there.+ method_label: How the messages name the method, such as its folder.++ Raises:+ MethodEntryError: The method holds no bundle, its manifest declares no `main_pipe` or one no single bundle+ defines, or, with no manifest, its bundles declare none or several.+ """+ bundle_paths = [candidate.tree_path for candidate in candidates]+ if not candidates:+ msg = f"The method {method_label} holds no .mthds bundle, so it has no pipe to start at."+ raise MethodEntryError(msg)+ if manifest is not None:+ return _manifest_entry(candidates=candidates, manifest=manifest, method_label=method_label, bundle_paths=bundle_paths)++ declaring = [candidate for candidate in candidates if candidate.blueprint.main_pipe]+ if not declaring:+ msg = (+ f"The method {method_label} has no {MANIFEST_NAME}, and none of its bundles declares a main_pipe "+ f"({_listed(names=bundle_paths)}), so it has no pipe to start at. Declare a main_pipe in one bundle, or name the pipe to run."+ )+ raise MethodEntryError(msg)+ if len(declaring) > 1:+ declarations = ", ".join(f"'{candidate.tree_path}' declares '{candidate.blueprint.main_pipe}'" for candidate in declaring)+ msg = (+ f"The method {method_label} has no {MANIFEST_NAME}, and several of its bundles declare a main_pipe: {declarations}. "+ f"A method starts at one pipe: keep one main_pipe, declare the method's main_pipe in a {MANIFEST_NAME} at its root, "+ "or name the pipe to run."+ )+ raise MethodEntryError(msg)+ entry_candidate = declaring[0]+ main_pipe = entry_candidate.blueprint.main_pipe+ if main_pipe is None:+ # Selected because it declares one: this is a bug of the selection, never the caller's method.+ msg = f"The bundle '{entry_candidate.tree_path}' was selected for its main_pipe, yet declares none."+ raise PipelexUnexpectedError(msg)+ return _entry_of(candidate=entry_candidate, pipe_code=main_pipe)+++def _manifest_entry(*, candidates: list[EntryCandidate], manifest: MethodsManifest, method_label: str, bundle_paths: list[str]) -> MethodEntry:+ """The pipe the manifest names as its `main_pipe`, in the one bundle of a domain exporting it that defines it.++ Raises:+ MethodEntryError: The manifest declares no `main_pipe`, or none or several bundles define it.+ """+ main_pipe = manifest.main_pipe+ if main_pipe is None:+ msg = (+ f"The {MANIFEST_NAME} of the method {method_label} declares no main_pipe, so the method has no pipe to start at. "+ f"Declare main_pipe under [package], or name the pipe to run. Its bundles: {_listed(names=bundle_paths)}."+ )+ raise MethodEntryError(msg)+ # The manifest refuses a main_pipe no domain exports, so the domains exporting it say which bundles may define it.+ exporting_domains = {domain for domain, domain_exports in manifest.exports.items() if main_pipe in domain_exports.pipes}+ defining = [+ candidate+ for candidate in candidates+ if candidate.blueprint.domain in exporting_domains and candidate.blueprint.pipe is not None and main_pipe in candidate.blueprint.pipe+ ]+ if not defining:+ msg = (+ f"The {MANIFEST_NAME} of the method {method_label} declares the main_pipe '{main_pipe}', exported by "+ f"{_listed(names=sorted(exporting_domains))}, which none of its bundles defines in that domain: {_listed(names=bundle_paths)}."+ )+ raise MethodEntryError(msg)+ if len(defining) > 1:+ definitions = ", ".join(f"'{candidate.tree_path}' in domain '{candidate.blueprint.domain}'" for candidate in defining)+ msg = (+ f"The {MANIFEST_NAME} of the method {method_label} declares the main_pipe '{main_pipe}', which several of its "+ f"bundles define: {definitions}. Name the pipe to run as domain.pipe_code."+ )+ raise MethodEntryError(msg)+ return _entry_of(candidate=defining[0], pipe_code=main_pipe)+++def select_method_entry(*, tree: MethodTree) -> MethodEntry:+ """The pipe a run of a method tree listed from this host's disk starts at when it names none.++ Each bundle of the tree is parsed to read what it declares, and the manifest at the tree's root, when the tree+ holds one, is read.++ Raises:+ MethodEntryError: The tree has no directory on this host, its manifest cannot be read, or `select_entry`+ refuses it.+ MthdsParserError: A bundle of the tree does not parse.+ """+ if tree.directory is None:+ msg = "A method tree read from a map of file entries has no folder on this host to read its bundles from."+ raise MethodEntryError(msg)+ method_label = f"at '{tree.directory}'"+ candidates = [+ EntryCandidate(tree_path=tree_bundle.tree_path, blueprint=MthdsParser.make_pipelex_bundle_blueprint(bundle_path=tree_bundle.location))+ for tree_bundle in tree_bundles_of(tree=tree)+ ]+ manifest: MethodsManifest | None = None+ if MANIFEST_NAME in tree.members:+ try:+ manifest = read_package_manifest_at(tree.directory)+ except ManifestError as exc:+ msg = f"The {MANIFEST_NAME} of the method {method_label} cannot be read: {exc}"+ raise MethodEntryError(msg) from exc+ if manifest is None:+ msg = f"The method {method_label} lists a {MANIFEST_NAME} at its root that could not be read back."+ raise MethodEntryError(msg)+ return select_entry(candidates=candidates, manifest=manifest, method_label=method_label)diff --git a/pipelex/pipeline/validate_bundle.py b/pipelex/pipeline/validate_bundle.pyindex 91bf6d6c4..8c9e008e9 100644--- a/pipelex/pipeline/validate_bundle.py+++ b/pipelex/pipeline/validate_bundle.py@@ -22,6 +22,7 @@ from pipelex.interpreter_hub import ()from pipelex.libraries.bundle_resources import check_bundle_resourcesfrom pipelex.libraries.library_utils import get_pipelex_mthds_files_from_dirs+from pipelex.libraries.method_trees import reporting_host_library_tree_faultsfrom pipelex.libraries.pipe.exceptions import EntryPipeNotFoundErrorfrom pipelex.mthds_parsing.parser import MthdsParserfrom pipelex.mthds_parsing.pipelex_bundle_blueprint import PipelexBundleBlueprint@@ -203,22 +204,27 @@ async def validate_bundle(# Beside submitted content, a host's library directories are withheld from the verdict: their files are paths# on the host. The caller's own directories, and every directory of a bundle file, keep their paths.host_library_withholding: AbstractContextManager[None]+ # A host's library directories whose trees cannot be listed are the host's fault, never the caller's verdict.+ host_library_tree_faults: AbstractContextManager[None]if mthds_contents is not None and not library_dirs_are_callers:host_library_withholding = withholding_host_library_files(library_dirs=effective_dirs,caller_sources=[source for source in mthds_sources or [] if source is not None],caller_contents=mthds_contents,)+ host_library_tree_faults = reporting_host_library_tree_faults()else:host_library_withholding = nullcontext()+ host_library_tree_faults = nullcontext()await asyncio.sleep(0) # Yield to event loop (keeps function async-compatible)with host_library_withholding, translate_to_validate_bundle_error():if effective_dirs:log.verbose(f"Loading libraries from {len(effective_dirs)} directory(ies) ({source_label}) for validation")- library_manager.load_libraries(- library_id=library_id,- library_dirs=effective_dirs,- )+ with host_library_tree_faults:+ library_manager.load_libraries(+ library_id=library_id,+ library_dirs=effective_dirs,+ )else:log.verbose(f"No library directories to load ({source_label})")if mthds_contents is not None:@@ -298,7 +304,9 @@ async def validate_bundle(async def validate_bundles_from_directory(directory: Path, *, allow_signatures: bool = False) -> ValidateBundleResult:- mthds_files = get_pipelex_mthds_files_from_dirs(dirs={directory})+ # A directory whose tree cannot be listed is the invalid verdict, as a bundle of it that does not load is.+ with translate_to_validate_bundle_error():+ mthds_files = get_pipelex_mthds_files_from_dirs(dirs={directory})all_blueprints: list[PipelexBundleBlueprint] = []library_manager = get_library_manager()diff --git a/pipelex/pipeline/validate_bundle_translation.py b/pipelex/pipeline/validate_bundle_translation.pyindex 11bbc96a1..0c9cf52ac 100644--- a/pipelex/pipeline/validate_bundle_translation.py+++ b/pipelex/pipeline/validate_bundle_translation.py@@ -31,8 +31,9 @@ from pipelex.core.pipes.exceptions import ()from pipelex.core.validation import report_validation_errorfrom pipelex.interpreter_hub import get_library_manager-from pipelex.libraries.exceptions import LibraryError, LibraryLoadingError+from pipelex.libraries.exceptions import LibraryError, LibraryLoadingError, MethodTreeLoadErrorfrom pipelex.libraries.library_utils import get_pipelex_mthds_files_from_dirs+from pipelex.libraries.method_trees import bundle_files_underfrom pipelex.libraries.pipe.exceptions import PipeNotFoundErrorfrom pipelex.mthds_parsing.exceptions import MthdsParserErrorfrom pipelex.mthds_parsing.handle_pipe_errors import (@@ -337,7 +338,15 @@ def _host_library_files(*, library_dirs: Sequence[Path]) -> _HostLibraryFiles:"""names: set[str] = set()host_texts: list[str] = []- for library_file in get_pipelex_mthds_files_from_dirs(set(library_dirs)):+ library_files: list[Path]+ try:+ library_files = get_pipelex_mthds_files_from_dirs(set(library_dirs))+ except MethodTreeLoadError:+ # This runs while a verdict is being withheld, and the load listed these trees a moment ago: a listing that+ # fails now must never take the verdict's place, so every bundle file under the directories is withheld, a+ # superset of the names the load gave.+ library_files = bundle_files_under(library_dirs=library_dirs)+ for library_file in library_files:names.add(str(library_file))host_text = _bundle_text_or_none(library_file=library_file)if host_text is not None:diff --git a/pipelex/system/configuration/configs.py b/pipelex/system/configuration/configs.pyindex fb032dd59..74175f5e5 100644--- a/pipelex/system/configuration/configs.py+++ b/pipelex/system/configuration/configs.py@@ -102,6 +102,19 @@ class ScanConfig(ConfigModel):excluded_dirs: frozenset[str] = Field(strict=False)+class MethodTreeLimitsConfig(ConfigModel):+ """`[interpreter.method_tree_limits]`: the limits this machine lists a method tree under before reading any file.++ A limit left unset is not checked. They are this machine's own policy, not the hosted plane's: the lister counts+ files and sums their sizes while it walks, and refuses a tree past one of them before hashing anything, so a+ directory handed over by mistake is refused in moments rather than read in full.+ """++ max_files: int | None = Field(default=None, ge=1)+ max_total_bytes: int | None = Field(default=None, ge=0)+ max_file_bytes: int | None = Field(default=None, ge=0)++class BuilderConfig(ConfigModel):fix_loop_max_attempts: intdefault_output_dir: str@@ -205,6 +218,7 @@ class InterpreterConfig(ConfigModel):pipe_func: PipeFuncConfigpipeline_execution: PipelineExecutionConfigscan: ScanConfig+ method_tree_limits: MethodTreeLimitsConfigbuilder: BuilderConfigdiff --git a/pyproject.toml b/pyproject.tomlindex 9fb999070..ae0fe81a2 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -31,7 +31,7 @@ dependencies = ["json2html>=1.3.0","kajson==0.7.1","markdown-it-py[linkify]>=3.0.0", # the Markdown concept, the `markdown` filter and the built-in PDF engine- "mthds==0.21.0",+ "mthds==0.22.0","networkx>=3.4.2","openai>=2.0.0","opentelemetry-api>=1.39.0",@@ -43,7 +43,7 @@ dependencies = [# The client for the hosted Pipelex API, behind hosted runs (pipelex/hosted/). Pinned exactly, like# mthds: each pipelex-sdk release pins mthds exactly too, so the release chosen here must pin the# same mthds as this file, and the two pins move together (the bump-mthds skill says so).- "pipelex-sdk==0.38.0",+ "pipelex-sdk==0.45.0","polyfactory>=2.21.0","portkey-ai>=2.1.0","posthog>=6.7.0",diff --git a/subject_grants.toml b/subject_grants.tomlindex 9a0f96f69..3abe53544 100644--- a/subject_grants.toml+++ b/subject_grants.toml@@ -4224,10 +4224,6 @@ rationale = "Reads the step's inputs out of one working memory."param = "named_contents"rationale = "Derives the document's title from its one mapping of named contents."-["pipelex/pipe_operators/doc_gen/pipe_doc_gen_factory.py::bundle_dir_of_source"]-param = "source"-rationale = "Resolves the bundle directory of one source."-["pipelex/pipe_operators/doc_gen/template_field_check.py::_describe_fields"]param = "model_class"rationale = "Lists the fields of one model class."diff --git a/tests/data/errors/error_identity.txt b/tests/data/errors/error_identity.txtindex 5a41aea20..0cb6ea2a5 100644--- a/tests/data/errors/error_identity.txt+++ b/tests/data/errors/error_identity.txt@@ -49,6 +49,7 @@ ConstructFieldBlueprintValueError | Construct field blueprint value | https://doContextProviderError | Context provider | https://docs.pipelex.com/latest/errors/context-provider-error/CoreUnconditionalPluginDisabledError | Core unconditional plugin disabled | https://docs.pipelex.com/latest/errors/core-unconditional-plugin-disabled-error/CostRegistryError | Cost registry | https://docs.pipelex.com/latest/errors/cost-registry-error/+CrateIntegrityError | Crate integrity error | https://docs.pipelex.com/latest/errors/crate-integrity-error/CrateNormalizationError | Crate normalization error | https://docs.pipelex.com/latest/errors/crate-normalization-error/CredentialsError | Missing or invalid credentials | https://docs.pipelex.com/latest/errors/credentials-error/CsvCoercionError | CSV coercion error | https://docs.pipelex.com/latest/errors/csv-coercion-error/@@ -114,6 +115,7 @@ GoogleImgGenWorkerError | Google img gen worker | https://docs.pipelex.com/latesGoogleLLMWorkerError | Google LLM worker | https://docs.pipelex.com/latest/errors/google-llm-worker-error/GraphSpecError | Graph spec | https://docs.pipelex.com/latest/errors/graph-spec-error/GraphSpecValidationError | Graph spec validation | https://docs.pipelex.com/latest/errors/graph-spec-validation-error/+HostLibraryTreeError | Host library tree | https://docs.pipelex.com/latest/errors/host-library-tree-error/HostedBaseUrlError | Hosted base url | https://docs.pipelex.com/latest/errors/hosted-base-url-error/HostedLocalFileUploadUnavailableError | Hosted local file upload unavailable | https://docs.pipelex.com/latest/errors/hosted-local-file-upload-unavailable-error/HostedMethodInvalidError | Hosted method invalid | https://docs.pipelex.com/latest/errors/hosted-method-invalid-error/@@ -183,6 +185,7 @@ LogSinkHeaderValueError | Log sink header value holds a line break | https://docLogSinkVariableError | Log sink variable did not resolve | https://docs.pipelex.com/latest/errors/log-sink-variable-error/MarkdownFormattingBudgetError | Markdown formatting budget exceeded | https://docs.pipelex.com/latest/errors/markdown-formatting-budget-error/MethodDependencyFetchError | Method dependency fetch | https://docs.pipelex.com/latest/errors/method-dependency-fetch-error/+MethodEntryError | Method entry | https://docs.pipelex.com/latest/errors/method-entry-error/MethodFetchDisabledError | Method fetch disabled | https://docs.pipelex.com/latest/errors/method-fetch-disabled-error/MethodFetchError | Method fetch | https://docs.pipelex.com/latest/errors/method-fetch-error/MethodInstallError | Method install | https://docs.pipelex.com/latest/errors/method-install-error/@@ -193,6 +196,7 @@ MethodPackageTooLargeError | Method package too large | https://docs.pipelex.comMethodRefError | Method reference error | https://docs.pipelex.com/latest/errors/method-ref-error/MethodRefParseError | Method ref parse | https://docs.pipelex.com/latest/errors/method-ref-parse-error/MethodStructuresRefusedError | Method structures refused | https://docs.pipelex.com/latest/errors/method-structures-refused-error/+MethodTreeLoadError | Method tree load | https://docs.pipelex.com/latest/errors/method-tree-load-error/MigrationError | Migration error | https://docs.pipelex.com/latest/errors/migration-error/MigrationGoldenError | Migration golden error | https://docs.pipelex.com/latest/errors/migration-golden-error/MigrationLedgerError | Migration ledger error | https://docs.pipelex.com/latest/errors/migration-ledger-error/diff --git a/tests/e2e/pipelex/cli/test_csv_run.py b/tests/e2e/pipelex/cli/test_csv_run.pyindex f55452617..6594c7705 100644--- a/tests/e2e/pipelex/cli/test_csv_run.py+++ b/tests/e2e/pipelex/cli/test_csv_run.py@@ -57,7 +57,8 @@ class TestCsvRun:mock_execute = mocker.patch("pipelex.cli.commands.run.bundle_cmd.execute_run")# Build the kwargs dynamically so this stays type-clean before `save_csv` exists# (Phase 4 adds it); the RED failure is a runtime TypeError, not a type-check error.- call_kwargs: dict[str, object] = {"path": str(BUNDLE_SRC), "save_csv": "out.csv", "dry_run": True}+ # The demo's bundle declares no main_pipe, so its folder runs from the pipe it is told to start at.+ call_kwargs: dict[str, object] = {"path": str(BUNDLE_SRC), "pipe": "summarize_people", "save_csv": "out.csv", "dry_run": True}run_bundle_cmd(**call_kwargs) # type: ignore[arg-type]assert mock_execute.call_args is not Noneassert mock_execute.call_args.kwargs.get("save_csv") == "out.csv"diff --git a/tests/integration/pipelex/pipeline/test_validate_bundle_library_lifecycle.py b/tests/integration/pipelex/pipeline/test_validate_bundle_library_lifecycle.pyindex b3cce95a4..1db442a32 100644--- a/tests/integration/pipelex/pipeline/test_validate_bundle_library_lifecycle.py+++ b/tests/integration/pipelex/pipeline/test_validate_bundle_library_lifecycle.py@@ -106,6 +106,18 @@ class TestValidateBundleLibraryLifecycle:opened_library_id, _ = open_library_spy.spy_returnassert latest_call.kwargs["library_id"] == opened_library_id+ async def test_validate_bundles_from_directory_refuses_a_tree_it_cannot_list_as_a_verdict(self, tmp_path: Path, mocker: MockerFixture) -> None:+ """A directory whose method tree is refused is the invalid verdict before any library opens, never a raw error."""+ open_library_spy = mocker.spy(get_library_manager(), "open_library")+ (tmp_path / "test.mthds").write_text(_VALID_MTHDS, encoding="utf-8")+ (tmp_path / "linked.mthds").symlink_to(tmp_path / "test.mthds")++ with pytest.raises(ValidateBundleError) as raised:+ await validate_bundles_from_directory(directory=tmp_path)++ assert "linked.mthds" in raised.value.message+ open_library_spy.assert_not_called()+async def test_validate_bundle_tears_down_on_base_exception_in_pre_try_window(self,load_empty_library: Callable[[], str],diff --git a/tests/integration/pipelex/pipeline/test_verdict_names_no_host_path.py b/tests/integration/pipelex/pipeline/test_verdict_names_no_host_path.pyindex 1d06bc5b3..0e410a069 100644--- a/tests/integration/pipelex/pipeline/test_verdict_names_no_host_path.py+++ b/tests/integration/pipelex/pipeline/test_verdict_names_no_host_path.py@@ -24,8 +24,9 @@ from typing import Any, NamedTupleimport pytestfrom pytest_mock import MockerFixture-from pipelex.base_exceptions import DisclosureMode, PipelexError, ValidationErrorItem+from pipelex.base_exceptions import DisclosureMode, ErrorDomain, PipelexError, ValidationErrorItemfrom pipelex.config import get_config+from pipelex.libraries.exceptions import HostLibraryTreeErrorfrom pipelex.pipeline.exceptions import ValidateBundleErrorfrom pipelex.pipeline.pipeline_run_setup import pipeline_run_setupfrom pipelex.pipeline.runner import PipelexMTHDSProtocol@@ -294,6 +295,54 @@ class TestVerdictNamesNoHostPath:assert f"'{_DEPENDENCY_ADDRESS}/notices/tide_notices_copy.mthds'" in item.message, surface.description_assert_names_no_host_path(payload=_strict_payload(verdict), tmp_path=tmp_path)+ @pytest.mark.parametrize("surface", _SURFACES, ids=[surface.surface_id for surface in _SURFACES])+ async def test_a_dependency_s_tree_refusal_names_the_package_by_its_address(+ self, surface: _Surface, installed_methods_dir: Path, tmp_path: Path+ ) -> None:+ package_dir = _install_dependency(methods_dir=installed_methods_dir, bundles={"notices/tide_notices.mthds": _DEPENDENCY_NOTICE_BUNDLE})+ # The membership rule refuses a symbolic link, so the package's tree cannot be listed.+ (package_dir / "notices" / "linked_notices.mthds").symlink_to(package_dir / "notices" / "tide_notices.mthds")++ verdict = await _verdict_on(surface=surface, caller_bundle=_CALLER_BUNDLE, tmp_path=tmp_path)++ assert "github.com/acme-tests/harbour-methods" in verdict.message, surface.description+ assert "notices/linked_notices.mthds" in verdict.message, surface.description+ assert "max_files" not in verdict.message, "no advice about this host's limits reaches the caller"+ _assert_names_no_host_path(payload=_strict_payload(verdict), tmp_path=tmp_path)++ @pytest.mark.parametrize(+ "surface",+ [surface for surface in _SURFACES if surface.surface_id.endswith("_beside_host_library")],+ ids=lambda surface: surface.surface_id,+ )+ async def test_a_host_library_tree_that_cannot_be_listed_is_the_host_s_fault(self, surface: _Surface, tmp_path: Path) -> None:+ """Beside submitted content, the host's own directories are no input of the caller's: no verdict, and no host path."""+ host_library_dir = _valid_host_library(tmp_path=tmp_path)+ (host_library_dir / "linked_board.mthds").symlink_to(host_library_dir / "harbour_board.mthds")++ with pytest.raises(HostLibraryTreeError) as raised:+ await _load_on(surface=surface, caller_bundle=_VALID_CALLER_BUNDLE, tmp_path=tmp_path)++ report = raised.value.to_error_report()+ assert report.error_domain == ErrorDomain.CONFIG+ assert report.caller_facing_message is False+ _assert_names_no_host_path(payload=report.to_dict(disclosure_mode=DisclosureMode.STRICT), tmp_path=tmp_path)++ async def test_a_callers_library_tree_that_cannot_be_listed_is_their_verdict(self, tmp_path: Path) -> None:+ library_dir = _valid_host_library(tmp_path=tmp_path)+ (library_dir / "linked_board.mthds").symlink_to(library_dir / "harbour_board.mthds")+ runtime = PipelexMTHDSProtocol(+ library_dirs=[str(library_dir)],+ library_dirs_are_callers=True,+ pipe_run_mode=PipeRunMode.DRY,+ execution_config=_execution_config(),+ )++ with pytest.raises(ValidateBundleError) as raised:+ await runtime.validate([_VALID_CALLER_BUNDLE])++ assert "linked_board.mthds" in raised.value.message+async def test_a_host_library_item_carries_no_source_on_submitted_content(self, tmp_path: Path) -> None:host_library_dir = tmp_path / "host" / "library"_write(path=host_library_dir / "harbour_board.mthds", content=_MISSPELLED_CONCEPT_LIBRARY_BUNDLE)diff --git a/tests/integration/pipelex/pipes/operator/pipe_doc_gen/doc_gen_helpers.py b/tests/integration/pipelex/pipes/operator/pipe_doc_gen/doc_gen_helpers.pyindex 5f4a8cf31..c285094d3 100644--- a/tests/integration/pipelex/pipes/operator/pipe_doc_gen/doc_gen_helpers.py+++ b/tests/integration/pipelex/pipes/operator/pipe_doc_gen/doc_gen_helpers.py@@ -160,10 +160,24 @@ async def refusal_report(*, step_fields: str) -> str:return str(exc_info.value.to_error_report().model_dump())-def write_bundle(*, directory: Path, step_fields: str) -> Path:- """Write the test bundle with the given PipeDocGen step into a directory, for a step that names a template file."""+METHOD_MANIFEST = """[package]+address = "github.com/pipelex-tests/invoice-printer"+version = "1.0.0"+description = "Prints an invoice from a template file"+"""+++def write_bundle(*, directory: Path, step_fields: str, as_method: bool = True) -> Path:+ """Write the test bundle with the given PipeDocGen step into a directory, for a step that names a template file.++ With `as_method`, the directory also holds a `METHODS.toml`, so the bundle file named alone is rooted at the+ directory and its step finds the files beside it in its method tree. Without one, the file named alone is the+ one-file tree of that file, from which a reference to any other file is refused.+ """bundle_path = directory / "invoice.mthds"bundle_path.write_text(PipeDocGenTestData.bundle(step_fields=step_fields), encoding="utf-8")+ if as_method:+ (directory / "METHODS.toml").write_text(METHOD_MANIFEST, encoding="utf-8")return bundle_pathdiff --git a/tests/integration/pipelex/pipes/operator/pipe_doc_gen/test_pipe_doc_gen_dependency_template_file.py b/tests/integration/pipelex/pipes/operator/pipe_doc_gen/test_pipe_doc_gen_dependency_template_file.pyindex 7a4cab3a7..1dc44e651 100644--- a/tests/integration/pipelex/pipes/operator/pipe_doc_gen/test_pipe_doc_gen_dependency_template_file.py+++ b/tests/integration/pipelex/pipes/operator/pipe_doc_gen/test_pipe_doc_gen_dependency_template_file.py@@ -58,5 +58,16 @@ class TestPipeDocGenDependencyTemplateFile:refusal = exc_info.valueassert refusal.source == f"{_PACKAGE_ADDRESS}/invoice/invoice.mthds"report = str(refusal.to_error_report().model_dump())- assert "'invoice.html', which does not exist beside its bundle" in report+ assert (+ "The file reference 'invoice.html' in 'invoice/invoice.mthds' resolves to 'invoice/invoice.html', which is not a file of the tree."+ in report+ )assert str(tmp_path) not in report++ def test_a_dependency_s_step_reads_no_file_its_package_keeps_out(self, mocker: MockerFixture, tmp_path: Path) -> None:+ """A file the package's own .gitignore keeps out of its tree is invisible to its steps, though it lies beside the bundle."""+ (tmp_path / ".gitignore").write_text("*.html\n", encoding="utf-8")+ with pytest.raises(PipeLoadRefusalError) as exc_info:+ self._load_dependency(mocker=mocker, package_root=tmp_path, template=_TEMPLATE)++ assert "it is kept out by a .gitignore pattern" in str(exc_info.value.to_error_report().model_dump())diff --git a/tests/integration/pipelex/pipes/operator/pipe_doc_gen/test_pipe_doc_gen_host_template_files.py b/tests/integration/pipelex/pipes/operator/pipe_doc_gen/test_pipe_doc_gen_host_template_files.pyindex 7f3db0b54..3b1850919 100644--- a/tests/integration/pipelex/pipes/operator/pipe_doc_gen/test_pipe_doc_gen_host_template_files.py+++ b/tests/integration/pipelex/pipes/operator/pipe_doc_gen/test_pipe_doc_gen_host_template_files.py@@ -76,7 +76,7 @@ class TestPipeDocGenHostTemplateFiles:verdict = await _caller_verdict(mthds_content=PipeDocGenTestData.CALLER_BUNDLE_MTHDS, host_dir=tmp_path)- assert "a path relative to the bundle's file, inside its bundle, and this one cannot be: it is absolute" in verdict+ assert "a path relative to the bundle's file, in its method, and this one cannot be: it is absolute" in verdictassert str(tmp_path.resolve()) not in verdictassert str(tmp_path) not in verdictdiff --git a/tests/integration/pipelex/pipes/operator/pipe_doc_gen/test_pipe_doc_gen_run_bundle_trees.py b/tests/integration/pipelex/pipes/operator/pipe_doc_gen/test_pipe_doc_gen_run_bundle_trees.pynew file mode 100644index 000000000..f82193697--- /dev/null+++ b/tests/integration/pipelex/pipes/operator/pipe_doc_gen/test_pipe_doc_gen_run_bundle_trees.py@@ -0,0 +1,83 @@+"""`pipelex-agent run bundle` reads a method only through its tree: a bundle named alone is a one-file tree, a folder its own."""++from __future__ import annotations++import json+from typing import TYPE_CHECKING, Any++import pytest+import typer++from pipelex.cli.agent_cli.commands.agent_output import CliOutputFormat, set_agent_cli_error_format+from pipelex.cli.agent_cli.commands.run.bundle_cmd import run_bundle_cmd as agent_run_bundle_cmd+from tests.integration.pipelex.pipes.operator.pipe_doc_gen.doc_gen_helpers import write_bundle++if TYPE_CHECKING:+ from pathlib import Path++ from pytest_mock import MockerFixture++_STEP_FIELDS = 'format = "docx"\ntemplate_file = "letter.docx"'+++def _run_agent_bundle(*, path: Path, pipe: str | None = None) -> dict[str, Any]:+ """Run `run bundle` on `path` as a dry run with JSON errors, and return the error envelope it exits with."""+ try:+ agent_run_bundle_cmd(+ path=str(path),+ pipe=pipe,+ inputs=None,+ dry_run=True,+ mock_inputs=True,+ graph=False,+ costs=False,+ library_dir=None,+ with_memory=False,+ output_format=CliOutputFormat.JSON,+ error_format=None,+ )+ finally:+ set_agent_cli_error_format(CliOutputFormat.JSON)+ msg = "The run was expected to be refused"+ raise AssertionError(msg)++[email protected]("stub_engines")+class TestPipeDocGenRunBundleTrees:+ @pytest.fixture(autouse=True)+ def _no_boot(self, mocker: MockerFixture) -> None:+ mocker.patch("pipelex.cli.agent_cli.commands.run.bundle_cmd.make_pipelex_for_agent_cli")+ mocker.patch("pipelex.cli.agent_cli.commands.run.bundle_cmd.Pipelex.teardown_if_needed")++ def test_a_bundle_named_alone_is_refused_the_file_beside_it(self, tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None:+ """With no METHODS.toml at or above it, the bundle's method is that one file, and the run says how to give it the rest."""+ (tmp_path / "letter.docx").write_bytes(b"docx")+ bundle_path = write_bundle(directory=tmp_path, step_fields=_STEP_FIELDS, as_method=False)++ with pytest.raises(typer.Exit) as exc_info:+ _run_agent_bundle(path=bundle_path)++ assert exc_info.value.exit_code == 1+ envelope: dict[str, Any] = json.loads(capsys.readouterr().err)+ assert envelope["error_type"] == "ValidateBundleError"+ assert envelope["error_domain"] == "input"+ messages = [item["message"] for item in envelope["validation_errors"]]+ assert any("letter.docx" in message and "Run the method's folder instead, or add a METHODS.toml to it." in message for message in messages)++ def test_a_folder_whose_bundles_declare_several_main_pipes_is_refused_naming_them(+ self, tmp_path: Path, capsys: pytest.CaptureFixture[str]+ ) -> None:+ (tmp_path / "letter.docx").write_bytes(b"docx")+ write_bundle(directory=tmp_path, step_fields=_STEP_FIELDS, as_method=False)+ (tmp_path / "other").mkdir()+ write_bundle(directory=tmp_path / "other", step_fields=_STEP_FIELDS, as_method=False)++ with pytest.raises(typer.Exit):+ _run_agent_bundle(path=tmp_path)++ envelope: dict[str, Any] = json.loads(capsys.readouterr().err)+ assert envelope["error_domain"] == "input"+ rendered = json.dumps(envelope)+ assert "several of its bundles declare a main_pipe" in rendered+ assert "'invoice.mthds' declares 'print_invoice'" in rendered+ assert "'other/invoice.mthds' declares 'print_invoice'" in rendereddiff --git a/tests/integration/pipelex/pipes/operator/pipe_doc_gen/test_pipe_doc_gen_template_files.py b/tests/integration/pipelex/pipes/operator/pipe_doc_gen/test_pipe_doc_gen_template_files.pyindex 8e34d299b..f87dfe739 100644--- a/tests/integration/pipelex/pipes/operator/pipe_doc_gen/test_pipe_doc_gen_template_files.py+++ b/tests/integration/pipelex/pipes/operator/pipe_doc_gen/test_pipe_doc_gen_template_files.py@@ -8,7 +8,7 @@ from pipelex.core.bundle_resources.bundle_resource import make_bundle_resourcesfrom pipelex.pipe_operators.doc_gen.pipe_doc_gen import PipeDocGenfrom pipelex.pipeline.exceptions import ValidateBundleErrorfrom pipelex.pipeline.validate_bundle import ValidateBundleResult, validate_bundle-from tests.integration.pipelex.pipes.operator.pipe_doc_gen.doc_gen_helpers import StubEngines, write_bundle+from tests.integration.pipelex.pipes.operator.pipe_doc_gen.doc_gen_helpers import METHOD_MANIFEST, StubEngines, write_bundlefrom tests.integration.pipelex.pipes.operator.pipe_doc_gen.test_data import PipeDocGenTestData_DOCX_TEMPLATE_STEP = 'format = "docx"\ntemplate_file = "invoice.docx"'@@ -34,28 +34,74 @@ class TestPipeDocGenTemplateFiles:@pytest.mark.parametrize(("template_file", "expected"),[- ("invoice.docx", "does not exist"),- ("../outside.docx", "it leaves the bundle through a '..' segment"),- ("linked.docx", "outside its bundle's directory"),+ ("invoice.docx", "resolves to 'invoice.docx', which is not a file of the tree."),+ ("../outside.docx", "climbs above the root of its method tree"),+ ("ignored.docx", "which is not a file of the tree: it is kept out by a .gitignore pattern"),+ (".templates/invoice.docx", "which is not a file of the tree: it is kept out by a path segment starting with a dot"),],- ids=["missing_file", "outside_the_bundle", "linked_outside_the_bundle"],+ ids=["missing_file", "outside_the_tree", "gitignored", "hidden"],)- async def test_a_template_file_is_found_inside_the_bundle_s_directory(self, tmp_path: Path, template_file: str, expected: str) -> None:- """A template file must exist beside the bundle, and a path out of the bundle's directory is refused even when the file exists.-- The blueprint refuses a path written out of the bundle, and the load one that leads out of it through a link.- """- bundle_dir = tmp_path / "bundle"- bundle_dir.mkdir()+ async def test_a_template_file_is_a_file_of_the_bundle_s_method_tree(self, tmp_path: Path, template_file: str, expected: str) -> None:+ """A template file must be a file of the method's tree: one outside it, or one its membership rule keeps out, is never read."""+ method_dir = tmp_path / "method"+ method_dir.mkdir()(tmp_path / "outside.docx").write_bytes(b"docx")- (bundle_dir / "linked.docx").symlink_to(tmp_path / "outside.docx")- bundle_path = write_bundle(directory=bundle_dir, step_fields=f'format = "docx"\ntemplate_file = "{template_file}"')+ (method_dir / "ignored.docx").write_bytes(b"docx")+ (method_dir / ".gitignore").write_text("ignored.docx\n", encoding="utf-8")+ (method_dir / ".templates").mkdir()+ (method_dir / ".templates" / "invoice.docx").write_bytes(b"docx")+ bundle_path = write_bundle(directory=method_dir, step_fields=f'format = "docx"\ntemplate_file = "{template_file}"')with pytest.raises(ValidateBundleError) as exc_info:await validate_bundle(mthds_file_path=bundle_path)assert expected in str(exc_info.value.to_error_report().model_dump())+ @pytest.mark.usefixtures("stub_engines")+ async def test_a_template_file_climbs_through_dot_dot_to_another_folder_of_its_tree(self, tmp_path: Path) -> None:+ """A `..` may climb out of the bundle's folder to a file elsewhere in its method tree, which the step reads."""+ (tmp_path / "templates").mkdir()+ (tmp_path / "templates" / "invoice.html").write_text("<h1>{{ invoice.number }}</h1>", encoding="utf-8")+ (tmp_path / "bundles").mkdir()+ bundle_path = write_bundle(+ directory=tmp_path / "bundles", step_fields='format = "pdf"\ntemplate_file = "../templates/invoice.html"', as_method=False+ )+ (tmp_path / "METHODS.toml").write_text(METHOD_MANIFEST, encoding="utf-8")++ result = await validate_bundle(mthds_file_path=bundle_path)++ (pipe,) = result.pipes+ assert isinstance(pipe, PipeDocGen)+ assert pipe.template == "<h1>{{ invoice.number }}</h1>"++ @pytest.mark.usefixtures("stub_engines")+ async def test_a_bundle_file_named_alone_reads_no_file_beside_it(self, tmp_path: Path) -> None:+ """With no METHODS.toml at or above its folder, a bundle named alone is a one-file tree: the file beside it is refused."""+ (tmp_path / "invoice.docx").write_bytes(b"docx")+ bundle_path = write_bundle(directory=tmp_path, step_fields=_DOCX_TEMPLATE_STEP, as_method=False)++ with pytest.raises(ValidateBundleError) as exc_info:+ await validate_bundle(mthds_file_path=bundle_path)++ report = str(exc_info.value.to_error_report().model_dump())+ assert "names the template file 'invoice.docx'" in report+ assert "Run the method's folder instead, or add a METHODS.toml to it." in report+ assert exc_info.value.to_error_report().error_domain == "input"++ @pytest.mark.usefixtures("stub_engines")+ async def test_a_symbolic_link_in_the_method_refuses_its_tree(self, tmp_path: Path) -> None:+ """A tree holds regular files only, so a link anywhere in the method refuses it before any step reads through it."""+ (tmp_path / "outside.docx").write_bytes(b"docx")+ method_dir = tmp_path / "method"+ method_dir.mkdir()+ (method_dir / "linked.docx").symlink_to(tmp_path / "outside.docx")+ bundle_path = write_bundle(directory=method_dir, step_fields='format = "docx"\ntemplate_file = "linked.docx"')++ with pytest.raises(ValidateBundleError) as exc_info:+ await validate_bundle(mthds_file_path=bundle_path)++ assert "holds a symbolic link, 'linked.docx'" in str(exc_info.value.to_error_report().model_dump())+@pytest.mark.usefixtures("stub_engines")async def test_a_template_file_named_by_an_absolute_path_is_refused_without_repeating_it(self, tmp_path: Path) -> None:"""Even an absolute path to a file inside the bundle's directory is refused, and the refusal never carries the host's path."""@@ -67,7 +113,7 @@ class TestPipeDocGenTemplateFiles:await validate_bundle(mthds_file_path=bundle_path)report = str(exc_info.value.to_error_report().model_dump())- assert "a path relative to the bundle's file, inside its bundle, and this one cannot be: it is absolute" in report+ assert "a path relative to the bundle's file, in its method, and this one cannot be: it is absolute" in reportassert str(absolute_template) not in report@pytest.mark.usefixtures("stub_engines")@@ -111,6 +157,7 @@ class TestPipeDocGenTemplateFiles:async def test_an_anything_input_reaches_the_checker_with_an_undeclared_shape(self, tmp_path: Path, stub_engines: StubEngines) -> None:"""An `Anything` input has no content class to describe, so the checker is told its shape is not declared."""(tmp_path / "invoice.docx").write_bytes(b"docx bytes")+ (tmp_path / "METHODS.toml").write_text(METHOD_MANIFEST, encoding="utf-8")bundle_path = tmp_path / "invoice.mthds"bundle_path.write_text(PipeDocGenTestData.bundle(step_fields=_DOCX_TEMPLATE_STEP).replace(diff --git a/tests/unit/pipelex/cli/agent_cli/test_run_core_absent_output.py b/tests/unit/pipelex/cli/agent_cli/test_run_core_absent_output.pyindex 5b19985e6..d546f092a 100644--- a/tests/unit/pipelex/cli/agent_cli/test_run_core_absent_output.py+++ b/tests/unit/pipelex/cli/agent_cli/test_run_core_absent_output.py@@ -58,7 +58,7 @@ class TestAgentRunCoreAbsentOutput:assert result["reason"] == "condition 'gate' evaluated to continue"# The on-disk output mirrors the envelope.- disk_payload = json.loads((tmp_path / "mthds-wip" / "live_run.json").read_text(encoding="utf-8"))+ disk_payload = json.loads((tmp_path / "mthds-wip" / "results" / "gate_output_01" / "live_run.json").read_text(encoding="utf-8"))assert disk_payload["absent"] is Truedef test_absent_main_output_with_memory_renders_markdown_absence_doc(diff --git a/tests/unit/pipelex/cli/agent_cli/test_run_core_output_folder.py b/tests/unit/pipelex/cli/agent_cli/test_run_core_output_folder.pynew file mode 100644index 000000000..120851e81--- /dev/null+++ b/tests/unit/pipelex/cli/agent_cli/test_run_core_output_folder.py@@ -0,0 +1,58 @@+"""`pipelex-agent run` writes its outputs into a numbered folder under `results/` beside its bundle, kept out of the method tree."""++from __future__ import annotations++import asyncio+import json+from types import SimpleNamespace+from typing import TYPE_CHECKING++from mthds.method_tree.lister import list_method_tree++from pipelex.cli.agent_cli.commands.run._run_core import run_pipeline_core+from pipelex.core.memory.absence import AbsenceKind, AbsenceRecord+from pipelex.core.memory.working_memory import WorkingMemory++if TYPE_CHECKING:+ from pathlib import Path++ from pytest_mock import MockerFixture++BUNDLE = 'domain = "folder_probe"\nmain_pipe = "gate"\n'+++class TestAgentRunCoreOutputFolder:+ def test_two_runs_beside_the_bundle_leave_its_method_tree_unchanged(self, mocker: MockerFixture, tmp_path: Path) -> None:+ """Each run gets its own ignored folder, so the second run reads the tree the first read, with the same digest."""+ memory = WorkingMemory()+ memory.record_new_main_absence(+ AbsenceRecord(variable_name="summary", kind=AbsenceKind.DECLARED_ABSENT, reason="nothing to say", producing_pipe="gate")+ )+ pipe_output = SimpleNamespace(working_memory=memory, graph_spec=None, tokens_usages=None)+ runner_mock = mocker.MagicMock()+ runner_mock.execute = mocker.AsyncMock(return_value=SimpleNamespace(pipe_output=pipe_output))+ mocker.patch("pipelex.cli.agent_cli.commands.run._run_core.PipelexMTHDSProtocol", return_value=runner_mock)+ method_dir = tmp_path / "method"+ method_dir.mkdir()+ bundle_path = method_dir / "bundle.mthds"+ bundle_path.write_text(BUNDLE, encoding="utf-8")+ tree_before = list_method_tree(method_dir)++ for _ in range(2):+ asyncio.run(+ run_pipeline_core(+ "folder_probe.gate",+ mthds_contents=[BUNDLE],+ bundle_uris=[str(bundle_path)],+ costs=False,+ method_tree=list_method_tree(method_dir),+ )+ )++ tree_after = list_method_tree(method_dir)+ assert tree_after.digest == tree_before.digest+ assert tree_after.tree_paths == ["bundle.mthds"]+ for run_folder_name in ("gate_output_01", "gate_output_02"):+ run_folder = method_dir / "results" / run_folder_name+ assert (run_folder / ".gitignore").is_file()+ assert json.loads((run_folder / "live_run.json").read_text(encoding="utf-8"))["method_tree_digest"] == tree_before.digestdiff --git a/tests/unit/pipelex/cli/agent_cli/test_run_hosted_envelope.py b/tests/unit/pipelex/cli/agent_cli/test_run_hosted_envelope.pyindex db0498697..a721371e8 100644--- a/tests/unit/pipelex/cli/agent_cli/test_run_hosted_envelope.py+++ b/tests/unit/pipelex/cli/agent_cli/test_run_hosted_envelope.py@@ -260,3 +260,21 @@ class TestRunHostedEnvelope:request = run_hosted.await_args.kwargs["request"]assert isinstance(request, HostedRunRequest)assert [mthds_file.source for mthds_file in request.mthds_files or []] == [str(bundle_path), str(shared_dir / "shared.mthds")]++ def test_a_folder_run_names_its_main_pipe_by_its_domain_when_two_domains_share_its_code(+ self, run_hosted: AsyncMock, tmp_path: Path, capsys: pytest.CaptureFixture[str]+ ) -> None:+ """The hosted API starts at the pipe the local run would, never at a same-coded pipe of another domain sent with it."""+ pipe_table = (+ '[pipe.summarize]\ntype = "PipeLLM"\ndescription = "Summarize"\ninputs = { text = "Text" }\noutput = "Text"\nprompt = "Summarize $text"\n'+ )+ (tmp_path / "a_first.mthds").write_text(f'domain = "probe_other"\n\n{pipe_table}', encoding="utf-8")+ (tmp_path / "z_entry.mthds").write_text(f'domain = "probe_entry"\nmain_pipe = "summarize"\n\n{pipe_table}', encoding="utf-8")++ run_bundle_cmd(path=str(tmp_path), runner=RunExecution.HOSTED, output_format=CliOutputFormat.JSON)++ capsys.readouterr()+ assert run_hosted.await_args is not None+ request = run_hosted.await_args.kwargs["request"]+ assert isinstance(request, HostedRunRequest)+ assert request.pipe_code == "probe_entry.summarize"diff --git a/tests/unit/pipelex/cli/commands/run/test_bundle_cmd_auto_inputs.py b/tests/unit/pipelex/cli/commands/run/test_bundle_cmd_auto_inputs.pyindex 45d1b6f1f..50136b000 100644--- a/tests/unit/pipelex/cli/commands/run/test_bundle_cmd_auto_inputs.py+++ b/tests/unit/pipelex/cli/commands/run/test_bundle_cmd_auto_inputs.py@@ -16,12 +16,24 @@ if TYPE_CHECKING:BUNDLE_CMD_MODULE = "pipelex.cli.commands.run.bundle_cmd"+_BUNDLE = """+domain = "auto_inputs_tests"+main_pipe = "summarize"++[pipe.summarize]+type = "PipeLLM"+description = "Summarize the text"+inputs = { text = "Text" }+output = "Text"+prompt = "Summarize $text"+"""+class TestBundleCmdAutoInputs:@pytest.fixturedef bundle_dir(self, tmp_path: Path) -> Path:- """A pipeline directory holding a default bundle file (contents never parsed: execute_run is mocked)."""- (tmp_path / "bundle.mthds").write_text("# bundle\n", encoding="utf-8")+ """A method's folder holding one bundle declaring its main pipe (execute_run is mocked, so nothing runs)."""+ (tmp_path / "bundle.mthds").write_text(_BUNDLE, encoding="utf-8")return tmp_pathdef test_auto_detects_inputs_toml(self, mocker: MockerFixture, bundle_dir: Path) -> None:diff --git a/tests/unit/pipelex/cli/commands/run/test_bundle_cmd_method_folder.py b/tests/unit/pipelex/cli/commands/run/test_bundle_cmd_method_folder.pynew file mode 100644index 000000000..14f6dbbd2--- /dev/null+++ b/tests/unit/pipelex/cli/commands/run/test_bundle_cmd_method_folder.py@@ -0,0 +1,77 @@+"""`pipelex run bundle <folder>` runs the method rooted at the folder, from the pipe the protocol's rule selects."""++from __future__ import annotations++from typing import TYPE_CHECKING++import pytest+import typer++from pipelex.cli.commands.run.bundle_cmd import run_bundle_cmd++if TYPE_CHECKING:+ from pathlib import Path++ from pytest_mock import MockerFixture++BUNDLE_CMD_MODULE = "pipelex.cli.commands.run.bundle_cmd"+++def _bundle(*, domain: str, pipe_code: str, main_pipe: str | None) -> str:+ main_pipe_line = f'main_pipe = "{main_pipe}"\n' if main_pipe else ""+ return f"""domain = "{domain}"+{main_pipe_line}+[pipe.{pipe_code}]+type = "PipeLLM"+description = "Summarize the text"+inputs = {{ text = "Text" }}+output = "Text"+prompt = "Summarize $text"+"""+++class TestBundleCmdMethodFolder:+ def test_a_folder_runs_from_its_manifest_s_main_pipe_and_records_its_tree(self, mocker: MockerFixture, tmp_path: Path) -> None:+ """No `bundle.mthds` is looked for: the manifest names the entry, and the run is handed the tree it reads."""+ mock_execute = mocker.patch(f"{BUNDLE_CMD_MODULE}.execute_run")+ (tmp_path / "METHODS.toml").write_text(+ '[package]\naddress = "github.com/pipelex-tests/folder"\nversion = "1.0.0"\ndescription = "d"\nmain_pipe = "extract"\n\n'+ '[exports.folder_steps]\npipes = ["extract"]\n',+ encoding="utf-8",+ )+ (tmp_path / "bundle.mthds").write_text(_bundle(domain="folder_report", pipe_code="write_report", main_pipe="write_report"), encoding="utf-8")+ (tmp_path / "steps").mkdir()+ (tmp_path / "steps" / "extract.mthds").write_text(_bundle(domain="folder_steps", pipe_code="extract", main_pipe=None), encoding="utf-8")++ run_bundle_cmd(path=str(tmp_path))++ kwargs = mock_execute.call_args.kwargs+ assert kwargs["pipe_code"] == "folder_steps.extract"+ assert kwargs["bundle_path"] == str((tmp_path / "steps" / "extract.mthds").resolve())+ assert kwargs["library_dir"] == [str(tmp_path)]+ assert kwargs["method_tree"].tree_paths == ["METHODS.toml", "bundle.mthds", "steps/extract.mthds"]++ def test_a_named_pipe_needs_no_entry(self, mocker: MockerFixture, tmp_path: Path) -> None:+ """With `--pipe`, bundles declaring several main pipes are no ambiguity: the run starts where it was told."""+ mock_execute = mocker.patch(f"{BUNDLE_CMD_MODULE}.execute_run")+ (tmp_path / "a.mthds").write_text(_bundle(domain="folder_a", pipe_code="first", main_pipe="first"), encoding="utf-8")+ (tmp_path / "b.mthds").write_text(_bundle(domain="folder_b", pipe_code="second", main_pipe="second"), encoding="utf-8")++ run_bundle_cmd(path=str(tmp_path), pipe="folder_b.second")++ kwargs = mock_execute.call_args.kwargs+ assert kwargs["pipe_code"] == "folder_b.second"+ assert kwargs["bundle_path"] is None++ def test_a_folder_with_no_single_entry_is_refused(self, mocker: MockerFixture, tmp_path: Path) -> None:+ mock_execute = mocker.patch(f"{BUNDLE_CMD_MODULE}.execute_run")+ mock_handler = mocker.patch(f"{BUNDLE_CMD_MODULE}.handle_validate_bundle_error", side_effect=typer.Exit(1))+ (tmp_path / "a.mthds").write_text(_bundle(domain="folder_a", pipe_code="first", main_pipe="first"), encoding="utf-8")+ (tmp_path / "b.mthds").write_text(_bundle(domain="folder_b", pipe_code="second", main_pipe="second"), encoding="utf-8")++ with pytest.raises(typer.Exit):+ run_bundle_cmd(path=str(tmp_path))++ verdict = mock_handler.call_args.args[0]+ assert "several of its bundles declare a main_pipe: 'a.mthds' declares 'first', 'b.mthds' declares 'second'" in verdict.message+ mock_execute.assert_not_called()diff --git a/tests/unit/pipelex/cli/commands/run/test_hosted_sources.py b/tests/unit/pipelex/cli/commands/run/test_hosted_sources.pyindex e17e584db..5fa5ea6a0 100644--- a/tests/unit/pipelex/cli/commands/run/test_hosted_sources.py+++ b/tests/unit/pipelex/cli/commands/run/test_hosted_sources.py@@ -58,6 +58,9 @@ class TestHostedSources:_write(path=tmp_path / "lib" / "nested" / "deep.mthds", content="deep")_write(path=tmp_path / "lib" / "notes.md", content="not a bundle")_write(path=tmp_path / "lib" / ".venv" / "vendored.mthds", content="excluded")+ # A file the library's method tree keeps out is never sent, as a local load never reads it.+ _write(path=tmp_path / "lib" / "draft.mthds", content="gitignored")+ _write(path=tmp_path / "lib" / ".gitignore", content="draft.mthds\n")single = _write(path=tmp_path / "single.mthds", content="single")files = collect_mthds_files(primary=bundle, library_dirs=[str(tmp_path / "lib"), str(single), str(tmp_path / "missing")])diff --git a/tests/unit/pipelex/cli/commands/run/test_run_hosted_cli.py b/tests/unit/pipelex/cli/commands/run/test_run_hosted_cli.pyindex af2697467..876317eb9 100644--- a/tests/unit/pipelex/cli/commands/run/test_run_hosted_cli.py+++ b/tests/unit/pipelex/cli/commands/run/test_run_hosted_cli.py@@ -117,7 +117,7 @@ class TestRunHostedCli:run_bundle_cmd(path=str(bundle_dir), hosted=True, output_dir=str(output_dir), no_pretty_print=True)request = _hosted_request(run_hosted=run_hosted)- assert request.pipe_code == "summarize"+ assert request.pipe_code == "probe.summarize"assert [mthds_file.content for mthds_file in request.mthds_files or []] == [BUNDLE, LIBRARY_BUNDLE]assert request.method_ref is Noneassert request.method_id is None@@ -264,6 +264,22 @@ class TestRunHostedCli:request = _hosted_request(run_hosted=run_hosted)assert [mthds_file.source for mthds_file in request.mthds_files or []] == [str(bundle_dir / "bundle.mthds"), str(shared_dir / "shared.mthds")]+ def test_a_folder_run_names_its_main_pipe_by_its_domain_when_two_domains_share_its_code(self, run_hosted: AsyncMock, tmp_path: Path) -> None:+ """The hosted API starts at the pipe the local run would, never at a same-coded pipe of another domain sent with it."""+ method_dir = tmp_path / "method"+ method_dir.mkdir()+ (method_dir / "a_first.mthds").write_text(+ BUNDLE.replace('domain = "probe"\nmain_pipe = "summarize"', 'domain = "probe_other"'), encoding="utf-8"+ )+ (method_dir / "z_entry.mthds").write_text(BUNDLE.replace('domain = "probe"', 'domain = "probe_entry"'), encoding="utf-8")+ output_dir = tmp_path / "results"++ run_bundle_cmd(path=str(method_dir), hosted=True, output_dir=str(output_dir), no_pretty_print=True)++ request = _hosted_request(run_hosted=run_hosted)+ assert request.pipe_code == "probe_entry.summarize"+ assert (output_dir / "summarize_output_01" / "main_stuff.json").is_file()+def test_a_named_text_output_is_saved_as_markdown_too(self, mocker: MockerFixture, bundle_dir: Path, tmp_path: Path) -> None:"""The working memory names the main output through its `main_stuff` alias when a step named it."""named = RunResults.model_validate(diff --git a/tests/unit/pipelex/cli/commands/run/test_run_output_folder.py b/tests/unit/pipelex/cli/commands/run/test_run_output_folder.pynew file mode 100644index 000000000..53f9900ac--- /dev/null+++ b/tests/unit/pipelex/cli/commands/run/test_run_output_folder.py@@ -0,0 +1,55 @@+import json+from pathlib import Path++import pytest+from mthds.method_tree.digest import tree_listing+from mthds.method_tree.lister import list_method_tree++from pipelex.cli.commands.run._run_output_folder import METHOD_TREE_RECORD_NAME, make_run_output_folder, run_output_label, write_method_tree_record+++class TestRunOutputFolder:+ def test_two_consecutive_runs_leave_the_method_tree_unchanged(self, tmp_path: Path) -> None:+ """A run writing its outputs inside the method's folder, as from the method's own directory, never adds to its tree."""+ (tmp_path / "main.mthds").write_text("domain = 'run_folder_tests'\n", encoding="utf-8")+ before = list_method_tree(tmp_path)++ for _ in range(2):+ output_path = make_run_output_folder(output_dir=tmp_path / "results", label="summarize")+ (output_path / "main_stuff.json").write_text("{}", encoding="utf-8")+ write_method_tree_record(output_path=output_path, method_tree=before)++ after = list_method_tree(tmp_path)+ assert sorted(path.name for path in (tmp_path / "results").iterdir()) == ["summarize_output_01", "summarize_output_02"]+ assert tree_listing(after.files) == tree_listing(before.files)+ assert after.digest == before.digest++ def test_the_record_names_the_tree_the_run_read(self, tmp_path: Path) -> None:+ method_dir = tmp_path / "method"+ method_dir.mkdir()+ (method_dir / "main.mthds").write_text("domain = 'run_folder_tests'\n", encoding="utf-8")+ method_tree = list_method_tree(method_dir)+ output_path = make_run_output_folder(output_dir=tmp_path / "results", label="summarize")++ record_path = write_method_tree_record(output_path=output_path, method_tree=method_tree)++ assert record_path == output_path / METHOD_TREE_RECORD_NAME+ record = json.loads(record_path.read_text(encoding="utf-8"))+ assert record["digest"] == method_tree.digest+ assert record["root"] == str(method_tree.directory)+ assert record["is_one_file"] is False+ assert record["files"] == [{"path": "main.mthds", "sha256": method_tree.files[0].sha256, "size": method_tree.files[0].size}]+ assert (output_path / ".gitignore").read_text(encoding="utf-8").splitlines()[-1] == "*"++ @pytest.mark.parametrize(+ ("pipe_ref", "expected_label"),+ [+ ("summarize", "summarize"),+ ("report.summarize", "summarize"),+ ("scoring->report.summarize", "summarize"),+ ("scoring->summarize", "summarize"),+ ],+ )+ def test_a_run_folder_is_labelled_by_the_pipe_s_own_code(self, pipe_ref: str, expected_label: str) -> None:+ """A pipe named with its domain, or through a dependency's alias, labels its folder as its bare code does, locally or hosted."""+ assert run_output_label(pipe_ref=pipe_ref) == expected_labeldiff --git a/tests/unit/pipelex/libraries/test_host_bundle_files.py b/tests/unit/pipelex/libraries/test_host_bundle_files.pyindex 8c614f5d5..e09f6931a 100644--- a/tests/unit/pipelex/libraries/test_host_bundle_files.py+++ b/tests/unit/pipelex/libraries/test_host_bundle_files.py@@ -1,5 +1,11 @@from pathlib import Path+import kajson+import pytest+from mthds.method_tree.lister import list_method_tree++from pipelex.core.method_trees.tree_bundle import TreeBundle, TreeBundleRef+from pipelex.libraries.exceptions import CrateIntegrityErrorfrom pipelex.libraries.library_crate import LibraryCratefrom pipelex.libraries.library_crate_factory import LibraryCrateFactoryfrom pipelex.libraries.library_manager import MTHDS_METHODS_DIRNAME, find_methods_dirs_from_blueprints@@ -42,23 +48,44 @@ class TestHostBundleFiles:assert from_content.host_bundle_file is Noneassert from_content.source == str(bundle_path)- def test_the_crate_carries_the_host_file_of_its_file_loaded_pipes_only(self, tmp_path: Path) -> None:- """A `source` the bundle's text writes on a pipe, or a caller's label, never becomes a file to read beside."""+ def test_the_crate_carries_the_tree_of_its_file_loaded_pipes_only(self, tmp_path: Path) -> None:+ """A `source` the bundle's text writes on a pipe, or a caller's label, never becomes a tree to read files from."""bundle_path = _write_bundle(directory=tmp_path)other_dir = tmp_path / "other"other_dir.mkdir()content_with_a_written_source = _bundle(domain="host_files_other", pipe_fields=f'source = "{bundle_path}"')+ from_file = MthdsParser.make_pipelex_bundle_blueprint(bundle_path=bundle_path)+ tree_bundle = TreeBundle(tree=list_method_tree(tmp_path), tree_path="bundle.mthds")+ from_file.place_in_tree(tree_bundle=tree_bundle)+ from_content = MthdsParser.make_pipelex_bundle_blueprint(mthds_content=content_with_a_written_source, mthds_source=str(other_dir / "x.mthds"))- crate = LibraryCrateFactory.make_from_blueprints(- blueprints=[- MthdsParser.make_pipelex_bundle_blueprint(bundle_path=bundle_path),- MthdsParser.make_pipelex_bundle_blueprint(mthds_content=content_with_a_written_source, mthds_source=str(other_dir / "x.mthds")),- ]- )+ crate = LibraryCrateFactory.make_from_blueprints(blueprints=[from_file, from_content])- assert crate.host_bundle_files == {"host_files_tests.summarize": str(bundle_path)}+ assert crate.tree_bundles == {"host_files_tests.summarize": TreeBundleRef(tree_key=tree_bundle.tree_key, tree_path="bundle.mthds")}+ assert list(crate.method_trees) == [tree_bundle.tree_key]+ assert crate.tree_bundles_by_pipe_ref() == {"host_files_tests.summarize": tree_bundle}assert crate.pipes["host_files_other.summarize"].source == str(bundle_path)- assert crate.fingerprint == LibraryCrate.model_validate_json(crate.model_dump_json()).fingerprint+ # The trees travel as text, so the crate crosses any boundary a Temporal payload does, kajson's included.+ for round_tripped in (LibraryCrate.model_validate_json(crate.model_dump_json()), kajson.loads(kajson.dumps(crate))):+ assert isinstance(round_tripped, LibraryCrate)+ assert round_tripped.fingerprint == crate.fingerprint+ assert round_tripped.tree_bundles_by_pipe_ref() == {"host_files_tests.summarize": tree_bundle}+ # The trees travel with the crate but are no part of what it is: the fingerprint ignores them.+ unplaced = LibraryCrateFactory.make_from_blueprints(+ blueprints=[MthdsParser.make_pipelex_bundle_blueprint(bundle_path=bundle_path), from_content],+ )+ assert unplaced.method_trees == {}+ assert unplaced.fingerprint == crate.fingerprint++ def test_a_crate_naming_a_tree_it_does_not_carry_is_refused(self, tmp_path: Path) -> None:+ bundle_path = _write_bundle(directory=tmp_path)+ crate = LibraryCrateFactory.make_from_blueprints(blueprints=[MthdsParser.make_pipelex_bundle_blueprint(bundle_path=bundle_path)])+ broken = crate.model_copy(+ update={"tree_bundles": {"host_files_tests.summarize": TreeBundleRef(tree_key="directory:/x:sha256:0", tree_path="bundle.mthds")}}+ )++ with pytest.raises(CrateIntegrityError, match="carries no such tree"):+ broken.tree_bundles_by_pipe_ref()def test_method_packages_are_searched_above_a_file_read_here_only(self, tmp_path: Path) -> None:"""Walking up from a caller's label would search this host's directories wherever the caller pointed."""diff --git a/tests/unit/pipelex/libraries/test_method_trees.py b/tests/unit/pipelex/libraries/test_method_trees.pynew file mode 100644index 000000000..e5357cee4--- /dev/null+++ b/tests/unit/pipelex/libraries/test_method_trees.py@@ -0,0 +1,207 @@+from pathlib import Path++import pytest+from mthds.method_tree.models import TreeLimits+from pytest_mock import MockerFixture++from pipelex.base_exceptions import ErrorDomain+from pipelex.libraries.exceptions import HostLibraryTreeError, MethodTreeLoadError+from pipelex.libraries.method_trees import (+ LIMITS_ADVICE,+ bundle_files_under,+ list_library_bundle_files,+ list_package_method_tree,+ reporting_host_library_tree_faults,+ tree_bundle_of_lone_file,+)++_MANIFEST = '[package]\naddress = "github.com/pipelex-tests/nested"\nversion = "1.0.0"\ndescription = "A nested package"\n'+++def _touch(*, directory: Path, tree_path: str, content: str = "domain = 'tree_tests'\n") -> Path:+ file_path = directory / tree_path+ file_path.parent.mkdir(parents=True, exist_ok=True)+ file_path.write_text(content, encoding="utf-8")+ return file_path+++class TestMethodTrees:+ def test_a_library_directory_lists_only_the_bundles_its_tree_holds(self, tmp_path: Path) -> None:+ """A bundle the membership rule keeps out, gitignored, hidden or in a run's output folder, is never loaded."""+ _touch(directory=tmp_path, tree_path="method/main.mthds")+ _touch(directory=tmp_path, tree_path="method/draft.mthds")+ _touch(directory=tmp_path, tree_path="method/.gitignore", content="draft.mthds\n")+ _touch(directory=tmp_path, tree_path=".archive/old.mthds")+ _touch(directory=tmp_path, tree_path="results/summarize_output_01/copy.mthds")+ _touch(directory=tmp_path, tree_path="results/summarize_output_01/.gitignore", content="*\n")++ bundle_files = list_library_bundle_files(library_dirs=[tmp_path])++ assert [bundle_file.tree_bundle.tree_path for bundle_file in bundle_files] == ["method/main.mthds"]+ assert bundle_files[0].path == tmp_path / "method" / "main.mthds"++ def test_a_nested_package_or_repository_loads_as_a_tree_of_its_own(self, tmp_path: Path) -> None:+ """A package or repository below a library directory is no part of its tree, yet its methods load, each reading its own files."""+ _touch(directory=tmp_path, tree_path="top.mthds")+ _touch(directory=tmp_path, tree_path="packages/invoicing/METHODS.toml", content=_MANIFEST)+ _touch(directory=tmp_path, tree_path="packages/invoicing/invoice.mthds")+ _touch(directory=tmp_path, tree_path="checkouts/reports/report.mthds")+ (tmp_path / "checkouts" / "reports" / ".git").mkdir()++ bundle_files = list_library_bundle_files(library_dirs=[tmp_path])++ by_location = {bundle_file.path: bundle_file.tree_bundle for bundle_file in bundle_files}+ assert set(by_location) == {+ tmp_path / "top.mthds",+ tmp_path / "packages" / "invoicing" / "invoice.mthds",+ tmp_path / "checkouts" / "reports" / "report.mthds",+ }+ invoice = by_location[tmp_path / "packages" / "invoicing" / "invoice.mthds"]+ assert invoice.tree_path == "invoice.mthds"+ assert invoice.tree.directory == (tmp_path / "packages" / "invoicing").resolve()+ report = by_location[tmp_path / "checkouts" / "reports" / "report.mthds"]+ assert report.tree.directory == (tmp_path / "checkouts" / "reports").resolve()+ assert by_location[tmp_path / "top.mthds"].tree.tree_paths == ["top.mthds"]++ def test_each_bundle_is_listed_once_as_a_member_of_the_first_tree_holding_it(self, tmp_path: Path) -> None:+ _touch(directory=tmp_path, tree_path="a.mthds")+ _touch(directory=tmp_path, tree_path="sub/b.mthds")++ bundle_files = list_library_bundle_files(library_dirs=[tmp_path, tmp_path / "sub"])++ assert [bundle_file.tree_bundle.tree_path for bundle_file in bundle_files] == ["a.mthds", "sub/b.mthds"]++ def test_a_relative_library_directory_names_its_files_relatively(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:+ _touch(directory=tmp_path, tree_path="lib/a.mthds")+ monkeypatch.chdir(tmp_path)++ (bundle_file,) = list_library_bundle_files(library_dirs=[Path("lib")])++ assert bundle_file.path == Path("lib") / "a.mthds"++ @pytest.mark.parametrize("contents", [{}, {".hidden.mthds": "x"}], ids=["missing", "empty_tree"])+ def test_a_directory_holding_no_tree_file_holds_no_bundle(self, tmp_path: Path, contents: dict[str, str]) -> None:+ library_dir = tmp_path / "library"+ for tree_path, content in contents.items():+ _touch(directory=library_dir, tree_path=tree_path, content=content)++ assert list_library_bundle_files(library_dirs=[library_dir]) == []++ def test_a_tree_past_this_machine_s_limits_is_refused_before_it_is_read(self, tmp_path: Path, mocker: MockerFixture) -> None:+ mocker.patch("pipelex.libraries.method_trees.local_tree_limits", return_value=TreeLimits(max_files=1))+ _touch(directory=tmp_path, tree_path="a.mthds")+ _touch(directory=tmp_path, tree_path="b.mthds")++ with pytest.raises(MethodTreeLoadError) as exc_info:+ list_library_bundle_files(library_dirs=[tmp_path])++ assert exc_info.value.message.endswith(LIMITS_ADVICE)++ def test_a_bundle_file_named_alone_is_rooted_at_the_manifest_above_it_else_alone(self, tmp_path: Path) -> None:+ """Named alone, a bundle reads the files of the method a METHODS.toml above it roots, and with none only itself."""+ lone = _touch(directory=tmp_path, tree_path="loose/lone.mthds")+ _touch(directory=tmp_path, tree_path="loose/notes.md")+ _touch(directory=tmp_path, tree_path="method/METHODS.toml", content=_MANIFEST)+ rooted = _touch(directory=tmp_path, tree_path="method/bundles/main.mthds")++ lone_bundle = tree_bundle_of_lone_file(bundle_file=lone)+ rooted_bundle = tree_bundle_of_lone_file(bundle_file=rooted)++ assert lone_bundle.tree.is_one_file+ assert lone_bundle.tree.tree_paths == ["lone.mthds"]+ assert not rooted_bundle.tree.is_one_file+ assert rooted_bundle.tree_path == "bundles/main.mthds"+ assert rooted_bundle.tree.tree_paths == ["METHODS.toml", "bundles/main.mthds"]++ def test_a_library_directory_holding_only_nested_methods_loads_them_all(self, tmp_path: Path) -> None:+ """Its own tree is empty, yet the packages and repositories nested in it load, and those nested in an empty one of them."""+ _touch(directory=tmp_path, tree_path=".hidden-notes.md", content="kept out")+ _touch(directory=tmp_path, tree_path="packages/invoicing/METHODS.toml", content=_MANIFEST)+ _touch(directory=tmp_path, tree_path="packages/invoicing/invoice.mthds")+ _touch(directory=tmp_path, tree_path="checkouts/reports/report.mthds")+ (tmp_path / "checkouts" / "reports" / ".git").mkdir()+ # A nested repository whose own tree is empty too, holding nothing but a nested package.+ (tmp_path / "checkouts" / "suite" / ".git").mkdir(parents=True)+ _touch(directory=tmp_path, tree_path="checkouts/suite/pkg/METHODS.toml", content=_MANIFEST)+ _touch(directory=tmp_path, tree_path="checkouts/suite/pkg/suite.mthds")+ # A directory the .gitignore keeps out is never entered, so the package under it is never reached, as in a listing.+ _touch(directory=tmp_path, tree_path=".gitignore", content="build/\n")+ _touch(directory=tmp_path, tree_path="build/stale/METHODS.toml", content=_MANIFEST)+ _touch(directory=tmp_path, tree_path="build/stale/stale.mthds")++ bundle_files = list_library_bundle_files(library_dirs=[tmp_path])++ by_location = {bundle_file.path: bundle_file.tree_bundle for bundle_file in bundle_files}+ assert set(by_location) == {+ tmp_path / "packages" / "invoicing" / "invoice.mthds",+ tmp_path / "checkouts" / "reports" / "report.mthds",+ tmp_path / "checkouts" / "suite" / "pkg" / "suite.mthds",+ }+ suite_tree = by_location[tmp_path / "checkouts" / "suite" / "pkg" / "suite.mthds"].tree+ assert suite_tree.directory == (tmp_path / "checkouts" / "suite" / "pkg").resolve()++ def test_a_nested_checkout_holding_no_bundle_is_skipped_before_it_is_listed(self, tmp_path: Path) -> None:+ """A nested repository with nothing to load never refuses the load, whatever it holds; one holding a bundle follows every rule."""+ _touch(directory=tmp_path, tree_path="top.mthds")+ _touch(directory=tmp_path, tree_path="vendor/tool/README.md", content="a tool")+ (tmp_path / "vendor" / "tool" / ".git").mkdir()+ (tmp_path / "vendor" / "tool" / "latest").symlink_to(tmp_path / "vendor" / "tool" / "README.md")++ assert [bundle_file.tree_bundle.tree_path for bundle_file in list_library_bundle_files(library_dirs=[tmp_path])] == ["top.mthds"]++ _touch(directory=tmp_path, tree_path="vendor/tool/tool.mthds")+ with pytest.raises(MethodTreeLoadError, match="symbolic link"):+ list_library_bundle_files(library_dirs=[tmp_path])++ def test_this_machine_s_limits_are_read_once_per_listing(self, tmp_path: Path, mocker: MockerFixture) -> None:+ limits_spy = mocker.patch("pipelex.libraries.method_trees.local_tree_limits", return_value=TreeLimits())+ _touch(directory=tmp_path, tree_path="top.mthds")+ for name in ("one", "two"):+ _touch(directory=tmp_path, tree_path=f"packages/{name}/METHODS.toml", content=_MANIFEST)+ _touch(directory=tmp_path, tree_path=f"packages/{name}/{name}.mthds")++ assert len(list_library_bundle_files(library_dirs=[tmp_path, tmp_path / "packages"])) == 3+ limits_spy.assert_called_once()++ @pytest.mark.parametrize("refusal", ["symbolic_link", "limit"])+ def test_a_dependency_package_refusal_names_its_address_never_this_host_s_path(self, tmp_path: Path, refusal: str) -> None:+ package_root = tmp_path / "installed" / "invoicing"+ _touch(directory=package_root, tree_path="METHODS.toml", content=_MANIFEST)+ _touch(directory=package_root, tree_path="invoice.mthds")+ limits: TreeLimits | None = None+ if refusal == "symbolic_link":+ (package_root / "link.mthds").symlink_to(package_root / "invoice.mthds")+ else:+ limits = TreeLimits(max_files=1)++ with pytest.raises(MethodTreeLoadError) as exc_info:+ list_package_method_tree(root=package_root, package_address="github.com/pipelex-tests/invoicing", limits=limits)++ message = exc_info.value.message+ assert "github.com/pipelex-tests/invoicing" in message+ assert str(tmp_path) not in message+ assert str(tmp_path.resolve()) not in message+ assert LIMITS_ADVICE not in message++ def test_a_host_library_tree_that_cannot_be_listed_is_the_host_s_fault(self, tmp_path: Path) -> None:+ """Beside a caller's bundles, a tree of the host's own directories is no input of theirs, and its message never reaches them."""+ _touch(directory=tmp_path, tree_path="lib/a.mthds")+ (tmp_path / "lib" / "link.mthds").symlink_to(tmp_path / "lib" / "a.mthds")++ with pytest.raises(HostLibraryTreeError) as exc_info, reporting_host_library_tree_faults():+ list_library_bundle_files(library_dirs=[tmp_path / "lib"])++ report = exc_info.value.to_error_report()+ assert report.error_domain == ErrorDomain.CONFIG+ assert report.caller_facing_message is False+ assert isinstance(exc_info.value.__cause__, MethodTreeLoadError)++ def test_every_bundle_file_under_a_directory_is_found_whatever_the_membership_rule(self, tmp_path: Path) -> None:+ """The superset a verdict's withholding falls back to when listing the trees fails: it never raises."""+ _touch(directory=tmp_path, tree_path="lib/a.mthds")+ _touch(directory=tmp_path, tree_path="lib/.hidden/b.mthds")+ (tmp_path / "lib" / "link.mthds").symlink_to(tmp_path / "lib" / "a.mthds")++ found = bundle_files_under(library_dirs=[tmp_path / "lib", tmp_path / "missing"])++ assert set(found) == {tmp_path / "lib" / "a.mthds", tmp_path / "lib" / ".hidden" / "b.mthds", tmp_path / "lib" / "link.mthds"}diff --git a/tests/unit/pipelex/pipe_operators/pipe_doc_gen/test_pipe_doc_gen_blueprint.py b/tests/unit/pipelex/pipe_operators/pipe_doc_gen/test_pipe_doc_gen_blueprint.pyindex c845eed11..c6d808aa7 100644--- a/tests/unit/pipelex/pipe_operators/pipe_doc_gen/test_pipe_doc_gen_blueprint.py+++ b/tests/unit/pipelex/pipe_operators/pipe_doc_gen/test_pipe_doc_gen_blueprint.py@@ -40,9 +40,8 @@ class TestPipeDocGenBlueprint:("unc_template_file", {"format": "docx", "template_file": "\\\\server\\share\\invoice.docx"}, "it holds a backslash"),("posix_unc_template_file", {"format": "docx", "template_file": "//server/share/invoice.docx"}, "it is absolute"),("colon_in_the_name", {"format": "docx", "template_file": "invoice:v2.docx"}, "it holds a colon"),+ ("url", {"format": "docx", "template_file": "https://example.com/invoice.docx"}, "it is a URL"),("backslash_in_the_name", {"format": "docx", "template_file": "templates\\invoice.docx"}, "it holds a backslash"),- ("leaves_the_bundle", {"format": "docx", "template_file": "../invoice.docx"}, "'..' segment"),- ("dot_dot_inside_the_bundle", {"format": "docx", "template_file": "templates/../invoice.docx"}, "'..' segment"),("empty_template_file", {"format": "docx", "template_file": " "}, "it is empty"),],)@@ -62,7 +61,13 @@ class TestPipeDocGenBlueprint:assert blueprint.template_file == template_fileassert canonical_resource_path_or_none(path=template_file) is not None- @pytest.mark.parametrize("template_file", ["/srv/methods/invoice.docx", "C:/srv/methods/invoice.docx", "../srv/methods/invoice.docx"])+ @pytest.mark.parametrize("template_file", ["../invoice.docx", "templates/../invoice.docx", "../shared/templates/invoice.docx"])+ def test_a_path_climbing_through_dot_dot_is_left_to_the_method_tree(self, template_file: str) -> None:+ """A `..` may climb to a file elsewhere in the method's tree, which only the load, holding the tree, can tell."""+ blueprint = _blueprint(format="docx", template_file=template_file)+ assert blueprint.template_file == template_file++ @pytest.mark.parametrize("template_file", ["/srv/methods/invoice.docx", "C:/srv/methods/invoice.docx", "srv\\methods\\invoice.docx"])def test_the_refusal_of_a_template_file_outside_its_bundle_never_repeats_it(self, template_file: str) -> None:"""A host library's step would otherwise put the host's own path in front of a caller."""with pytest.raises(ValidationError) as exc_info:diff --git a/tests/unit/pipelex/pipeline/fixes/test_fix_loop_tree_refusal.py b/tests/unit/pipelex/pipeline/fixes/test_fix_loop_tree_refusal.pynew file mode 100644index 000000000..990459acb--- /dev/null+++ b/tests/unit/pipelex/pipeline/fixes/test_fix_loop_tree_refusal.py@@ -0,0 +1,41 @@+from pathlib import Path++import pytest+from pytest_mock import MockerFixture++from pipelex.pipeline.fixes.fix_loop import fix_bundle_file++_BUNDLE = """domain = "fix_tree_refusal"+main_pipe = "write_notice"++[pipe.write_notice]+type = "PipeLLM"+description = "Write a notice"+inputs = { topic = "Text" }+output = "Text"+prompt = "Write about $topic"+"""++[email protected](loop_scope="class")+class TestFixLoopTreeRefusal:+ async def test_a_library_tree_that_cannot_be_listed_is_the_invalid_verdict_with_nothing_to_fix(+ self, tmp_path: Path, mocker: MockerFixture+ ) -> None:+ """The snapshot lists the method's trees before validating: a refused tree ends the loop with its message, never an exception."""+ library_dir = tmp_path / "library"+ library_dir.mkdir()+ bundle_path = library_dir / "notice.mthds"+ bundle_path.write_text(_BUNDLE, encoding="utf-8")+ (library_dir / "linked_notice.mthds").symlink_to(bundle_path)+ validate_mock = mocker.patch("pipelex.pipeline.fixes.fix_loop.validate_bundle")++ result = await fix_bundle_file(bundle_path, library_dirs=[library_dir], max_iterations=3)++ assert result.is_valid is False+ assert result.iterations == 0+ assert result.files_written == []+ (item,) = result.remaining_errors+ assert "linked_notice.mthds" in item.message+ validate_mock.assert_not_called()+ assert bundle_path.read_text(encoding="utf-8") == _BUNDLEdiff --git a/tests/unit/pipelex/pipeline/test_method_entry.py b/tests/unit/pipelex/pipeline/test_method_entry.pynew file mode 100644index 000000000..49ad75ccf--- /dev/null+++ b/tests/unit/pipelex/pipeline/test_method_entry.py@@ -0,0 +1,134 @@+from pathlib import Path++import pytest+from mthds.method_tree.lister import list_method_tree++from pipelex.pipeline.exceptions import MethodEntryError+from pipelex.pipeline.method_entry import select_method_entry+++def _bundle(*, domain: str, pipe_code: str, main_pipe: str | None) -> str:+ main_pipe_line = f'main_pipe = "{main_pipe}"\n' if main_pipe else ""+ return f"""domain = "{domain}"+{main_pipe_line}+[pipe.{pipe_code}]+type = "PipeLLM"+description = "Summarize the text"+inputs = {{ text = "Text" }}+output = "Text"+prompt = "Summarize $text"+"""+++def _manifest(*, main_pipe: str | None, exports: dict[str, list[str]]) -> str:+ main_pipe_line = f'main_pipe = "{main_pipe}"\n' if main_pipe else ""+ exports_text = "".join(f"\n[exports.{domain}]\npipes = {pipes!r}\n".replace("'", '"') for domain, pipes in exports.items())+ return f"""[package]+address = "github.com/pipelex-tests/entry"+version = "1.0.0"+description = "A method whose entry is selected"+{main_pipe_line}{exports_text}"""+++def _write(*, directory: Path, files: dict[str, str]) -> Path:+ for tree_path, content in files.items():+ file_path = directory / tree_path+ file_path.parent.mkdir(parents=True, exist_ok=True)+ file_path.write_text(content, encoding="utf-8")+ return directory+++class TestMethodEntry:+ def test_a_method_with_no_manifest_starts_at_its_one_main_pipe(self, tmp_path: Path) -> None:+ """A bundle declaring no main_pipe is no rival: the one that declares it is where the method starts."""+ _write(+ directory=tmp_path,+ files={+ "steps/extract.mthds": _bundle(domain="entry_steps", pipe_code="extract", main_pipe=None),+ "report.mthds": _bundle(domain="entry_report", pipe_code="write_report", main_pipe="write_report"),+ },+ )++ entry = select_method_entry(tree=list_method_tree(tmp_path))++ assert entry.pipe_ref == "entry_report.write_report"+ assert entry.pipe_code == "write_report"+ assert entry.tree_path == "report.mthds"++ def test_a_manifest_names_the_entry_whatever_the_bundles_declare(self, tmp_path: Path) -> None:+ """With a METHODS.toml at the root, its main_pipe wins over every bundle's, however many declare one."""+ _write(+ directory=tmp_path,+ files={+ "METHODS.toml": _manifest(main_pipe="extract", exports={"entry_steps": ["extract"]}),+ "steps/extract.mthds": _bundle(domain="entry_steps", pipe_code="extract", main_pipe="extract"),+ "report.mthds": _bundle(domain="entry_report", pipe_code="write_report", main_pipe="write_report"),+ },+ )++ entry = select_method_entry(tree=list_method_tree(tmp_path))++ assert entry.pipe_ref == "entry_steps.extract"+ assert entry.tree_path == "steps/extract.mthds"++ @pytest.mark.parametrize(+ ("files", "expected"),+ [+ (+ {+ "a.mthds": _bundle(domain="entry_a", pipe_code="first", main_pipe=None),+ "b/b.mthds": _bundle(domain="entry_b", pipe_code="second", main_pipe=None),+ },+ "none of its bundles declares a main_pipe ('a.mthds', 'b/b.mthds')",+ ),+ (+ {+ "a.mthds": _bundle(domain="entry_a", pipe_code="first", main_pipe="first"),+ "b/b.mthds": _bundle(domain="entry_b", pipe_code="second", main_pipe="second"),+ },+ "several of its bundles declare a main_pipe: 'a.mthds' declares 'first', 'b/b.mthds' declares 'second'",+ ),+ (+ {+ "METHODS.toml": _manifest(main_pipe=None, exports={}),+ "a.mthds": _bundle(domain="entry_a", pipe_code="first", main_pipe="first"),+ },+ (+ "declares no main_pipe, so the method has no pipe to start at. Declare main_pipe under [package], or name the pipe to run. "+ "Its bundles: 'a.mthds'."+ ),+ ),+ (+ {+ "METHODS.toml": _manifest(main_pipe="missing", exports={"entry_a": ["missing"]}),+ "a.mthds": _bundle(domain="entry_a", pipe_code="first", main_pipe="first"),+ },+ "declares the main_pipe 'missing', exported by 'entry_a', which none of its bundles defines in that domain: 'a.mthds'.",+ ),+ (+ {"notes.md": "# Notes\n"},+ "holds no .mthds bundle, so it has no pipe to start at",+ ),+ ],+ ids=["no_main_pipe", "several_main_pipes", "manifest_without_main_pipe", "manifest_main_pipe_undefined", "no_bundle"],+ )+ def test_a_method_with_no_single_entry_is_refused_naming_its_bundles_by_tree_path(+ self, tmp_path: Path, files: dict[str, str], expected: str+ ) -> None:+ _write(directory=tmp_path, files=files)++ with pytest.raises(MethodEntryError) as exc_info:+ select_method_entry(tree=list_method_tree(tmp_path))++ assert expected in exc_info.value.message+ assert exc_info.value.to_error_report().error_domain == "input"++ def test_a_manifest_that_does_not_validate_is_an_entry_refusal(self, tmp_path: Path) -> None:+ """A manifest the lister can read but the standard refuses, here one with no address, gives no entry."""+ _write(+ directory=tmp_path,+ files={"METHODS.toml": '[package]\nversion = "1.0.0"\n', "a.mthds": _bundle(domain="entry_a", pipe_code="first", main_pipe="first")},+ )++ with pytest.raises(MethodEntryError, match=r"The METHODS\.toml of the method at .* cannot be read"):+ select_method_entry(tree=list_method_tree(tmp_path))diff --git a/tests/unit/pipelex/pipeline/test_validate_bundle_error_withholding.py b/tests/unit/pipelex/pipeline/test_validate_bundle_error_withholding.pyindex 9dbfff282..bcc834cf6 100644--- a/tests/unit/pipelex/pipeline/test_validate_bundle_error_withholding.py+++ b/tests/unit/pipelex/pipeline/test_validate_bundle_error_withholding.py@@ -124,3 +124,21 @@ class TestValidateBundleErrorWithholding:assert raised.value.message == f"Pipe 'print_notice' in '{HOST_LIBRARY_FILE_PLACEHOLDER}' could not read '{HOST_LIBRARY_FILE_PLACEHOLDER}'"assert walk_spy.call_count == 1++ def test_a_host_tree_that_cannot_be_listed_while_withholding_never_replaces_the_verdict(self, tmp_path: Path) -> None:+ """The host's files are still withheld, from every bundle file under its directories, and the verdict stays the verdict."""+ host_library_file = tmp_path / "harbour_board.mthds"+ host_library_file.write_text('domain = "harbour_board"\n', encoding="utf-8")+ hidden_host_file = tmp_path / ".drafts" / "tide_board.mthds"+ hidden_host_file.parent.mkdir()+ hidden_host_file.write_text('domain = "tide_board"\n', encoding="utf-8")+ # A symbolic link is refused by the membership rule, so listing the host's tree fails.+ (tmp_path / "linked.mthds").symlink_to(host_library_file)+ message = f"Pipe 'post_notice' in '{host_library_file}' could not be loaded, nor '{hidden_host_file}'"++ with pytest.raises(ValidateBundleError) as raised, withholding_host_library_files(library_dirs=[tmp_path]):+ raise ValidateBundleError(message=message)++ assert raised.value.message == (+ f"Pipe 'post_notice' in '{HOST_LIBRARY_FILE_PLACEHOLDER}' could not be loaded, nor '{HOST_LIBRARY_FILE_PLACEHOLDER}'"+ )diff --git a/uv.lock b/uv.lockindex 81330a85a..328db3291 100644--- a/uv.lock+++ b/uv.lock@@ -2754,7 +2754,7 @@ wheels = [[[package]]name = "mthds"-version = "0.21.0"+version = "0.22.0"source = { registry = "https://pypi.org/simple" }dependencies = [{ name = "httpx" },@@ -2763,9 +2763,9 @@ dependencies = [{ name = "tomlkit" },{ name = "typing-extensions" },]-sdist = { url = "https://files.pythonhosted.org/packages/58/99/ebe133c36b3db8cf7bb4abf60c3330f980fbd378822bcc62cddc09a989c9/mthds-0.21.0.tar.gz", hash = "sha256:c52f13b58471b07583e29d15aece925c03d3509a0ea1204b13e549d20a0b1e33", size = 282110, upload-time = "2026-10-10T12:10:30.802Z" }+sdist = { url = "https://files.pythonhosted.org/packages/d7/e7/d765bd459f47d7c1773c1af847eefc6e1e1cc5e7e7b9266eb99d54945375/mthds-0.22.0.tar.gz", hash = "sha256:0ece0af23f2e412d3fdfa08c2b07881a837292b69c2df721137bf1b319916d1b", size = 365343, upload-time = "2026-10-15T22:04:58.59Z" }wheels = [- { url = "https://files.pythonhosted.org/packages/e4/1d/97001f8f2d42e904f2ce2f8a005cddb596efcdf103b413a0d22bbcd5bf67/mthds-0.21.0-py3-none-any.whl", hash = "sha256:c2ac9762980936e63864941c2107fa589c17ea367ca00dbb6a17bf4022c6beaa", size = 111488, upload-time = "2026-10-10T12:10:29.298Z" },+ { url = "https://files.pythonhosted.org/packages/ac/f3/2ce468930b48bac1f750a97426f141cded85702c83b02ba3801b11fdefcd/mthds-0.22.0-py3-none-any.whl", hash = "sha256:bed9d09dac9c76c07cdb26675bbcf4eca79c4b926bfff8c0c02bdaa7b643e2e7", size = 149743, upload-time = "2026-10-15T22:04:56.956Z" },][[package]]@@ -3735,7 +3735,7 @@ requires-dist = [{ name = "mkdocs-meta-manager", marker = "extra == 'docs'", specifier = ">=1.1.0" },{ name = "mkdocs-redirects", marker = "extra == 'docs'", specifier = "==1.2.2" },{ name = "moto", extras = ["dynamodb", "s3"], marker = "extra == 'dev'", specifier = ">=5.0.0" },- { name = "mthds", specifier = "==0.21.0" },+ { name = "mthds", specifier = "==0.22.0" },{ name = "mypy", marker = "extra == 'dev'", specifier = "==1.19.1" },{ name = "networkx", specifier = ">=3.4.2" },{ name = "openai", specifier = ">=2.0.0" },@@ -3744,7 +3744,7 @@ requires-dist = [{ name = "opentelemetry-sdk", specifier = ">=1.39.0" },{ name = "opentelemetry-semantic-conventions", specifier = ">=0.59b0" },{ name = "pillow", specifier = ">=12.1.1" },- { name = "pipelex-sdk", specifier = "==0.38.0" },+ { name = "pipelex-sdk", specifier = "==0.45.0" },{ name = "pipelex-tools", marker = "extra == 'dev'", specifier = ">=0.7.2" },{ name = "pipelex-tools-py", specifier = ">=0.1.3" },{ name = "polyfactory", specifier = ">=2.21.0" },@@ -3856,7 +3856,7 @@ provides-extras = ["dev"][[package]]name = "pipelex-sdk"-version = "0.38.0"+version = "0.45.0"source = { registry = "https://pypi.org/simple" }dependencies = [{ name = "httpx" },@@ -3864,9 +3864,9 @@ dependencies = [{ name = "pydantic" },{ name = "typing-extensions" },]-sdist = { url = "https://files.pythonhosted.org/packages/42/96/ec90e611b2fb8944e95f8fcf917093d7a1b3c9dcf60994b8c7553d0ae394/pipelex_sdk-0.38.0.tar.gz", hash = "sha256:6a2e5000408c6dbcdf06cc662df8f13accbebb1a3293616ddd90dcceefef0377", size = 533494, upload-time = "2026-10-10T13:41:06.803Z" }+sdist = { url = "https://files.pythonhosted.org/packages/f8/31/50cb829da5577dbfdaca032b0c4dc2285daa5ceb8c05b299341190ce82a1/pipelex_sdk-0.45.0.tar.gz", hash = "sha256:8e9de14ff0c4fa8787a03ef3f93130dcafd32b97b157cd519460ec5edaba19e3", size = 622581, upload-time = "2026-10-15T22:28:11.191Z" }wheels = [- { url = "https://files.pythonhosted.org/packages/71/14/01a43fc54d87e883e92ca317e0a1b8d23dfe3ef0251dbcbf9a29e101b72c/pipelex_sdk-0.38.0-py3-none-any.whl", hash = "sha256:291296c4ea5e03cd3328626c6ee68acf669f7c60d60f41ca26d489297e2820df", size = 213968, upload-time = "2026-10-10T13:41:04.946Z" },+ { url = "https://files.pythonhosted.org/packages/b9/7c/9ec2d03af8aabe69ac272153f36543809f58b4d5de4243b41ad1767b0545/pipelex_sdk-0.45.0-py3-none-any.whl", hash = "sha256:b037e34aec048b85b60e70dac6098f8753a0e1472ed71b9274dfef805a8280c2", size = 241187, upload-time = "2026-10-15T22:28:09.638Z" },][[package]]