Skip to content

Build AI methods with your coding agent, then run them anywhere: as an MCP for chatbots, as a webapp for people, or via API for your software.

feature/Report-names-step-files · L-261020-cb904c (#1464)

Louis Choquel9bd4d6ded981
diff --git a/.drift/acks/cli-docs.toml b/.drift/acks/cli-docs.toml
index e643c6306..c53336bc4 100644
--- a/.drift/acks/cli-docs.toml
+++ b/.drift/acks/cli-docs.toml
@@ -1,8 +1,8 @@
contract = "cli-docs"
-digest = "sha256:e2503f6e69a587a26fd9c9c8e856e158b359b2cbbc5285f680e1c427ea5486f4"
-reviewed_by = "Louis Choquel"
-reviewed_at = "2026-10-19T11:41:17Z"
-rationale = "clean-pass: merging #1458's head (Runner-problem-params, 9578055b3) into this branch, whose own _run_hosted.py and bundle_cmd.py changes are the triggers, keeps both edits to docs/tools/cli/agent-cli.md, #1458's validate-bundle verdict paragraph and this branch's run-on-a-runner paragraph, which touch different passages; pipelex/cli/agent_cli/CLAUDE.md, which #1458 did not change, still describes the hosted caller-view refusals as this branch wrote them, and nothing else under docs/tools/cli/ is stale."
+digest = "sha256:8e42deecf8cc65eebf459049b3eb92d8d3b00c711db16e359080f7630ef788c4"
+reviewed_by = "Claude Opus 5.5"
+reviewed_at = "2026-10-20T04:12:48Z"
+rationale = "real-catch: _validate_core.py now splats tree_files_read and tree_file_readers into the validate bundle and validate method envelopes; docs/tools/cli/agent-cli.md gained a 'The files each step reads' note and pipelex/cli/agent_cli/CLAUDE.md's validate bullet names both members, their Markdown section and that validate all and validate pipe carry neither; nothing else under docs/tools/cli/ describes the validate envelope's members."
[trigger_files]
"pipelex/cli/__init__.py" = "blob:8b137891791fe96927ad78e64b0aad7bded08bdc"
@@ -45,7 +45,7 @@ rationale = "clean-pass: merging #1458's head (Runner-problem-params, 9578055b3)
"pipelex/cli/agent_cli/commands/run/pipe_cmd.py" = "blob:1a5a2f86ac80fb86cb53376202bad05ffd19e9f6"
"pipelex/cli/agent_cli/commands/run/stdin_resolver.py" = "blob:db0176d7de1e074fd6eefd787c10e71b891eeb32"
"pipelex/cli/agent_cli/commands/validate/__init__.py" = "blob:e69de29bb2d1d6434b8b29ae775ad8c2e48c5391"
-"pipelex/cli/agent_cli/commands/validate/_validate_core.py" = "blob:b15fb5ee5b3d84cb229f3eb3353df8354cfd488d"
+"pipelex/cli/agent_cli/commands/validate/_validate_core.py" = "blob:b30afc76d88860d0d34c00eb8bebbad7d54bd498"
"pipelex/cli/agent_cli/commands/validate/app.py" = "blob:1982684a2aae81e0d634bb3b262457a5f187d979"
"pipelex/cli/agent_cli/commands/validate/bundle_cmd.py" = "blob:3c7045c77c15b2ab5fdb1629a4203e53231b764d"
"pipelex/cli/agent_cli/commands/validate/method_cmd.py" = "blob:19cbf94ca5027983e5fd95b73bc346efaf195c37"
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 72f1da10f..31525b042 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -2,6 +2,10 @@
## [Unreleased]
+### Added
+
+- **The validation report names the files each step reads**: `PipelexValidationReport`, and so the valid verdict of `/v1/validate`, carries `tree_files_read`, mapping each validated pipe's `pipe_ref` to the files of the method's tree it reads, each a `TreeFileRead` with its `tree_path` and its `use` (`template` for a `PipeDocGen` step's `template_file`, `template_asset` for a file its HTML template names by a literal relative URL, `python_module` for the module holding a `PipeFunc` step's function), an empty list for a pipe reading none, and `tree_file_readers`, mapping each of those files to the pipes that read it. `pipelex-agent validate bundle` and `validate method` carry both in their envelope, and the Markdown of a valid verdict lists them in a "Files the steps read" section.
+
### Changed
- **The API server's examples send a method in `files`**: the server's README and the API Server pages send a method to `/v1/execute` and `/v1/validate` in `files`, a map from each file's path in the method's folder to its content, where they sent `mthds_contents`. The reference pages and the OpenAPI artifact now say that `mthds_contents`, with `bundle_b64`, `resources` and `/v1/validate`'s `mthds_sources`, is being retired and will be refused by a later release, so a client sends `files` now.
diff --git a/api/tests/unit/test_bundle_resources_routes.py b/api/tests/unit/test_bundle_resources_routes.py
index a0a20c431..e98a3e6b1 100644
--- a/api/tests/unit/test_bundle_resources_routes.py
+++ b/api/tests/unit/test_bundle_resources_routes.py
@@ -521,6 +521,37 @@ class TestBundleResourcesRoutes:
else:
_assert_the_verdict_names_the_missing_file(verdict=verdict, file="logo.png", reason="not_in_tree")
+ def test_the_valid_verdict_of_a_tree_names_the_files_each_step_reads(self, mocker: MockerFixture) -> None:
+ """Per step, the files of the tree it reads by tree path and use; per file, the steps reading it; a carried file in neither."""
+ _stub_the_document_engine(mocker)
+ html_mthds = _INVOICE_MTHDS.replace('format = "docx"', 'format = "pdf"').replace(_TEMPLATE_PATH, "templates/invoice.html")
+ files: dict[str, Any] = {
+ "invoicing.mthds": html_mthds,
+ "templates/invoice.html": '<img src="logo.png" alt="Acme"><h1>{{ invoice.number }}</h1>',
+ "templates/logo.png": {"base64": _b64(b"\x89PNG\r\n\x1a\n a logo")},
+ "README.md": "# Invoicing, read by people and by no step\n",
+ }
+
+ response = _build_client().post("/v1/validate", json={"files": files, "render": ["markdown"]})
+
+ assert response.status_code == 200, response.text
+ verdict = response.json()
+ assert verdict["is_valid"] is True, response.text
+ assert verdict["tree_files_read"] == {
+ "invoicing.print_invoice": [
+ {"tree_path": "templates/invoice.html", "use": "template"},
+ {"tree_path": "templates/logo.png", "use": "template_asset"},
+ ]
+ }
+ assert verdict["tree_file_readers"] == {
+ "templates/invoice.html": ["invoicing.print_invoice"],
+ "templates/logo.png": ["invoicing.print_invoice"],
+ }
+ assert (
+ "- `invoicing.print_invoice` reads `templates/invoice.html` (template), `templates/logo.png` (template asset)"
+ in verdict["rendered_markdown"]
+ )
+
def test_two_bundles_of_a_method_each_read_the_file_beside_them(self, mocker: MockerFixture) -> None:
"""A step names its file from its bundle's own tree path, so two bundles shipping a template each under one relative path both run."""
_stub_the_document_engine(mocker)
diff --git a/docs/api-server/openapi/pipelex-api.openapi.yaml b/docs/api-server/openapi/pipelex-api.openapi.yaml
index 6e77cf4b7..a33d0e446 100644
--- a/docs/api-server/openapi/pipelex-api.openapi.yaml
+++ b/docs/api-server/openapi/pipelex-api.openapi.yaml
@@ -8647,6 +8647,32 @@ components:
``nb_tokens_by_category`` keys) are open sets on the wire, typed as plain strings —
runtime enum churn is non-breaking for clients.'
+ TreeFileRead:
+ properties:
+ tree_path:
+ type: string
+ title: Tree Path
+ description: The file's path from the root of the step's method tree, such as `templates/invoice.html`.
+ use:
+ $ref: '#/components/schemas/TreeFileUse'
+ description: 'What the step reads the file as: `template`, the template a PipeDocGen step fills; `template_asset`,
+ an image, a stylesheet or a font its HTML template names; `python_module`, the module of the function a PipeFunc
+ step calls.'
+ additionalProperties: false
+ type: object
+ required:
+ - tree_path
+ - use
+ title: TreeFileRead
+ description: 'A file of its method tree a step reads: its tree path, and what the step reads it as.'
+ TreeFileUse:
+ type: string
+ enum:
+ - template
+ - template_asset
+ - python_module
+ title: TreeFileUse
+ description: What a step reads a file of its method tree as.
UnexplainedPath:
properties:
path:
@@ -8866,6 +8892,26 @@ components:
$ref: '#/components/schemas/LiftablePipeEntry'
type: array
title: Liftable Pipes
+ tree_files_read:
+ additionalProperties:
+ items:
+ $ref: '#/components/schemas/TreeFileRead'
+ type: array
+ type: object
+ title: Tree Files Read
+ description: 'Per pipe, the files of the method''s tree the step reads, keyed by namespaced `pipe_ref` like `pipe_io_contracts`:
+ each file by its tree path, with what the step reads it as. A PipeDocGen step reads its template and the images,
+ stylesheets and fonts its HTML template names by literal relative URLs; a PipeFunc step reads the module of its
+ function. A pipe that reads no file of the tree, a controller included, has an empty list.'
+ tree_file_readers:
+ additionalProperties:
+ items:
+ type: string
+ type: array
+ type: object
+ title: Tree File Readers
+ description: 'The per-file view of `tree_files_read`: each tree path a step reads, in sorted order, to the namespaced
+ refs of the pipes that read it. A file of the method''s tree absent here is read by no step.'
graph_spec:
anyOf:
- $ref: '#/components/schemas/GraphSpec'
diff --git a/docs/api-server/pipe-validate.md b/docs/api-server/pipe-validate.md
index e336c3349..b5b1269fc 100644
--- a/docs/api-server/pipe-validate.md
+++ b/docs/api-server/pipe-validate.md
@@ -61,6 +61,10 @@ The 200 body is one of two arms, discriminated on the mandatory `is_valid` field
],
"pending_signatures": [],
"is_runnable": true,
+ "tree_files_read": {
+ "my_domain.my_pipe": []
+ },
+ "tree_file_readers": {},
"mthds_contents": ["..."],
"message": "MTHDS content validated successfully",
"default_pipe_ref": "my_domain.my_pipe"
@@ -77,6 +81,8 @@ The 200 body is one of two arms, discriminated on the mandatory `is_valid` field
- `warnings` (list): advisory findings that never flip `is_valid`, each a problem item with the same shape as an item of the invalid arm's `errors`, so one parser reads both. Four families, always in this order: the useless-`!` lint (`method.optional_force_redundant`), the entry-pipe vacuous-presence lint (`method.input_presence_vacuous`), the intent-hint lints (`method.hint_invalid`, whose `reason` is `unknown_key`, `unknown_intent` or `inapplicable_intent`, whose `hint` names the hint, and whose `location` names the declaration the hint is on: `field` `concept.<Concept>` for a concept, `concept.<Concept>.structure.<field>` for a field of its structure, `pipe` and `input` for a pipe's input), and the warnings a `PipeDocGen` step's template check found in its office template file during the dry run (`pipelex.docgen.template_warning`, whose `template` names the file and `finding` says what the check found, located at the step by its `pipe`). A member an item does not have is absent, never `null`. Empty when there is nothing to report
- `pending_signatures` (list[str]): namespaced refs of pipes still declared as signatures (contract-only pipes — `inputs`/`output` with no `type` and no implementation) in the assembled library — what remains to implement
- `is_runnable` (boolean): `pending_signatures` is empty — whether the validated library is complete enough to run
+- `tree_files_read` (object): per step, the files of the method's tree it reads, keyed by the same namespaced `pipe_ref` as `pipe_io_contracts`. Each entry is a list of `{tree_path, use}` items, `tree_path` being the file's path from the root of the method's tree with forward slashes, never a path on the server, and `use` one of `template` (a `PipeDocGen` step's `template_file`), `template_asset` (a file an HTML template names by a literal relative URL, such as an image, a stylesheet or a font, or a file a stylesheet it names refers to) and `python_module` (the module holding a `PipeFunc` step's function, `module.py` or a package's `__init__.py`). A step that reads no file, a controller or a step whose template is written inline with no asset, has an empty list, and so does a `PipeFunc` whose function is found on the runtime's import path rather than in the tree. See [Validating a method tree](#validating-a-method-tree)
+- `tree_file_readers` (object): the same facts read the other way, keyed by tree path in sorted order, each listing the `pipe_ref`s of the steps that read that file. A file of the tree that no step reads, the bundles themselves, a README or a package's `__init__.py` beside its modules, has no entry
**Response Fields (wire extras, valid arm only, this server only):**
@@ -173,6 +179,7 @@ A method sent in `files`, or fetched by `method_ref`, is validated whole, as it
- **The default pipe and the graph follow the tree's entry pipe.** `default_pipe_ref` is the pipe the protocol's rule names, the `main_pipe` of the root `METHODS.toml` or the one `main_pipe` its bundles declare, and the graph is drawn from it. A tree with no entry pipe validates with neither `default_pipe_ref` nor `graph_spec`.
- **Refusals of the tree itself produce no verdict**: a key the tree's rule refuses is a **422** `method.package_invalid` naming the rule as its `reason`, or `security.input_path_refused` for a key leaving the tree's root, a malformed entry a **422** `request.invalid`, a hash the runner does not hold a **422** `request.blobs_missing`, and a tree past its limits a **422** `method.package_invalid` naming the limit, as on a [run](pipe-run.md#sending-a-method-as-a-tree).
- **The `mthds_contents` echo** holds the text of the tree's own bundles.
+- **The report names the files each step reads.** `tree_files_read` lists, for each step, the files of the tree it reads, each by its tree path and its use, and `tree_file_readers` lists, for each such file, the steps that read it, so an editor can show which step uses which file and warn before a file a step reads is renamed or removed. The paths are derived from what each step already names, a `PipeDocGen` step's `template_file` and the literal relative URLs of its HTML template, and a `PipeFunc` step's `function_name` located in the tree, with no reference syntax of their own. A file sent in `resources` is named by its key, collapsed as the key is.
## Sending the files the steps read (`resources`)
diff --git a/docs/tools/cli/agent-cli.md b/docs/tools/cli/agent-cli.md
index f5dd40b36..493b6e367 100644
--- a/docs/tools/cli/agent-cli.md
+++ b/docs/tools/cli/agent-cli.md
@@ -96,6 +96,9 @@ For `bundle`, additional options are available:
On a successful run, the envelope also carries `pending_signatures` — the library-wide list of pipes still declared as `PipeSignature` (unimplemented forward declarations), each namespaced by `pipe_ref` (`domain.code`). In JSON it is a `pending_signatures` array, in markdown a "Pending signatures" section. A top-down build reads it to see exactly which headers remain to implement. The envelope also carries a derived `is_runnable` boolean (`true` ⇔ `pending_signatures` is empty), and the markdown states the runnability verdict in plain English — runnable when complete, NOT yet runnable above the "Pending signatures" section otherwise. `validate bundle`, `validate method`, and `validate pipe --all` carry them and gate on them: without `--allow-signatures`, the command exits non-zero when `is_runnable` is false. Bare `validate pipe <code>` omits them, and a `--pipe` slice surfaces them for information without gating.
+!!! note "The files each step reads"
+ `validate bundle` and `validate method` also carry `tree_files_read` and `tree_file_readers`, the members of the same name in the protocol's validation report (see [Pipe Validate](../../api-server/pipe-validate.md)). `tree_files_read` maps each validated pipe's `pipe_ref` to the files of the method's tree it reads, each a `{tree_path, use}` object whose `use` is `template`, `template_asset` or `python_module`, and an empty list for a pipe reading none; `tree_file_readers` maps each of those files, by tree path, to the pipes that read it. A tree path is the file's path from the root of the method's tree, never a path on your disk. In markdown they render as a "Files the steps read" section listing each step that reads a file. `validate pipe`, which may span several trees, carries neither.
+
!!! note "Every refusal of the bundle is an invalid verdict"
A refusal of your input raised while the bundle is loaded or validated is answered with the invalid verdict — the problem `method.invalid` with `is_valid: false`, its faults in `errors` — and exit code `1`, never a no-verdict failure and exit code `2`, which is kept for failures of the tool or its environment. Each item carries its own `code`, the sentence that code renders as its `detail`, its `params`, its `location` (its `pipe`, its `field`, its `file`, and for a TOML syntax error its 1-based `line` and `column`) and, when there is one, its `suggested_fix`. A pipe whose dry run fails gives one item per failing pipe, coded by its failure and located at the innermost pipe that failed and is not allowed to fail (its file whenever the pipe is in your files; a validator of submitted content beside a host's library directories names only that content's files), with the sentence the failure's code renders for a caller, never the failure's own text, on `validate bundle`, `validate pipe <code>` and `validate pipe --all` alike; a controller that failed because a pipe it runs failed is reported once, at that pipe. Every error found while parsing is its own item, so a misspelled field is reported beside the categorized errors as an item of its own, located at its pipe and its file. An undeclared concept gives a `method.concept_unresolved` item whose `params.concept` names it. A pipe code or an input name that breaks the naming rules gives a `method.identifier_invalid` item whose `params.name` names it, and an item whose check knows facts its location does not hold, such as the concept a pipe declares and those that would fit, or the name two bundles both declare, gives them in its `params`, by the names the error registry gives them. A pipe naming a model the model deck does not define gives one `model.unknown` item located at the pipe's `model` field, its `params` naming the reference as written and its `suggestions` the deck's close matches, plus a `rename-model` `suggested_fix` when there is exactly one suggestion. That fix's `safety` is `unsafe`, because a close name can still be a different model: `fix bundle` never applies it, and the Markdown labels it `unsafe, confirm before applying` beside its fix code, where a safe fix is labelled `safe`. Any other refusal of your input with no code of its own gives one item with the refusal's code, located on its pipe when it was raised while that pipe was built. A pipe factory's refusal, such as a `PipeExtract` whose input is neither an image nor a document, is one of these items, located on its pipe. In Markdown the items are listed one per line under `### Errors`, each with its code, its sentence, its location, its suggestions and its suggested fix. See [Error Model](../../under-the-hood/error-model.md).
diff --git a/pipelex/cli/agent_cli/CLAUDE.md b/pipelex/cli/agent_cli/CLAUDE.md
index a3f1fb7b3..4cb19a993 100644
--- a/pipelex/cli/agent_cli/CLAUDE.md
+++ b/pipelex/cli/agent_cli/CLAUDE.md
@@ -18,7 +18,7 @@ Only the error format is backed by module-level `ContextVar`s in `agent_output.p
Markdown structure per command:
- `run`: `# Pipeline run complete`, a `## Result` section (the rendered `main_stuff` markdown with `--with-memory`, otherwise the concept JSON in a fenced block), and output / graph file paths. With the graph on, the `graph_files` object of the `--with-memory` envelope names the viewer (`graph_html`), the graph on a live run (`graph_spec`, the graphspec renamed `live_run_graph.json`) and the graph's three companions written beside it under their canonical names (`pipe_io_contracts`, `input_form`, `output_form` — the validate artifacts describing the graph's data, keyed by `pipe_ref`); the agent CLI settles its graph inclusion (graphspec on, viewer on) before the run rather than at render time, since the companions are built inside the run's library window under that same flag; a viewer resolves them from the graph file's directory, so the rename never moves them. Cost is machine-first: on the local runner, when costs are on (`--costs`, the default) and the run did reportable work (any tokens or any cost), a best-effort `cost_report` object (`{total_cost, by_model: [...]}`, real USD) rides the JSON `--with-memory` envelope — there is **no** Rich cost table on stderr (unlike the human `pipelex run` CLI). The markdown renderer ignores the `cost_report` key, so it is JSON-only. A free/zero-price model still reports its token usage (with `total_cost` 0). The `cost_report` is **absent** when the run did no reportable work (dry runs), when `--no-costs` is passed, on a hosted run (which assembles no local usage), or if cost-summary aggregation fails (caught and skipped so it never fails an otherwise-successful run) — so consumers must treat it as optional. A local `run bundle` also carries `method_tree_digest`, the digest of the method tree it read, in its output file and its `--with-memory` envelope.
-- `validate`: `# Validation passed`, the bundle path when relevant (a method's folder, for a folder judged whole), and a list of validated pipes with their status. `validate bundle <folder> --graph` / `--view` on a folder that decides no pipe to start at adds `graph_unavailable`, a sentence saying why no graph was drawn, rendered as a `## Graph` section. Every entry in `validated_pipes` (markdown and JSON) identifies its pipe by the namespaced `pipe_ref` (`domain.code`) — the same unambiguous identity across `validate all`, `validate bundle`, and `validate pipe`, so the same pipe is never reported under two identifiers. `validate bundle`, `validate method`, and `validate all` carry `pending_signatures` — the library-wide list of pipes still declared as `PipeSignature` (unimplemented forward declarations), namespaced by `pipe_ref` — rendered as a "Pending signatures" section in markdown. They also carry a derived `is_runnable` boolean (`true` ⇔ `pending_signatures` is empty), and the markdown adds an explicit plain-English runnability verdict on those surfaces: a complete bundle/library states it is runnable, while one with pending signatures states it is NOT yet runnable immediately above the verbatim "Pending signatures" heading. These whole-bundle/whole-library surfaces are **strict by default**: they exit non-zero on `not is_runnable` unless `--allow-signatures` is passed (which both tolerates the placeholders for the exit code and mock-runs the signature pipes in the sweep). Single-pipe surfaces make no library-wide runnability claim and never gate the exit code on it: bare `validate pipe <code>` omits the key and emits no verdict, while `validate bundle`/`validate method` with `--pipe` still surface the library-wide `pending_signatures` for information but do not gate on it (the requested slice can be fully implemented even when unrelated placeholders remain elsewhere). The whole-bundle and whole-library surfaces also carry a `warnings` array — the advisory lints, then what the sweep's dry runs found without failing (a `PipeDocGen` template check's `template_check_warning`s), rendered as a "Warnings" section in markdown — assembled by `pipelex/pipeline/advisory_warnings.py`, the ONE composition point every advisory-bearing channel calls (this CLI, the bare CLI and the protocol report). Bare `validate pipe <code>` carries the dry-run findings alone (`pipelex/pipeline/dry_run_warnings.py`), since one pipe has no flow to lint but what its dry run found is about it; `validate bundle` / `validate method` with `--pipe` carry every family, because they validate the whole bundle and only narrow the dry run, so the array stays bundle-wide there. Add a new advisory family there, never at a call site: the sites used to assemble their own and disagreed about which families they carried.
+- `validate`: `# Validation passed`, the bundle path when relevant (a method's folder, for a folder judged whole), and a list of validated pipes with their status. `validate bundle <folder> --graph` / `--view` on a folder that decides no pipe to start at adds `graph_unavailable`, a sentence saying why no graph was drawn, rendered as a `## Graph` section. Every entry in `validated_pipes` (markdown and JSON) identifies its pipe by the namespaced `pipe_ref` (`domain.code`) — the same unambiguous identity across `validate all`, `validate bundle`, and `validate pipe`, so the same pipe is never reported under two identifiers. `validate bundle`, `validate method`, and `validate all` carry `pending_signatures` — the library-wide list of pipes still declared as `PipeSignature` (unimplemented forward declarations), namespaced by `pipe_ref` — rendered as a "Pending signatures" section in markdown. They also carry a derived `is_runnable` boolean (`true` ⇔ `pending_signatures` is empty), and the markdown adds an explicit plain-English runnability verdict on those surfaces: a complete bundle/library states it is runnable, while one with pending signatures states it is NOT yet runnable immediately above the verbatim "Pending signatures" heading. These whole-bundle/whole-library surfaces are **strict by default**: they exit non-zero on `not is_runnable` unless `--allow-signatures` is passed (which both tolerates the placeholders for the exit code and mock-runs the signature pipes in the sweep). Single-pipe surfaces make no library-wide runnability claim and never gate the exit code on it: bare `validate pipe <code>` omits the key and emits no verdict, while `validate bundle`/`validate method` with `--pipe` still surface the library-wide `pending_signatures` for information but do not gate on it (the requested slice can be fully implemented even when unrelated placeholders remain elsewhere). The whole-bundle and whole-library surfaces also carry a `warnings` array — the advisory lints, then what the sweep's dry runs found without failing (a `PipeDocGen` template check's `template_check_warning`s), rendered as a "Warnings" section in markdown — assembled by `pipelex/pipeline/advisory_warnings.py`, the ONE composition point every advisory-bearing channel calls (this CLI, the bare CLI and the protocol report). Bare `validate pipe <code>` carries the dry-run findings alone (`pipelex/pipeline/dry_run_warnings.py`), since one pipe has no flow to lint but what its dry run found is about it; `validate bundle` / `validate method` with `--pipe` carry every family, because they validate the whole bundle and only narrow the dry run, so the array stays bundle-wide there. Add a new advisory family there, never at a call site: the sites used to assemble their own and disagreed about which families they carried. `validate bundle` and `validate method` also carry `tree_files_read` and `tree_file_readers`, the protocol report's members of the same names, splatted from `tree_files_envelope_fields` in `pipelex/pipeline/tree_files_read.py` so the envelope and the report share their shapes, and rendered as a "Files the steps read" section in markdown; `validate all` and `validate pipe`, which may span several trees, carry neither.
- `fix`: `# Fix applied - bundle is valid` after one or more fixes, `# Bundle already valid` when the loop had nothing to apply, or `# Bundle valid but not runnable` when pending `PipeSignature` placeholders remain. The JSON success envelope carries `is_valid`, `bundle_path`, `iterations`, `fixes_applied`, `files_written`, `remaining_errors`, and, on successful validation, `pending_signatures` plus `is_runnable`. Whole-bundle fix runs are strict by default: they emit the success envelope but exit 1 on `not is_runnable` unless `--allow-signatures` is passed. A still-invalid verdict exits 1 with the problem `method.invalid` on stderr (`agent_error_fix_bundle`), its `errors` the faults left, beside the result's fields (`is_valid`, `bundle_path`, `iterations`, `fixes_applied`, `files_written`, any `bail_reason`); its Markdown ends with the fix progress (`format_fix_progress_markdown`). `fix function-names <method folder>` is the migration of a method's flat PipeFunc `function_name`s into import paths from the method tree's root (`pipelex/migration/function_names.py`): `# Function names of the method at <folder>` then the rewrites, the `@pipe_func(...)` registration names dropped, the names left unresolved with their reason and candidates, and the markers whose argument it kept (one registering a name a step still holds unresolved, or one whose name is not a string literal); the JSON envelope carries `directory`, `is_dry_run`, `is_complete`, `changed_files`, `rewrites`, `marker_removals`, `unresolved` and `unresolved_markers`. It exits 1 after the envelope when a name or a marker is left unresolved, and 2 via `FunctionNamesMigrationError` when the folder is not a method tree.
- `migrate`: `# Configuration migration`, the mode (applied / dry run) and verdict, the directories walked, a files-walked/changed/written count and, when a former release left something, its file count, then one `## <path>` section per file the former-release cleanup touches, listing its changes and its backup, a `## Pipelex still cannot start` section when the cleanup's check, run on every run, found something, and one per file that is not clean listing the entries applied, the entries blocked with their reason and guidance, and any unexplained paths. Paths, operation kinds, ledger-supplied values and the names of retired backends and profiles only — no value read from the user's file is ever rendered, in either format.
- `init`: `# Pipelex initialized` with target directory and execution, then enabled backends and routing profile for a local setup, or whether a Pipelex API key is set for a hosted one.
diff --git a/pipelex/cli/agent_cli/commands/validate/_validate_core.py b/pipelex/cli/agent_cli/commands/validate/_validate_core.py
index b15fb5ee5..b30afc76d 100644
--- a/pipelex/cli/agent_cli/commands/validate/_validate_core.py
+++ b/pipelex/cli/agent_cli/commands/validate/_validate_core.py
@@ -19,6 +19,7 @@ from pipelex.pipeline.blueprint_selection import collect_entry_pipe_refs
from pipelex.pipeline.bundle_validator import BundleValidator
from pipelex.pipeline.dry_run_warnings import build_dry_run_warnings
from pipelex.pipeline.execution_seams import acquire_library
+from pipelex.pipeline.tree_files_read import tree_files_envelope_fields
from pipelex.pipeline.usage_stats_reporting import MethodValidationReport
from pipelex.pipeline.validate_bundle import build_pending_signatures, build_validated_pipes, validate_bundle
from pipelex.pipeline.validate_bundle_translation import translate_to_validate_bundle_error
@@ -151,6 +152,8 @@ async def validate_bundle_core(
"pending_signatures": result.pending_signatures,
"is_runnable": not result.pending_signatures,
"warnings": [warning.model_dump(exclude_none=True) for warning in warnings],
+ # The files of the method's tree each loaded pipe reads, and who reads each, as the canonical report states them.
+ **tree_files_envelope_fields(pipes=result.pipes),
}
@@ -206,6 +209,8 @@ async def validate_method_folder_core(
"pending_signatures": result.pending_signatures,
"is_runnable": not result.pending_signatures,
"warnings": [warning.model_dump(exclude_none=True) for warning in warnings],
+ # The files of the method's tree each loaded pipe reads, and who reads each, as the canonical report states them.
+ **tree_files_envelope_fields(pipes=result.pipes),
}
@@ -341,4 +346,6 @@ async def validate_pipe_in_bundle_core(
"pending_signatures": result.pending_signatures,
"is_runnable": not result.pending_signatures,
"warnings": [warning.model_dump(exclude_none=True) for warning in warnings],
+ # The files of the method's tree each loaded pipe reads, and who reads each, as the canonical report states them.
+ **tree_files_envelope_fields(pipes=result.pipes),
}
diff --git a/pipelex/core/method_trees/tree_file_reads.py b/pipelex/core/method_trees/tree_file_reads.py
new file mode 100644
index 000000000..68c177f05
--- /dev/null
+++ b/pipelex/core/method_trees/tree_file_reads.py
@@ -0,0 +1,52 @@
+"""A file of its method tree that a step reads, named by its tree path, with what the step reads it as.
+
+A step names the files of its tree it reads in the fields the MTHDS standard gives it (`mthds/docs/spec/method-tree.md`):
+a `PipeDocGen` step's `template_file`, by a file reference, and the relative URLs of an HTML template, by URL
+references; and a `PipeFunc` step's `function_name`, by an import path whose module is a file of the tree. Each pipe
+resolves those names to tree paths as it is built and validated, so a validation can say which step reads which file,
+and which files of the tree no step reads (`wip/extra-files/design.md`, "Validation explains the method's files").
+"""
+
+from enum import StrEnum
+
+from pydantic import BaseModel, ConfigDict, Field
+
+
+class TreeFileUse(StrEnum):
+ """What a step reads a file of its method tree as."""
+
+ TEMPLATE = "template"
+ """The template a `PipeDocGen` step fills, named by its `template_file`."""
+
+ TEMPLATE_ASSET = "template_asset"
+ """An image, a stylesheet or a font that an HTML template names by a literal relative URL, or that one of its
+ stylesheets names, which the step's print reads."""
+
+ PYTHON_MODULE = "python_module"
+ """The module defining the function a `PipeFunc` step calls, named by the import path of its `function_name`."""
+
+ @property
+ def label(self) -> str:
+ """How a sentence names the use: "template", "template asset", "Python module"."""
+ match self:
+ case TreeFileUse.TEMPLATE:
+ return "template"
+ case TreeFileUse.TEMPLATE_ASSET:
+ return "template asset"
+ case TreeFileUse.PYTHON_MODULE:
+ return "Python module"
+
+
+class TreeFileRead(BaseModel):
+ """A file of its method tree a step reads: its tree path, and what the step reads it as."""
+
+ model_config = ConfigDict(frozen=True, extra="forbid")
+
+ tree_path: str = Field(description="The file's path from the root of the step's method tree, such as `templates/invoice.html`.")
+ use: TreeFileUse = Field(
+ strict=False,
+ description=(
+ "What the step reads the file as: `template`, the template a PipeDocGen step fills; `template_asset`, an image, "
+ "a stylesheet or a font its HTML template names; `python_module`, the module of the function a PipeFunc step calls."
+ ),
+ )
diff --git a/pipelex/pipe_machinery/pipe_abstract.py b/pipelex/pipe_machinery/pipe_abstract.py
index 129a95baf..1cb8db622 100644
--- a/pipelex/pipe_machinery/pipe_abstract.py
+++ b/pipelex/pipe_machinery/pipe_abstract.py
@@ -15,6 +15,7 @@ from pipelex.core.concepts.concept import Concept
from pipelex.core.concepts.native.concept_native import NativeConceptCode
from pipelex.core.memory.absence import AbsenceKind, AbsenceRecord
from pipelex.core.memory.working_memory import MAIN_STUFF_NAME, WorkingMemory
+from pipelex.core.method_trees.tree_file_reads import TreeFileRead
from pipelex.core.pipes.exceptions import PipeValidationError
from pipelex.core.pipes.inputs.exceptions import OptionalValueAbsentError, PipeRunInputsError
from pipelex.core.pipes.inputs.input_stuff_specs import InputStuffSpecs, NamedStuffSpec
@@ -201,6 +202,14 @@ class PipeAbstract(ABC, BaseModel):
"""
return set()
+ def tree_files_read(self) -> list[TreeFileRead]:
+ """The files of its method tree this pipe reads itself, each by its tree path, as its building and validation resolved them.
+
+ None by default: a controller reads no file, its steps do. A `PipeDocGen` reads its template and the files its
+ HTML template names, and a `PipeFunc` the module of its function, when it lies in the step's tree.
+ """
+ return []
+
@property
def concept_dependencies(self) -> list[Concept]:
"""Return all unique concept dependencies (output + inputs) without duplicates."""
diff --git a/pipelex/pipe_operators/doc_gen/pipe_doc_gen.py b/pipelex/pipe_operators/doc_gen/pipe_doc_gen.py
index cf13555cc..2fc53eefd 100644
--- a/pipelex/pipe_operators/doc_gen/pipe_doc_gen.py
+++ b/pipelex/pipe_operators/doc_gen/pipe_doc_gen.py
@@ -17,6 +17,7 @@ from pipelex.cogt.models.model_deck_check import check_doc_gen_choice_with_deck
from pipelex.cogt.templating.template_preprocessor import rewrite_template_sigils
from pipelex.core.memory.working_memory import WorkingMemory
from pipelex.core.method_trees.tree_bundle import MethodFileRef, TreeBundleRef
+from pipelex.core.method_trees.tree_file_reads import TreeFileRead, TreeFileUse
from pipelex.core.pipes.inputs.input_stuff_specs import InputStuffSpecs
from pipelex.core.pipes.pipe_output import PipeOutput
from pipelex.core.stuffs.document_content import DocumentContent
@@ -92,11 +93,32 @@ class PipeDocGen(PipeOperator[PipeDocGenOutput]):
# step built from a crate by digest alone, which reads no file, and which a validation refuses rather than pass
# unchecked (`validate_inputs_with_library`).
_template_file_text: str | None = PrivateAttr(default=None)
+ # The images, stylesheets and fonts of its method tree the step's HTML template names by literal relative URLs, by
+ # tree path, as the load resolved them when it checked the template in a tree this process holds. Empty for a step
+ # built from a crate by digest alone, whose URLs resolve at print, and for a template naming no file.
+ _template_asset_tree_paths: list[str] = PrivateAttr(default_factory=list)
def hold_template_file_text(self, *, template_file_text: str) -> None:
"""Keep the text of the step's HTML template file, read at load, for the load's checks."""
self._template_file_text = template_file_text
+ def hold_template_asset_tree_paths(self, *, asset_tree_paths: list[str]) -> None:
+ """Keep the files of the tree the step's HTML template names by literal relative URLs, as the load resolved them."""
+ self._template_asset_tree_paths = list(asset_tree_paths)
+
+ @override
+ def tree_files_read(self) -> list[TreeFileRead]:
+ """The step's template file, then the images, stylesheets and fonts its HTML template names by literal relative URLs.
+
+ A URL the template computes from the run's inputs names a file only once the step prints, so it is not here.
+ """
+ files_read: list[TreeFileRead] = []
+ if self.template_file_ref is not None:
+ files_read.append(TreeFileRead(tree_path=self.template_file_ref.tree_path, use=TreeFileUse.TEMPLATE))
+ for asset_tree_path in self._template_asset_tree_paths:
+ files_read.append(TreeFileRead(tree_path=asset_tree_path, use=TreeFileUse.TEMPLATE_ASSET))
+ return files_read
+
@property
def template_file_text(self) -> str | None:
"""The text of the step's HTML template file as the load read and checked it, or None when the load read none."""
diff --git a/pipelex/pipe_operators/doc_gen/pipe_doc_gen_factory.py b/pipelex/pipe_operators/doc_gen/pipe_doc_gen_factory.py
index 2755b8f65..9fe4de5f2 100644
--- a/pipelex/pipe_operators/doc_gen/pipe_doc_gen_factory.py
+++ b/pipelex/pipe_operators/doc_gen/pipe_doc_gen_factory.py
@@ -1,4 +1,4 @@
-from typing import Any
+from typing import Any, NamedTuple
from mthds.method_tree.exceptions import MethodTreeRefusedError
from mthds.method_tree.references import is_file_reference
@@ -18,10 +18,22 @@ from pipelex.pipe_machinery.pipe_factory import PipeFactoryProtocol
from pipelex.pipe_operators.doc_gen.exceptions import PipeDocGenFactoryError
from pipelex.pipe_operators.doc_gen.pipe_doc_gen import PipeDocGen
from pipelex.pipe_operators.doc_gen.pipe_doc_gen_blueprint import PipeDocGenBlueprint, template_required_roots
-from pipelex.pipe_operators.doc_gen.template_asset_check import find_template_asset_refusal, literal_template_urls
+from pipelex.pipe_operators.doc_gen.template_asset_check import check_template_assets, literal_template_urls
from pipelex.tools.jinja2.template_category import TemplateCategory
+class _InlineTemplateTree(NamedTuple):
+ """Where an inline HTML template sits in its method tree, and the files of the tree its literal relative URLs name.
+
+ `bundle_ref` is None for bundles parsed from content strings with no file submitted beside them, and
+ `asset_tree_paths` is empty when this process does not hold the tree, the URLs then resolving at print.
+ """
+
+ bundle_ref: TreeBundleRef | None
+ is_one_file: bool
+ asset_tree_paths: list[str]
+
+
class PipeDocGenFactory(PipeFactoryProtocol[PipeDocGenBlueprint, PipeDocGen]):
@classmethod
@override
@@ -50,8 +62,14 @@ class PipeDocGenFactory(PipeFactoryProtocol[PipeDocGenBlueprint, PipeDocGen]):
template_bundle_ref: TreeBundleRef | None = None
template_tree_is_one_file = False
template_file_text: str | None = None
+ # The images, stylesheets and fonts of its tree the step's HTML template names by literal relative URLs, resolved
+ # when the template is checked here, which its print reads.
+ template_asset_tree_paths: list[str] = []
if blueprint.template is not None:
- template_bundle_ref, template_tree_is_one_file = cls._inline_template_tree(pipe_code=pipe_code, template=blueprint.template)
+ inline_template_tree = cls._inline_template_tree(pipe_code=pipe_code, template=blueprint.template)
+ template_bundle_ref = inline_template_tree.bundle_ref
+ template_tree_is_one_file = inline_template_tree.is_one_file
+ template_asset_tree_paths = inline_template_tree.asset_tree_paths
if blueprint.template_file is not None:
template_file = blueprint.template_file
match get_building_tree_bundle():
@@ -70,7 +88,7 @@ class PipeDocGenFactory(PipeFactoryProtocol[PipeDocGenBlueprint, PipeDocGen]):
data=cls._read_tree_file(tree_bundle=tree_bundle, tree_path=resolved_tree_path),
)
cls._check_html_template_file(pipe_code=pipe_code, template=template_file_text, inputs=inputs, template_file=template_file)
- cls._check_template_assets(
+ template_asset_tree_paths = cls._check_template_assets(
pipe_code=pipe_code,
template=template_file_text,
template_label=f"the template file '{resolved_tree_path}'",
@@ -112,32 +130,36 @@ class PipeDocGenFactory(PipeFactoryProtocol[PipeDocGenBlueprint, PipeDocGen]):
)
if template_file_text is not None:
pipe_doc_gen.hold_template_file_text(template_file_text=template_file_text)
+ pipe_doc_gen.hold_template_asset_tree_paths(asset_tree_paths=template_asset_tree_paths)
return pipe_doc_gen
@classmethod
- def _inline_template_tree(cls, *, pipe_code: str, template: str) -> tuple[TreeBundleRef | None, bool]:
+ def _inline_template_tree(cls, *, pipe_code: str, template: str) -> _InlineTemplateTree:
"""The bundle of an inline template, as its method tree names it, which its relative URLs resolve against.
Returns the bundle's reference, None for bundles parsed from content strings with no file submitted beside them,
- and whether its tree is the one-file tree of a bundle named alone. The URLs the template writes as they are are
- checked now in a tree this process holds, and refused in none, where a relative URL names no file; the rest are
- the rendered HTML's, resolved in the tree when the print's engine asks for each.
+ whether its tree is the one-file tree of a bundle named alone, and the files of the tree its literal relative URLs
+ name. The URLs the template writes as they are are checked now in a tree this process holds, and refused in none,
+ where a relative URL names no file; the rest are the rendered HTML's, resolved in the tree when the print's engine
+ asks for each.
Raises:
PipeDocGenFactoryError: A literal relative URL of the template names no image, stylesheet or font of its tree.
"""
match get_building_tree_bundle():
case TreeBundle() as tree_bundle:
- cls._check_template_assets(
+ asset_tree_paths = cls._check_template_assets(
pipe_code=pipe_code,
template=template,
template_label=f"the inline template of PipeDocGen '{pipe_code}'",
holder=tree_bundle.tree_path,
tree_bundle=tree_bundle,
)
- return tree_bundle.make_ref(), tree_bundle.tree.is_one_file
+ return _InlineTemplateTree(
+ bundle_ref=tree_bundle.make_ref(), is_one_file=tree_bundle.tree.is_one_file, asset_tree_paths=asset_tree_paths
+ )
case TreeBundleRef() as tree_bundle_ref:
- return tree_bundle_ref, False
+ return _InlineTemplateTree(bundle_ref=tree_bundle_ref, is_one_file=False, asset_tree_paths=[])
case None:
for url in literal_template_urls(template=template):
if is_file_reference(url, context=ReferenceContext.URL):
@@ -149,24 +171,28 @@ class PipeDocGenFactory(PipeFactoryProtocol[PipeDocGenBlueprint, PipeDocGen]):
raise PipeDocGenFactoryError(msg, facts=_file_facts(pipe_code=pipe_code, file=url)).with_error_code(
CoreCode.METHOD_RESOURCE_MISSING
)
- return None, False
+ return _InlineTemplateTree(bundle_ref=None, is_one_file=False, asset_tree_paths=[])
@classmethod
- def _check_template_assets(cls, *, pipe_code: str, template: str, template_label: str, holder: str | None, tree_bundle: TreeBundle) -> None:
+ def _check_template_assets(cls, *, pipe_code: str, template: str, template_label: str, holder: str | None, tree_bundle: TreeBundle) -> list[str]:
"""Check that each literal relative URL of an HTML template names an image, a stylesheet or a font of its tree.
+ Returns:
+ The images, stylesheets and fonts of the tree the URLs name, by tree path.
+
Raises:
PipeDocGenFactoryError: One names no such file; the message names the URL, where it is written and why.
"""
- refusal = find_template_asset_refusal(
+ asset_check = check_template_assets(
template=template,
template_label=template_label,
holder=holder,
tree=tree_bundle.tree,
read_file=lambda tree_path: cls._read_tree_file(tree_bundle=tree_bundle, tree_path=tree_path),
)
+ refusal = asset_check.refusal
if refusal is None:
- return
+ return asset_check.asset_tree_paths
msg = f"PipeDocGen '{pipe_code}' names '{refusal.reference}' in {refusal.holder_label}, which its method cannot give it. {refusal.reason}"
facts = _file_facts(pipe_code=pipe_code, file=refusal.reference)
raise PipeDocGenFactoryError(msg, facts=facts).with_error_code(CoreCode.METHOD_RESOURCE_MISSING)
diff --git a/pipelex/pipe_operators/doc_gen/template_asset_check.py b/pipelex/pipe_operators/doc_gen/template_asset_check.py
index 220d0b8aa..906e1a179 100644
--- a/pipelex/pipe_operators/doc_gen/template_asset_check.py
+++ b/pipelex/pipe_operators/doc_gen/template_asset_check.py
@@ -10,6 +10,9 @@ Only literal URLs are checked. A value holding a template expression or a sigil,
`href="$link"`, is the run's, and is checked when the document prints. What is read is what a printer reads from HTML:
an image's `src`, an embedded object, a linked stylesheet or attachment, an SVG image, and every `url()` and `@import`
of a stylesheet, an inline one or a `style` attribute included. A Jinja comment is skipped, and so is a CSS comment.
+
+The check also says which files of the tree those literal URLs name, so the step can tell a validation the files its
+print reads beside its template (`pipelex.core.method_trees.tree_file_reads`).
"""
import re
@@ -57,6 +60,17 @@ class TemplateAssetRefusal(NamedTuple):
reason: str
+class TemplateAssetCheck(NamedTuple):
+ """What checking an HTML template's literal relative URLs found: the files of the tree they name, and the first refusal.
+
+ `asset_tree_paths` holds each image, stylesheet and font the template or one of its stylesheets names, by tree path,
+ once, in the order the check reached it; when the check refuses a URL, it holds those reached before it.
+ """
+
+ asset_tree_paths: list[str]
+ refusal: TemplateAssetRefusal | None
+
+
class _HtmlUrls(HTMLParser):
"""The URLs an HTML template names that a printer reads, as written."""
@@ -109,10 +123,10 @@ def literal_template_urls(*, template: str) -> list[str]:
return [url for url in parser.urls if not any(marker in url for marker in _TEMPLATE_VALUE_MARKERS)]
-def find_template_asset_refusal(
+def check_template_assets(
*, template: str, template_label: str, holder: str | None, tree: MethodTree, read_file: Callable[[str], bytes | None]
-) -> TemplateAssetRefusal | None:
- """The first literal relative URL of an HTML template that names no image, stylesheet or font of its method tree.
+) -> TemplateAssetCheck:
+ """Resolve each literal relative URL of an HTML template in its method tree, and find the first that names no asset.
Args:
template: The template's text, as written.
@@ -123,8 +137,10 @@ def find_template_asset_refusal(
read_file: Reads a file of the tree by its tree path, for a stylesheet's own URLs; None when it cannot.
Returns:
- The refusal, or None when every literal relative URL names an image, a stylesheet or a font of the tree.
+ The images, stylesheets and fonts the URLs name, by tree path, and the refusal of the first URL that names no
+ image, stylesheet or font of the tree, None when every one does.
"""
+ asset_tree_paths: list[str] = []
checked_stylesheets: set[str] = set()
pending: list[tuple[str, str | None, str]] = [(url, holder, template_label) for url in literal_template_urls(template=template)]
while pending:
@@ -132,12 +148,16 @@ def find_template_asset_refusal(
try:
tree_path = _resolve(reference=reference, holder=url_holder, tree=tree)
except MethodTreeRefusedError as exc:
- return TemplateAssetRefusal(reference=reference, holder_label=holder_label, reason=exc.message)
+ refusal = TemplateAssetRefusal(reference=reference, holder_label=holder_label, reason=exc.message)
+ return TemplateAssetCheck(asset_tree_paths=asset_tree_paths, refusal=refusal)
if tree_path is None:
continue
asset_refusal = template_asset_refusal(tree_path=tree_path)
if asset_refusal is not None:
- return TemplateAssetRefusal(reference=reference, holder_label=holder_label, reason=asset_refusal)
+ refusal = TemplateAssetRefusal(reference=reference, holder_label=holder_label, reason=asset_refusal)
+ return TemplateAssetCheck(asset_tree_paths=asset_tree_paths, refusal=refusal)
+ if tree_path not in asset_tree_paths:
+ asset_tree_paths.append(tree_path)
if not tree_path.lower().endswith(_STYLESHEET_SUFFIX) or tree_path in checked_stylesheets:
continue
checked_stylesheets.add(tree_path)
@@ -145,7 +165,7 @@ def find_template_asset_refusal(
if stylesheet is not None:
stylesheet_label = f"the stylesheet '{tree_path}'"
pending.extend((url, tree_path, stylesheet_label) for url in stylesheet_urls(stylesheet=stylesheet))
- return None
+ return TemplateAssetCheck(asset_tree_paths=asset_tree_paths, refusal=None)
def _resolve(*, reference: str, holder: str | None, tree: MethodTree) -> str | None:
diff --git a/pipelex/pipe_operators/func/pipe_func.py b/pipelex/pipe_operators/func/pipe_func.py
index f9652ad5a..094cab1c5 100644
--- a/pipelex/pipe_operators/func/pipe_func.py
+++ b/pipelex/pipe_operators/func/pipe_func.py
@@ -19,6 +19,7 @@ from pipelex.core.method_trees.building_bundle import BuildingBundle
from pipelex.core.method_trees.exceptions import TreePythonError
from pipelex.core.method_trees.method_file_store import LOCAL_METHOD_FILES
from pipelex.core.method_trees.tree_bundle import TreeBundle, TreeBundleRef
+from pipelex.core.method_trees.tree_file_reads import TreeFileRead, TreeFileUse
from pipelex.core.method_trees.tree_python import check_tree_module_names, top_level_module_names, tree_label_holding
from pipelex.core.pipes.exceptions import PipeRunError, PipeValidationError
from pipelex.core.pipes.inputs.input_stuff_specs import InputStuffSpecs, TypedNamedStuffSpec
@@ -94,6 +95,10 @@ class PipeFunc(PipeOperator[PipeFuncOutput]):
# Whether validation located the function and proved its form, which a dependency package's step, validated as it is
# built and again with its consumer's library, then need not do twice.
_is_function_proven: bool = PrivateAttr(default=False)
+ # The tree path of the function's module, once validation located it among the files of the step's method tree. None
+ # until then, and when the module lies on this process's import path rather than in a tree, or the step was built
+ # from a crate that names its tree by digest alone.
+ _module_tree_path: str | None = PrivateAttr(default=None)
@override
def required_variables(self) -> set[str]:
@@ -140,7 +145,7 @@ class PipeFunc(PipeOperator[PipeFuncOutput]):
if self._is_function_proven:
return
try:
- self._check_function_statically()
+ self._module_tree_path = self._check_function_statically()
except (PipeFuncFunctionError, TreePythonError) as exc:
msg = f"PipeFunc '{self.code}' cannot call its function '{self.function_name}': {exc.message}"
raise PipeValidationError(
@@ -157,7 +162,7 @@ class PipeFunc(PipeOperator[PipeFuncOutput]):
"""Locate the function and prove its form: a step whose function cannot be found is one that cannot be built."""
self.validate_output_with_library()
- def _check_function_statically(self) -> None:
+ def _check_function_statically(self) -> str | None:
"""Locate the module where the step's import root puts it, prove the function's form, and check the tree's module names.
A sandbox-hosted deployment imports the function in a sandbox, whose packages are not this process's, so there
@@ -165,6 +170,9 @@ class PipeFunc(PipeOperator[PipeFuncOutput]):
digest alone is not checked again: its tree holds no file here, and the host that assembled the crate proved the
function before the run was dispatched.
+ Returns:
+ The module's tree path when it was located among the files of the step's method tree, else None.
+
Raises:
PipeFuncFunctionError: The module is not found, or the form is not proven.
TreeModuleShadowingError: A top-level module of the tree, or of the submitted Python, would shadow another.
@@ -173,7 +181,7 @@ class PipeFunc(PipeOperator[PipeFuncOutput]):
module_source: PipeFuncModuleSource
match self._tree_bundle:
case TreeBundleRef():
- return
+ return None
case TreeBundle(tree=MethodTree(directory=Path() as tree_directory)) as tree_bundle:
tree = tree_bundle.tree
try:
@@ -189,6 +197,8 @@ class PipeFunc(PipeOperator[PipeFuncOutput]):
root=tree_directory.resolve(),
checks_this_process=imports_in_this_process,
)
+ # Located in the tree, the module is labelled by its tree path.
+ return module_source.module_label
case TreeBundle() as tree_bundle if not imports_in_this_process:
# The files submitted with bundles parsed from content strings, which this process holds in memory: a
# sandbox imports the function from them.
@@ -201,15 +211,30 @@ class PipeFunc(PipeOperator[PipeFuncOutput]):
root=None,
checks_this_process=False,
)
+ # Located among the submitted files, the module is labelled by its tree path in the tree they make.
+ return module_source.module_label
case None if not imports_in_this_process:
# No file was submitted with the bundle, so a sandbox would have no module to import.
module_source = locate_in_files(function_name=self.function_name, files={})
check_pipe_func_definition(module_source=module_source)
+ return None
case TreeBundle() | None:
# A bundle parsed from a content string imports its function from this process's import path, where
- # the library directories loaded beside it put their Python.
+ # the library directories loaded beside it put their Python: no file of a tree.
module_source = locate_on_import_path(function_name=self.function_name)
check_pipe_func_definition(module_source=module_source)
+ return None
+
+ @override
+ def tree_files_read(self) -> list[TreeFileRead]:
+ """The module of the step's function, when validation located it among the files of the step's method tree.
+
+ A function found on this process's import path is no file of a tree, and a step built from a crate that names its
+ tree by digest alone was proven where the crate was assembled: neither names a file here.
+ """
+ if self._module_tree_path is None:
+ return []
+ return [TreeFileRead(tree_path=self._module_tree_path, use=TreeFileUse.PYTHON_MODULE)]
@staticmethod
def _tree_label(*, tree_bundle: TreeBundle) -> str:
diff --git a/pipelex/pipeline/tree_files_read.py b/pipelex/pipeline/tree_files_read.py
new file mode 100644
index 000000000..ddfc17b02
--- /dev/null
+++ b/pipelex/pipeline/tree_files_read.py
@@ -0,0 +1,69 @@
+"""The files of the method's tree each validated pipe reads (`tree_files_read`), and the per-file view of them (`tree_file_readers`).
+
+A validation says, for each pipe it judged, which files of the method's tree the step reads, each by its tree path with
+what it reads it as (`pipelex.core.method_trees.tree_file_reads`): a `PipeDocGen` step's template and the files its HTML
+template names, a `PipeFunc` step's module. Each pipe resolved those names as it was built and validated, so this only
+collects them. A client then shows which step uses each file, and an agent tells a file no step reads from one a step
+needs (`wip/extra-files/design.md`, "Validation explains the method's files").
+
+Both views are keyed the way every other validate artifact is: `tree_files_read` by namespaced `pipe_ref`, with the same
+key set as `pipe_io_contracts`, a pipe that reads no file holding an empty list; `tree_file_readers` by tree path. The
+pipes a validation judges are the ones of the method it was handed, which is one tree, so a tree path names one file.
+"""
+
+from collections.abc import Iterable, Mapping
+from typing import Any, TypeAlias
+
+from pipelex.core.method_trees.tree_file_reads import TreeFileRead
+from pipelex.pipe_machinery.pipe_abstract import PipeAbstract
+
+#: The files of its method tree each pipe reads, keyed by namespaced `pipe_ref`.
+TreeFilesRead: TypeAlias = dict[str, list[TreeFileRead]]
+
+
+def build_tree_files_read(*, pipes: Iterable[PipeAbstract]) -> TreeFilesRead:
+ """The files of its method tree each pipe reads, keyed by namespaced `pipe_ref`, in the pipes' order.
+
+ Args:
+ pipes: The validated pipes, the same ones `pipe_io_contracts` describes.
+
+ Returns:
+ One entry per pipe, an empty list for a pipe that reads no file of its tree.
+ """
+ return {pipe.pipe_ref: pipe.tree_files_read() for pipe in pipes}
+
+
+def build_tree_file_readers(*, tree_files_read: Mapping[str, Iterable[TreeFileRead]]) -> dict[str, list[str]]:
+ """The per-file view of `tree_files_read`: each tree path a pipe reads, to the refs of the pipes that read it.
+
+ Args:
+ tree_files_read: The files each pipe reads, keyed by namespaced `pipe_ref`.
+
+ Returns:
+ The tree paths in sorted order, each with its readers in the order `tree_files_read` lists them, each once.
+ """
+ readers: dict[str, list[str]] = {}
+ for pipe_ref, files_read in tree_files_read.items():
+ for file_read in files_read:
+ pipe_refs = readers.setdefault(file_read.tree_path, [])
+ if pipe_ref not in pipe_refs:
+ pipe_refs.append(pipe_ref)
+ return {tree_path: readers[tree_path] for tree_path in sorted(readers)}
+
+
+def tree_files_envelope_fields(*, pipes: Iterable[PipeAbstract]) -> dict[str, Any]:
+ """The two views as the JSON-ready members of an agent-CLI validate envelope, `tree_files_read` and `tree_file_readers`.
+
+ Args:
+ pipes: The validated pipes.
+
+ Returns:
+ The envelope's two members, in the canonical report's shapes.
+ """
+ tree_files_read = build_tree_files_read(pipes=pipes)
+ return {
+ "tree_files_read": {
+ pipe_ref: [file_read.model_dump(mode="json") for file_read in files_read] for pipe_ref, files_read in tree_files_read.items()
+ },
+ "tree_file_readers": build_tree_file_readers(tree_files_read=tree_files_read),
+ }
diff --git a/pipelex/pipeline/validate_in_process.py b/pipelex/pipeline/validate_in_process.py
index ed1373571..3490a7e9a 100644
--- a/pipelex/pipeline/validate_in_process.py
+++ b/pipelex/pipeline/validate_in_process.py
@@ -38,6 +38,7 @@ from pipelex.pipeline.exceptions import MethodEntryError
from pipelex.pipeline.input_form import qualify_current_library_crate
from pipelex.pipeline.liftable_pipes import LiftablePipeEntry, build_liftable_pipes
from pipelex.pipeline.method_entry import select_method_entry
+from pipelex.pipeline.tree_files_read import TreeFilesRead, build_tree_files_read
from pipelex.pipeline.usage_stats_reporting import MethodValidationReport
from pipelex.pipeline.validate_bundle import validate_bundle
from pipelex.pipeline.validation_report import PipelexValidationReport, build_validation_report
@@ -195,6 +196,8 @@ async def _validate_bundles_in_scope(
# One taint walk per validate pass — both report projections read the same analyses.
taint_analyses = collect_controller_taint_analyses(result.pipes)
liftable_pipes: list[LiftablePipeEntry] = build_liftable_pipes(taint_analyses)
+ # The files of the method's tree each pipe reads, as building and validating it resolved them.
+ tree_files_read: TreeFilesRead = build_tree_files_read(pipes=result.pipes)
warnings: list[ValidationErrorItem] = build_advisory_warnings(
taint_analyses=taint_analyses,
input_form=artifacts.input_form,
@@ -244,6 +247,7 @@ async def _validate_bundles_in_scope(
liftable_pipes=liftable_pipes,
dry_run_result=result.dry_run_result,
pending_signatures=result.pending_signatures,
+ tree_files_read=tree_files_read,
graph_spec=graph_spec,
warnings=warnings,
)
diff --git a/pipelex/pipeline/validation_render.py b/pipelex/pipeline/validation_render.py
index d4dd0de8b..d98507528 100644
--- a/pipelex/pipeline/validation_render.py
+++ b/pipelex/pipeline/validation_render.py
@@ -24,6 +24,7 @@ import shlex
from pathlib import Path
from typing import TYPE_CHECKING, Any, cast
+from pipelex.core.method_trees.tree_file_reads import TreeFileRead
from pipelex.errors.error_codes import error_code_spec, parse_error_code
from pipelex.interpreter_hub import resolve_library_dirs
from pipelex.pipeline.fixes.applicability import is_safe_fix_for_load_scope, is_target_in_write_scope
@@ -47,7 +48,8 @@ def format_validate_markdown(result: dict[str, Any]) -> str:
also merge ``graph_files`` / ``graphspec`` into the result — those are
surfaced too so markdown mode does not silently drop them — or, for a
method's folder with no single pipe to draw, ``graph_unavailable``, the
- reason no graph was drawn.
+ reason no graph was drawn. ``tree_files_read``, when a step reads files of
+ the method's tree, lists them under each such step.
Args:
result: The JSON-mode result dict for the validation.
@@ -110,6 +112,8 @@ def format_validate_markdown(result: dict[str, Any]) -> str:
for warning in warnings:
lines.append(f"- **{warning.get('error_type') or warning.get('category')}** — {warning.get('message')}")
+ lines += _tree_files_read_lines(tree_files_read=result.get("tree_files_read") or {})
+
graph_files = result.get("graph_files")
if isinstance(graph_files, dict):
lines += ["", "## Graph files", ""]
@@ -126,6 +130,26 @@ def format_validate_markdown(result: dict[str, Any]) -> str:
return "\n".join(lines)
+def _tree_files_read_lines(*, tree_files_read: dict[str, list[dict[str, Any]]]) -> list[str]:
+ """The section naming, for each step that reads files of the method's tree, those files and what it reads each as.
+
+ Nothing when no step reads a file of the tree, or the result carries no `tree_files_read`: the steps that read none
+ are left out, since a file of the tree no step names is read by none.
+ """
+ reading_steps = {pipe_ref: files_read for pipe_ref, files_read in tree_files_read.items() if files_read}
+ if not reading_steps:
+ return []
+ lines = ["", "## Files the steps read", ""]
+ for pipe_ref, files_read in reading_steps.items():
+ described = ", ".join(_described_file_read(file_read=TreeFileRead.model_validate(file_read)) for file_read in files_read)
+ lines.append(f"- `{pipe_ref}` reads {described}")
+ return lines
+
+
+def _described_file_read(*, file_read: TreeFileRead) -> str:
+ return f"`{file_read.tree_path}` ({file_read.use.label})"
+
+
def render_invalid_verdict_markdown(*, items: Sequence[ProblemItem], message: str) -> str:
"""Render an invalid validation verdict (the InvalidReport arm) as agent-readable markdown, from its problem items.
diff --git a/pipelex/pipeline/validation_report.py b/pipelex/pipeline/validation_report.py
index 72cce4792..01f43da9b 100644
--- a/pipelex/pipeline/validation_report.py
+++ b/pipelex/pipeline/validation_report.py
@@ -18,6 +18,7 @@ from mthds.protocol.models import ValidationReport
from pydantic import Field
from pipelex.base_exceptions import ValidationErrorItem
+from pipelex.core.method_trees.tree_file_reads import TreeFileRead
from pipelex.graph.graphspec import GraphSpec
from pipelex.mthds_parsing.pipelex_bundle_blueprint import PipelexBundleBlueprint
from pipelex.pipeline.blueprint_selection import select_primary_blueprint
@@ -25,6 +26,7 @@ from pipelex.pipeline.bundle_validator import DryRunOutput
from pipelex.pipeline.input_form import InputForm, OutputForm
from pipelex.pipeline.liftable_pipes import LiftablePipeEntry
from pipelex.pipeline.pipe_io_contracts import PipeIOContracts
+from pipelex.pipeline.tree_files_read import TreeFilesRead, build_tree_file_readers
from pipelex.pipeline.validate_bundle import ValidatedPipeEntry, build_validated_pipes
from pipelex.tools.typing.pydantic_utils import empty_list_factory_of
@@ -69,6 +71,25 @@ class PipelexValidationReport(ValidationReport[Any]):
"""Pipes that may be skipped (lifted) when an optional slot resolves absent — the
build-time visibility that the implicit-lifting design (D3) commits to."""
+ tree_files_read: dict[str, list[TreeFileRead]] = Field(
+ default_factory=dict,
+ description=(
+ "Per pipe, the files of the method's tree the step reads, keyed by namespaced `pipe_ref` like `pipe_io_contracts`: "
+ "each file by its tree path, with what the step reads it as. A PipeDocGen step reads its template and the images, "
+ "stylesheets and fonts its HTML template names by literal relative URLs; a PipeFunc step reads the module of its "
+ "function. A pipe that reads no file of the tree, a controller included, has an empty list."
+ ),
+ )
+
+ # Derived from `tree_files_read` by `build_validation_report`, never assembled apart from it.
+ tree_file_readers: dict[str, list[str]] = Field(
+ default_factory=dict,
+ description=(
+ "The per-file view of `tree_files_read`: each tree path a step reads, in sorted order, to the namespaced refs of "
+ "the pipes that read it. A file of the method's tree absent here is read by no step."
+ ),
+ )
+
graph_spec: GraphSpec | None = None
"""Best-effort execution graph of the declared main pipe; `None` when the batch
declares no `main_pipe` or the graph dry-run degrades."""
@@ -96,6 +117,7 @@ def build_validation_report(
output_form: OutputForm,
dry_run_result: dict[str, DryRunOutput],
pending_signatures: list[str],
+ tree_files_read: TreeFilesRead,
graph_spec: GraphSpec | None = None,
liftable_pipes: list[LiftablePipeEntry] | None = None,
warnings: list[ValidationErrorItem] | None = None,
@@ -104,7 +126,8 @@ def build_validation_report(
The single constructor every backend calls — local protocol `validate` and the hosted
API's Temporal mapping. Selects the primary blueprint, projects the dry-run status map
- into `validated_pipes`, and derives the `is_runnable` verdict.
+ into `validated_pipes`, derives the per-file `tree_file_readers` from `tree_files_read`,
+ and derives the `is_runnable` verdict.
Args:
blueprints: The validated batch's blueprints, in declaration order (non-empty).
@@ -113,6 +136,8 @@ def build_validation_report(
output_form: Per-pipe output-form descriptors from `build_output_form`, keyed the same way.
dry_run_result: The sweep's per-pipe status map (`ValidateBundleResult.dry_run_result`).
pending_signatures: Library-wide unsatisfied signature refs.
+ tree_files_read: The files of the method's tree each pipe reads, from `build_tree_files_read`, keyed like
+ `pipe_io_contracts`. Required, so a backend that forgets it fails loudly instead of reporting no file read.
graph_spec: Best-effort graph of the declared main pipe, when one was produced.
liftable_pipes: Entries from `build_liftable_pipes`, when the caller computed them.
warnings: Advisory items from `build_advisory_warnings`, when the caller computed them.
@@ -126,6 +151,8 @@ def build_validation_report(
input_form=input_form,
output_form=output_form,
liftable_pipes=liftable_pipes or [],
+ tree_files_read=tree_files_read,
+ tree_file_readers=build_tree_file_readers(tree_files_read=tree_files_read),
graph_spec=graph_spec,
validated_pipes=build_validated_pipes(dry_run_result),
pending_signatures=pending_signatures,
diff --git a/tests/integration/pipelex/cli/test_agent_validate_tree_files_read.py b/tests/integration/pipelex/cli/test_agent_validate_tree_files_read.py
new file mode 100644
index 000000000..e6c432957
--- /dev/null
+++ b/tests/integration/pipelex/cli/test_agent_validate_tree_files_read.py
@@ -0,0 +1,105 @@
+import json
+from collections.abc import Awaitable
+from pathlib import Path
+from typing import Any
+
+import pytest
+
+from pipelex.cli.agent_cli.commands.validate._validate_core import validate_bundle_core, validate_method_folder_core
+from pipelex.interpreter_hub import clear_current_library, get_current_library_id_or_none, get_library_manager
+from pipelex.pipeline.validation_render import format_validate_markdown
+
+# No `from __future__ import annotations`: the function a `PipeFunc` calls has its annotations read when it is bound.
+
+_METHOD_MTHDS = """\
+domain = "agent_tree_reads"
+main_pipe = "bill"
+
+[pipe.bill]
+type = "PipeSequence"
+description = "Total the invoice, then thank the customer"
+inputs = { text = "Text" }
+output = "Text"
+steps = [{ pipe = "compute_total", result = "total" }, { pipe = "thank", result = "thanks" }]
+
+[pipe.compute_total]
+type = "PipeFunc"
+description = "Total the invoice"
+function_name = "agent_tree_reads_funcs.totals.compute_total"
+output = "Text"
+
+[pipe.thank]
+type = "PipeCompose"
+description = "Thank the customer"
+inputs = { text = "Text" }
+output = "Text"
+template = "Thank you, $text"
+"""
+
+_TOTALS_MODULE = """\
+from pipelex.core.memory.working_memory import WorkingMemory
+from pipelex.core.stuffs.text_content import TextContent
+from pipelex.system.registries.func_registry import pipe_func
+
+
+@pipe_func()
+async def compute_total(working_memory: WorkingMemory) -> TextContent:
+ return TextContent(text="42")
+"""
+
+_EXPECTED_FIELDS = {
+ "tree_files_read": {
+ "agent_tree_reads.bill": [],
+ "agent_tree_reads.compute_total": [{"tree_path": "agent_tree_reads_funcs/totals.py", "use": "python_module"}],
+ "agent_tree_reads.thank": [],
+ },
+ "tree_file_readers": {"agent_tree_reads_funcs/totals.py": ["agent_tree_reads.compute_total"]},
+}
+
+
[email protected](name="method_dir")
+def method_dir_fixture(tmp_path: Path) -> Path:
+ """A method folder whose `PipeFunc` names a module of its tree, under a manifest."""
+ method_dir = tmp_path / "method"
+ (method_dir / "agent_tree_reads_funcs").mkdir(parents=True)
+ (method_dir / "invoicing.mthds").write_text(_METHOD_MTHDS, encoding="utf-8")
+ (method_dir / "agent_tree_reads_funcs" / "__init__.py").write_text("", encoding="utf-8")
+ (method_dir / "agent_tree_reads_funcs" / "totals.py").write_text(_TOTALS_MODULE, encoding="utf-8")
+ (method_dir / "METHODS.toml").write_text(
+ '[package]\naddress = "github.com/pipelex-tests/agent-tree-reads"\nversion = "1.0.0"\n', encoding="utf-8"
+ )
+ return method_dir
+
+
+async def _released(validation: Awaitable[dict[str, Any]]) -> dict[str, Any]:
+ """Await an agent-CLI validation, then tear down the library it leaves open and current for the process's exit.
+
+ That library holds the tree's Python, which a later load of another tree's Python in this process would be refused
+ beside. It is current in the test's own context, so it is torn down here rather than by a fixture.
+ """
+ try:
+ return await validation
+ finally:
+ library_id = get_current_library_id_or_none()
+ if library_id is not None:
+ get_library_manager().teardown(library_id=library_id)
+ clear_current_library()
+
+
[email protected](loop_scope="class")
+class TestAgentValidateTreeFilesRead:
+ async def test_the_method_folder_envelope_names_the_files_each_step_reads(self, method_dir: Path) -> None:
+ """The envelope carries both views as JSON, in the canonical report's shapes, and its Markdown names the module."""
+ result = await _released(
+ validate_method_folder_core(method_dir=method_dir, library_dirs=[method_dir], allow_signatures=False, pipe_code=None)
+ )
+
+ assert {key: result[key] for key in _EXPECTED_FIELDS} == _EXPECTED_FIELDS
+ assert json.loads(json.dumps(result))["tree_files_read"] == _EXPECTED_FIELDS["tree_files_read"]
+ assert "- `agent_tree_reads.compute_total` reads `agent_tree_reads_funcs/totals.py` (Python module)" in format_validate_markdown(result)
+
+ async def test_the_bundle_envelope_names_them_from_the_bundle_s_tree(self, method_dir: Path) -> None:
+ """A bundle file named alone under a manifest is rooted at the manifest's folder, so its step's module is found there."""
+ result = await _released(validate_bundle_core(bundle_path=method_dir / "invoicing.mthds"))
+
+ assert {key: result[key] for key in _EXPECTED_FIELDS} == _EXPECTED_FIELDS
diff --git a/tests/integration/pipelex/pipeline/test_validate_tree_files_read.py b/tests/integration/pipelex/pipeline/test_validate_tree_files_read.py
new file mode 100644
index 000000000..f5b63f6aa
--- /dev/null
+++ b/tests/integration/pipelex/pipeline/test_validate_tree_files_read.py
@@ -0,0 +1,116 @@
+from pathlib import Path
+
+import pytest
+
+from pipelex.core.memory.working_memory import WorkingMemory
+from pipelex.core.method_trees.tree_file_reads import TreeFileRead, TreeFileUse
+from pipelex.core.stuffs.text_content import TextContent
+from pipelex.pipeline.validate_in_process import validate_bundles_in_process
+from pipelex.system.registries.func_registry import pipe_func
+
+# No `from __future__ import annotations`: the function a `PipeFunc` calls has its annotations read when it is bound.
+
+_INVOICING_MTHDS = """\
+domain = "tree_reads"
+main_pipe = "bill"
+
+[pipe.bill]
+type = "PipeSequence"
+description = "Total the invoice, then thank the customer"
+inputs = { text = "Text" }
+output = "Text"
+steps = [{ pipe = "compute_total", result = "total" }, { pipe = "thank", result = "thanks" }]
+
+[pipe.compute_total]
+type = "PipeFunc"
+description = "Total the invoice"
+function_name = "tree_reads_funcs.totals.compute_total"
+output = "Text"
+
+[pipe.thank]
+type = "PipeCompose"
+description = "Thank the customer"
+inputs = { text = "Text" }
+output = "Text"
+template = "Thank you, $text"
+"""
+
+_TOTALS_MODULE = """\
+from pipelex.core.memory.working_memory import WorkingMemory
+from pipelex.core.stuffs.text_content import TextContent
+from pipelex.system.registries.func_registry import pipe_func
+
+
+@pipe_func()
+async def compute_total(working_memory: WorkingMemory) -> TextContent:
+ return TextContent(text="42")
+"""
+
+
+# A bundle sent as a string, whose `PipeFunc` finds its function on this process's import path: this very module.
+_IMPORT_PATH_MTHDS = """\
+domain = "tree_reads_import_path"
+
+[pipe.compute_total]
+type = "PipeFunc"
+description = "Total the invoice"
+function_name = "tests.integration.pipelex.pipeline.test_validate_tree_files_read.tree_reads_compute_total"
+output = "Text"
+"""
+
+
+@pipe_func()
+def tree_reads_compute_total(working_memory: WorkingMemory) -> TextContent: # ruff: ignore[unused-function-argument] # pyright: ignore[reportUnusedParameter]
+ return TextContent(text="42")
+
+
+def _write_method(*, method_dir: Path, module_tree_path: str) -> None:
+ """A method folder whose `PipeFunc` names its module by import path, beside a file no step reads."""
+ files = {
+ "invoicing.mthds": _INVOICING_MTHDS,
+ "tree_reads_funcs/__init__.py": "",
+ module_tree_path: _TOTALS_MODULE,
+ "README.md": "# Invoicing\n",
+ }
+ for tree_path, text in files.items():
+ (method_dir / tree_path).parent.mkdir(parents=True, exist_ok=True)
+ (method_dir / tree_path).write_text(text, encoding="utf-8")
+
+
[email protected](loop_scope="class")
+class TestValidateTreeFilesRead:
+ @pytest.mark.parametrize(
+ "module_tree_path",
+ [
+ pytest.param("tree_reads_funcs/totals.py", id="module_file"),
+ pytest.param("tree_reads_funcs/totals/__init__.py", id="package"),
+ ],
+ )
+ async def test_the_report_names_the_module_of_each_pipe_func_and_an_empty_list_for_every_other_pipe(
+ self, tmp_path: Path, module_tree_path: str
+ ) -> None:
+ """The module is the file the import path resolves to in the tree; a controller and a compose step read no file."""
+ method_dir = tmp_path / "method"
+ _write_method(method_dir=method_dir, module_tree_path=module_tree_path)
+
+ report = await validate_bundles_in_process(method_dir=method_dir, library_dirs_are_callers=True)
+
+ assert report.tree_files_read == {
+ "tree_reads.bill": [],
+ "tree_reads.compute_total": [TreeFileRead(tree_path=module_tree_path, use=TreeFileUse.PYTHON_MODULE)],
+ "tree_reads.thank": [],
+ }
+ # Keyed exactly like the contracts: every validated pipe, whether or not it reads a file.
+ assert set(report.tree_files_read) == set(report.pipe_io_contracts)
+ # The package's `__init__.py`, the bundle and the README are files no step reads.
+ assert report.tree_file_readers == {module_tree_path: ["tree_reads.compute_total"]}
+ assert report.model_dump(mode="json")["tree_files_read"]["tree_reads.compute_total"] == [
+ {"tree_path": module_tree_path, "use": "python_module"}
+ ]
+
+ async def test_a_function_found_on_the_import_path_is_no_file_of_a_tree(self) -> None:
+ """A bundle sent as a string with no file beside it belongs to no tree, so its step reads no file a report can name."""
+ report = await validate_bundles_in_process(mthds_contents=[_IMPORT_PATH_MTHDS])
+
+ assert report.tree_files_read == {"tree_reads_import_path.compute_total": []}
+ assert report.tree_file_readers == {}
diff --git a/tests/integration/pipelex/pipes/operator/pipe_doc_gen/test_pipe_doc_gen_tree_files_read.py b/tests/integration/pipelex/pipes/operator/pipe_doc_gen/test_pipe_doc_gen_tree_files_read.py
new file mode 100644
index 000000000..3f9da6982
--- /dev/null
+++ b/tests/integration/pipelex/pipes/operator/pipe_doc_gen/test_pipe_doc_gen_tree_files_read.py
@@ -0,0 +1,94 @@
+from pathlib import Path
+
+import pytest
+
+from pipelex.core.method_trees.tree_file_reads import TreeFileRead, TreeFileUse
+from pipelex.pipeline.validate_in_process import validate_bundles_in_process
+from tests.integration.pipelex.pipes.operator.pipe_doc_gen.doc_gen_helpers import METHOD_MANIFEST, write_bundle
+from tests.integration.pipelex.pipes.operator.pipe_doc_gen.test_data import PipeDocGenTestData
+
+_PIPE_REF = "doc_gen_tests.print_invoice"
+_TEMPLATE = """<link rel="stylesheet" href="../styles/print.css">
+<style>h1 { background: url('../images/band.png'); }</style>
+<img src="logo.png"><img src="logo.png"><img src="data:image/png;base64,iVBORw0KGgo="><img src="https://cdn.example.com/x.png">
+<img src="{{ invoice.customer }}">
+<h1>{{ invoice.number }}</h1>
+"""
+_FILES = {
+ "templates/logo.png": b"\x89PNG logo",
+ "images/band.png": b"\x89PNG band",
+ "styles/print.css": b'@font-face { font-family: Brand; src: url("../fonts/brand.woff2"); }\n',
+ "fonts/brand.woff2": b"wOF2 brand",
+ "notes/README.md": b"# Read by people, by no step\n",
+}
+
+
+def _write_method(*, method_dir: Path) -> None:
+ """A method folder whose step prints from `templates/invoice.html`, with the files its template names and one no step reads."""
+ method_dir.mkdir(parents=True)
+ for tree_path, data in _FILES.items():
+ (method_dir / tree_path).parent.mkdir(parents=True, exist_ok=True)
+ (method_dir / tree_path).write_bytes(data)
+ (method_dir / "templates" / "invoice.html").write_text(_TEMPLATE, encoding="utf-8")
+ write_bundle(directory=method_dir, step_fields='format = "pdf"\ntemplate_file = "templates/invoice.html"')
+
+
[email protected]("stub_engines")
[email protected](loop_scope="class")
+class TestPipeDocGenTreeFilesRead:
+ async def test_the_report_names_the_template_file_and_each_file_its_html_names(self, tmp_path: Path) -> None:
+ """The template first, then its literal URLs' files, a stylesheet's own after it, each once; a computed URL is the print's."""
+ method_dir = tmp_path / "method"
+ _write_method(method_dir=method_dir)
+
+ report = await validate_bundles_in_process(method_dir=method_dir, library_dirs_are_callers=True)
+
+ assert report.tree_files_read == {
+ _PIPE_REF: [
+ TreeFileRead(tree_path="templates/invoice.html", use=TreeFileUse.TEMPLATE),
+ TreeFileRead(tree_path="styles/print.css", use=TreeFileUse.TEMPLATE_ASSET),
+ TreeFileRead(tree_path="images/band.png", use=TreeFileUse.TEMPLATE_ASSET),
+ TreeFileRead(tree_path="templates/logo.png", use=TreeFileUse.TEMPLATE_ASSET),
+ TreeFileRead(tree_path="fonts/brand.woff2", use=TreeFileUse.TEMPLATE_ASSET),
+ ]
+ }
+ # A file no step reads, the README and the bundle itself, is in no reader's list.
+ assert report.tree_file_readers == {
+ "fonts/brand.woff2": [_PIPE_REF],
+ "images/band.png": [_PIPE_REF],
+ "styles/print.css": [_PIPE_REF],
+ "templates/invoice.html": [_PIPE_REF],
+ "templates/logo.png": [_PIPE_REF],
+ }
+ # Files are named by their tree paths, never by where the method sits on this host.
+ assert str(tmp_path) not in report.model_dump_json(include={"tree_files_read", "tree_file_readers"})
+
+ async def test_an_inline_template_s_files_resolve_against_its_bundle(self, tmp_path: Path) -> None:
+ method_dir = tmp_path / "method"
+ (method_dir / "bundles").mkdir(parents=True)
+ (method_dir / "assets").mkdir()
+ (method_dir / "assets" / "logo.png").write_bytes(b"\x89PNG inline")
+ (method_dir / "METHODS.toml").write_text(METHOD_MANIFEST, encoding="utf-8")
+ step_fields = "format = \"pdf\"\ntemplate = '''<img src=\"../assets/logo.png\"><h1>{{ invoice.number }}</h1>'''"
+ write_bundle(directory=method_dir / "bundles", step_fields=step_fields, as_method=False)
+
+ report = await validate_bundles_in_process(method_dir=method_dir, library_dirs_are_callers=True)
+
+ assert report.tree_files_read == {_PIPE_REF: [TreeFileRead(tree_path="assets/logo.png", use=TreeFileUse.TEMPLATE_ASSET)]}
+ assert report.tree_file_readers == {"assets/logo.png": [_PIPE_REF]}
+
+ async def test_a_file_submitted_with_content_is_named_by_the_path_it_is_keyed_by(self) -> None:
+ """Bundles from content strings sit at the root of the tree their files make, so the template's tree path is its key, collapsed."""
+ report = await validate_bundles_in_process(
+ mthds_contents=[PipeDocGenTestData.bundle(step_fields='format = "docx"\ntemplate_file = "templates/invoice.docx"')],
+ submitted_files={"./templates/invoice.docx": b"PK docx template bytes"},
+ )
+
+ assert report.tree_files_read == {_PIPE_REF: [TreeFileRead(tree_path="templates/invoice.docx", use=TreeFileUse.TEMPLATE)]}
+ assert report.tree_file_readers == {"templates/invoice.docx": [_PIPE_REF]}
+
+ async def test_a_step_laying_its_inputs_out_reads_no_file(self) -> None:
+ report = await validate_bundles_in_process(mthds_contents=[PipeDocGenTestData.bundle(step_fields=PipeDocGenTestData.PDF_LAYOUT_STEP)])
+
+ assert report.tree_files_read == {_PIPE_REF: []}
+ assert report.tree_file_readers == {}
diff --git a/tests/unit/pipelex/pipe_operators/pipe_doc_gen/test_template_asset_check.py b/tests/unit/pipelex/pipe_operators/pipe_doc_gen/test_template_asset_check.py
index 4ea8b67a1..e5acf5b36 100644
--- a/tests/unit/pipelex/pipe_operators/pipe_doc_gen/test_template_asset_check.py
+++ b/tests/unit/pipelex/pipe_operators/pipe_doc_gen/test_template_asset_check.py
@@ -1,7 +1,7 @@
import pytest
from pipelex.core.method_trees.method_file_store import HeldFile, tree_of_held_files
-from pipelex.pipe_operators.doc_gen.template_asset_check import find_template_asset_refusal, literal_template_urls, stylesheet_urls
+from pipelex.pipe_operators.doc_gen.template_asset_check import check_template_assets, literal_template_urls, stylesheet_urls
_FILES = {
"METHODS.toml": b'[package]\naddress = "github.com/acme/letters"\n',
@@ -47,12 +47,39 @@ class TestTemplateAssetCheck:
def test_images_stylesheets_and_the_urls_they_name_pass(self) -> None:
template = '<link rel="stylesheet" href="../styles/print.css"><img src="logo.png?v=2"><img src="https://cdn.example.com/x.png">'
- refusal = find_template_asset_refusal(
+ refusal = check_template_assets(
template=template, template_label="the template file", holder="templates/letter.html", tree=_TREE, read_file=_read_file
- )
+ ).refusal
assert refusal is None
+ def test_the_check_names_each_file_the_urls_reach_once_by_tree_path(self) -> None:
+ """The template's own URLs first, then each stylesheet's, as the check reaches them; an external URL names no file."""
+ template = (
+ '<link rel="stylesheet" href="../styles/print.css"><img src="logo.png?v=2"><img src="./logo.png">'
+ '<img src="https://cdn.example.com/x.png"><img src="data:image/png;base64,iVBORw0KGgo=">'
+ )
+
+ asset_check = check_template_assets(
+ template=template, template_label="the template file", holder="templates/letter.html", tree=_TREE, read_file=_read_file
+ )
+
+ assert asset_check.refusal is None
+ assert asset_check.asset_tree_paths == ["styles/print.css", "templates/logo.png", "fonts/brand.woff2", "styles/base.css", "images/band.png"]
+
+ def test_a_refused_url_keeps_the_files_reached_before_it(self) -> None:
+ asset_check = check_template_assets(
+ template='<img src="logo.png"><img src="missing.png">',
+ template_label="the template",
+ holder="templates/letter.html",
+ tree=_TREE,
+ read_file=_read_file,
+ )
+
+ assert asset_check.refusal is not None
+ assert asset_check.refusal.reference == "missing.png"
+ assert asset_check.asset_tree_paths == ["templates/logo.png"]
+
@pytest.mark.parametrize(
("template", "reference", "holder_label", "reason"),
[
@@ -72,9 +99,9 @@ class TestTemplateAssetCheck:
],
)
def test_a_literal_url_naming_no_asset_of_the_tree_is_refused(self, template: str, reference: str, holder_label: str, reason: str) -> None:
- refusal = find_template_asset_refusal(
+ refusal = check_template_assets(
template=template, template_label="the template", holder="templates/letter.html", tree=_TREE, read_file=_read_file
- )
+ ).refusal
assert refusal is not None
assert (refusal.reference, refusal.holder_label) == (reference, holder_label)
@@ -83,13 +110,13 @@ class TestTemplateAssetCheck:
def test_a_url_of_a_linked_stylesheet_is_checked_against_the_stylesheet(self) -> None:
files = {**_FILES, "styles/base.css": b"body { background: url(../images/missing.png); }\n"}
- refusal = find_template_asset_refusal(
+ refusal = check_template_assets(
template='<link rel="stylesheet" href="../styles/print.css">',
template_label="the template",
holder="templates/letter.html",
tree=_TREE,
read_file=files.get,
- )
+ ).refusal
assert refusal is not None
assert refusal.reference == "../images/missing.png"
@@ -97,13 +124,13 @@ class TestTemplateAssetCheck:
assert "'images/missing.png', which is not a file" in refusal.reason
def test_an_inline_template_of_bundles_from_strings_resolves_from_the_root(self) -> None:
- refusal = find_template_asset_refusal(
+ refusal = check_template_assets(
template='<img src="templates/logo.png"><img src="../logo.png">',
template_label="the template",
holder=None,
tree=_TREE,
read_file=_read_file,
- )
+ ).refusal
assert refusal is not None
assert refusal.reference == "../logo.png"
@@ -112,9 +139,9 @@ class TestTemplateAssetCheck:
def test_a_bundle_named_alone_is_told_to_hand_over_its_folder(self) -> None:
one_file_tree = tree_of_held_files(files={"letters.mthds": HeldFile.of_bytes(data=b"x")}).model_copy(update={"is_one_file": True})
- refusal = find_template_asset_refusal(
+ refusal = check_template_assets(
template='<img src="logo.png">', template_label="the template", holder="letters.mthds", tree=one_file_tree, read_file=_read_file
- )
+ ).refusal
assert refusal is not None
assert "Hand over the method's directory instead" in refusal.reason
diff --git a/tests/unit/pipelex/pipeline/test_validation_render.py b/tests/unit/pipelex/pipeline/test_validation_render.py
index 7d9b4936c..ee00bd4bc 100644
--- a/tests/unit/pipelex/pipeline/test_validation_render.py
+++ b/tests/unit/pipelex/pipeline/test_validation_render.py
@@ -109,6 +109,48 @@ class TestValidationRender:
assert markdown.startswith("# Validation passed")
assert markdown.endswith(f"## Graph\n\n{reason}")
+ def test_names_the_files_each_step_reads_under_the_steps_that_read_any(self):
+ """Each step reading files of the method's tree is listed with them and their use; a step reading none is not."""
+ result: dict[str, Any] = {
+ "success": True,
+ "bundle_path": "/fake/method",
+ "validated_pipes": [{"pipe_ref": "invoicing.print_invoice", "status": "SUCCESS"}],
+ "total_pipes": 3,
+ "tree_files_read": {
+ "invoicing.print_invoice": [
+ {"tree_path": "templates/invoice.html", "use": "template"},
+ {"tree_path": "templates/logo.png", "use": "template_asset"},
+ ],
+ "invoicing.compute_totals": [{"tree_path": "funcs/totals.py", "use": "python_module"}],
+ "invoicing.summarize": [],
+ },
+ "tree_file_readers": {
+ "funcs/totals.py": ["invoicing.compute_totals"],
+ "templates/invoice.html": ["invoicing.print_invoice"],
+ "templates/logo.png": ["invoicing.print_invoice"],
+ },
+ }
+
+ markdown = format_validate_markdown(result)
+
+ assert "## Files the steps read" in markdown
+ assert "- `invoicing.print_invoice` reads `templates/invoice.html` (template), `templates/logo.png` (template asset)" in markdown
+ assert "- `invoicing.compute_totals` reads `funcs/totals.py` (Python module)" in markdown
+ assert "invoicing.summarize" not in markdown
+
+ @pytest.mark.parametrize(
+ "tree_files_read",
+ [pytest.param(None, id="absent"), pytest.param({"invoicing.summarize": []}, id="no_step_reads_a_file")],
+ )
+ def test_omits_the_files_section_when_no_step_reads_a_file(self, tree_files_read: dict[str, Any] | None):
+ result: dict[str, Any] = {"success": True, "validated_pipes": [], "total_pipes": 1}
+ if tree_files_read is not None:
+ result["tree_files_read"] = tree_files_read
+
+ markdown = format_validate_markdown(result)
+
+ assert "Files the steps read" not in markdown
+
def test_omits_runnability_verdict_when_key_absent(self):
"""The `validate all` / `validate pipe` shape omits the `pending_signatures` key entirely —
so no runnability verdict (positive or negative) leaks onto those non-bundle surfaces.
diff --git a/tests/unit/pipelex/pipeline/test_validation_report.py b/tests/unit/pipelex/pipeline/test_validation_report.py
index 9368f80ac..386dfb7a1 100644
--- a/tests/unit/pipelex/pipeline/test_validation_report.py
+++ b/tests/unit/pipelex/pipeline/test_validation_report.py
@@ -4,12 +4,14 @@ The canonical ``PipelexValidationReport`` is constructed in exactly one place
function — by every backend (local protocol ``validate``, hosted direct, hosted Temporal). The
test pins what the assembly derives from its ingredients: primary-blueprint selection for
``bundle_blueprint`` (first declaring ``main_pipe``, else first), the ``validated_pipes``
-projection from the dry-run status map, and ``is_runnable = not pending_signatures``.
+projection from the dry-run status map, the per-file ``tree_file_readers`` view derived from
+``tree_files_read``, and ``is_runnable = not pending_signatures``.
Pure unit test (no Pipelex boot): blueprints from the interpreter, hand-built status map.
"""
from pipelex.base_exceptions import ValidationErrorCategory, ValidationErrorItem
+from pipelex.core.method_trees.tree_file_reads import TreeFileRead, TreeFileUse
from pipelex.mthds_parsing.parser import MthdsParser
from pipelex.pipeline.bundle_validator import DryRunOutput, DryRunStatus
from pipelex.pipeline.pipe_io_contracts import IOMultiplicity, PipeIOContract, PipeOutputContract
@@ -69,6 +71,7 @@ class TestBuildValidationReport:
output_form={},
dry_run_result=dry_run_result,
pending_signatures=["beta.still_pending"],
+ tree_files_read={"beta.do_it": []},
)
# is_valid is the always-True discriminant of the valid arm of the HTTP response union.
@@ -80,6 +83,8 @@ class TestBuildValidationReport:
assert report.pending_signatures == ["beta.still_pending"]
assert report.is_runnable is False
assert report.graph_spec is None
+ assert report.tree_files_read == {"beta.do_it": []}
+ assert report.tree_file_readers == {}
def test_runnable_when_nothing_pending(self) -> None:
blueprints = [MthdsParser.make_pipelex_bundle_blueprint(mthds_content=_MAIN_PIPE_MTHDS)]
@@ -91,6 +96,7 @@ class TestBuildValidationReport:
output_form={},
dry_run_result={},
pending_signatures=[],
+ tree_files_read={},
)
assert report.bundle_blueprint is blueprints[0]
@@ -118,8 +124,41 @@ class TestBuildValidationReport:
output_form={},
dry_run_result={},
pending_signatures=[],
+ tree_files_read={},
warnings=[warning_item],
)
assert report.is_valid is True
assert report.warnings == [warning_item]
+
+ def test_the_per_file_view_is_derived_from_the_files_each_pipe_reads(self) -> None:
+ """Each tree path a pipe reads, sorted, names every pipe reading it once, in the order the per-pipe view lists them."""
+ blueprints = [MthdsParser.make_pipelex_bundle_blueprint(mthds_content=_MAIN_PIPE_MTHDS)]
+ letterhead = TreeFileRead(tree_path="templates/letterhead.html", use=TreeFileUse.TEMPLATE)
+ logo = TreeFileRead(tree_path="templates/logo.png", use=TreeFileUse.TEMPLATE_ASSET)
+ totals_module = TreeFileRead(tree_path="funcs/totals.py", use=TreeFileUse.PYTHON_MODULE)
+ tree_files_read = {
+ "beta.write_letter": [letterhead, logo],
+ "beta.write_reminder": [letterhead, logo],
+ "beta.compute_totals": [totals_module],
+ "beta.do_it": [],
+ }
+
+ report = build_validation_report(
+ blueprints=blueprints,
+ pipe_io_contracts={},
+ input_form={},
+ output_form={},
+ dry_run_result={},
+ pending_signatures=[],
+ tree_files_read=tree_files_read,
+ )
+
+ assert report.tree_files_read == tree_files_read
+ assert report.tree_file_readers == {
+ "funcs/totals.py": ["beta.compute_totals"],
+ "templates/letterhead.html": ["beta.write_letter", "beta.write_reminder"],
+ "templates/logo.png": ["beta.write_letter", "beta.write_reminder"],
+ }
+ assert list(report.tree_file_readers) == sorted(report.tree_file_readers)
+ assert report.model_dump(mode="json")["tree_files_read"]["beta.compute_totals"] == [{"tree_path": "funcs/totals.py", "use": "python_module"}]