Build AI methods with your coding agent, then run them anywhere: as an MCP for chatbots, as a webapp for people, or via API for your software.
feature/Fetched-trees-checked-against-lock · L-261016-ea2a97 (#1468)
Louis Choquel049bee3eab2e
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 0ff09daad..5e32e8b94 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -5,6 +5,7 @@### Added- **The validation report names the files each step reads**: `PipelexValidationReport`, and so the valid verdict of `/v1/validate`, carries `tree_files_read`, mapping each validated pipe's `pipe_ref` to the files of the method's tree it reads, each a `TreeFileRead` with its `tree_path` and its `use` (`template` for a `PipeDocGen` step's `template_file`, `template_asset` for a file its HTML template names by a literal relative URL, `python_module` for the module holding a `PipeFunc` step's function), an empty list for a pipe reading none, and `tree_file_readers`, mapping each of those files to the pipes that read it. `pipelex-agent validate bundle` and `validate method` carry both in their envelope, and the Markdown of a valid verdict lists them in a "Files the steps read" section.+- **A fetched package is checked against the calling method's `methods.lock`**: when a method carries a `methods.lock` at the root of its tree, a package it calls by address that the runtime fetched itself, for this load or by an earlier fetch whose installed copy records its provenance, must have the tree digest the lock records for its address, whatever version the lock names, else the load is refused with the core code `security.integrity_mismatch` (domain `method`, a `422` with its security event on the API server), whose message names the locked and fetched versions and digests and says to pin the reference to the locked version or to lock the method again, or, when an installed copy answered a reference pinning another tag, says so and to remove that copy. A lock not naming the address, or no lock, checks nothing, and packages shipped with the method or installed by another tool are not checked. The lock is read only when a dependency resolves to such a copy, and one the runtime cannot read, one recording a `hash` from before tree digests included, then refuses the load with `method.package_invalid` and the new reason `lock_invalid`.### Changeddiff --git a/api/tests/unit/test_fetched_dependency_lock.py b/api/tests/unit/test_fetched_dependency_lock.pynew file mode 100644index 000000000..941eca146--- /dev/null+++ b/api/tests/unit/test_fetched_dependency_lock.py@@ -0,0 +1,175 @@+"""A run checks a dependency the runner fetches on a miss against the `methods.lock` the calling method carries.++The calling method is sent as `files` or fetched by `method_ref`, its lock at the root of its tree. Its bundle calls the+invented probe package by address, which nothing ships and no store holds, so the runner fetches it on a miss, the fetch+mocked to a local copy of the package. The installed store is an empty temporary directory, and nothing reaches the+network.+"""++import json+import logging+from collections.abc import Callable+from pathlib import Path+from typing import Any++import pytest+from fastapi import Depends, FastAPI+from fastapi.testclient import TestClient+from mthds.method_tree.lister import list_method_tree_at_root+from mthds.package.lock_file import LOCK_FILENAME+from mthds.package.manifest.parser import parse_methods_toml+from mthds.protocol.problem.registry import CoreCode+from pipelex.errors.security_event import SECURITY_EVENT_MESSAGE+from pipelex.methods.fetching import FetchedMethodPackage+from pipelex.methods.method_ref import parse_method_ref+from pytest_mock import MockerFixture, MockType++from pipelex_api.api_config import ApiConfig+from pipelex_api.exception_handlers import register_exception_handlers+from pipelex_api.problem_view import mark_operator_request+from pipelex_api.routes import router as api_router+from tests.unit._constants import (+ STUB_METHOD_ADDRESS,+ UNRESOLVED_PACKAGE_MTHDS,+ VENDORED_PROBE_ADDRESS,+ VENDORED_PROBE_MANIFEST,+ VENDORED_PROBE_MARKER,+ VENDORED_PROBE_MTHDS,+)+from tests.unit._problems import assert_problem++_PIPELINE_NS = "pipelex_api.routes.pipelex.pipeline"+_METHOD_REF = f"{STUB_METHOD_ADDRESS}@v0.1.0"+_OTHER_DIGEST = "sha256:" + "0" * 64+_SECURITY_EVENT_LOGGER = "pipelex.errors.security_event"+++def _lock_naming(*, address: str, digest: str) -> str:+ """A `methods.lock` holding one entry, for `address` at the probe's version with the tree digest `digest`."""+ return f'["{address}"]\nversion = "1.0.0"\ndigest = "{digest}"\nsource = "https://github.com/invented/probe-lib.git"\n'+++def _mock_probe_fetch(*, mocker: MockerFixture, tmp_path: Path) -> tuple[MockType, str]:+ """Serve a fetch of the probe package from a local copy; return the mock and the copy's tree digest."""+ package_dir = tmp_path / "fetched-probe"+ package_dir.mkdir(parents=True)+ (package_dir / "METHODS.toml").write_text(VENDORED_PROBE_MANIFEST, encoding="utf-8")+ (package_dir / "probe.mthds").write_text(VENDORED_PROBE_MTHDS, encoding="utf-8")+ fetched = FetchedMethodPackage(+ ref=parse_method_ref(VENDORED_PROBE_ADDRESS),+ full_address=VENDORED_PROBE_ADDRESS,+ commit_sha="c" * 40,+ clone_dir=tmp_path,+ package_dir=package_dir,+ manifest=parse_methods_toml(VENDORED_PROBE_MANIFEST),+ )+ fetch_mock = mocker.patch("pipelex.methods.fetch_on_miss.fetch_method_package", return_value=fetched)+ return fetch_mock, list_method_tree_at_root(package_dir).digest+++def _build_client(mocker: MockerFixture, *, tmp_path: Path, as_operator: bool = False) -> TestClient:+ """The routes behind the production handlers, with fetch-on-miss on; `as_operator` lets each request read `diagnostics`."""+ mocker.patch("pipelex.cli.installed_methods.GLOBAL_METHODS_DIR", tmp_path / "global-methods")+ mocker.patch("pipelex.cli.installed_methods.PROJECT_METHODS_DIR", tmp_path / "project-methods")+ mocker.patch("pipelex.methods.fetch_on_miss.is_method_fetch_on_miss_enabled", return_value=True)+ mocker.patch(+ f"{_PIPELINE_NS}.get_api_config", return_value=ApiConfig(orchestration_mode="direct", allow_request_orchestration_mode_override=False)+ )+ app = FastAPI()+ app.include_router(api_router, prefix="/v1", dependencies=[Depends(mark_operator_request)] if as_operator else [])+ register_exception_handlers(app)+ return TestClient(app)+++def _execute_files(client: TestClient, *, lock: str | None) -> Any:+ files = {"bundle.mthds": UNRESOLVED_PACKAGE_MTHDS}+ if lock is not None:+ files[LOCK_FILENAME] = lock+ return client.post("/v1/execute", json={"files": files, "inputs": {"text": "hello"}})+++class TestFetchedDependencyLock:+ @pytest.mark.parametrize("locks_the_fetched_tree", [True, False])+ def test_a_sent_tree_s_lock_checks_the_dependency_fetched_for_it(+ self, mocker: MockerFixture, tmp_path: Path, caplog: pytest.LogCaptureFixture, locks_the_fetched_tree: bool+ ):+ fetch_mock, digest = _mock_probe_fetch(mocker=mocker, tmp_path=tmp_path)+ client = _build_client(mocker, tmp_path=tmp_path, as_operator=True)+ locked_digest = digest if locks_the_fetched_tree else _OTHER_DIGEST++ with caplog.at_level(logging.WARNING, logger=_SECURITY_EVENT_LOGGER):+ response = _execute_files(client, lock=_lock_naming(address=VENDORED_PROBE_ADDRESS, digest=locked_digest))++ assert fetch_mock.call_count == 1+ events = [vars(record) for record in caplog.records if record.getMessage() == SECURITY_EVENT_MESSAGE]+ if locks_the_fetched_tree:+ assert response.status_code == 200, response.text+ assert f"{VENDORED_PROBE_MARKER}: hello" in json.dumps(response.json()["pipe_output"]["working_memory"])+ assert events == []+ return+ body = assert_problem(response, status=422, code=CoreCode.SECURITY_INTEGRITY_MISMATCH)+ assert body["domain"] == "method"+ assert body["params"] == {"dependency": VENDORED_PROBE_ADDRESS}+ # The operator reads the digest the lock records and the digest of the tree that was fetched.+ message = body["diagnostics"]["message"]+ assert _OTHER_DIGEST in message+ assert digest in message+ assert str(tmp_path) not in json.dumps(body)+ # The refusal is a security rule's, so the deployment logs its security event, once, naming the kind of target.+ (event,) = events+ assert event["error.code"] == CoreCode.SECURITY_INTEGRITY_MISMATCH+ assert event["security.target"] == "dependency"++ @pytest.mark.parametrize("locks_the_fetched_tree", [True, False])+ def test_validation_checks_the_dependency_fetched_for_a_sent_tree(self, mocker: MockerFixture, tmp_path: Path, locks_the_fetched_tree: bool):+ """A validation loads what a run loads, so a lock refusal is no verdict: the security refusal answers, as on a run."""+ _fetch_mock, digest = _mock_probe_fetch(mocker=mocker, tmp_path=tmp_path)+ client = _build_client(mocker, tmp_path=tmp_path)+ files = {+ "bundle.mthds": UNRESOLVED_PACKAGE_MTHDS,+ LOCK_FILENAME: _lock_naming(address=VENDORED_PROBE_ADDRESS, digest=digest if locks_the_fetched_tree else _OTHER_DIGEST),+ }++ response = client.post("/v1/validate", json={"files": files})++ if locks_the_fetched_tree:+ assert response.status_code == 200, response.text+ assert response.json()["is_valid"] is True+ return+ body = assert_problem(response, status=422, code=CoreCode.SECURITY_INTEGRITY_MISMATCH)+ assert body["params"] == {"dependency": VENDORED_PROBE_ADDRESS}++ def test_a_sent_tree_without_a_lock_runs_the_fetched_dependency(self, mocker: MockerFixture, tmp_path: Path):+ _mock_probe_fetch(mocker=mocker, tmp_path=tmp_path)+ client = _build_client(mocker, tmp_path=tmp_path)++ response = _execute_files(client, lock=None)++ assert response.status_code == 200, response.text++ def test_a_method_ref_package_s_lock_checks_the_dependency_fetched_for_it(+ self, mocker: MockerFixture, tmp_path: Path, install_method_package: Callable[..., Path]+ ):+ _mock_probe_fetch(mocker=mocker, tmp_path=tmp_path)+ install_method_package(+ files={"smoke.mthds": UNRESOLVED_PACKAGE_MTHDS, LOCK_FILENAME: _lock_naming(address=VENDORED_PROBE_ADDRESS, digest=_OTHER_DIGEST)}+ )+ client = _build_client(mocker, tmp_path=tmp_path)++ response = client.post("/v1/execute", json={"method_ref": _METHOD_REF, "inputs": {"text": "hello"}})++ body = assert_problem(response, status=422, code=CoreCode.SECURITY_INTEGRITY_MISMATCH)+ assert body["params"] == {"dependency": VENDORED_PROBE_ADDRESS}+ assert "diagnostics" not in body++ def test_a_lock_the_runner_cannot_read_refuses_the_fetched_dependency(self, mocker: MockerFixture, tmp_path: Path):+ """A lock written before tree digests records a `hash`: the method's own fault, refused as an invalid method."""+ _mock_probe_fetch(mocker=mocker, tmp_path=tmp_path)+ client = _build_client(mocker, tmp_path=tmp_path)++ response = _execute_files(client, lock=f'["{VENDORED_PROBE_ADDRESS}"]\nversion = "1.0.0"\nhash = "sha256:{"c" * 64}"\n')++ body = assert_problem(response, status=422, code=CoreCode.METHOD_INVALID)+ assert [(item["code"], item["params"]) for item in body["errors"]] == [+ (CoreCode.METHOD_PACKAGE_INVALID, {"reason": "lock_invalid", "path": LOCK_FILENAME})+ ], bodydiff --git a/docs/api-server/error-responses.md b/docs/api-server/error-responses.mdindex a52aeab8a..61129959a 100644--- a/docs/api-server/error-responses.md+++ b/docs/api-server/error-responses.md@@ -232,7 +232,7 @@ A few runtime refusals worth knowing by their code:- **404** — `method.not_found`: the `method_ref` repository was fetched, but no package in it matches the requested address by manifest identity, or it holds no such tag. The problem's sentences name the reference.- **409** — `pipelex.run.already_exists`: the submitted `pipeline_run_id` is already registered for a run that is still in flight on this server. Completed and failed runs free their id, so this only fires for genuinely concurrent duplicates — resubmit after the in-flight run finishes, or pick a fresh id. Only `POST /v1/start` accepts a client-supplied `pipeline_run_id`, so only `/start` can produce it.-- **422** — the `method_ref` resolution refusals the caller can fix: `request.invalid` for a reference that does not parse, `method.ambiguous`, and `method.package_invalid` for a package past its limits or carrying a symbolic link. `pipelex.deployment.python_structures_refused` is a **422** too: a method, fetched by `method_ref` or sent as a tree, declares in-process Python structure classes (`StructuredContent` subclasses), which hosted execution always refuses, sandbox or not — express the types as MTHDS concepts with inline structures instead (see [Pipe Run](pipe-run.md#running-a-method-by-address-method_ref)).+- **422** — the `method_ref` resolution refusals the caller can fix: `request.invalid` for a reference that does not parse, `method.ambiguous`, and `method.package_invalid` for a package past its limits or carrying a symbolic link. `security.integrity_mismatch` is a **422** too: a package the method calls by address, which the server fetched, differs from the tree digest the method's `methods.lock` records for it, and its `dependency` names the address (see [Pipe Run](pipe-run.md#sending-the-method-packages-the-method-calls)). `pipelex.deployment.python_structures_refused` is a **422** too: a method, fetched by `method_ref` or sent as a tree, declares in-process Python structure classes (`StructuredContent` subclasses), which hosted execution always refuses, sandbox or not — express the types as MTHDS concepts with inline structures instead (see [Pipe Run](pipe-run.md#running-a-method-by-address-method_ref)).The runner's own refusals answer these codes:@@ -256,7 +256,7 @@ A pipe selection the server cannot make is an input **422** named for its failurThe run routes' extension fields are validated before any method is fetched or any library loaded, and a failure there is a **422** `request.invalid` naming the field that failed, `callback_urls`, `storage_scope`, `read_scope` or `analytics_groups`, or a path into it (see [Pipe Run](pipe-run.md#host-supplied-run-context-storage_scope-and-analytics_groups)). A storage scope that does not lie under the run's read scope, whether either was sent or defaulted, names `read_scope`. A body failing on several fields at once carries one item per refused field in `errors`. A callback URL whose scheme is not `http` or `https`, or whose host the SSRF rules refuse, is the security refusal `security.input_url_refused`, which logs its security event and names the class of destination it refused, never the URL. `POST /v1/validate` takes `analytics_groups` too and refuses it the same way.-A method sent as a tree, in `files` or `bundle_b64`, is read under the MTHDS standard's tree rule before anything loads, on the run routes and `/validate` alike (see [Pipe Run](pipe-run.md#sending-a-method-as-a-tree)). A key or an entry the rule refuses, two shipped packages a reference could not tell apart, or a tree past `MAX_METHOD_TREE_FILES`, `MAX_METHOD_TREE_FILE_KIB` or `MAX_METHOD_TREE_TOTAL_KIB`, or a method past `MAX_METHOD_DEPENDENCIES` or `MAX_METHOD_REQUEST_TOTAL_KIB` across its trees, is a **422** `method.package_invalid` whose `reason` names the rule: the standard's tree refusal codes, such as `invalid_tree_path`, `duplicate_key`, `excluded_name` or `path_collision`, with the `path` it refuses, or the runner's own, one per rule and one per limit: `too_many_files`, `file_too_large` and `tree_too_large` for a tree, `too_many_dependencies` and `method_too_large` for a request's trees together, `bundle_too_many_files` and `bundle_too_large` for a `bundle_b64` archive, `manifest_missing`, `manifest_invalid`, `package_name_invalid`, `dependency_address_collision` and `no_bundle`, with the `path` or the `limit` it names. A key that is not a tree path without leaving the tree, empty or holding a backslash or a colon, is `invalid_tree_path`. A path of the tree that would leave its root is the security refusal `security.input_path_refused`, naming the `rule` it broke, `absolute` or `parent_segment`, never the path. A malformed entry, or `dependencies` that do not fit the request, is a **422** `request.invalid` naming the field. A hash entry this runner does not hold is a **422** `request.blobs_missing` whose `missing_blobs` parameter lists the SHA-256 hashes to upload through the blob routes before sending the request again, and one it holds at another size than the entry states a **422** `request.invalid`: the entry's size is the one to correct, since uploading the bytes again would not change the size they are held at. The blob routes answer in the same vocabulary: a body whose SHA-256 is not the one its path names is a **422** `request.invalid` naming the field `body`, one hash listed with two sizes a **422** `request.invalid` naming `blobs`, and a blob past the per-file limit a **413** `request.too_large`. None of them invites a retry, since the same request is refused again.+A method sent as a tree, in `files` or `bundle_b64`, is read under the MTHDS standard's tree rule before anything loads, on the run routes and `/validate` alike (see [Pipe Run](pipe-run.md#sending-a-method-as-a-tree)). A key or an entry the rule refuses, two shipped packages a reference could not tell apart, or a tree past `MAX_METHOD_TREE_FILES`, `MAX_METHOD_TREE_FILE_KIB` or `MAX_METHOD_TREE_TOTAL_KIB`, or a method past `MAX_METHOD_DEPENDENCIES` or `MAX_METHOD_REQUEST_TOTAL_KIB` across its trees, is a **422** `method.package_invalid` whose `reason` names the rule: the standard's tree refusal codes, such as `invalid_tree_path`, `duplicate_key`, `excluded_name` or `path_collision`, with the `path` it refuses, or the runner's own, one per rule and one per limit: `too_many_files`, `file_too_large` and `tree_too_large` for a tree, `too_many_dependencies` and `method_too_large` for a request's trees together, `bundle_too_many_files` and `bundle_too_large` for a `bundle_b64` archive, `manifest_missing`, `manifest_invalid`, `package_name_invalid`, `dependency_address_collision` and `no_bundle`, with the `path` or the `limit` it names. A key that is not a tree path without leaving the tree, empty or holding a backslash or a colon, is `invalid_tree_path`. A path of the tree that would leave its root is the security refusal `security.input_path_refused`, naming the `rule` it broke, `absolute` or `parent_segment`, never the path. A malformed entry, or `dependencies` that do not fit the request, is a **422** `request.invalid` naming the field. A hash entry this runner does not hold is a **422** `request.blobs_missing` whose `missing_blobs` parameter lists the SHA-256 hashes to upload through the blob routes before sending the request again, and one it holds at another size than the entry states a **422** `request.invalid`: the entry's size is the one to correct, since uploading the bytes again would not change the size they are held at. The blob routes answer in the same vocabulary: a body whose SHA-256 is not the one its path names is a **422** `request.invalid` naming the field `body`, one hash listed with two sizes a **422** `request.invalid` naming `blobs`, and a blob past the per-file limit a **413** `request.too_large`. None of them invites a retry, since the same request is refused again. A `methods.lock` at the tree's root is read only when a package the method calls by address resolves to a copy the runner fetched, for this request or an earlier one: a lock the runner cannot read, such as one written before tree digests that records a `hash`, then refuses the method with the **422** verdict `method.invalid`, whose item is `method.package_invalid` with the reason `lock_invalid` and the `path` `methods.lock`, and a lock recording another tree digest for the fetched package's address is the **422** `security.integrity_mismatch`.The files a request sends in `resources` are checked before the bundle loads and again once it is parsed (see [Pipe Validate](pipe-validate.md#sending-the-files-the-steps-read-resources)). A key that would leave the bundle is the security refusal `security.input_path_refused`, naming the `rule` it broke; any other key that cannot name a file inside the bundle, two keys naming the same file, more files than `MAX_RESOURCES_PER_REQUEST`, a file none of the bundle's steps names, and files that cannot make one method tree, such as two keys equal once case-folded, are **422** `request.invalid` naming the field `resources`; and a value that is not base64 is a **422** `request.malformed` with the reason `bad_base64`. They answer the same on every route, the diagnostic ones included: they are the request's fault, not the bundle's.diff --git a/docs/api-server/pipe-run.md b/docs/api-server/pipe-run.mdindex abd621ac6..cbda13d8f 100644--- a/docs/api-server/pipe-run.md+++ b/docs/api-server/pipe-run.md@@ -306,6 +306,8 @@ Two shipped packages a reference could not tell apart, whose manifests declare tThe validation of a method sent as a tree, or fetched by `method_ref`, resolves the packages it calls as the run does, its shipped packages first, then the methods installed on the server and a fetch on a miss, so `/validate` resolves a dependency the run would resolve (see [Validating a method tree](pipe-validate.md#validating-a-method-tree)). Both write the method to disk and load it in worker threads of the bounded method-load pool, off the server's event loop, so a large tree or a slow load never holds up the server's other requests.+**A package the server fetches is checked against the method's `methods.lock`.** When the method, sent as `files` or in a `bundle_b64` archive or fetched by `method_ref`, carries a `methods.lock` at the root of its tree, a package it calls by address that the server fetches on a miss, or finds installed by an earlier fetch of its own, must have the tree digest the lock records for that address. A different digest refuses the run with a `422` `security.integrity_mismatch`, whose `dependency` names the address, and the operator's `diagnostics` name the digest and version the lock records and the version and digest that were fetched; `/validate` answers the same `422`, as a security refusal is never a verdict, and the deployment logs the refusal's security event. A lock naming another version than the one fetched refuses the run as well, since the version is part of the package's tree, and the server never fetches the locked version in place of the reference. A lock that does not name the address, or no lock, checks nothing, and a package shipped with the request is not checked, as it travels with the lock. The lock is read only when a dependency the method calls resolves to a copy the server fetched, for this request or an earlier one. A lock the server then cannot read, such as one written before tree digests that records a `hash`, refuses the run with a `422` verdict whose item is `method.package_invalid` with the reason `lock_invalid` and the `path` `methods.lock`. See [Include by Address: Fetch-on-Miss](../building-methods/packages.md#include-by-address-fetch-on-miss).+---## Running a method by address (`method_ref`)@@ -339,6 +341,7 @@ Both `/execute` and `/start` accept **`method_ref`** — a globally resolvable a| More than one package matches | 422 | `method.ambiguous` || Package exceeds the fetched-package ceilings, or contains a symlink (file or directory) | 422 | `method.package_invalid` || Package declares Python structure classes | 422 | `pipelex.deployment.python_structures_refused` |+| A package it calls, fetched by the server, differs from the tree digest its `methods.lock` records | 422 | `security.integrity_mismatch` || Package ships `.py` on a non-sandbox deployment | 501 | `pipelex.deployment.custom_code_requires_sandbox` |**The clone cache.** Fetched repositories are cached on local disk **keyed by resolved commit SHA** — never by tag. Each request first resolves the reference to its current SHA with a cheap `git ls-remote` (no clone); a cached SHA is reused, a new one (a moved tag, an advanced HEAD) triggers a fresh clone. The cache is per server instance and bounded by count, total bytes, and age, oldest first; a clone a request is still reading is never evicted, though it counts toward the bounds. That protection holds within one process, so a deployment running several worker processes gives each its own `METHOD_CACHE_DIR`.diff --git a/docs/building-methods/packages.md b/docs/building-methods/packages.mdindex 9eeaef5cc..081fed068 100644--- a/docs/building-methods/packages.md+++ b/docs/building-methods/packages.md@@ -135,6 +135,7 @@ An address-based reference resolves against the **installed methods** — `~/.mtA few behaviors worth knowing:- **Installed wins over a fetch.** Once a method is installed, it is used as-is — including when a reference pins a different `@<tag>` than what is installed (a warning tells you so). Remove the installed directory to re-fetch.+- **A fetched package is checked against the calling method's `methods.lock`.** When the method calling a package by address carries a `methods.lock` at the root of its tree, beside its `METHODS.toml`, and Pipelex resolves that package from a copy it fetched itself, whether for this load or by an earlier fetch whose copy carries a `.provenance.json` sidecar, the tree digest of that copy is compared with the digest the lock records for the package's address (matched without regard to case, any `@<tag>` stripped). A different digest refuses the load with `security.integrity_mismatch`, whose message names the address, the digest and version the lock records, and the version and digest that were fetched. The digest is compared whatever version the lock names: the version is part of the tree, in `METHODS.toml`, so a lock recording another version than the one fetched refuses the load too, and the message says to pin the reference to the locked version or to lock the method again. When the reference pins a `@<tag>` but an installed copy fetched at another tag or at the default branch answered it (installed wins, above), the message says so instead, naming both, and says to remove that installed copy so the pinned tag is fetched. A bundle loaded in a later batch of the same library, such as the contents a run submits beside its library directories, is checked against the tree an earlier batch fetched and loaded for the same reference, since it runs that tree. Pipelex never fetches the locked version in place of the reference, since a lock verifies what was fetched and never chooses it. A lock that does not name the address, or no lock at all, checks nothing. A package shipped with the method (under `.mthds/methods/<name>/` or in a request's `dependencies`) is not checked, as it travels with the lock from the same author, and neither is a copy another installer wrote without a provenance sidecar, such as `mthds install`, which checks the lock itself. The lock is read only when a dependency the method calls resolves to a copy Pipelex fetched, for this load or an earlier one; a load whose dependencies are all shipped, or installed otherwise, never reads it. A lock Pipelex then cannot read, including one written before tree digests that records a `hash`, one whose bytes cannot be read, and one that changed since the method's tree was listed, refuses the load with `method.package_invalid`, reason `lock_invalid`, path `methods.lock`.- **A package can travel with the method.** A method sent over HTTP to a Pipelex API server as `files` can carry the packages it calls in `dependencies`, each a map of file entries keyed by its address, and one sent as a `bundle_b64` archive under `.mthds/methods/<name>/` at its root, each with its `METHODS.toml`. The server searches them before its installed copies and before fetch-on-miss, for that request alone, so a shipped package wins over an installed copy and runs where fetching is disabled; see [Sending the method packages the method calls](../api-server/pipe-run.md#sending-the-method-packages-the-method-calls). On disk, the `.mthds/methods/<name>/` layout beside the bundle is one of the local stores above.- **A package bundle that does not parse refuses the load.** When a bundle of a referenced package does not parse, the bundle calling it is refused with the parser's items, each naming the file as `<address>/<path inside the package>`, never by its path on the machine, whether the package was shipped with the bundle, installed or fetched.- **A miss that cannot be bridged refuses the bundle, never a silent pass.** Fetch disabled, an address this runtime cannot fetch, or a failed fetch (a repository that does not exist, a `@tag` that is not a tag, a repository holding no package at that address) refuses the load with one `unresolved_package_dependency` item per package, located on the first pipe that calls it, with the reference as written in `missing_pipe_code` and the reason and the remedy in its message. It is the same invalid verdict as any other mistake in the bundle: `pipelex validate` reports it, `POST /v1/validate` answers `200` with `is_valid: false`, and a run is refused with a `422` before any pipe runs. Every package of the bundle is tried before the refusal, so two bad references give two items.diff --git a/pipelex/libraries/library_manager.py b/pipelex/libraries/library_manager.pyindex 56267e59f..360876b84 100644--- a/pipelex/libraries/library_manager.py+++ b/pipelex/libraries/library_manager.py@@ -73,7 +73,8 @@ from pipelex.libraries.visibility_utils import make_visibility_checkerfrom pipelex.libraries.written_trees import WRITTEN_METHOD_TREESfrom pipelex.methods.address_dependencies import AddressDependencySources, unshipped_address_dependency_messagefrom pipelex.methods.exceptions import MethodDependencyFetchError, MethodDependencyNotShippedError, MethodFetchDisabledError-from pipelex.methods.fetch_on_miss import find_vendored_method, resolve_address_based_method+from pipelex.methods.fetch_on_miss import find_vendored_method, resolve_address_based_method, split_address_reference+from pipelex.methods.method_lock import FetchedCopy, FetchedTree, ensure_fetched_tree_lockedfrom pipelex.methods.structures_check import ensure_no_structured_content_in_library_sourcesfrom pipelex.mthds_parsing.exceptions import MthdsParserErrorfrom pipelex.mthds_parsing.handle_pipe_errors import categorize_pipe_validation_error@@ -379,6 +380,9 @@ class LibraryManager(LibraryManagerAbstract):# library_id -> the files its library directories submit, joined only when a bundle parsed from a content# string loads: a library that loads none never has two directories' files judged as one tree.self._library_dir_submissions: dict[str, list[dict[str, HeldFile]]] = {}+ # library_id -> alias -> the tree loaded for a dependency from a copy this runtime fetched, against which a later+ # load batch calling the same alias, which runs that tree, checks the locks of its own callers.+ self._fetched_dependency_trees: dict[str, dict[str, FetchedTree]] = {}############################################################# Manager lifecycle@@ -417,6 +421,7 @@ class LibraryManager(LibraryManagerAbstract):self._submitted_files.pop(library_id, None)self._submitted_trees.pop(library_id, None)self._library_dir_submissions.pop(library_id, None)+ self._fetched_dependency_trees.pop(library_id, None)# The library's runs are over: its hold on a method tree's Python, if a PipeFunc step took one, is released,# and so are the trees it held for its runs to read.TREE_PYTHON_HOLD.release(holder=library_id)@@ -445,6 +450,7 @@ class LibraryManager(LibraryManagerAbstract):self._submitted_files = {}self._submitted_trees = {}self._library_dir_submissions = {}+ self._fetched_dependency_trees = {}TREE_PYTHON_HOLD.release_all()LOCAL_METHOD_FILES.release_all()with ExitStack() as teardown_stack:@@ -1602,6 +1608,8 @@ class LibraryManager(LibraryManagerAbstract):resolved_dep: ResolvedDependency,package_address: str,tree_limits: TreeLimits | None = None,+ fetched_copy: FetchedCopy | None = None,+ caller_trees: tuple[MethodTree, ...] = (),) -> None:"""Load a single resolved dependency into an isolated child library.@@ -1624,11 +1632,18 @@ class LibraryManager(LibraryManagerAbstract):resolved_dep: The resolved dependency infopackage_address: The dependency's address as a reference names it, without any ``@<tag>``tree_limits: The limits the package's tree is listed under, read from the configuration when not given+ fetched_copy: When the package is a copy this runtime fetched, that copy, whose listed tree is checked against+ the locks of ``caller_trees`` before any of its files is read, and recorded for a later load batch calling+ it; None for a package shipped or installed otherwise, which no lock is checked against here+ caller_trees: The trees of the bundles calling the package, whose locks a fetched copy is checked againstRaises:MthdsParserError: A bundle of the dependency does not parse, its items named by the bundle's source.MethodRequirementsError: The package's requirements.txt holds a line the standard refuses, or namespackages its PipeFunc environment does not hold, named by the package's address.+ MethodLockMismatchError: The package is a fetched copy whose tree digest differs from the one a calling+ method's lock records for its address.+ MethodLockInvalidError: A calling method's lock, needed to check the fetched copy, cannot be read."""alias = resolved_dep.alias@@ -1639,6 +1654,12 @@ class LibraryManager(LibraryManagerAbstract):tree_bundles_by_source: dict[str, TreeBundle] = {}# Listed under the package's address: a refusal never names the directory it was installed or fetched into.package_tree = list_package_method_tree(root=resolved_dep.package_root, package_address=package_address, limits=tree_limits)+ fetched_tree: FetchedTree | None = None+ if fetched_copy is not None:+ # The tree this load lists, and so loads, is the one checked: a fetched copy that is not the tree its callers'+ # locks record is refused before any of its files is read.+ fetched_tree = FetchedTree(copy=fetched_copy, tree_digest=package_tree.digest)+ ensure_fetched_tree_locked(fetched_tree=fetched_tree, caller_trees=caller_trees)# The package's own requirements.txt is judged before any of its bundles loads, as the consumer's tree is.ensure_package_requirements_met(tree=package_tree, package_address=package_address)# Held for the consumer's runs, which read a file of the package's tree, such as one of its steps' template@@ -1799,6 +1820,9 @@ class LibraryManager(LibraryManagerAbstract):# Register child library for isolationlibrary.dependency_libraries[alias] = child_library+ if fetched_tree is not None:+ # A later load batch calling this alias runs this tree, so the locks of its callers are checked against it.+ self._fetched_dependency_trees.setdefault(library_id, {})[alias] = fetched_tree# Add aliased entries to main library for backward-compatible cross-package lookupsfor concept in dep_concepts:@@ -1840,8 +1864,11 @@ class LibraryManager(LibraryManagerAbstract):A package that cannot be resolved is the caller's to fix, since the reference is part of their bundle: everyaddress is tried, then the load is refused with one ``unresolved_package_dependency`` item per address that- failed, located on the first pipe naming it. A failure of the host's own store (``MethodInstallError``) and the- refusal of a package's Python structures (``MethodStructuresRefusedError``) propagate as they are.+ failed, located on the first pipe naming it. A failure of the host's own store (``MethodInstallError``), the+ refusal of a package's Python structures (``MethodStructuresRefusedError``) and the refusal of a fetched copy+ its callers' locks do not record (``MethodLockMismatchError``, ``MethodLockInvalidError``) propagate as they are. An+ address an earlier batch of the library already loaded is not loaded again, but when it was loaded from a copy+ this runtime fetched, the locks of this batch's callers are checked against the tree it loaded, which they run.Also searches for .mthds/methods/ directories relative to the bundlesource path, walking up ancestor directories to find installed methods@@ -1860,12 +1887,18 @@ class LibraryManager(LibraryManagerAbstract):# Each address-based alias, with the first reference naming it and the bundle that makes it, in bundle order.first_references: dict[str, tuple[LocatedPipeReference, PipelexBundleBlueprint]] = {}+ # The trees of the bundles calling each alias, whose locks a copy of it this runtime fetched is checked against.+ caller_trees: dict[str, list[MethodTree]] = {}for blueprint in blueprints:for reference in blueprint.collect_located_pipe_references():- if QualifiedRef.has_cross_package_prefix(reference.pipe_ref):- alias, _remainder = QualifiedRef.split_cross_package_ref(reference.pipe_ref)- if QualifiedRef.is_address_based_alias(alias) and alias not in first_references:- first_references[alias] = (reference, blueprint)+ if not QualifiedRef.has_cross_package_prefix(reference.pipe_ref):+ continue+ alias, _remainder = QualifiedRef.split_cross_package_ref(reference.pipe_ref)+ if not QualifiedRef.is_address_based_alias(alias):+ continue+ first_references.setdefault(alias, (reference, blueprint))+ if blueprint.tree_bundle is not None:+ caller_trees.setdefault(alias, []).append(blueprint.tree_bundle.tree)if not first_references:return@@ -1881,6 +1914,11 @@ class LibraryManager(LibraryManagerAbstract):unresolved_items: list[PipesAndConceptValidationErrorData] = []for full_address, (reference, blueprint) in first_references.items():if full_address in library.dependency_libraries:+ # An earlier batch of this library loaded the package, and this batch runs the tree it loaded: when that+ # tree is a copy this runtime fetched, the locks of this batch's callers are checked against it too.+ loaded_fetched_tree = self._fetched_dependency_trees.get(library_id, {}).get(full_address)+ if loaded_fetched_tree is not None:+ ensure_fetched_tree_locked(fetched_tree=loaded_fetched_tree, caller_trees=caller_trees.get(full_address, []))continuetry:self._load_address_based_dependency(@@ -1890,6 +1928,7 @@ class LibraryManager(LibraryManagerAbstract):extra_search_dirs=extra_search_dirs,methods_dirs=methods_dirs,address_dependency_sources=address_dependency_sources,+ caller_trees=tuple(caller_trees.get(full_address, [])),)except (MethodFetchDisabledError, MethodDependencyFetchError, MethodDependencyNotShippedError) as exc:# The item carries the sentence its code renders; the host's log keeps the whole cause chain, git's raw output@@ -1926,6 +1965,7 @@ class LibraryManager(LibraryManagerAbstract):extra_search_dirs: list[Path] | None = None,methods_dirs: list[Path] | None = None,address_dependency_sources: AddressDependencySources = AddressDependencySources.ANYWHERE,+ caller_trees: tuple[MethodTree, ...] = (),) -> None:"""Load a method package on demand using its full address, fetching it on a miss.@@ -1943,12 +1983,17 @@ class LibraryManager(LibraryManagerAbstract):methods_dirs: The load's own methods directories, searched before everything else and never writtenaddress_dependency_sources: ``SHIPPED`` refuses an address ``methods_dirs`` do not answer, reading nothinginstalled on this host and fetching nothing; ``ANYWHERE`` falls back to the installed methods and a fetch+ caller_trees: The trees of the bundles calling the package, whose locks a copy this runtime fetched, for+ this load or an earlier one, is checked againstRaises:MethodRefError: The address resolves to no vendored or installed method and could not befetched — fetch-on-miss disabled, an unfetchable address, a failed fetch,or a failed install. Never a silent pass.MethodDependencyNotShippedError: Under ``SHIPPED``, no package of ``methods_dirs`` answers the address.+ MethodLockMismatchError: The package is a copy this runtime fetched, and a calling method's lock records+ another tree digest for its address.+ MethodLockInvalidError: A calling method's lock, needed to check a fetched copy, cannot be read."""vendored = find_vendored_method(full_address=full_address, methods_dirs=methods_dirs) if methods_dirs else Nonematch address_dependency_sources:@@ -1961,6 +2006,20 @@ class LibraryManager(LibraryManagerAbstract):exported_pipe_codes = determine_exported_pipes(manifest=installed.manifest)+ # A copy this runtime fetched, for this load or for an earlier one whose install recorded the fetch's provenance, is+ # checked against the locks of the methods calling it. A shipped copy travels with the lock, from the same author,+ # and a copy another installer wrote, such as `mthds install`, which checks the lock itself, is not this runtime's+ # fetch: neither is checked here.+ fetched_copy: FetchedCopy | None = None+ if vendored is None and installed.provenance is not None:+ reference = split_address_reference(full_address=full_address)+ fetched_copy = FetchedCopy(+ address=reference.address,+ version=installed.manifest.version,+ pinned_tag=reference.tag,+ fetched_tag=installed.provenance.tag,+ )+resolved_dep = ResolvedDependency(alias=full_address,address=installed.manifest.address,@@ -1976,6 +2035,8 @@ class LibraryManager(LibraryManagerAbstract):resolved_dep=resolved_dep,# The address the lookup matched (the manifest's address and the method's name), never a tag.package_address=f"{installed.manifest.address}/{installed.name}",+ fetched_copy=fetched_copy,+ caller_trees=caller_trees,)def _remove_pipes_from_blueprint(self, blueprint: PipelexBundleBlueprint) -> None:diff --git a/pipelex/methods/exceptions.py b/pipelex/methods/exceptions.pyindex 77f9c553f..4daac2d33 100644--- a/pipelex/methods/exceptions.py+++ b/pipelex/methods/exceptions.py@@ -1,5 +1,6 @@from typing import Any, ClassVar, Self+from mthds.package.lock_file import LOCK_FILENAMEfrom mthds.protocol.problem.registry import CoreCodefrom typing_extensions import override@@ -119,3 +120,19 @@ class MethodDependencyNotShippedError(MethodRefError):"""An address-referenced method the caller did not ship, under a resolution that reads nothing installed and fetches nothing."""error_code = CoreCode.METHOD_DEPENDENCY_MISSING+++class MethodLockInvalidError(MethodRefError):+ """The `methods.lock` of a method calling a dependency the runtime fetched cannot be read, so it cannot verify the dependency."""++ error_code = CoreCode.METHOD_PACKAGE_INVALID++ @override+ def problem_params(self) -> dict[str, Any]:+ return {"reason": MethodPackageReason.LOCK_INVALID, "path": LOCK_FILENAME}+++class MethodLockMismatchError(MethodRefError):+ """A dependency the runtime fetched holds a tree whose digest is not the one the calling method's `methods.lock` records."""++ error_code = CoreCode.SECURITY_INTEGRITY_MISMATCHdiff --git a/pipelex/methods/fetch_on_miss.py b/pipelex/methods/fetch_on_miss.pyindex c82840ead..90bd56226 100644--- a/pipelex/methods/fetch_on_miss.py+++ b/pipelex/methods/fetch_on_miss.py@@ -61,6 +61,21 @@ def _lookup_address(*, full_address: str) -> _LookupAddress:return _LookupAddress(address=ref.address, ref=ref, parse_error=None)+class AddressReference(NamedTuple):+ """An address-based reference split into the address it names and the tag it pins."""++ address: str+ tag: str | None+++def split_address_reference(*, full_address: str) -> AddressReference:+ """Split a reference into its address, any ``@<tag>`` stripped, which is how the stores and a method's lock key a+ package, and the tag it pins, None when it pins none or does not parse.+ """+ lookup = _lookup_address(full_address=full_address)+ return AddressReference(address=lookup.address, tag=lookup.ref.tag if lookup.ref is not None else None)++def find_vendored_method(*, full_address: str, methods_dirs: list[Path]) -> InstalledMethod | None:"""Look an address-based reference up among the packages a load was handed, and nowhere else.diff --git a/pipelex/methods/method_lock.py b/pipelex/methods/method_lock.pynew file mode 100644index 000000000..efd0c2d6e--- /dev/null+++ b/pipelex/methods/method_lock.py@@ -0,0 +1,167 @@+"""Check a dependency the runtime fetched against the `methods.lock` of the method that calls it.++A method's `methods.lock` (`mthds/docs/spec/lock-format.md`) records, for each remote dependency, the version it was+locked at and the tree digest of the package's tree (`mthds/docs/spec/method-tree.md`, "The Tree Digest"). It lies at the+root of the method's tree, beside `METHODS.toml`, and it is a file of that tree, so it travels with the method wherever+the method goes: in a tree a caller sends, in a package fetched by `method_ref`, and in a method's directory on disk.++When a load resolves a package a bundle calls by address to a copy the runtime fetched, whether it fetched it for this+load or an earlier fetch installed it (the copy carries the fetch's provenance), the tree the load lists for that package+is checked against the lock at the root of every tree whose bundles call it. A later batch of the same library calling a+package an earlier batch already loaded from such a copy is checked against the same tree, since it runs that tree. A+lock that does not name the package's address, or no lock at all, checks nothing; a lock naming it with another digest+refuses the load (`MethodLockMismatchError`, the core code `security.integrity_mismatch`), whatever version it names,+since the lock verifies what was fetched and never chooses it. A package the method ships with itself is not checked: it+travels with the lock, from the same author, so there is nothing fetched to verify.++The lock is read with the `mthds` package's own parser, and only when a dependency resolves to a copy the runtime+fetched, so a lock that cannot be read refuses only a load that needs it (`MethodLockInvalidError`,+`method.package_invalid` with the reason `lock_invalid`): a lock the tree lists whose bytes cannot be read, or changed+since the tree was listed, and a lock written before tree digests among them, as the standard requires.+"""++from collections.abc import Iterable+from typing import NamedTuple++from mthds.method_tree.digest import sha256_hex+from mthds.method_tree.models import MethodTree+from mthds.package.exceptions import LockFileError+from mthds.package.lock_file import LOCK_FILENAME, LockedPackage, parse_lock_file++from pipelex.core.method_trees.method_file_store import LOCAL_METHOD_FILES+from pipelex.methods.exceptions import MethodLockInvalidError, MethodLockMismatchError+++class FetchedCopy(NamedTuple):+ """A copy of a dependency the runtime fetched, for this load or an earlier one, as the reference calling it found it.++ Attributes:+ address: The package's address as a lock keys it, without any ``@<tag>``.+ version: The version the copy's manifest declares.+ pinned_tag: The tag the reference pins, None for a reference pinning none.+ fetched_tag: The tag the copy was fetched at, as its provenance records it, None for a fetch of the default branch.+ """++ address: str+ version: str+ pinned_tag: str | None+ fetched_tag: str | None+++class FetchedTree(NamedTuple):+ """The tree a load listed, and so loaded, for a fetched copy of a dependency."""++ copy: FetchedCopy+ tree_digest: str+++def _unreadable_lock(*, address: str, reason: str) -> MethodLockInvalidError:+ return MethodLockInvalidError(f"The method's {LOCK_FILENAME} {reason}, so it cannot verify the dependency '{address}' this runtime fetched.")+++def _lock_bytes(*, tree: MethodTree, address: str) -> bytes | None:+ """The bytes of the lock at the root of `tree`, as the tree was listed, or None when the tree holds none.++ A tree listed from a directory is read there, and its bytes are checked against the hash the listing recorded; a tree+ of files held in memory, such as the files submitted beside a method's content strings, is read from the method file+ store, which holds it for the load and checks the same hash.++ Raises:+ MethodLockInvalidError: The tree lists a lock whose bytes cannot be read, or changed since the tree was listed.+ """+ lock_file = next((tree_file for tree_file in tree.files if tree_file.path == LOCK_FILENAME), None)+ if lock_file is None:+ return None+ if tree.directory is None:+ held_bytes = LOCAL_METHOD_FILES.read_held_file(tree_digest=tree.digest, tree_path=LOCK_FILENAME)+ if held_bytes is None:+ raise _unreadable_lock(address=address, reason="is listed in the method's tree but not held for this load, or changed since")+ return held_bytes+ try:+ lock_bytes = tree.file_location(LOCK_FILENAME).read_bytes()+ except OSError as exc:+ raise _unreadable_lock(address=address, reason="cannot be read") from exc+ if sha256_hex(lock_bytes) != lock_file.sha256:+ raise _unreadable_lock(address=address, reason="changed since the method's tree was listed")+ return lock_bytes+++def _locked_entries(*, tree: MethodTree, address: str) -> list[LockedPackage]:+ """The entries the lock at the root of `tree` records for `address`, compared without regard to case.++ Raises:+ MethodLockInvalidError: The lock cannot be read, is not UTF-8 text, does not parse, or holds an entry the lock+ format refuses.+ """+ lock_bytes = _lock_bytes(tree=tree, address=address)+ if lock_bytes is None:+ return []+ try:+ lock_text = lock_bytes.decode("utf-8")+ except UnicodeDecodeError as exc:+ raise _unreadable_lock(address=address, reason="is not UTF-8 text") from exc+ try:+ lock = parse_lock_file(lock_text)+ except LockFileError as exc:+ msg = f"The method's {LOCK_FILENAME} cannot verify the dependency '{address}' this runtime fetched: {exc.message}"+ raise MethodLockInvalidError(msg) from exc+ folded = address.casefold()+ return [locked for locked_address, locked in lock.packages.items() if locked_address.casefold() == folded]+++def _unique_trees(*, trees: Iterable[MethodTree]) -> list[MethodTree]:+ """The trees once each, by digest: two trees of one digest hold the same lock."""+ unique: dict[str, MethodTree] = {}+ for tree in trees:+ unique.setdefault(tree.digest, tree)+ return list(unique.values())+++def _tag_names_version(*, tag: str, version: str) -> bool:+ """Whether a tag names a version, written bare or with a leading ``v``, as a shipped copy's tag is matched."""+ return tag in {version, f"v{version}"}+++def _mismatch_message(*, fetched_tree: FetchedTree, locked: LockedPackage) -> str:+ """The operator's account of a fetched tree a lock does not record, with the remedy that fits how the copy was found."""+ copy = fetched_tree.copy+ msg = (+ f"The dependency '{copy.address}' differs from the method's {LOCK_FILENAME}: the lock records the tree digest {locked.digest} at "+ f"version {locked.version}, and this runtime fetched version {copy.version}, whose tree digest is {fetched_tree.tree_digest}."+ )+ if copy.pinned_tag is not None and copy.fetched_tag != copy.pinned_tag:+ # The reference pins a tag, but an installed copy an earlier fetch made at another tag answered it.+ fetched_at = f"the tag {copy.fetched_tag}" if copy.fetched_tag is not None else "the repository's default branch"+ msg += (+ f" The reference pins the tag {copy.pinned_tag}, but an installed copy of version {copy.version}, fetched earlier at "+ f"{fetched_at}, was used in place of the pinned version: remove that installed copy so the pinned tag is fetched."+ )+ if not _tag_names_version(tag=copy.pinned_tag, version=locked.version):+ msg += f" The pinned tag does not name the locked version {locked.version} either: pin the reference to it, or lock the method again."+ elif copy.version != locked.version:+ if copy.pinned_tag is None:+ msg += " The lock records another version than the one fetched"+ else:+ msg += f" The reference pins the tag {copy.pinned_tag}, another version than the lock records"+ msg += f": pin the reference to the locked version, or lock the method again to record version {copy.version}."+ return msg+++def ensure_fetched_tree_locked(*, fetched_tree: FetchedTree, caller_trees: Iterable[MethodTree]) -> None:+ """Refuse a fetched dependency's tree when a lock of a method calling it records another tree digest for its address.++ Args:+ fetched_tree: The tree the load listed for the copy the runtime fetched.+ caller_trees: The trees of the bundles calling the dependency, whose locks are checked.++ Raises:+ MethodLockInvalidError: A calling method's lock cannot be read.+ MethodLockMismatchError: A calling method's lock records another tree digest for the dependency's address.+ """+ address = fetched_tree.copy.address+ for caller_tree in _unique_trees(trees=caller_trees):+ for locked in _locked_entries(tree=caller_tree, address=address):+ if locked.digest == fetched_tree.tree_digest:+ continue+ msg = _mismatch_message(fetched_tree=fetched_tree, locked=locked)+ raise MethodLockMismatchError(msg).with_params(dependency=address)diff --git a/pipelex/methods/package_reason.py b/pipelex/methods/package_reason.pyindex a3c373c6a..a2d7207eb 100644--- a/pipelex/methods/package_reason.py+++ b/pipelex/methods/package_reason.py@@ -42,3 +42,5 @@ class MethodPackageReason(StrEnum):VERSION_INVALID = "version_invalid"# The method's tree cannot be listed: a path it names is missing, one of its files cannot be read, or its manifest does not parse.TREE_UNREADABLE = "tree_unreadable"+ # The method's `methods.lock` is not UTF-8 text, does not parse, or records an entry the lock format refuses.+ LOCK_INVALID = "lock_invalid"diff --git a/tests/data/errors/error_identity.txt b/tests/data/errors/error_identity.txtindex 6187c3a8a..a8d601568 100644--- a/tests/data/errors/error_identity.txt+++ b/tests/data/errors/error_identity.txt@@ -203,6 +203,8 @@ MethodFetchDisabledError | method.dependency_missingMethodFetchError | one of method.fetch_failed, pipelex.config.dependency_missing, method.not_found, method.package_invalidMethodFileUnavailableError | one of runtime.unexpected, method.resource_missingMethodInstallError | one of pipelex.config.invalid, method.package_invalid+MethodLockInvalidError | method.package_invalid+MethodLockMismatchError | security.integrity_mismatchMethodNotFoundError | method.not_foundMethodPackageAmbiguityError | method.ambiguousMethodPackageNotFoundError | method.not_founddiff --git a/tests/integration/pipelex/libraries/test_data.py b/tests/integration/pipelex/libraries/test_data.pyindex 41fcf9304..8e23ca9b6 100644--- a/tests/integration/pipelex/libraries/test_data.py+++ b/tests/integration/pipelex/libraries/test_data.py@@ -502,3 +502,60 @@ steps = [{ from = "filing.invoice.supplier.name", result = "supplier_name" },]"""+++class FetchedDependencyLockTestData:+ """A calling method, the package it calls by address and fetches on a miss, and the locks it could carry; nothing here is real."""++ DEP_ALIAS: ClassVar[str] = "github.com/pipelex-tests/fom-lock-methods/scoring"++ DEP_MANIFEST: ClassVar[str] = """[package]+name = "scoring"+address = "github.com/pipelex-tests/fom-lock-methods"+version = "0.1.0"+description = "An invented scoring package a locked method fetches"++[exports.fom_lock_scoring]+pipes = ["compute"]+"""++ DEP_BUNDLE: ClassVar[str] = """domain = "fom_lock_scoring"+description = "An invented scoring package"++[pipe.compute]+type = "PipeLLM"+description = "Compute a score"+inputs = { item = "Text" }+output = "Text"+prompt = "Score: $item"+"""++ CALLER_MANIFEST: ClassVar[str] = """[package]+address = "github.com/pipelex-tests/locked-caller"+version = "1.0.0"+description = "An invented method whose lock pins what it fetches"+"""++ @classmethod+ def caller_calling(cls, *, pipe_ref: str, domain: str = "fom_lock_caller") -> str:+ """The calling method's bundle, in `domain`, whose single step names `pipe_ref`."""+ return f"""domain = "{domain}"+description = "A method calling a fetched package"++[pipe.go]+type = "PipeSequence"+description = "Call the fetched package"+inputs = {{ item = "Text" }}+output = "Text"+steps = [{{ pipe = "{pipe_ref}", result = "score" }}]+"""++ @classmethod+ def lock_naming(cls, *, address: str, digest: str) -> str:+ """A `methods.lock` holding one entry, for `address` at the package's version with the tree digest `digest`."""+ return f'["{address}"]\nversion = "0.1.0"\ndigest = "{digest}"\nsource = "https://github.com/pipelex-tests/fom-lock-methods.git"\n'++ @classmethod+ def legacy_lock_naming(cls, *, address: str) -> str:+ """A `methods.lock` written before tree digests, its entry recording a `hash`."""+ return f'["{address}"]\nversion = "0.1.0"\nhash = "sha256:{"c" * 64}"\nsource = "https://github.com/pipelex-tests/fom-lock-methods.git"\n'diff --git a/tests/integration/pipelex/libraries/test_fetched_dependency_lock.py b/tests/integration/pipelex/libraries/test_fetched_dependency_lock.pynew file mode 100644index 000000000..c700c118b--- /dev/null+++ b/tests/integration/pipelex/libraries/test_fetched_dependency_lock.py@@ -0,0 +1,216 @@+"""A dependency the runtime fetched is checked against the lock of the method that calls it.++The calling method is a directory on disk, its `methods.lock` at its root beside its `METHODS.toml`. Its bundle calls a+package by address, which no store holds, so it is fetched on a miss, mocked to a local fixture package, and installed+into the (redirected) global methods directory. Nothing reaches the network.+"""++from __future__ import annotations++from typing import TYPE_CHECKING++import pytest+from mthds.method_tree.lister import list_method_tree_at_root+from mthds.package.lock_file import LOCK_FILENAME+from mthds.package.manifest.parser import parse_methods_toml+from mthds.protocol.problem.registry import CoreCode++from pipelex.cli.installed_methods import PROVENANCE_FILENAME+from pipelex.core.method_trees.file_references import BundleFileOrigin+from pipelex.errors.problem import ProblemView+from pipelex.interpreter_hub import get_library_manager+from pipelex.methods.exceptions import MethodLockInvalidError, MethodLockMismatchError+from pipelex.methods.fetching import FetchedMethodPackage+from pipelex.methods.method_ref import parse_method_ref+from pipelex.mthds_parsing.parser import MthdsParser+from tests.integration.pipelex.libraries.test_data import FetchedDependencyLockTestData++if TYPE_CHECKING:+ from pathlib import Path++ from pytest_mock import MockerFixture, MockType++DEP_ALIAS = FetchedDependencyLockTestData.DEP_ALIAS+OTHER_DIGEST = "sha256:" + "0" * 64++[email protected](name="global_methods_dir")+def global_methods_dir_fixture(tmp_path: Path, mocker: MockerFixture) -> Path:+ """Point the installed stores at empty temporary directories and enable fetch-on-miss."""+ global_dir = tmp_path / "global-methods"+ mocker.patch("pipelex.cli.installed_methods.GLOBAL_METHODS_DIR", global_dir)+ mocker.patch("pipelex.cli.installed_methods.PROJECT_METHODS_DIR", tmp_path / "project-methods")+ mocker.patch("pipelex.methods.fetch_on_miss.is_method_fetch_on_miss_enabled", return_value=True)+ return global_dir+++def _write_dependency(*, package_dir: Path) -> str:+ """Write the dependency package into `package_dir` and return its tree digest."""+ package_dir.mkdir(parents=True)+ (package_dir / "METHODS.toml").write_text(FetchedDependencyLockTestData.DEP_MANIFEST, encoding="utf-8")+ (package_dir / "scoring.mthds").write_text(FetchedDependencyLockTestData.DEP_BUNDLE, encoding="utf-8")+ return list_method_tree_at_root(package_dir).digest+++def _mock_fetch(*, tmp_path: Path, mocker: MockerFixture) -> tuple[MockType, str]:+ """Serve every fetch of the dependency from a local fixture package; return the mock and the fixture tree's digest."""+ package_dir = tmp_path / "fetched-dep"+ digest = _write_dependency(package_dir=package_dir)+ fetched = FetchedMethodPackage(+ ref=parse_method_ref(DEP_ALIAS),+ full_address=DEP_ALIAS,+ commit_sha="c" * 40,+ clone_dir=tmp_path,+ package_dir=package_dir,+ manifest=parse_methods_toml(FetchedDependencyLockTestData.DEP_MANIFEST),+ )+ return mocker.patch("pipelex.methods.fetch_on_miss.fetch_method_package", return_value=fetched), digest+++def _write_caller(*, method_dir: Path, lock: str | None, pipe_ref: str = f"{DEP_ALIAS}->fom_lock_scoring.compute") -> Path:+ """Write the calling method, with its lock at its root when one is given, and return its directory."""+ method_dir.mkdir(parents=True)+ (method_dir / "METHODS.toml").write_text(FetchedDependencyLockTestData.CALLER_MANIFEST, encoding="utf-8")+ (method_dir / "caller.mthds").write_text(FetchedDependencyLockTestData.caller_calling(pipe_ref=pipe_ref), encoding="utf-8")+ if lock is not None:+ (method_dir / LOCK_FILENAME).write_text(lock, encoding="utf-8")+ return method_dir+++def _load(*, method_dir: Path, methods_dirs: list[Path] | None = None) -> set[str]:+ """Load the calling method from its directory, as a run or a validation does, and return the dependency aliases loaded."""+ library_manager = get_library_manager()+ library_id, library = library_manager.open_library()+ try:+ library_manager.load_libraries(library_id=library_id, library_dirs=[method_dir], methods_dirs=methods_dirs)+ return set(library.dependency_libraries)+ finally:+ library_manager.teardown(library_id=library_id)+++class TestFetchedDependencyLock:+ @pytest.mark.usefixtures("global_methods_dir")+ @pytest.mark.parametrize("lock_kind", ["no lock", "lock naming another address", "lock recording the fetched tree"])+ def test_a_fetched_dependency_the_lock_does_not_contradict_loads(self, tmp_path: Path, mocker: MockerFixture, lock_kind: str) -> None:+ fetch_mock, digest = _mock_fetch(tmp_path=tmp_path, mocker=mocker)+ lock: str | None+ match lock_kind:+ case "no lock":+ lock = None+ case "lock naming another address":+ lock = FetchedDependencyLockTestData.lock_naming(address="github.com/invented/other-lib/ranking", digest=OTHER_DIGEST)+ case _:+ lock = FetchedDependencyLockTestData.lock_naming(address=DEP_ALIAS, digest=digest)+ method_dir = _write_caller(method_dir=tmp_path / "caller", lock=lock)++ assert _load(method_dir=method_dir) == {DEP_ALIAS}+ assert fetch_mock.call_count == 1++ @pytest.mark.usefixtures("global_methods_dir")+ @pytest.mark.parametrize("pipe_ref", [f"{DEP_ALIAS}->fom_lock_scoring.compute", f"{DEP_ALIAS}@v0.1.0->fom_lock_scoring.compute"])+ def test_a_fetched_dependency_whose_tree_the_lock_does_not_record_is_refused(self, tmp_path: Path, mocker: MockerFixture, pipe_ref: str) -> None:+ """A reference pinning a tag is looked up in the lock by its address alone, as the installed stores are."""+ _fetch_mock, digest = _mock_fetch(tmp_path=tmp_path, mocker=mocker)+ lock = FetchedDependencyLockTestData.lock_naming(address=DEP_ALIAS, digest=OTHER_DIGEST)+ method_dir = _write_caller(method_dir=tmp_path / "caller", lock=lock, pipe_ref=pipe_ref)++ with pytest.raises(MethodLockMismatchError) as exc_info:+ _load(method_dir=method_dir)++ problem = exc_info.value.to_problem(view=ProblemView.CALLER)+ assert problem.code == CoreCode.SECURITY_INTEGRITY_MISMATCH+ assert problem.params is not None+ assert problem.params.as_wire() == {"dependency": DEP_ALIAS}+ assert OTHER_DIGEST in exc_info.value.message+ assert digest in exc_info.value.message++ @pytest.mark.parametrize("pinned_tag", [None, "v0.1.0"])+ def test_the_copy_an_earlier_fetch_installed_is_checked_too(+ self, global_methods_dir: Path, tmp_path: Path, mocker: MockerFixture, pinned_tag: str | None+ ) -> None:+ """A first load with no lock installs the fetched copy; a second, locked to another tree, finds it installed and is refused.++ The first fetch pinned no tag, so a second reference pinning one is answered by that installed copy, which the refusal+ says, rather than telling the author to pin a reference that already pins.+ """+ fetch_mock, _digest = _mock_fetch(tmp_path=tmp_path, mocker=mocker)+ assert _load(method_dir=_write_caller(method_dir=tmp_path / "unlocked-caller", lock=None)) == {DEP_ALIAS}+ assert (global_methods_dir / "scoring" / PROVENANCE_FILENAME).is_file()++ lock = FetchedDependencyLockTestData.lock_naming(address=DEP_ALIAS, digest=OTHER_DIGEST)+ pinned_alias = DEP_ALIAS if pinned_tag is None else f"{DEP_ALIAS}@{pinned_tag}"+ with pytest.raises(MethodLockMismatchError) as exc_info:+ _load(method_dir=_write_caller(method_dir=tmp_path / "locked-caller", lock=lock, pipe_ref=f"{pinned_alias}->fom_lock_scoring.compute"))+ assert fetch_mock.call_count == 1+ message = exc_info.value.message+ if pinned_tag is None:+ assert "installed copy" not in message+ else:+ assert f"The reference pins the tag {pinned_tag}, but an installed copy of version 0.1.0" in message+ assert "remove that installed copy so the pinned tag is fetched" in message++ @pytest.mark.usefixtures("global_methods_dir")+ @pytest.mark.parametrize("locks_the_fetched_tree", [True, False])+ def test_a_later_batch_calling_a_dependency_an_earlier_batch_fetched_is_checked(+ self, tmp_path: Path, mocker: MockerFixture, locks_the_fetched_tree: bool+ ) -> None:+ """A library loads its directories, then the contents submitted beside them, as a run does: the dependency the+ directories' batch fetched is not loaded again, and the lock submitted with the contents is checked against its tree.+ """+ fetch_mock, digest = _mock_fetch(tmp_path=tmp_path, mocker=mocker)+ host_dir = _write_caller(method_dir=tmp_path / "host-library", lock=None)+ lock = FetchedDependencyLockTestData.lock_naming(address=DEP_ALIAS, digest=digest if locks_the_fetched_tree else OTHER_DIGEST)+ submitted = MthdsParser.make_pipelex_bundle_blueprint(+ mthds_content=FetchedDependencyLockTestData.caller_calling(+ pipe_ref=f"{DEP_ALIAS}->fom_lock_scoring.compute", domain="fom_lock_submitted"+ ),+ file_origin=BundleFileOrigin.SUBMITTED_FILES,+ )+ library_manager = get_library_manager()+ library_id, library = library_manager.open_library()+ try:+ library_manager.load_libraries(library_id=library_id, library_dirs=[host_dir])+ assert set(library.dependency_libraries) == {DEP_ALIAS}+ library_manager.add_submitted_files(library_id=library_id, submitted_files={LOCK_FILENAME: lock.encode("utf-8")})+ if locks_the_fetched_tree:+ library_manager.load_from_blueprints(library_id=library_id, blueprints=[submitted])+ else:+ with pytest.raises(MethodLockMismatchError) as exc_info:+ library_manager.load_from_blueprints(library_id=library_id, blueprints=[submitted])+ assert OTHER_DIGEST in exc_info.value.message+ assert digest in exc_info.value.message+ finally:+ library_manager.teardown(library_id=library_id)+ assert fetch_mock.call_count == 1++ @pytest.mark.usefixtures("global_methods_dir")+ def test_a_lock_that_cannot_be_read_refuses_a_fetched_dependency(self, tmp_path: Path, mocker: MockerFixture) -> None:+ _mock_fetch(tmp_path=tmp_path, mocker=mocker)+ method_dir = _write_caller(method_dir=tmp_path / "caller", lock=FetchedDependencyLockTestData.legacy_lock_naming(address=DEP_ALIAS))++ with pytest.raises(MethodLockInvalidError):+ _load(method_dir=method_dir)++ @pytest.mark.usefixtures("global_methods_dir")+ def test_a_shipped_dependency_is_not_checked(self, tmp_path: Path, mocker: MockerFixture) -> None:+ """A package the method ships travels with its lock, from its author: the runtime fetched nothing to verify."""+ fetch_mock, _digest = _mock_fetch(tmp_path=tmp_path, mocker=mocker)+ methods_dir = tmp_path / "shipped-methods"+ _write_dependency(package_dir=methods_dir / "scoring")+ lock = FetchedDependencyLockTestData.lock_naming(address=DEP_ALIAS, digest=OTHER_DIGEST)+ method_dir = _write_caller(method_dir=tmp_path / "caller", lock=lock)++ assert _load(method_dir=method_dir, methods_dirs=[methods_dir]) == {DEP_ALIAS}+ fetch_mock.assert_not_called()++ def test_a_copy_installed_without_a_fetch_provenance_is_not_checked(+ self, global_methods_dir: Path, tmp_path: Path, mocker: MockerFixture+ ) -> None:+ """A copy another installer wrote, such as `mthds install`, which verifies the lock itself, is not the runtime's fetch."""+ fetch_mock, _digest = _mock_fetch(tmp_path=tmp_path, mocker=mocker)+ _write_dependency(package_dir=global_methods_dir / "scoring")+ lock = FetchedDependencyLockTestData.lock_naming(address=DEP_ALIAS, digest=OTHER_DIGEST)+ method_dir = _write_caller(method_dir=tmp_path / "caller", lock=lock)++ assert _load(method_dir=method_dir) == {DEP_ALIAS}+ fetch_mock.assert_not_called()diff --git a/tests/unit/pipelex/methods/test_data.py b/tests/unit/pipelex/methods/test_data.pynew file mode 100644index 000000000..9ec308754--- /dev/null+++ b/tests/unit/pipelex/methods/test_data.py@@ -0,0 +1,67 @@+from typing import ClassVar+++class MethodLockTestData:+ """An invented locked dependency, and the `methods.lock` files a calling method could carry for it; nothing here is real."""++ ADDRESS: ClassVar[str] = "github.com/invented/locked-lib/scoring"+ ADDRESS_IN_OTHER_CASE: ClassVar[str] = "github.com/Invented/Locked-Lib/scoring"+ OTHER_ADDRESS: ClassVar[str] = "github.com/invented/other-lib/ranking"+ LOCKED_VERSION: ClassVar[str] = "1.0.0"+ FETCHED_VERSION: ClassVar[str] = "1.2.0"+ LOCKED_DIGEST: ClassVar[str] = "sha256:" + "a" * 64+ FETCHED_DIGEST: ClassVar[str] = "sha256:" + "b" * 64++ BUNDLE: ClassVar[str] = 'domain = "lock_tests"\n'++ # Each case: a topic, the fetched copy as (its version, the tag the reference pins, the tag the copy was fetched at),+ # the fragments the refusal's message holds, and those it must not hold. The lock records LOCKED_VERSION.+ MISMATCH_MESSAGE_CASES: ClassVar[list[tuple[str, tuple[str, str | None, str | None], list[str], list[str]]]] = [+ (+ "an unpinned reference fetched another version",+ ("1.2.0", None, None),+ ["The lock records another version than the one fetched", "pin the reference to the locked version", "record version 1.2.0"],+ ["installed copy"],+ ),+ (+ "the reference pins another version, fetched at that tag",+ ("1.2.0", "v1.2.0", "v1.2.0"),+ ["The reference pins the tag v1.2.0, another version than the lock records", "pin the reference to the locked version"],+ ["installed copy"],+ ),+ (+ "the reference pins the locked version, an installed copy of another was used",+ ("1.2.0", "v1.0.0", None),+ [+ "The reference pins the tag v1.0.0, but an installed copy of version 1.2.0, fetched earlier at the repository's default branch",+ "remove that installed copy so the pinned tag is fetched",+ ],+ ["pin the reference to the locked version", "pin the reference to it"],+ ),+ (+ "the reference pins a third version, an installed copy fetched at another tag was used",+ ("1.2.0", "v2.0.0", "v1.2.0"),+ [+ "The reference pins the tag v2.0.0, but an installed copy of version 1.2.0, fetched earlier at the tag v1.2.0",+ "remove that installed copy so the pinned tag is fetched",+ "The pinned tag does not name the locked version 1.0.0 either",+ ],+ [],+ ),+ (+ "the locked version, another tree",+ ("1.0.0", None, None),+ [],+ ["pin the reference", "installed copy"],+ ),+ ]++ @classmethod+ def lock_naming(cls, *, address: str, digest: str, version: str = LOCKED_VERSION) -> str:+ """A `methods.lock` holding one entry, for `address` at `version` with the tree digest `digest`."""+ return f'["{address}"]\nversion = "{version}"\ndigest = "{digest}"\nsource = "https://github.com/invented/locked-lib.git"\n'++ @classmethod+ def legacy_lock_naming(cls, *, address: str) -> str:+ """A `methods.lock` written before tree digests, its entry recording a `hash`."""+ return f'["{address}"]\nversion = "1.0.0"\nhash = "sha256:{"c" * 64}"\nsource = "https://github.com/invented/locked-lib.git"\n'diff --git a/tests/unit/pipelex/methods/test_method_lock.py b/tests/unit/pipelex/methods/test_method_lock.pynew file mode 100644index 000000000..b08d033e1--- /dev/null+++ b/tests/unit/pipelex/methods/test_method_lock.py@@ -0,0 +1,193 @@+from collections.abc import Generator+from pathlib import Path++import pytest+from mthds.method_tree.lister import list_method_tree_at_root+from mthds.method_tree.models import MethodTree+from mthds.package.lock_file import LOCK_FILENAME+from mthds.protocol.problem.registry import CoreCode, ProblemDomain++from pipelex.core.method_trees.method_file_store import LOCAL_METHOD_FILES, HeldFile+from pipelex.errors.problem import ProblemView+from pipelex.methods.exceptions import MethodLockInvalidError, MethodLockMismatchError+from pipelex.methods.method_lock import FetchedCopy, FetchedTree, ensure_fetched_tree_locked+from pipelex.methods.package_reason import MethodPackageReason+from tests.unit.pipelex.methods.test_data import MethodLockTestData++_HOLDER = "method-lock-tests"++[email protected](name="held_files")+def held_files_fixture() -> Generator[None, None, None]:+ yield+ LOCAL_METHOD_FILES.release(holder=_HOLDER)+++def _tree(*, root: Path, lock: str | bytes | None) -> MethodTree:+ """The tree of a calling method on disk: a bundle, and the lock at its root when one is given."""+ root.mkdir(parents=True, exist_ok=True)+ (root / "main.mthds").write_text(MethodLockTestData.BUNDLE, encoding="utf-8")+ if isinstance(lock, str):+ (root / LOCK_FILENAME).write_text(lock, encoding="utf-8")+ elif isinstance(lock, bytes):+ (root / LOCK_FILENAME).write_bytes(lock)+ return list_method_tree_at_root(root)+++def _fetched(+ *,+ version: str = MethodLockTestData.FETCHED_VERSION,+ pinned_tag: str | None = None,+ fetched_tag: str | None = None,+) -> FetchedTree:+ """The tree listed for a copy of the dependency this runtime fetched, at `version`, as a reference pinning `pinned_tag` found it."""+ copy = FetchedCopy(address=MethodLockTestData.ADDRESS, version=version, pinned_tag=pinned_tag, fetched_tag=fetched_tag)+ return FetchedTree(copy=copy, tree_digest=MethodLockTestData.FETCHED_DIGEST)+++class TestMethodLock:+ def test_a_caller_without_a_lock_checks_nothing(self, tmp_path: Path) -> None:+ tree = _tree(root=tmp_path / "caller", lock=None)++ ensure_fetched_tree_locked(fetched_tree=_fetched(), caller_trees=[tree])++ def test_a_lock_that_does_not_name_the_address_checks_nothing(self, tmp_path: Path) -> None:+ lock = MethodLockTestData.lock_naming(address=MethodLockTestData.OTHER_ADDRESS, digest=MethodLockTestData.LOCKED_DIGEST)+ tree = _tree(root=tmp_path / "caller", lock=lock)++ ensure_fetched_tree_locked(fetched_tree=_fetched(), caller_trees=[tree])++ @pytest.mark.parametrize("locked_address", [MethodLockTestData.ADDRESS, MethodLockTestData.ADDRESS_IN_OTHER_CASE])+ def test_a_fetched_tree_the_lock_records_passes(self, tmp_path: Path, locked_address: str) -> None:+ """The address is compared without regard to case, as the installed stores and a fetch match it."""+ lock = MethodLockTestData.lock_naming(address=locked_address, digest=MethodLockTestData.FETCHED_DIGEST)+ tree = _tree(root=tmp_path / "caller", lock=lock)++ ensure_fetched_tree_locked(fetched_tree=_fetched(), caller_trees=[tree])++ def test_a_fetched_tree_the_lock_does_not_record_is_refused(self, tmp_path: Path) -> None:+ lock = MethodLockTestData.lock_naming(+ address=MethodLockTestData.ADDRESS, digest=MethodLockTestData.LOCKED_DIGEST, version=MethodLockTestData.FETCHED_VERSION+ )+ tree = _tree(root=tmp_path / "caller", lock=lock)++ with pytest.raises(MethodLockMismatchError) as exc_info:+ ensure_fetched_tree_locked(fetched_tree=_fetched(), caller_trees=[tree])++ problem = exc_info.value.to_problem(view=ProblemView.OPERATOR)+ assert problem.code == CoreCode.SECURITY_INTEGRITY_MISMATCH+ assert problem.domain == ProblemDomain.METHOD+ assert problem.params is not None+ assert problem.params.as_wire() == {"dependency": MethodLockTestData.ADDRESS}+ # The operator reads the address, the digest the lock records and the digest of what was fetched.+ message = exc_info.value.message+ assert MethodLockTestData.ADDRESS in message+ assert MethodLockTestData.LOCKED_DIGEST in message+ assert MethodLockTestData.FETCHED_DIGEST in message+ assert str(tmp_path) not in message++ @pytest.mark.parametrize(+ ("topic", "fetched", "present", "absent"),+ MethodLockTestData.MISMATCH_MESSAGE_CASES,+ )+ def test_the_refusal_names_the_remedy_that_fits_how_the_copy_was_found(+ self, tmp_path: Path, topic: str, fetched: tuple[str, str | None, str | None], present: list[str], absent: list[str]+ ) -> None:+ """The lock verifies what was fetched, never chooses it: another version than the locked one is refused, both named,+ and the remedy is the one that fits, an installed copy used in place of a pinned tag being named as such.+ """+ lock = MethodLockTestData.lock_naming(address=MethodLockTestData.ADDRESS, digest=MethodLockTestData.LOCKED_DIGEST)+ tree = _tree(root=tmp_path / "caller", lock=lock)+ version, pinned_tag, fetched_tag = fetched++ with pytest.raises(MethodLockMismatchError) as exc_info:+ ensure_fetched_tree_locked(fetched_tree=_fetched(version=version, pinned_tag=pinned_tag, fetched_tag=fetched_tag), caller_trees=[tree])++ message = exc_info.value.message+ assert f"at version {MethodLockTestData.LOCKED_VERSION}" in message, topic+ assert f"fetched version {version}" in message, topic+ for fragment in present:+ assert fragment in message, (topic, fragment, message)+ for fragment in absent:+ assert fragment not in message, (topic, fragment, message)++ def test_every_calling_tree_s_lock_is_checked(self, tmp_path: Path) -> None:+ """Two methods of one load call the dependency: the lock of either refuses a tree it does not record."""+ matching = MethodLockTestData.lock_naming(address=MethodLockTestData.ADDRESS, digest=MethodLockTestData.FETCHED_DIGEST)+ differing = MethodLockTestData.lock_naming(address=MethodLockTestData.ADDRESS, digest=MethodLockTestData.LOCKED_DIGEST)+ trees = [_tree(root=tmp_path / "first", lock=matching), _tree(root=tmp_path / "second", lock=differing)]++ with pytest.raises(MethodLockMismatchError):+ ensure_fetched_tree_locked(fetched_tree=_fetched(), caller_trees=trees)++ @pytest.mark.usefixtures("held_files")+ def test_the_lock_of_a_tree_held_in_memory_is_read_from_the_store(self) -> None:+ """Files submitted beside content strings make a tree with no directory, whose lock the method file store holds."""+ lock = MethodLockTestData.lock_naming(address=MethodLockTestData.ADDRESS, digest=MethodLockTestData.LOCKED_DIGEST)+ tree = LOCAL_METHOD_FILES.hold_files(+ holder=_HOLDER,+ files={LOCK_FILENAME: HeldFile.of_bytes(data=lock.encode()), "notes.md": HeldFile.of_bytes(data=b"notes\n")},+ )+ assert tree.directory is None++ with pytest.raises(MethodLockMismatchError):+ ensure_fetched_tree_locked(fetched_tree=_fetched(), caller_trees=[tree])++ def test_a_lock_listed_in_a_tree_no_longer_held_is_refused(self) -> None:+ """A lock the tree lists is never skipped as absent: when the store no longer holds its bytes, the check refuses."""+ lock = MethodLockTestData.lock_naming(address=MethodLockTestData.ADDRESS, digest=MethodLockTestData.FETCHED_DIGEST)+ tree = LOCAL_METHOD_FILES.hold_files(holder=_HOLDER, files={LOCK_FILENAME: HeldFile.of_bytes(data=lock.encode())})+ LOCAL_METHOD_FILES.release(holder=_HOLDER)++ with pytest.raises(MethodLockInvalidError) as exc_info:+ ensure_fetched_tree_locked(fetched_tree=_fetched(), caller_trees=[tree])++ assert "not held for this load" in exc_info.value.message++ @pytest.mark.parametrize(+ ("is_removed", "finding"),+ [+ (True, "cannot be read"),+ (False, "changed since the method's tree was listed"),+ ],+ )+ def test_a_lock_on_disk_that_moved_after_the_listing_is_refused(self, tmp_path: Path, is_removed: bool, finding: str) -> None:+ """The lock read is the one the tree listed: removed or rewritten since, it is refused, never skipped or trusted."""+ lock = MethodLockTestData.lock_naming(address=MethodLockTestData.ADDRESS, digest=MethodLockTestData.FETCHED_DIGEST)+ tree = _tree(root=tmp_path / "caller", lock=lock)+ lock_path = tmp_path / "caller" / LOCK_FILENAME+ if is_removed:+ lock_path.unlink()+ else:+ lock_path.write_text(+ MethodLockTestData.lock_naming(address=MethodLockTestData.OTHER_ADDRESS, digest=MethodLockTestData.LOCKED_DIGEST),+ encoding="utf-8",+ )++ with pytest.raises(MethodLockInvalidError) as exc_info:+ ensure_fetched_tree_locked(fetched_tree=_fetched(), caller_trees=[tree])++ assert finding in exc_info.value.message+ assert str(tmp_path) not in exc_info.value.message+ problem = exc_info.value.to_problem(view=ProblemView.CALLER)+ assert problem.code == CoreCode.METHOD_PACKAGE_INVALID++ @pytest.mark.parametrize(+ ("lock", "finding"),+ [+ (MethodLockTestData.legacy_lock_naming(address=MethodLockTestData.ADDRESS), "predates tree digests"),+ ('["github.com/invented/locked-lib/scoring"\nversion = "1.0.0"\n', "Invalid TOML"),+ (b'["github.com/invented/locked-lib/scoring"]\nversion = "\xff"\n', "not UTF-8"),+ ],+ )+ def test_a_lock_that_cannot_be_read_is_refused(self, tmp_path: Path, lock: str | bytes, finding: str) -> None:+ tree = _tree(root=tmp_path / "caller", lock=lock)++ with pytest.raises(MethodLockInvalidError) as exc_info:+ ensure_fetched_tree_locked(fetched_tree=_fetched(), caller_trees=[tree])++ assert finding in exc_info.value.message+ problem = exc_info.value.to_problem(view=ProblemView.CALLER)+ assert problem.code == CoreCode.METHOD_PACKAGE_INVALID+ assert problem.params is not None+ assert problem.params.as_wire() == {"reason": MethodPackageReason.LOCK_INVALID, "path": LOCK_FILENAME}