Build AI methods with your coding agent, then run them anywhere: as an MCP for chatbots, as a webapp for people, or via API for your software.
feature/Requirements-check · L-261015-96da9b (#1433)
Louis Choquel2f31e654b3c0
diff --git a/.drift/acks/hub-layering-convention.toml b/.drift/acks/hub-layering-convention.tomlindex d015409ec..47f7c9c51 100644--- a/.drift/acks/hub-layering-convention.toml+++ b/.drift/acks/hub-layering-convention.toml@@ -1,10 +1,10 @@contract = "hub-layering-convention"-digest = "sha256:8ad6535ebd13f9a08da05c84f082e11f9dec80718a9666d9bcaba5c3c4a55747"+digest = "sha256:72fac07cc613e7b8eefe3ffa16d368fcfd1c883e9ca67fa42521189cd29509f0"reviewed_by = "Louis Choquel"-reviewed_at = "2026-10-15T23:18:44Z"-rationale = "clean-pass: runtime_hub.py changes only the scoped_event_log docstring, which now says usage events honour the override too; no symbol moves between the hubs, no import crosses the boundary, and the tracing row of the partition table already names scoped_event_log and get_event_log_override."+reviewed_at = "2026-10-16T01:21:40Z"+rationale = "clean-pass: interpreter_hub.py gains the PipeFuncPackageOfferRegistry setter and getters and the module accessor get_pipe_func_package_offer_registry; this change already lists it in the pipe func row of the partition table and names the registry's install beside the PipeFuncExecutorRegistry in Pipelex.setup; no import crosses the boundary."[trigger_files]"pipelex/cli/dev_cli/commands/hub_layering_guard.py" = "blob:fda2237457fe0b6830ff2ae8f720698735bec4c3"-"pipelex/interpreter_hub.py" = "blob:023f8452daf76dde409afc7c0966b149b11edabb"+"pipelex/interpreter_hub.py" = "blob:4efc29ca60c42ca0fe0533c402d7bb246b0bb2c2""pipelex/runtime_hub.py" = "blob:2af9f62d93b2005daa55a7c61e0390601e430c00"diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 20fb84092..e0ca5fae1 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -1,5 +1,15 @@# Changelog+## [Unreleased]++### Added++- **`PluginRegistrar.add_pipe_func_package_offer`**: the plugin providing an out-of-process PipeFunc execution mode states, keyed by the same mode, the Python packages and the marker environment its sandbox offers (`PipeFuncPackageOffer`), which a method's `requirements.txt` is checked against; a mode stating none refuses every method declaring requirements with `PipeFuncPackageSetUnstatedError`, a configuration fault. `MethodRequirementsFileError`, `MethodRequirementsNotMetError`, `HostLibraryRequirementsError`, `PipeFuncPackageSetUnstatedError` and `DuplicatePipeFuncPackageOfferError` are new `error_type`s.++### Changed++- **A method's `requirements.txt` is checked when it loads (Breaking)**: a method tree holding Python and a `requirements.txt` at its root is refused before any of its Python is imported, naming each requirement, when the environment its PipeFunc steps run in does not meet it, or when the file holds a line the MTHDS standard refuses (a URL, an option such as `-r` or `-e`, a comment after a requirement, a continued line). A package a method calls by its address is checked the same way, named by its address. Locally the environment is the interpreter Pipelex runs in and the refusal gives the `uv pip install` command that fixes it; on a sandbox-hosted deployment it is the package set the deployment states for its sandbox, a run is refused with a `422` and a validation by `method_ref` with an `is_valid: false` verdict located on the line, naming no path on the server. A method of a server's own library directories whose requirements are not met is the server's fault, a `500` `HostLibraryRequirementsError` whose message never reaches the caller. Pipelex never installs anything. A project root loaded as a library directory is such a tree, so hand over the folder holding the methods instead when the root's `requirements.txt` is an application's.+## [v0.86.1] - 2026-10-16### Addeddiff --git a/api/pipelex_api/routes/pipelex/pipeline.py b/api/pipelex_api/routes/pipelex/pipeline.pyindex cfbe4c43c..21034b0d9 100644--- a/api/pipelex_api/routes/pipelex/pipeline.py+++ b/api/pipelex_api/routes/pipelex/pipeline.py@@ -1,5 +1,6 @@from __future__ import annotations+import asyncioimport hashlibimport hmacfrom contextlib import ExitStack, contextmanager@@ -11,13 +12,16 @@ from fastapi.responses import JSONResponsefrom mthds.protocol.exceptions import PipelineRequestErrorfrom pipelex.config import get_config, is_pipe_func_sandbox_hostedfrom pipelex.core.pipes.pipe_output import PipeOutput+from pipelex.libraries.method_requirements import ensure_served_method_requirements_metfrom pipelex.methods.structures_check import ensure_no_structured_content_pythonfrom pipelex.pipe_run.delivery_assignment import DeliveryAssignment, StorageTarget, WebhookTargetfrom pipelex.pipe_run.pipe_run_protocol import PipeRunProtocolfrom pipelex.pipe_run.remote_dispatch import prepare_job_for_remote_dispatch+from pipelex.pipeline.exceptions import ValidateBundleErrorfrom pipelex.pipeline.pipeline_response import PipelexRunResultExecute, PipelexRunResultStart, RunStatefrom pipelex.pipeline.pipeline_run_setup import pipeline_run_setupfrom pipelex.pipeline.runner import PipelexMTHDSProtocol+from pipelex.pipeline.validate_bundle_translation import translate_to_validate_bundle_errorfrom pipelex.reporting.usage_records import apply_tokens_usage_wire_shapefrom pipelex.runtime_bridge.exceptions import MissingBundleValidatorError, MissingOrchestratorErrorfrom pipelex.runtime_bridge.orchestration_mode import DIRECT_ORCHESTRATION_MODE@@ -325,6 +329,7 @@ class ApiRunner(PipelexMTHDSProtocol):orchestrator = get_orchestrator_registry().get_optional(mode=orchestration_mode)if orchestrator is None:raise MissingOrchestratorError(mode=orchestration_mode)+ await self._ensure_served_method_requirements_met()# Dispatch the run through the mode-selected orchestrator by injecting it as this runner's# PipeRun, then delegate to the base execute, which owns the full run lifecycle. The@@ -407,6 +412,7 @@ class ApiRunner(PipelexMTHDSProtocol):f"async-capable orchestration, and this deployment has none. Use /execute (synchronous) instead.")raise_bad_request(msg, error_type=ErrorType.START_REQUIRES_ASYNC_ORCHESTRATION)+ await self._ensure_served_method_requirements_met()created_at = get_current_iso_timestamp()pipelex_inputs: PipelineInputs | WorkingMemory | None = cast("PipelineInputs | WorkingMemory | None", inputs)@@ -484,6 +490,26 @@ class ApiRunner(PipelexMTHDSProtocol):workflow_id=dispatch_ack.workflow_id,)+ async def _ensure_served_method_requirements_met(self) -> None:+ """Refuse the request's method when its requirements.txt is not met, before anything loads or is dispatched.++ This runner's `library_dirs` are the directories it made for the request, holding the method's non-`.mthds`+ files, which the load reads as a host's library directories: there a requirements refusal is reported as the+ host's fault, so the caller's own is judged here first, naming no path on this host. A directory that cannot be+ listed stays the host's fault, as the load reports it. The listing reads and hashes files, so it runs off the+ event loop.++ Raises:+ MethodRequirementsFileError: The method's requirements.txt holds a line the standard refuses.+ MethodRequirementsNotMetError: The PipeFunc environment does not meet the method's requirements.+ HostLibraryTreeError: A directory of the request could not be listed as a method tree.+ PipeFuncPackageSetUnstatedError: The deployment states no package set for its PipeFunc sandbox.+ """+ if not self.library_dirs:+ return+ library_dirs = [Path(library_dir) for library_dir in self.library_dirs]+ await asyncio.to_thread(ensure_served_method_requirements_met, library_dirs=library_dirs)+async def validate_verdict(self,*,@@ -513,8 +539,10 @@ class ApiRunner(PipelexMTHDSProtocol):and the `bundle_blueprint` on success — carry a real `source` instead of `None`. The routemaps a length mismatch to a 422 before we get here; `None` keeps the sourceless behavior.`library_dirs` is host context the in-process arm needs; a dispatched arm ignores it (the- worker loads its own library). A bundle without a declared `main_pipe` validates fine and- simply carries `graph_spec=None` (D2 — no precondition).+ worker loads its own library), so the requirements.txt of the methods they hold is checked here+ first, for every arm, and a method whose requirements are not met is an invalid verdict. A+ bundle without a declared `main_pipe` validates fine and simply carries `graph_spec=None`+ (D2 — no precondition).`graph_pipe_code` is the pipe the best-effort graph is drawn from, resolved the way a runresolves its entry pipe; `None` keeps the runtime's default, the primary blueprint's@@ -528,6 +556,14 @@ class ApiRunner(PipelexMTHDSProtocol):if validator is None:raise MissingBundleValidatorError(mode=orchestration_mode)library_dirs = [Path(library_dir) for library_dir in self.library_dirs] if self.library_dirs else None+ # The method's requirements.txt is judged here, on the runner, which holds the method's files and the package set+ # its PipeFunc steps run with, before any dispatch: a dispatched validator never reads these directories. A+ # refusal is the method's invalid verdict, as a run of it is refused before its files load.+ try:+ with translate_to_validate_bundle_error():+ await self._ensure_served_method_requirements_met()+ except ValidateBundleError as invalid_method:+ return invalid_method.to_error_report()verdict = await validator.validate_bundles(mthds_contents=mthds_contents,mthds_sources=mthds_sources,diff --git a/api/tests/unit/test_method_requirements_refusal.py b/api/tests/unit/test_method_requirements_refusal.pynew file mode 100644index 000000000..77f07e712--- /dev/null+++ b/api/tests/unit/test_method_requirements_refusal.py@@ -0,0 +1,262 @@+"""A sandbox-hosted deployment refuses a method whose requirements.txt its PipeFunc sandbox does not meet, naming the packages.++The package set is what the plugin providing the execution mode states; here a stub registered for the neutral+`sandbox` mode. A run is refused on the runner before the method's files load, before any dispatch, and a validation by+address is refused on the runner as an invalid verdict, before the validator is reached, since a dispatched validator+never reads the method's Python. A method of the runner's own library directories whose requirements are not met is+the deployment's fault, never the caller's.+"""++import asyncio+import sys+from collections.abc import Callable, Generator, Sequence+from pathlib import Path++import pytest+from fastapi import FastAPI+from fastapi.testclient import TestClient+from pipelex.base_exceptions import DisclosureMode, ErrorReport+from pipelex.config import get_config+from pipelex.core.bundle_resources.bundle_resource import BundleResources+from pipelex.interpreter_hub import get_interpreter_hub+from pipelex.libraries.exceptions import MethodRequirementsNotMetError+from pipelex.pipe_run.delivery_assignment import DeliveryAssignment+from pipelex.pipe_run.pipe_job import PipeJob+from pipelex.plugins.bundle_validator_registry import BundleValidatorRegistry+from pipelex.plugins.orchestrator_registry import OrchestratorRegistry+from pipelex.plugins.pipe_func_package_offer_registry import PipeFuncPackageOffer, PipeFuncPackageOfferRegistry+from pipelex.runtime_bridge.payloads import PipelexPipeDispatchAck, PipelexPipeRunOutput+from pipelex.system.caller_identity import CallerIdentity+from pipelex.system.configuration.pipe_func_config import PipeFuncConfig+from pytest_mock import MockerFixture++from pipelex_api.api_config import ApiConfig+from pipelex_api.exception_handlers import register_exception_handlers+from pipelex_api.routes import router as api_router+from tests.unit._constants import STUB_METHOD_ADDRESS++_PIPELINE_NS = "pipelex_api.routes.pipelex.pipeline"+_SANDBOX_ENVIRONMENT = "the Daytona sandbox snapshot 'pipelex-base-test'"+_ABSENT_PACKAGE = "pipelex-requirements-test-absent"++_PLAIN_MTHDS = """\+domain = "plain_demo"+main_pipe = "echo"++[pipe.echo]+type = "PipeCompose"+description = "Echo the data"+inputs = { data = "Text" }+output = "Text"+template = "$data"+"""++_PIPE_FUNC_MTHDS = """\+domain = "crunch_demo"+main_pipe = "crunch"++[pipe.crunch]+type = "PipeFunc"+description = "Crunch numbers in caller code"+inputs = { data = "Text" }+output = "Text"+function_name = "crunch"+"""++_PIPE_FUNC_PY = """\+from pipelex.core.memory.working_memory import WorkingMemory+from pipelex.core.stuffs.text_content import TextContent+from pipelex.system.registries.func_registry import pipe_func+++@pipe_func(name="crunch")+async def crunch(working_memory: WorkingMemory) -> TextContent:+ return TextContent(text="crunched")+"""+++class _RecordingOrchestrator:+ supports_fire_and_forget = True++ def __init__(self) -> None:+ self.dispatches: list[str] = []++ async def execute(self, *, pipe_job: PipeJob, delivery_assignment: DeliveryAssignment | None) -> PipelexPipeRunOutput: # noqa: ARG002+ self.dispatches.append(pipe_job.pipe.code)+ msg = "These tests dispatch through /start only."+ raise AssertionError(msg)++ async def start(self, *, pipe_job: PipeJob, delivery_assignment: DeliveryAssignment | None) -> PipelexPipeDispatchAck: # noqa: ARG002+ self.dispatches.append(pipe_job.pipe.code)+ return PipelexPipeDispatchAck(pipeline_run_id="run-1", workflow_id="wf-1")+++class _UnreachableValidator:+ """A dispatched validator the requirements check must never let a refused method reach."""++ def __init__(self) -> None:+ self.calls = 0++ async def validate_bundles(+ self,+ *,+ mthds_contents: list[str], # noqa: ARG002+ mthds_sources: list[str] | None, # noqa: ARG002+ allow_signatures: bool, # noqa: ARG002+ library_dirs: Sequence[Path] | None, # noqa: ARG002+ caller_identity: CallerIdentity | None, # noqa: ARG002+ graph_pipe_code: str | None, # noqa: ARG002+ bundle_resources: BundleResources | None, # noqa: ARG002+ ) -> ErrorReport:+ self.calls += 1+ msg = "A method refused for its requirements must not reach the validator."+ raise AssertionError(msg)++[email protected](name="sandbox_hosted_mode")+def sandbox_hosted_mode_fixture(mocker: MockerFixture) -> Generator[None, None, None]:+ """Run PipeFunc steps in the neutral `sandbox` mode, whose plugin states a small package set."""+ offer = PipeFuncPackageOffer(+ environment=_SANDBOX_ENVIRONMENT,+ distributions={"pandas": "2.2.3", "openpyxl": "3.1.5"},+ marker_environment={"python_version": "3.12", "sys_platform": "linux"},+ )++ def stated_offer(_config: PipeFuncConfig) -> PipeFuncPackageOffer | None:+ return offer++ mocker.patch(+ "pipelex.libraries.package_environments.get_pipe_func_package_offer_registry",+ return_value=PipeFuncPackageOfferRegistry({"sandbox": stated_offer}),+ )+ pipe_func_config = get_config().interpreter.pipe_func+ previous = pipe_func_config.execution_mode+ pipe_func_config.execution_mode = "sandbox"+ try:+ yield+ finally:+ pipe_func_config.execution_mode = previous+++def _build_client(mocker: MockerFixture, *, orchestrator: _RecordingOrchestrator, validator: _UnreachableValidator) -> TestClient:+ mocker.patch(+ f"{_PIPELINE_NS}.get_api_config", return_value=ApiConfig(orchestration_mode="temporal", allow_request_orchestration_mode_override=False)+ )+ mocker.patch(f"{_PIPELINE_NS}.get_orchestrator_registry", return_value=OrchestratorRegistry({"temporal": orchestrator}))+ mocker.patch(f"{_PIPELINE_NS}.get_bundle_validator_registry", return_value=BundleValidatorRegistry({"temporal": validator}))+ app = FastAPI()+ app.include_router(api_router, prefix="/v1")+ register_exception_handlers(app, disclosure_mode=DisclosureMode.STRICT)+ return TestClient(app)+++class TestMethodRequirementsRefusal:+ @pytest.mark.usefixtures("sandbox_hosted_mode")+ def test_a_run_whose_requirements_the_sandbox_does_not_meet_is_refused_before_dispatch(self, mocker: MockerFixture) -> None:+ orchestrator = _RecordingOrchestrator()+ client = _build_client(mocker, orchestrator=orchestrator, validator=_UnreachableValidator())++ files = {"main.mthds": _PIPE_FUNC_MTHDS, "funcs/pipe_func.py": _PIPE_FUNC_PY, "requirements.txt": "pandas>=2\npandas>=3\npolars\n"}+ response = client.post("/v1/start", json={"files": files, "inputs": {"data": "1,2,3"}})++ assert response.status_code == 422, response.text+ assert response.headers["content-type"].startswith("application/problem+json")+ body = response.json()+ assert body["error_type"] == "MethodRequirementsNotMetError"+ # Caller-facing under STRICT disclosure: it names each package and the sandbox, and no directory of the runner.+ assert "line 2 'pandas>=3' (pandas 2.2.3 is there); line 3 'polars' (polars is missing)" in body["detail"]+ assert body["detail"].startswith(f"The method needs Python packages that {_SANDBOX_ENVIRONMENT} does not have")+ assert "rooted at" not in body["detail"]+ assert orchestrator.dispatches == []++ @pytest.mark.usefixtures("sandbox_hosted_mode")+ def test_a_run_whose_requirements_the_sandbox_meets_dispatches(self, mocker: MockerFixture) -> None:+ orchestrator = _RecordingOrchestrator()+ client = _build_client(mocker, orchestrator=orchestrator, validator=_UnreachableValidator())++ files = {"main.mthds": _PIPE_FUNC_MTHDS, "funcs/pipe_func.py": _PIPE_FUNC_PY, "requirements.txt": "# data\npandas>=2.2,<3\nopenpyxl\n"}+ response = client.post("/v1/start", json={"files": files, "inputs": {"data": "1,2,3"}})++ assert response.status_code in {200, 202}, response.text+ assert orchestrator.dispatches == ["crunch"]++ @pytest.mark.usefixtures("sandbox_hosted_mode")+ def test_a_refused_requirements_line_refuses_the_run(self, mocker: MockerFixture) -> None:+ orchestrator = _RecordingOrchestrator()+ client = _build_client(mocker, orchestrator=orchestrator, validator=_UnreachableValidator())++ files = {"main.mthds": _PIPE_FUNC_MTHDS, "funcs/pipe_func.py": _PIPE_FUNC_PY, "requirements.txt": "-r base.txt\n"}+ response = client.post("/v1/start", json={"files": files, "inputs": {"data": "1,2,3"}})++ assert response.status_code == 422, response.text+ body = response.json()+ assert body["error_type"] == "MethodRequirementsFileError"+ assert "line 1 '-r base.txt'" in body["detail"]+ assert orchestrator.dispatches == []++ @pytest.mark.usefixtures("sandbox_hosted_mode")+ def test_a_validation_by_address_is_an_invalid_verdict_located_on_the_requirement(+ self, mocker: MockerFixture, install_method_package: Callable[..., Path]+ ) -> None:+ validator = _UnreachableValidator()+ client = _build_client(mocker, orchestrator=_RecordingOrchestrator(), validator=validator)+ install_method_package(files={"main.mthds": _PIPE_FUNC_MTHDS, "funcs/pipe_func.py": _PIPE_FUNC_PY, "requirements.txt": "pandas>=3\n"})++ response = client.post("/v1/validate", json={"method_ref": f"{STUB_METHOD_ADDRESS}@v0.1.0"})++ assert response.status_code == 200, response.text+ body = response.json()+ assert body["is_valid"] is False+ (item,) = body["validation_errors"]+ assert item["source"] == "requirements.txt"+ assert item["line"] == 1+ assert "pandas 2.2.3 is there" in item["message"]+ assert validator.calls == 0++ @pytest.mark.usefixtures("sandbox_hosted_mode")+ def test_a_validation_checks_the_method_off_the_event_loop(self, mocker: MockerFixture, install_method_package: Callable[..., Path]) -> None:+ """The check lists and hashes the method's files, so it runs in a worker thread, never on the loop serving requests."""+ threads_holding_a_loop: list[bool] = []++ def refuse_after_noting_the_thread(*, library_dirs: Sequence[Path]) -> None: # noqa: ARG001+ try:+ asyncio.get_running_loop()+ except RuntimeError:+ threads_holding_a_loop.append(False)+ else:+ threads_holding_a_loop.append(True)+ msg = "The method needs Python packages."+ raise MethodRequirementsNotMetError(msg, items=[])++ mocker.patch(f"{_PIPELINE_NS}.ensure_served_method_requirements_met", side_effect=refuse_after_noting_the_thread)+ validator = _UnreachableValidator()+ client = _build_client(mocker, orchestrator=_RecordingOrchestrator(), validator=validator)+ install_method_package(files={"main.mthds": _PIPE_FUNC_MTHDS, "funcs/pipe_func.py": _PIPE_FUNC_PY, "requirements.txt": "pandas>=3\n"})++ response = client.post("/v1/validate", json={"method_ref": f"{STUB_METHOD_ADDRESS}@v0.1.0"})++ assert response.status_code == 200, response.text+ assert response.json()["is_valid"] is False+ assert threads_holding_a_loop == [False]+ assert validator.calls == 0++ def test_a_host_library_whose_requirements_are_not_met_is_the_deployments_fault(self, mocker: MockerFixture, tmp_path: Path) -> None:+ """In the direct mode, a method of the runner's own library directories is judged against its interpreter, and its refusal is withheld."""+ host_library = tmp_path / "host-library"+ (host_library / "funcs").mkdir(parents=True)+ (host_library / "host.mthds").write_text('domain = "host_demo"\n', encoding="utf-8")+ (host_library / "funcs" / "helpers.py").write_text("VALUE = 1\n", encoding="utf-8")+ (host_library / "requirements.txt").write_text(f"{_ABSENT_PACKAGE}>=1\n", encoding="utf-8")+ mocker.patch.object(get_interpreter_hub(), "get_default_library_dirs", return_value=[host_library])+ orchestrator = _RecordingOrchestrator()+ client = _build_client(mocker, orchestrator=orchestrator, validator=_UnreachableValidator())++ response = client.post("/v1/start", json={"mthds_contents": [_PLAIN_MTHDS], "inputs": {"data": "hello"}})++ assert response.status_code == 500, response.text+ body = response.json()+ assert body["error_type"] == "HostLibraryRequirementsError"+ assert body["error_domain"] == "config"+ for host_detail in (str(tmp_path), sys.executable, _ABSENT_PACKAGE):+ assert host_detail not in response.text, host_detail+ assert orchestrator.dispatches == []diff --git a/docs/api-server/pipe-run.md b/docs/api-server/pipe-run.mdindex c1eb3c626..723af4ea4 100644--- a/docs/api-server/pipe-run.md+++ b/docs/api-server/pipe-run.md@@ -243,6 +243,8 @@ The server materializes the bundle into a temporary library directory for the ru**No Python structure classes.** A structure class (a `StructuredContent` subclass) can only be used by importing its module into the runner's own process, which a sandbox-hosted deployment never does, so a bundle whose Python declares one is refused with a `403` (`MethodStructuresRefusedError`) naming each file and class, before anything is loaded. The rule is the same however the method arrives: inline as a bundle, or fetched by `method_ref`. Declare the types as MTHDS concepts with inline structures instead; a PipeFunc returns them by importing the classes the sandbox generates from those concepts (`from structures import <domain>__<Concept>`). The `structures` module that `pipelex build structures` writes is accepted as long as it is left as generated, and it is not sent to the sandbox, which generates its own.+**Requirements are checked against the sandbox.** A `requirements.txt` at the root of the bundle declares the packages its Python imports, one PEP 508 requirement per line (see [Method Trees](../building-methods/method-trees.md#python-packages-requirementstxt)). Nothing is installed for a run: the packages of the deployment's PipeFunc sandbox are the whole offer, and a bundle asking for a package the sandbox does not hold, or for a version it does not hold, is refused with a `422` (`MethodRequirementsNotMetError`) naming each requirement and the version the sandbox holds, before the bundle's files load and before anything is dispatched. The refusal names the file by its path in the bundle, `requirements.txt`, and no path on the server. A line the standard refuses, such as `-r base.txt`, a URL or a comment after a requirement, is a `422` (`MethodRequirementsFileError`) naming the line. A deployment that states no package set for its sandbox refuses every bundle declaring requirements with a `500` (`PipeFuncPackageSetUnstatedError`), a fault of its configuration, and so does a method of the server's own library directories whose requirements are not met (`HostLibraryRequirementsError`), whose message is withheld from the caller. A `method_ref` package is checked the same way, on a run as on [`/validate`](pipe-validate.md), where the refusal is an `is_valid: false` verdict located on the requirement's line.+### Shipping the method packages the bundle callsA bundle can call a pipe of another method package by its address, such as `github.com/Pipelex/methods/documents->documents.extract_page_contents_and_views`. A `files` or `bundle_b64` request can carry that package with the bundle, so the run needs no copy installed on the server and no fetch. Put each package under `.mthds/methods/<name>/` at the bundle's root, with its `METHODS.toml` at `.mthds/methods/<name>/METHODS.toml`, which is the layout of a local method store:diff --git a/docs/api-server/pipe-validate.md b/docs/api-server/pipe-validate.mdindex fb78dfa54..64ad706e3 100644--- a/docs/api-server/pipe-validate.md+++ b/docs/api-server/pipe-validate.md@@ -17,7 +17,7 @@ Validate MTHDS content by parsing, loading, and dry-running pipes without execut**Request Fields:**- `mthds_contents` (list[str], optional): MTHDS contents to validate (always an array, even for a single file). Exactly one of `mthds_contents` / `method_ref` must be provided-- `method_ref` (string, optional): **Pipelex-API extension.** Validate a published method by address — `github.com/<owner>/<repo>` plus an optional package selector and `@<tag>` — resolved by this server through the same fetch path as a [`method_ref` run](pipe-run.md#running-a-method-by-address-method_ref): the repository is fetched at the tag, the package located by manifest identity, and its `.mthds` files validated with their **real relative paths** feeding the per-file sources (so `validation_errors[].source` names the owning file). The method packages the package ships under its own `.mthds/methods/<name>/` are not part of the validated content, and validation does not search them: a call into one resolves here through the server's installed copies and fetch-on-miss only, while a [run](pipe-run.md#shipping-the-method-packages-the-bundle-calls) uses the shipped copy first. Exactly one of `mthds_contents` / `method_ref`; `mthds_sources` cannot accompany it+- `method_ref` (string, optional): **Pipelex-API extension.** Validate a published method by address — `github.com/<owner>/<repo>` plus an optional package selector and `@<tag>` — resolved by this server through the same fetch path as a [`method_ref` run](pipe-run.md#running-a-method-by-address-method_ref): the repository is fetched at the tag, the package located by manifest identity, and its `.mthds` files validated with their **real relative paths** feeding the per-file sources (so `validation_errors[].source` names the owning file). The method packages the package ships under its own `.mthds/methods/<name>/` are not part of the validated content, and validation does not search them: a call into one resolves here through the server's installed copies and fetch-on-miss only, while a [run](pipe-run.md#shipping-the-method-packages-the-bundle-calls) uses the shipped copy first. Exactly one of `mthds_contents` / `method_ref`; `mthds_sources` cannot accompany it. A package holding Python and a `requirements.txt` has its requirements checked on this server against the package set of the deployment's PipeFunc sandbox before anything is validated, and a requirement the sandbox does not meet, or a line the standard refuses, makes the verdict `is_valid: false` with one item per line, whose `source` is `requirements.txt` and whose `line` is the line's number- `allow_signatures` (boolean, optional, default `false`): controls only the **sweep mechanics** for `PipeSignature` placeholders — whether signature pipes are mock-run during the dry-run sweep and therefore listed in `validated_pipes`. It does **not** change the verdict: an unimplemented signature is never a rejection, it is a *runnability fact* reported via `pending_signatures` + `is_runnable` in both modes- `mthds_sources` (list[str] | null, optional): per-file sources, parallel to `mthds_contents` — see [Sourcing submitted files](#sourcing-submitted-files). When present it must match `mthds_contents` in length (a mismatch is a 422 request error)- `resources` (object | null, optional): **Pipelex-API extension.** The files the bundle's steps read, such as a `PipeDocGen` step's `template_file`, as a map from the path the step names to the file's bytes in standard base64 — see [Sending the files the steps read](#sending-the-files-the-steps-read-resources). It accompanies `mthds_contents` only; beside `method_ref` it is a 422diff --git a/docs/building-methods/method-trees.md b/docs/building-methods/method-trees.mdindex 71cbc9711..58ee64eba 100644--- a/docs/building-methods/method-trees.md+++ b/docs/building-methods/method-trees.md@@ -36,6 +36,26 @@ A file a bundle names, such as a [`PipeDocGen`](pipes/pipe-operators/PipeDocGen.A library directory, such as a `-L` directory or one of `PIPELEXPATH`, is a place methods are found rather than one method. Pipelex loads its tree, then the tree of each nested package and nested repository under it, each as a method of its own whose steps read only its own files, so every method in the directory loads. A `.mthds` file kept out by the membership rule, a hidden one, a gitignored one or one inside a virtual environment, is not loaded.+## Python packages: `requirements.txt`++A method whose own Python, such as the module of a [`PipeFunc`](pipes/pipe-operators/PipeFunc.md) step, imports a package declares it in a `requirements.txt` at the root of its tree. Only the root's file declares anything: a `requirements.txt` in a subfolder is an ordinary file of the tree. Each line of the file is blank, a comment whose first character other than a space or a tab is `#`, or exactly one [PEP 508](https://peps.python.org/pep-0508/) requirement, markers and extras included:++```text+# Reading and writing spreadsheets+pandas>=2.2,<3+openpyxl==3.1.5+tzdata; python_version >= "3.12"+```++Pipelex refuses, naming each such line, a requirement naming a URL (`name @ https://…`), an option of any kind (a line starting with `-`, such as `-r`, `-e`, `-c` or `--index-url`), a comment after a requirement on the same line, and a line continued with a backslash. A comment after a requirement is refused as a comment whatever it says, a URL in it included. The file must be UTF-8 text.++When a tree holding Python and a root `requirements.txt` loads, Pipelex checks every requirement against the Python environment its PipeFunc steps run in, before importing any of the tree's Python and before building any of its pipes, and refuses the method when one is not met, naming each requirement and why. A requirement whose marker is false in that environment is not needed there and is not checked. A tree with no Python is not checked, since nothing in it could import a package. A package your method calls by its address is checked the same way when it loads, its refusal naming the package by its address. Pipelex never installs anything.++- **Locally**, a PipeFunc runs in the interpreter Pipelex runs in, so the requirements are checked against the packages installed there, extras included. The refusal names the method's folder and gives the command that installs what is missing into that interpreter, such as `uv pip install --python /path/to/python 'pandas>=2.2,<3'`. When Pipelex was installed as a uv tool, add the packages to the tool instead, which keeps them across upgrades: `uv tool install pipelex --with 'pandas>=2.2,<3'`.+- **On a sandbox-hosted deployment**, such as the hosted API, a PipeFunc runs in the deployment's sandbox, whose packages are the whole offer: nothing is installed for a method. The requirements are checked against the package set the deployment states for its sandbox, and a method asking for a package it does not hold, or for a version it does not hold, is refused when it loads and when it is validated, naming each requirement. A requirement with extras cannot be checked there and is refused, so list the packages the extra brings directly.++Python declaring a structure class is refused on a sandbox-hosted deployment whatever its requirements: see [Python classes](concepts/python-classes.md).+## Where a folder's run starts`pipelex run bundle <folder>` with no `--pipe` starts where the MTHDS protocol says a run naming no pipe starts, since the bundles of a folder have no order:diff --git a/docs/building-methods/pipes/pipe-operators/PipeFunc.md b/docs/building-methods/pipes/pipe-operators/PipeFunc.mdindex 7d0178250..3b72a5e4d 100644--- a/docs/building-methods/pipes/pipe-operators/PipeFunc.md+++ b/docs/building-methods/pipes/pipe-operators/PipeFunc.md@@ -108,6 +108,10 @@ async def summarize(working_memory: WorkingMemory) -> TextContent:Prefixing names with the domain they serve is the simplest way to keep them distinct as a project grows.+## Declaring the packages your functions need++When your functions import a package beyond Pipelex and the standard library, such as pandas or openpyxl, declare it in a `requirements.txt` at the root of the method's folder, one PEP 508 requirement per line. Pipelex checks it when the method loads and refuses the method, naming each requirement that is not met, rather than letting the function fail at its first import. Locally the check reads the interpreter Pipelex runs in and the refusal gives the `uv pip install` command that fixes it; on a sandbox-hosted deployment it reads the packages the sandbox holds, which are all a PipeFunc gets there. Pipelex never installs anything. The rules of the file are in [Method Trees](../../method-trees.md#python-packages-requirementstxt).+## ConfigurationOnce the function is registered, you can use it in your `.mthds` file.diff --git a/docs/configuration/config-technical/library-config.md b/docs/configuration/config-technical/library-config.mdindex 9081250ca..d37752157 100644--- a/docs/configuration/config-technical/library-config.md+++ b/docs/configuration/config-technical/library-config.md@@ -285,6 +285,8 @@ A library directory whose own tree is empty, one holding nothing but nested packThe structure classes and `@pipe_func()` functions of a library directory are discovered by scanning its Python files, which skips the directories listed in the `excluded_dirs` array under `[interpreter.scan]` in `pipelex.toml` (virtual environments, `.git`, caches, `node_modules`, `results` and the like); a value that is not an array, such as a single string, fails the boot. That list no longer decides which `.mthds` files load.+Before any of that Python is imported, each tree of the load that holds Python and a `requirements.txt` at its root has its requirements checked against the interpreter Pipelex runs in, and a method whose requirements are not met, or whose file holds a line the MTHDS standard refuses, such as `-r base.txt` or `-e .`, refuses the load (see [Method Trees](../../building-methods/method-trees.md#python-packages-requirementstxt)). When a server loads its own library directories beside a caller's bundles, such a refusal is a `HostLibraryRequirementsError`, a fault of the server's configuration whose message, naming its paths, never reaches the caller, as a tree it cannot list is a `HostLibraryTreeError`. A project root handed over as a library directory is such a tree, so its own `requirements.txt` is read as its methods' declaration: hand over the folder holding your methods rather than the project root, or keep the root's file to plain PEP 508 requirements that the interpreter meets.+## Project Organization**Golden rule:** Put `.mthds` files where they make sense in YOUR project. Pipelex finds them automatically.diff --git a/docs/contribute/hub-layering.md b/docs/contribute/hub-layering.mdindex 31addb953..0717af29c 100644--- a/docs/contribute/hub-layering.md+++ b/docs/contribute/hub-layering.md@@ -63,7 +63,7 @@ There is deliberately **no** `pipelex.hub`. It was deleted rather than kept as a| library dirs | `resolve_library_dirs`, `get_default_library_dirs` || pipe router | `get_pipe_router`, `set_pipe_router`, `teardown_current_pipe_router`, `scoped_pipe_router` || run | `get_pipe_run`, `does_pipe_run_leave_process`, `get_pipeline_manager`, `get_pipeline` |-| pipe func | `get_pipe_func_executor`, `scoped_pipe_func_executor`, `get_pipe_func_executor_registry` |+| pipe func | `get_pipe_func_executor`, `scoped_pipe_func_executor`, `get_pipe_func_executor_registry`, `get_pipe_func_package_offer_registry` |Setters follow their getters onto the matching container.@@ -164,7 +164,7 @@ The second split *is* the layer boundary, and the same rule of thumb divides it.External plugins are unaffected by the mechanism/adapter split: they import `pipelex.plugins.contract` and `pipelex.plugins.registrar`, both of which stayed put. They are, however, affected by the *layer* split — they are discovered through the two `pipelex.plugins.kernel` / `pipelex.plugins.interpreter` entry-point groups, which are group identifiers rather than module paths, and a plugin declares its layer by choosing one. (Careful: `pipelex.plugins` is also the name of the kernel-layer *package* holding the mechanism, and that is the sense every other mention of it on this page carries — the declared-package list, the transitive-breach worked example, the closure-test entry points. A package name and an entry-point group are unrelated things that happen to share a spelling. The single pre-split group of that name is retired and now fails startup.)-The registrar's slots are *ports* and they stay kernel-layer — `OrchestratorProtocol`, `BundleValidatorProtocol` and `PipeFuncExecutorFactoryFn` all live under `plugins/*_registry.py`. Only the *adapters* are interpreter-layer. That is ordinary ports-and-adapters, and it is what the layer rule's own remedy text prescribes: have the interpreter layer install it downward at boot. Where each derived registry is *installed* still follows the hub partition rather than the port's home: `RuntimeBoot.setup` puts the orchestrator, bundle-validator, backend, lister, storage and secrets registries and the service error vocabulary on the `RuntimeHub`, and `Pipelex.setup` puts the `PipeFuncExecutorRegistry` on the `InterpreterHub` — the one placement flagged [above](#which-hub-does-a-new-symbol-belong-to). Since [the boot split](#where-the-boot-splits) the two installs sit in the two halves that match their hubs, so the partition is now visible in the code rather than only in this sentence.+The registrar's slots are *ports* and they stay kernel-layer — `OrchestratorProtocol`, `BundleValidatorProtocol` and `PipeFuncExecutorFactoryFn` all live under `plugins/*_registry.py`. Only the *adapters* are interpreter-layer. That is ordinary ports-and-adapters, and it is what the layer rule's own remedy text prescribes: have the interpreter layer install it downward at boot. Where each derived registry is *installed* still follows the hub partition rather than the port's home: `RuntimeBoot.setup` puts the orchestrator, bundle-validator, backend, lister, storage and secrets registries and the service error vocabulary on the `RuntimeHub`, and `Pipelex.setup` puts the `PipeFuncExecutorRegistry` on the `InterpreterHub` — the one placement flagged [above](#which-hub-does-a-new-symbol-belong-to) — with the `PipeFuncPackageOfferRegistry` beside it, whose offers the library load reads to check a method's `requirements.txt` in a sandbox execution mode. Since [the boot split](#where-the-boot-splits) the two installs sit in the two halves that match their hubs, so the partition is now visible in the code rather than only in this sentence.Composition happens where welding is legal. `pipelex/providers/builtins.py` exports `KERNEL_BUILTIN_PLUGINS`; `pipelex/interpreter_plugins/builtins.py` imports that half — downward, which the interpreter layer may do — and exports the composed `BUILTIN_PLUGINS` and `CORE_UNCONDITIONAL_PLUGIN_NAMES`. Both are **parameters** of `build_registrar`, never module globals it imports, so `plugins/discovery.py` stays kernel-layer: `pipelex.py` (the interpreter boot) and the `pipelex plugins list` diagnostic pass the composed lists in, while `runtime_boot.py` defaults to the kernel half alone. Exactly one place still answers "what are the built-in plugins" — it just lives in the layer permitted to say it.diff --git a/docs/errors/authoring-and-language.md b/docs/errors/authoring-and-language.mdindex e0a092b8c..a04617bb0 100644--- a/docs/errors/authoring-and-language.md+++ b/docs/errors/authoring-and-language.md@@ -122,10 +122,15 @@ own page. Classes are grouped by subsystem.- [`DomainLoadingError`](domain-loading-error.md) — Domain loading- [`EntryPipeAmbiguousError`](entry-pipe-ambiguous-error.md) — Entry pipe ambiguous- [`EntryPipeNotFoundError`](entry-pipe-not-found-error.md) — Entry pipe not found+- [`HostLibraryRequirementsError`](host-library-requirements-error.md) — Host library requirements- [`HostLibraryTreeError`](host-library-tree-error.md) — Host library tree- [`LibraryError`](library-error.md) — Library error- [`LibraryLoadingError`](library-loading-error.md) — Library loading+- [`MethodRequirementsError`](method-requirements-error.md) — Method requirements+- [`MethodRequirementsFileError`](method-requirements-file-error.md) — Method requirements file+- [`MethodRequirementsNotMetError`](method-requirements-not-met-error.md) — Method requirements not met- [`MethodTreeLoadError`](method-tree-load-error.md) — Method tree load+- [`PipeFuncPackageSetUnstatedError`](pipe-func-package-set-unstated-error.md) — Pipe func package set unstated- [`PipeLibraryError`](pipe-library-error.md) — Pipe library- [`PipeLoadingError`](pipe-loading-error.md) — Pipe loading- [`PipeNotFoundError`](pipe-not-found-error.md) — Pipe not founddiff --git a/docs/errors/duplicate-pipe-func-package-offer-error.md b/docs/errors/duplicate-pipe-func-package-offer-error.mdnew file mode 100644index 000000000..4fa8efa5c--- /dev/null+++ b/docs/errors/duplicate-pipe-func-package-offer-error.md@@ -0,0 +1,21 @@+---+title: "Duplicate pipe func package offer"+description: "Reference for the `DuplicatePipeFuncPackageOfferError` Pipelex error class."+---++<!-- pipelex:generated -->++# Duplicate pipe func package offer++Two plugins stated the package offer of the same PipeFunc execution mode.++| Field | Value |+|---|---|+| `error_type` | `DuplicatePipeFuncPackageOfferError` |+| `title` | Duplicate pipe func package offer |+| `type_uri` | `https://docs.pipelex.com/latest/errors/duplicate-pipe-func-package-offer-error/` |+| `error_domain` | _(inherited from parent)_ |+| Defined in | `pipelex.plugins.exceptions` |+| Parent class | [`PluginError`](plugin-error.md) |++[Back to Error Reference](index.md)diff --git a/docs/errors/host-library-requirements-error.md b/docs/errors/host-library-requirements-error.mdnew file mode 100644index 000000000..0cb13688a--- /dev/null+++ b/docs/errors/host-library-requirements-error.md@@ -0,0 +1,22 @@+---+title: "Host library requirements"+description: "Reference for the `HostLibraryRequirementsError` Pipelex error class."+---++<!-- pipelex:generated -->++# Host library requirements++Raised when a method of this host's own library directories, loaded beside a caller's bundles, is refused for its requirements.++| Field | Value |+|---|---|+| `error_type` | `HostLibraryRequirementsError` |+| `title` | Host library requirements |+| `type_uri` | `https://docs.pipelex.com/latest/errors/host-library-requirements-error/` |+| `error_domain` | `config` |+| Defined in | `pipelex.libraries.exceptions` |+| Parent class | [`PipelexError`](pipelex-error.md) |+| `retryable` | `false` |++[Back to Error Reference](index.md)diff --git a/docs/errors/inference-and-providers.md b/docs/errors/inference-and-providers.mdindex c9a6c1882..6c2714d40 100644--- a/docs/errors/inference-and-providers.md+++ b/docs/errors/inference-and-providers.md@@ -132,6 +132,7 @@ own page. Classes are grouped by subsystem.- [`DuplicateModelListerError`](duplicate-model-lister-error.md) — Duplicate model lister- [`DuplicateOrchestratorError`](duplicate-orchestrator-error.md) — Duplicate orchestrator- [`DuplicatePipeFuncExecutorError`](duplicate-pipe-func-executor-error.md) — Duplicate pipe func executor+- [`DuplicatePipeFuncPackageOfferError`](duplicate-pipe-func-package-offer-error.md) — Duplicate pipe func package offer- [`DuplicateSecretsProviderError`](duplicate-secrets-provider-error.md) — Duplicate secrets provider- [`DuplicateServiceErrorCodeError`](duplicate-service-error-code-error.md) — Duplicate service error code- [`DuplicateStorageProviderError`](duplicate-storage-provider-error.md) — Duplicate storage providerdiff --git a/docs/errors/method-requirements-error.md b/docs/errors/method-requirements-error.mdnew file mode 100644index 000000000..2151a3467--- /dev/null+++ b/docs/errors/method-requirements-error.md@@ -0,0 +1,22 @@+---+title: "Method requirements"+description: "Reference for the `MethodRequirementsError` Pipelex error class."+---++<!-- pipelex:generated -->++# Method requirements++Raised when a method's `requirements.txt` refuses the method as it loads.++| Field | Value |+|---|---|+| `error_type` | `MethodRequirementsError` |+| `title` | Method requirements |+| `type_uri` | `https://docs.pipelex.com/latest/errors/method-requirements-error/` |+| `error_domain` | `input` |+| Defined in | `pipelex.libraries.exceptions` |+| Parent class | [`LibraryLoadingError`](library-loading-error.md) |+| `retryable` | `false` |++[Back to Error Reference](index.md)diff --git a/docs/errors/method-requirements-file-error.md b/docs/errors/method-requirements-file-error.mdnew file mode 100644index 000000000..f97445a0f--- /dev/null+++ b/docs/errors/method-requirements-file-error.md@@ -0,0 +1,21 @@+---+title: "Method requirements file"+description: "Reference for the `MethodRequirementsFileError` Pipelex error class."+---++<!-- pipelex:generated -->++# Method requirements file++Raised when a method's `requirements.txt` holds a line the MTHDS standard refuses, or cannot be read as UTF-8 text.++| Field | Value |+|---|---|+| `error_type` | `MethodRequirementsFileError` |+| `title` | Method requirements file |+| `type_uri` | `https://docs.pipelex.com/latest/errors/method-requirements-file-error/` |+| `error_domain` | _(inherited from parent)_ |+| Defined in | `pipelex.libraries.exceptions` |+| Parent class | [`MethodRequirementsError`](method-requirements-error.md) |++[Back to Error Reference](index.md)diff --git a/docs/errors/method-requirements-not-met-error.md b/docs/errors/method-requirements-not-met-error.mdnew file mode 100644index 000000000..fdbca69cb--- /dev/null+++ b/docs/errors/method-requirements-not-met-error.md@@ -0,0 +1,21 @@+---+title: "Method requirements not met"+description: "Reference for the `MethodRequirementsNotMetError` Pipelex error class."+---++<!-- pipelex:generated -->++# Method requirements not met++Raised when the Python environment a method's PipeFunc steps run in does not satisfy its `requirements.txt`.++| Field | Value |+|---|---|+| `error_type` | `MethodRequirementsNotMetError` |+| `title` | Method requirements not met |+| `type_uri` | `https://docs.pipelex.com/latest/errors/method-requirements-not-met-error/` |+| `error_domain` | _(inherited from parent)_ |+| Defined in | `pipelex.libraries.exceptions` |+| Parent class | [`MethodRequirementsError`](method-requirements-error.md) |++[Back to Error Reference](index.md)diff --git a/docs/errors/pipe-func-package-set-unstated-error.md b/docs/errors/pipe-func-package-set-unstated-error.mdnew file mode 100644index 000000000..b6efd70c2--- /dev/null+++ b/docs/errors/pipe-func-package-set-unstated-error.md@@ -0,0 +1,22 @@+---+title: "Pipe func package set unstated"+description: "Reference for the `PipeFuncPackageSetUnstatedError` Pipelex error class."+---++<!-- pipelex:generated -->++# Pipe func package set unstated++Raised when a method declaring requirements loads on a deployment that states no package set for its PipeFunc sandbox.++| Field | Value |+|---|---|+| `error_type` | `PipeFuncPackageSetUnstatedError` |+| `title` | Pipe func package set unstated |+| `type_uri` | `https://docs.pipelex.com/latest/errors/pipe-func-package-set-unstated-error/` |+| `error_domain` | `config` |+| Defined in | `pipelex.libraries.exceptions` |+| Parent class | [`PipelexError`](pipelex-error.md) |+| `retryable` | `false` |++[Back to Error Reference](index.md)diff --git a/docs/under-the-hood/index.md b/docs/under-the-hood/index.mdindex 4fd946dbd..63ed46826 100644--- a/docs/under-the-hood/index.md+++ b/docs/under-the-hood/index.md@@ -21,6 +21,7 @@ Welcome to the technical deep-dives of Pipelex. This section is for contributors- **Test Profile Configuration** - How to configure which models are used in tests- **Dry Run Mock Generation** - How mock objects satisfy field validation constraints- **Init CLI Flows** - How `pipelex init` sets up the configuration directory+- **PipeFunc Requirements** - How a method's `requirements.txt` is checked against the environment its PipeFunc steps run in, and how a deployment states its sandbox's packages- **Runtime Bridge & Transport** - The backend-neutral seam that carries a pipe run across process boundaries — boundary DTOs, LibraryCrate propagation, deferred hydration, and per-call isolation- **Content Generation Across Boundaries** - How dynamic classes and large payloads cross a worker boundary- **Technical Design Decisions** - Why we chose X over Y@@ -48,6 +49,7 @@ Welcome to the technical deep-dives of Pipelex. This section is for contributors- [:material-flask-outline: Dry Run Mock Generation](./dry-run-mock-generation.md){ .md-button }- [:material-console: Init CLI Flows](./init-cli-flows.md){ .md-button }- [:material-pipe: Pipe Routing & Execution](./pipe-routing-and-execution.md){ .md-button }+- [:material-package-variant: PipeFunc Requirements](./pipe-func-requirements.md){ .md-button }- [:material-alert-circle-outline: Error Model](./error-model.md){ .md-button }- [:material-transit-connection-variant: Runtime Bridge & Transport](./runtime-bridge-and-transport.md){ .md-button }- [:material-swap-horizontal: Content Generation Across Boundaries](./distributed-content-generation.md){ .md-button }diff --git a/docs/under-the-hood/inference-backend-plugins.md b/docs/under-the-hood/inference-backend-plugins.mdindex f7b2ce243..22d80b5d9 100644--- a/docs/under-the-hood/inference-backend-plugins.md+++ b/docs/under-the-hood/inference-backend-plugins.md@@ -200,7 +200,7 @@ Declare an entry point in the `pipelex.plugins.kernel` group; discovery finds itacme = "my_pkg.acme_plugin:AcmePlugin"```-**The group declares your layer, and the choice is not cosmetic.** An inference backend is kernel-layer — it constructs a worker, never a `Pipe` — so it belongs in `pipelex.plugins.kernel`, and a kernel-only boot reads that group and finds it. Publish the same plugin under `pipelex.plugins.interpreter` and a kernel-only boot never even *imports* your module, so your backend is simply absent. The reverse mistake fails loudly instead: a `pipelex.plugins.kernel` plugin whose `register` reaches an interpreter-layer capability — `add_orchestrator`, `add_bundle_validator`, `add_pipe_func_executor`, or the three hub slots that hand back a `Pipe`-aware object (`claim_pipe_router`, `claim_pipe_run`, `claim_pipe_func_executor`) — raises `PluginLayerViolationError` naming the capability and the group to move to. The other three slots are kernel-tier and a kernel-group plugin may claim them freely: `claim_content_generator`, `claim_task_manager`, `claim_isolated_execution_probe`. The restriction is one-directional — an interpreter-group plugin may contribute kernel-tier capabilities too.+**The group declares your layer, and the choice is not cosmetic.** An inference backend is kernel-layer — it constructs a worker, never a `Pipe` — so it belongs in `pipelex.plugins.kernel`, and a kernel-only boot reads that group and finds it. Publish the same plugin under `pipelex.plugins.interpreter` and a kernel-only boot never even *imports* your module, so your backend is simply absent. The reverse mistake fails loudly instead: a `pipelex.plugins.kernel` plugin whose `register` reaches an interpreter-layer capability — `add_orchestrator`, `add_bundle_validator`, `add_pipe_func_executor`, `add_pipe_func_package_offer`, or the three hub slots that hand back a `Pipe`-aware object (`claim_pipe_router`, `claim_pipe_run`, `claim_pipe_func_executor`) — raises `PluginLayerViolationError` naming the capability and the group to move to. The other three slots are kernel-tier and a kernel-group plugin may claim them freely: `claim_content_generator`, `claim_task_manager`, `claim_isolated_execution_probe`. The restriction is one-directional — an interpreter-group plugin may contribute kernel-tier capabilities too.The single pre-split `pipelex.plugins` group is retired. It is not ignored: a plugin still published there fails startup with `RetiredPluginEntryPointGroupError`, because the alternative — being silently undiscovered — is the expensive failure to diagnose.diff --git a/docs/under-the-hood/pipe-func-requirements.md b/docs/under-the-hood/pipe-func-requirements.mdnew file mode 100644index 000000000..8ac9844d9--- /dev/null+++ b/docs/under-the-hood/pipe-func-requirements.md@@ -0,0 +1,100 @@+---+title: "PipeFunc Requirements"+description: "How Pipelex checks a method's requirements.txt against the Python environment its PipeFunc steps run in, where the check runs on a sandbox-hosted deployment, and the plugin seam through which a deployment states its sandbox's packages."+---++# PipeFunc Requirements++A method declares the Python packages its own Python imports in a `requirements.txt` at the root of its method tree, under the rules of the MTHDS standard: each line is blank, a comment, or exactly one PEP 508 requirement. The author's view of the file is in [Method Trees](../building-methods/method-trees.md#python-packages-requirementstxt). This page is about where Pipelex checks it and how a deployment tells Pipelex what its PipeFunc sandbox holds.++## What is checked, and when++A method tree is checked when it holds a `requirements.txt` at its root and a `.py` file anywhere in it. Python is the proxy for "has code that could import a package" rather than "has a PipeFunc pipe", because a server receives a method's Python apart from its bundles: the runner loads a bundle's `.py` files from a directory it makes for the request, while the `.mthds` files arrive as content, so the directory holding the Python holds no pipe to judge it by.++The check is `pipelex/libraries/method_requirements.py`, and a load runs it at three places, each before any Python of the tree is imported or captured and before any of its pipes is built:++- `LibraryManager.load_libraries` checks every tree of its library directories, and the tree of each bundle file it is handed, once all of them are listed.+- `LibraryManager.load_from_blueprints` checks the tree of every bundle read from this host's disk, on every load: a blueprint an earlier load placed in its tree, a refused one included, is judged again rather than taken as judged.+- A dependency package, one a bundle calls by its address, is checked as soon as its tree is listed, before any of its bundles loads.++A requirement whose marker is false in the environment is not needed there and is skipped. One is met when the environment holds a distribution of its name, compared normalized, at a version its specifier contains, pre-releases included, and when, for each extra it names, the requirements the distribution's metadata declares under that extra are met in turn. A marker that cannot be evaluated, such as one comparing as versions a value that is not one (which releases of `packaging` before 22 refuse), refuses its requirement rather than the load.++## The environment the check reads++`resolve_pipe_func_package_environment` in `pipelex/libraries/package_environments.py` picks the environment from `interpreter.pipe_func.execution_mode`:++| Execution mode | Environment | What it reads |+| --- | --- | --- |+| `direct` | `InterpreterPackageEnvironment` | The distributions installed in the interpreter Pipelex runs in, through `importlib.metadata`, and this interpreter's own marker environment. Extras are followed through each distribution's `Requires-Dist`. |+| any other mode | `OfferedPackageEnvironment` | The package set the plugin providing the mode states for its sandbox, and nothing else. A marker variable the offer does not state cannot be judged, and a requirement with extras cannot be followed, so both are refused rather than judged against this process's own values. |++In the `direct` mode, a method's Python is imported into the process that loads it, which is the process the check judges. A distributed orchestrator, such as our Temporal plugin, never carries a `direct`-mode method's Python to its worker: the library crate carries Python sources only in a sandbox mode, where they are captured rather than imported. A worker in the `direct` mode therefore runs only the functions its own deployment installed, whose packages are that deployment's to install, and the API server refuses a caller's Python outright unless it is sandbox-hosted (`403 CustomCodeRequiresSandbox`). So no worker needs to repeat the check.++## How a refusal names the method++The file is always named by its path in the method's tree, `requirements.txt`, on every item of a refusal. Paths on the host, the method's root and this interpreter's executable, are named only where the refusal may name them:++- **A check inside a load names them**, since its refusal belongs either to the caller's own disk, as on a local run, or to the host, around a server's own library directories, where `reporting_host_library_tree_faults` turns it into the host's fault and withholds it from the caller. Locally, the refusal therefore names the method's folder, each requirement and why, and the `uv pip install --python <this interpreter>` command that installs them.+- **A dependency package is named by its address**, as its bundles are, and its file by `<address>/requirements.txt`, never by the directory it was installed or fetched into.+- **A server checking a caller's method it holds in a directory of its own names no path on the host at all** (`ensure_served_method_requirements_met`): the method is "the method", the interpreter is "the Python interpreter Pipelex runs in", and the install command names no interpreter path.++## A host's own library directories++A server loads its own library directories (`PIPELEXPATH`, the instance defaults, installed libraries) beside a caller's bundles. A method of those directories whose requirements are not met is the deployment's fault, not the caller's: around such a load, `reporting_host_library_tree_faults` turns the refusal into a `HostLibraryRequirementsError`, a `config`-domain error answered with a `500` whose message, naming paths on the host, never reaches the caller under STRICT disclosure, as it turns a tree it cannot list into a `HostLibraryTreeError`.++## Where the check runs on a sandbox-hosted deployment++**On the runner, never on the worker or in the sandbox.** The runner is the process that holds the method's files, and the place a refusal reaches the caller before anything is dispatched. The runner loads the directory it makes for a request's files as one of its own library directories, where a refusal is the host's fault, so it judges the caller's method first, through `ensure_served_method_requirements_met`, run off the event loop since it lists and hashes the method's files:++- **A run** (`/execute`, `/start`) with a `files` or `bundle_b64` bundle, or by `method_ref`, is refused there as a `422` naming each requirement, before the method's files load and before any dispatch.+- **A validation by `method_ref`** (`/validate`) is dispatched whole to a validator that never reads the method's Python, a Temporal worker for instance, so `ApiRunner.validate_verdict` returns a refusal as the invalid verdict, one item per line, before the validator is reached. Inline `mthds_contents` carry no Python and are not checked.+- **A directory of the request that cannot be listed** stays the host's fault, a `500` `HostLibraryTreeError`, as the load reports it.+- **The worker** loads the library crate the runner built, which carries the method's Python sources but not its `requirements.txt`, so it never checks anything, and needs to know nothing of the sandbox's packages.++## The seam: stating the sandbox's packages++Pipelex cannot read a sandbox it never runs in, and it must not depend on the deployment that provides one, so the plugin providing an execution mode's executor states that mode's package set through its registrar, keyed by the same mode token:++```python+from pipelex.plugins.pipe_func_package_offer_registry import PipeFuncPackageOffer+from pipelex.system.configuration.pipe_func_config import PipeFuncConfig+++def _sandbox_package_offer(pipe_func_config: PipeFuncConfig) -> PipeFuncPackageOffer | None:+ package_set = read_my_sandbox_package_set() # the plugin's own reading of its image's lock, cached+ if package_set is None:+ return None+ return PipeFuncPackageOffer(+ environment=f"the Example sandbox image '{package_set.image}'",+ distributions={package.name: package.version for package in package_set.packages},+ marker_environment=package_set.marker_environment,+ )+++class ExampleSandboxPlugin:+ def register(self, registrar: PluginRegistrar) -> None:+ registrar.add_pipe_func_executor(mode="example", factory=_make_executor)+ registrar.add_pipe_func_package_offer(mode="example", factory=_sandbox_package_offer)+```++- **`PipeFuncPackageOffer`** (`pipelex/plugins/pipe_func_package_offer_registry.py`) is closed, frozen plain data: `environment`, the place as a refusal names it to the method's author; `distributions`, each distribution's name with its exact installed version, the names compared normalized; and `marker_environment`, the PEP 508 marker values of the sandbox's interpreter. A variable left out of `marker_environment`, such as `python_full_version` when the image pins only its minor version, makes every requirement whose marker reads it refused.+- **The factory** receives the `PipeFuncConfig` and is called each time a method declaring requirements loads, never at registration, so it may read the plugin's own configuration and should cache what it computes. It answers `None` when it cannot state the offer.+- **It fails closed.** A mode with no registered offer, or whose factory answers `None`, meets no requirement: a method declaring one is refused with `PipeFuncPackageSetUnstatedError`, a `config`-domain fault of the deployment whose message is not caller-facing, rather than loading and failing inside the sandbox at its first import. Its message says which of the two happened: no plugin registers an offer for the mode, or the registered one could not state the packages, whose remedy is in the plugin's configuration. A method declaring none still loads.+- **It is interpreter-layer**, like `add_pipe_func_executor`: a plugin published under `pipelex.plugins.kernel` that calls it is refused with `PluginLayerViolationError`. A second plugin stating the same mode is refused with `DuplicatePipeFuncPackageOfferError`, naming both.++`Pipelex.setup` builds the `PipeFuncPackageOfferRegistry` from the registrar beside the `PipeFuncExecutorRegistry` and puts it on the `InterpreterHub` (`get_pipe_func_package_offer_registry`).++### What our Daytona plugin supplies++For the hosted API, the Daytona sandbox plugin registers the offer for its `daytona` mode from the package set of the Daytona sandbox snapshot it boots, the lock its snapshot image is built from: the snapshot's distributions and versions, and the marker environment of its base image. The runner, where the check runs, must resolve the same snapshot name the worker's sandboxes boot from, since the worker is the process that creates them; a runner that cannot resolve it answers `None` and refuses every method declaring requirements.++## Errors++| Error | Domain | When |+| --- | --- | --- |+| [`MethodRequirementsFileError`](../errors/method-requirements-file-error.md) | `input` | The file holds a line the standard refuses, naming each, or is not UTF-8 text. |+| [`MethodRequirementsNotMetError`](../errors/method-requirements-not-met-error.md) | `input` | The environment does not meet a requirement, naming each and why. |+| [`HostLibraryRequirementsError`](../errors/host-library-requirements-error.md) | `config` | A method of a host's own library directories, loaded beside a caller's bundles, is refused for its requirements. |+| [`PipeFuncPackageSetUnstatedError`](../errors/pipe-func-package-set-unstated-error.md) | `config` | A sandbox-hosted deployment states no package set for its execution mode. |++The first two are `LibraryLoadingError`s carrying one blueprint validation item per line, whose `source` is the file's path in the tree and whose `line` is the line's number, so a validation reports them as an invalid verdict located on the line. They are deliberately not `MethodTreeLoadError`s, which a server reports as its own fault around the library directories it loads.diff --git a/mkdocs.yml b/mkdocs.ymlindex bb474e731..b1e643a2c 100644--- a/mkdocs.yml+++ b/mkdocs.yml@@ -353,6 +353,7 @@ plugins:- under-the-hood/test-profile-configuration.md: "Test Profile Configuration"- under-the-hood/init-cli-flows.md: "Init CLI Flows"- under-the-hood/pipe-routing-and-execution.md: "Pipe Routing & Execution"+ - under-the-hood/pipe-func-requirements.md: "PipeFunc Requirements"- under-the-hood/inference-backend-plugins.md: "Inference Backend Plugins"- under-the-hood/writing-an-inference-plugin.md: "Writing an Inference Plugin"- under-the-hood/service-error-codes.md: "Service Error Codes"@@ -499,6 +500,7 @@ nav:- Overview: building-methods/pipes/index.md- Libraries: building-methods/libraries.md- Packages: building-methods/packages.md+ - Method Trees: building-methods/method-trees.md- Executing Pipelines: building-methods/pipes/executing-pipelines.md- Running on the Hosted API: building-methods/pipes/running-on-the-hosted-api.md- Run Modes & Backends: building-methods/pipes/run-modes-and-backends.md@@ -614,6 +616,7 @@ nav:- Test Profile Configuration: under-the-hood/test-profile-configuration.md- Init CLI Flows: under-the-hood/init-cli-flows.md- Pipe Routing & Execution: under-the-hood/pipe-routing-and-execution.md+ - PipeFunc Requirements: under-the-hood/pipe-func-requirements.md- Inference Backend Plugins: under-the-hood/inference-backend-plugins.md- Writing an Inference Plugin: under-the-hood/writing-an-inference-plugin.md- Service Error Codes: under-the-hood/service-error-codes.mddiff --git a/pipelex/interpreter_hub.py b/pipelex/interpreter_hub.pyindex 023f8452d..4efc29ca6 100644--- a/pipelex/interpreter_hub.py+++ b/pipelex/interpreter_hub.py@@ -39,6 +39,7 @@ if TYPE_CHECKING:from pipelex.pipe_run.pipe_router_protocol import PipeRouterProtocolfrom pipelex.pipe_run.pipe_run_protocol import PipeRunProtocolfrom pipelex.plugins.pipe_func_executor_registry import PipeFuncExecutorRegistry+ from pipelex.plugins.pipe_func_package_offer_registry import PipeFuncPackageOfferRegistryclass InterpreterHub:@@ -64,6 +65,7 @@ class InterpreterHub:self._pipe_run_leaves_process = Falseself._pipe_func_executor: PipeFuncExecutorProtocol | None = Noneself._pipe_func_executor_registry: PipeFuncExecutorRegistry | None = None+ self._pipe_func_package_offer_registry: PipeFuncPackageOfferRegistry | None = None# pipelineself._pipeline_manager: PipelineManagerAbstract | None = None@@ -123,6 +125,9 @@ class InterpreterHub:def set_pipe_func_executor_registry(self, pipe_func_executor_registry: "PipeFuncExecutorRegistry"):self._pipe_func_executor_registry = pipe_func_executor_registry+ def set_pipe_func_package_offer_registry(self, *, pipe_func_package_offer_registry: "PipeFuncPackageOfferRegistry"):+ self._pipe_func_package_offer_registry = pipe_func_package_offer_registry+def set_pipeline_manager(self, pipeline_manager: PipelineManagerAbstract):self._pipeline_manager = pipeline_manager@@ -197,6 +202,12 @@ class InterpreterHub:raise RuntimeError(msg)return self._pipe_func_executor_registry+ def get_pipe_func_package_offer_registry(self) -> "PipeFuncPackageOfferRegistry":+ if self._pipe_func_package_offer_registry is None:+ msg = "PipeFuncPackageOfferRegistry is not initialized"+ raise RuntimeError(msg)+ return self._pipe_func_package_offer_registry+def get_required_pipeline_manager(self) -> PipelineManagerAbstract:if self._pipeline_manager is None:msg = "PipelineManager is not initialized"@@ -495,6 +506,10 @@ def get_pipe_func_executor_registry() -> "PipeFuncExecutorRegistry":return get_interpreter_hub().get_pipe_func_executor_registry()+def get_pipe_func_package_offer_registry() -> "PipeFuncPackageOfferRegistry":+ return get_interpreter_hub().get_pipe_func_package_offer_registry()++# rundiff --git a/pipelex/libraries/exceptions.py b/pipelex/libraries/exceptions.pyindex 238780658..351c799f2 100644--- a/pipelex/libraries/exceptions.py+++ b/pipelex/libraries/exceptions.py@@ -119,3 +119,82 @@ class HostLibraryTreeError(PipelexError):error_domain = ErrorDomain.CONFIGretryable = False+++class MethodRequirementsError(LibraryLoadingError):+ """Raised when a method's `requirements.txt` refuses the method as it loads.++ The method's own declaration, so the caller can fix it: the message names the file's lines and the packages, and+ one structured item per line rides the validation verdict, its `source` the file's path in the method's tree. Around+ a load of a host's own library directories it leaves as a `HostLibraryRequirementsError` instead, the host's fault+ (`reporting_host_library_tree_faults`).+ """++ error_domain = ErrorDomain.INPUT+ retryable = False+ _authors_caller_facing_message = True++ def __init__(self, message: str, *, items: list[PipelexBundleBlueprintValidationErrorData]):+ super().__init__(message, blueprint_validation_errors=items)+++class MethodRequirementsFileError(MethodRequirementsError):+ """Raised when a method's `requirements.txt` holds a line the MTHDS standard refuses, or cannot be read as UTF-8 text.++ Each line is blank, a comment, or exactly one PEP 508 requirement; a requirement naming a URL, an option such as+ `-r`, `-e` or `--index-url`, a comment after a requirement and a line continued with a backslash are refused.+ """+++class MethodRequirementsNotMetError(MethodRequirementsError):+ """Raised when the Python environment a method's PipeFunc steps run in does not satisfy its `requirements.txt`.++ Locally that environment is the interpreter Pipelex runs in, which Pipelex never installs anything into; on a+ sandbox-hosted deployment it is the package set the deployment states for its PipeFunc sandbox, which is the whole+ offer. The message names each requirement not met and why.+ """+++class HostLibraryRequirementsError(PipelexError):+ """Raised when a method of this host's own library directories, loaded beside a caller's bundles, is refused for its requirements.++ Such a directory is the host's own (installed libraries, the defaults, `PIPELEXPATH` on a server), so its+ `requirements.txt` holding a refused line, or naming packages the environment does not hold, is a fault of the+ host's configuration, not of the caller's method: it is not an `INPUT` error, and its message, which names paths on+ the host, is not caller-facing. Like `HostLibraryTreeError`, it is deliberately no `LibraryError`.+ """++ error_domain = ErrorDomain.CONFIG+ retryable = False+++class PipeFuncPackageSetUnstatedError(PipelexError):+ """Raised when a method declaring requirements loads on a deployment that states no package set for its PipeFunc sandbox.++ PipeFunc steps run out of this process, in the configured execution mode's sandbox, and either no plugin registers+ a statement of the packages found there (`PluginRegistrar.add_pipe_func_package_offer`), or the one registered could+ not state them, so no requirement can be judged and the method is refused rather than left to fail inside the+ sandbox. A fault of the deployment, not of the method: it is no `LibraryError`, which a validation would report as+ the caller's invalid bundle, and its message is not caller-facing.+ """++ error_domain = ErrorDomain.CONFIG+ retryable = False++ def __init__(self, *, execution_mode: str, offer_registered: bool):+ self.execution_mode = execution_mode+ self.offer_registered = offer_registered+ if offer_registered:+ message = (+ f"PipeFunc steps run in the '{execution_mode}' execution mode, whose plugin registers a package offer that "+ "could not state the Python packages they run with on this deployment, so a method declaring requirements "+ "in its requirements.txt cannot load. Check the plugin's configuration, such as the sandbox image or "+ "snapshot it reads its package set from."+ )+ else:+ message = (+ f"PipeFunc steps run in the '{execution_mode}' execution mode, for which no plugin registers the Python "+ "packages they run with, so a method declaring requirements in its requirements.txt cannot load. The plugin "+ "providing the mode must register its package offer with PluginRegistrar.add_pipe_func_package_offer."+ )+ super().__init__(message)diff --git a/pipelex/libraries/library_manager.py b/pipelex/libraries/library_manager.pyindex 73177292b..81b893d06 100644--- a/pipelex/libraries/library_manager.py+++ b/pipelex/libraries/library_manager.py@@ -46,9 +46,11 @@ from pipelex.libraries.library_crate import LibraryCratefrom pipelex.libraries.library_crate_factory import LibraryCrateFactoryfrom pipelex.libraries.library_factory import LibraryFactoryfrom pipelex.libraries.library_manager_abstract import LibraryManagerAbstract+from pipelex.libraries.method_requirements import ensure_method_requirements_met, ensure_package_requirements_metfrom pipelex.libraries.method_trees import (LibraryBundleFile,- list_library_bundle_files,+ library_bundle_files_of,+ list_library_dir_trees,list_package_method_tree,local_tree_limits,tree_bundle_of_lone_file,@@ -583,7 +585,8 @@ class LibraryManager(LibraryManagerAbstract):# in it, then the tree of each file named alone. A file the membership rule keeps out is never read.# This machine's limits are read once for the whole load.tree_limits = local_tree_limits()- all_bundle_files: list[LibraryBundleFile] = list_library_bundle_files(library_dirs=library_dirs, limits=tree_limits)+ library_trees = list_library_dir_trees(library_dirs=library_dirs, limits=tree_limits)+ all_bundle_files: list[LibraryBundleFile] = library_bundle_files_of(library_trees=library_trees)for library_file_path in library_file_paths or []:all_bundle_files.append(LibraryBundleFile(@@ -592,6 +595,19 @@ class LibraryManager(LibraryManagerAbstract):))+ # A method declaring Python packages in its requirements.txt is refused here when the environment its+ # PipeFunc steps run in does not meet them: before any of its Python is imported or captured, and before+ # any of its pipes is built. Every listed tree counts, one holding no bundle included, such as the Python a+ # server received beside a method's bundles. A refusal may name host paths: these are the caller's own+ # directories, or the host's, whose refusal `reporting_host_library_tree_faults` withholds as its own fault.+ ensure_method_requirements_met(+ trees=[+ *(library_tree.tree for library_tree in library_trees),+ *(bundle_file.tree_bundle.tree for bundle_file in all_bundle_files),+ ],+ names_host_paths=True,+ )+# Each file once, as the member of the first tree that holds it.seen_locations: set[Path] = set()bundle_files: list[LibraryBundleFile] = []@@ -873,6 +889,12 @@ class LibraryManager(LibraryManagerAbstract):for blueprint in unplaced_host_blueprints:if blueprint.host_bundle_file is not None:blueprint.place_in_tree(tree_bundle=tree_bundle_of_lone_file(bundle_file=blueprint.host_bundle_file, limits=tree_limits))+ # Every bundle read from this host's disk loads under its tree's requirements.txt, on every load: a blueprint+ # placed by an earlier load, a refused one included, is judged again rather than taken as judged.+ ensure_method_requirements_met(+ trees=[blueprint.tree_bundle.tree for blueprint in blueprints if blueprint.tree_bundle is not None],+ names_host_paths=True,+ )# Accumulate blueprints for later crate construction via get_crate()self._blueprints.setdefault(library_id, []).extend(blueprints)@@ -1343,6 +1365,8 @@ class LibraryManager(LibraryManagerAbstract):Raises:MthdsParserError: A bundle of the dependency does not parse, its items named by the bundle's source.+ MethodRequirementsError: The package's requirements.txt holds a line the standard refuses, or names+ packages its PipeFunc environment does not hold, named by the package's address."""alias = resolved_dep.alias@@ -1353,6 +1377,8 @@ class LibraryManager(LibraryManagerAbstract):tree_bundles_by_source: dict[str, TreeBundle] = {}# Listed under the package's address: a refusal never names the directory it was installed or fetched into.package_tree = list_package_method_tree(root=resolved_dep.package_root, package_address=package_address, limits=tree_limits)+ # The package's own requirements.txt is judged before any of its bundles loads, as the consumer's tree is.+ ensure_package_requirements_met(tree=package_tree, package_address=package_address)for tree_bundle in tree_bundles_of(tree=package_tree):# The package's address and the bundle's tree path: never the host's path to the package.bundle_source = f"{package_address}/{tree_bundle.tree_path}"diff --git a/pipelex/libraries/method_requirements.py b/pipelex/libraries/method_requirements.pynew file mode 100644index 000000000..7c5892dc8--- /dev/null+++ b/pipelex/libraries/method_requirements.py@@ -0,0 +1,447 @@+"""Checking a method's `requirements.txt` against the Python environment its PipeFunc steps run in, when the method loads.++The MTHDS standard (`mthds/docs/spec/method-tree.md`, "Python Dependencies: `requirements.txt`") lets a method declare+the Python packages its own Python needs in a `requirements.txt` at the root of its method tree: each line is blank, a+comment whose first character other than a space or a tab is `#`, or exactly one PEP 508 requirement, markers+included. A requirement naming a URL, an option of any kind (a line starting with `-`), a comment after a requirement+and a line continued with a backslash are refused, naming the line. Only the root's file declares anything.++Pipelex refuses a method whose requirements are not met when it loads, naming each package it cannot satisfy, and+never installs anything:++- A tree is checked when it holds `requirements.txt` at its root and holds Python, a `.py` file anywhere in it: a tree+ with no Python has nothing that could import a package. The check runs before any of the tree's Python is imported+ or captured, and before any of its pipes is built.+- The environment is the one the PipeFunc steps run in (`pipelex/libraries/package_environments.py`): this interpreter+ in the `direct` execution mode, and in a sandbox-hosted mode the package set the deployment states for its sandbox.+- A requirement whose marker is false in that environment is not needed there and is not checked. One is met when the+ environment holds a distribution of its name at a version its specifier contains, pre-releases included, and, for+ each extra it names, when what the extra needs is met in turn.++A refusal names the file by its path in the method's tree, and names paths on this host, the method's root and this+interpreter, only where it may: a check inside a load does, since its refusal is either the caller's own disk's or,+around a host's own library directories, the host's fault, which `reporting_host_library_tree_faults` withholds from+the caller. A dependency package is named by its address, and a server checking a caller's method it holds in a+directory of its own names no path on the host at all (`ensure_served_method_requirements_met`).+"""++import re+from collections.abc import Iterable, Sequence+from pathlib import Path+from typing import NamedTuple++from mthds.method_tree.models import MethodTree, TreeLimits+from packaging.markers import Marker, UndefinedComparison, UndefinedEnvironmentName+from packaging.requirements import InvalidRequirement, Requirement+from packaging.utils import canonicalize_name+from packaging.version import InvalidVersion, Version++from pipelex.core.exceptions import PipelexBundleBlueprintValidationErrorData+from pipelex.core.method_trees.tree_bundle import tree_key_of+from pipelex.libraries.exceptions import MethodRequirementsFileError, MethodRequirementsNotMetError+from pipelex.libraries.method_trees import list_library_dir_trees, reporting_host_library_tree_faults+from pipelex.libraries.package_environments import PythonPackageEnvironment, resolve_pipe_func_package_environment++REQUIREMENTS_FILE_NAME = "requirements.txt"+PYTHON_SUFFIX = ".py"++_UTF8_BOM = "\ufeff"++# The variables a PEP 508 marker can read. `extra` is not the environment's to state: a line of a requirements file+# names no extra of its own, so it reads as empty, as pip reads it.+_MARKER_VARIABLES = frozenset(+ {+ "implementation_name",+ "implementation_version",+ "os_name",+ "platform_machine",+ "platform_python_implementation",+ "platform_release",+ "platform_system",+ "platform_version",+ "python_full_version",+ "python_version",+ "sys_platform",+ "extras",+ "dependency_groups",+ }+)+_QUOTED_MARKER_VALUE = re.compile(r"\"[^\"]*\"|'[^']*'")+# A comment inside a line starts at a `#` that opens the line or follows whitespace, as pip reads one, so the fragment of+# a URL, such as `#sha256=…`, is not one.+_INLINE_COMMENT = re.compile(r"(^|\s+)#.*$")+_MARKER_WORD = re.compile(r"[A-Za-z_][A-Za-z0-9_]*")+++class RequirementLine(NamedTuple):+ """A requirement of a `requirements.txt`, with its 1-based line number and its text as written, stripped."""++ line_number: int+ text: str+ requirement: Requirement+++class RefusedRequirementsLine(NamedTuple):+ """A line of a `requirements.txt` the standard refuses, with why."""++ line_number: int+ text: str+ reason: str+++class ParsedRequirements(NamedTuple):+ """What a `requirements.txt` declares: its requirements, and the lines it holds that the standard refuses."""++ requirements: list[RequirementLine]+ refused_lines: list[RefusedRequirementsLine]+++class UnmetRequirement(NamedTuple):+ """A requirement the environment does not meet, with why."""++ requirement_line: RequirementLine+ reason: str+++def _refusal_reason(*, stripped: str) -> tuple[Requirement | None, str | None]:+ """The requirement a non-blank, non-comment line declares, or why the standard refuses the line."""+ if stripped.endswith("\\"):+ return None, "a line continued with a backslash is refused; write each requirement on a line of its own"+ if stripped.startswith("-"):+ return None, "an option line, such as -r, -e, -c or --index-url, is refused; write each package as a PEP 508 requirement"+ # The requirement is judged without the comment after it, so a comment that mentions a URL is not read as one.+ remainder = _INLINE_COMMENT.sub("", stripped).rstrip()+ try:+ requirement = Requirement(remainder)+ except InvalidRequirement as exc:+ if "://" in remainder:+ return None, "a requirement naming a URL is refused; name the package and the versions it accepts instead"+ parser_message = str(exc).splitlines()[0] if str(exc) else "it does not parse"+ return None, f"the line is not a PEP 508 requirement: {parser_message}"+ if requirement.url is not None:+ return None, "a requirement naming a URL is refused; name the package and the versions it accepts instead"+ if remainder != stripped:+ return None, "a comment after a requirement is refused; write the comment on a line of its own"+ return requirement, None+++def parse_requirements_text(*, text: str) -> ParsedRequirements:+ """Read the lines of a `requirements.txt` as the MTHDS standard does, keeping every refused line rather than the first.++ A leading byte order mark is skipped, and a line may end with a carriage return before its line feed.+ """+ requirements: list[RequirementLine] = []+ refused_lines: list[RefusedRequirementsLine] = []+ text = text.removeprefix(_UTF8_BOM)+ for line_index, raw_line in enumerate(text.split("\n")):+ line_number = line_index + 1+ line = raw_line.removesuffix("\r")+ if not line.strip():+ continue+ if line.lstrip(" \t").startswith("#"):+ continue+ stripped = line.strip()+ requirement, reason = _refusal_reason(stripped=stripped)+ if requirement is not None:+ requirements.append(RequirementLine(line_number=line_number, text=stripped, requirement=requirement))+ elif reason is not None:+ refused_lines.append(RefusedRequirementsLine(line_number=line_number, text=stripped, reason=reason))+ return ParsedRequirements(requirements=requirements, refused_lines=refused_lines)+++def marker_variables_of(*, marker: Marker) -> set[str]:+ """The PEP 508 variables a marker reads, its quoted values set aside so a value spelled like a variable is never one."""+ unquoted = _QUOTED_MARKER_VALUE.sub(" ", str(marker))+ return {word for word in _MARKER_WORD.findall(unquoted) if word in _MARKER_VARIABLES or word == "extra"}+++def _evaluate_marker(*, marker: Marker, environment: PythonPackageEnvironment, extra: str) -> bool:+ """A marker's value in the environment, reading `extra` as given.++ Raises:+ UndefinedComparison: The marker compares values that are not versions.+ UndefinedEnvironmentName: The marker reads a variable the evaluation does not define.+ InvalidVersion: The marker compares, as versions, a value that is not one; releases of `packaging` before 22+ raise it where later ones compare the values as strings.+ """+ return marker.evaluate(environment={**environment.marker_environment, "extra": extra})+++def _unmet_reason(+ *,+ requirement: Requirement,+ environment: PythonPackageEnvironment,+ followed_extras: set[tuple[str, str]],+) -> str | None:+ """Why the environment does not meet a requirement whose marker it already judged true, or None when it does."""+ installed_version = environment.version_of(distribution_name=requirement.name)+ if installed_version is None:+ return f"{requirement.name} is missing"+ try:+ version = Version(installed_version)+ except InvalidVersion:+ return f"{requirement.name} {installed_version} is there, a version no specifier can be compared with"+ if not requirement.specifier.contains(version, prereleases=True):+ return f"{requirement.name} {installed_version} is there"+ for extra in sorted(requirement.extras):+ extra_reason = _unmet_extra_reason(distribution_name=requirement.name, extra=extra, environment=environment, followed_extras=followed_extras)+ if extra_reason is not None:+ return extra_reason+ return None+++def _unmet_extra_reason(+ *,+ distribution_name: str,+ extra: str,+ environment: PythonPackageEnvironment,+ followed_extras: set[tuple[str, str]],+) -> str | None:+ """Why the environment does not meet what an extra of a distribution it holds needs, or None when it does.++ An extra needs each requirement of the distribution's metadata whose marker holds with the extra and not without+ it. Each extra is followed once, so extras that name one another end.+ """+ key = (canonicalize_name(distribution_name), canonicalize_name(extra))+ if key in followed_extras:+ return None+ followed_extras.add(key)+ dependencies = environment.dependencies_of(distribution_name=distribution_name)+ if dependencies is None:+ return (+ f"{environment.description} does not state what the extra '{extra}' of {distribution_name} needs; "+ "list the packages it needs in requirements.txt instead"+ )+ for dependency_text in dependencies:+ try:+ dependency = Requirement(dependency_text)+ except InvalidRequirement:+ # Metadata of an installed distribution that no installer could have read either; it needs nothing here.+ continue+ if dependency.marker is None:+ continue+ try:+ needed_by_extra = _evaluate_marker(marker=dependency.marker, environment=environment, extra=extra)+ needed_anyway = _evaluate_marker(marker=dependency.marker, environment=environment, extra="")+ except (UndefinedComparison, UndefinedEnvironmentName, InvalidVersion):+ continue+ if not needed_by_extra or needed_anyway:+ continue+ dependency_reason = _unmet_reason(requirement=dependency, environment=environment, followed_extras=followed_extras)+ if dependency_reason is not None:+ return f"the extra '{extra}' of {distribution_name} needs '{dependency_text}', and {dependency_reason}"+ return None+++def unmet_requirements(*, requirements: Iterable[RequirementLine], environment: PythonPackageEnvironment) -> list[UnmetRequirement]:+ """The requirements the environment does not meet, in the order given, each with why."""+ unmet: list[UnmetRequirement] = []+ for requirement_line in requirements:+ reason = _requirement_line_unmet_reason(requirement_line=requirement_line, environment=environment)+ if reason is not None:+ unmet.append(UnmetRequirement(requirement_line=requirement_line, reason=reason))+ return unmet+++def _requirement_line_unmet_reason(*, requirement_line: RequirementLine, environment: PythonPackageEnvironment) -> str | None:+ marker = requirement_line.requirement.marker+ if marker is not None:+ unstated = sorted(marker_variables_of(marker=marker) - set(environment.marker_environment) - {"extra"})+ if unstated:+ return f"its marker reads {', '.join(unstated)}, which {environment.description} does not state"+ try:+ if not _evaluate_marker(marker=marker, environment=environment, extra=""):+ return None+ except (UndefinedComparison, UndefinedEnvironmentName, InvalidVersion) as exc:+ return f"its marker cannot be evaluated: {exc}"+ return _unmet_reason(requirement=requirement_line.requirement, environment=environment, followed_extras=set())+++def holds_requirements_to_check(*, tree: MethodTree) -> bool:+ """Whether a tree's requirements are checked when it loads: it lies on disk, declares them at its root, and holds Python."""+ if tree.directory is None or REQUIREMENTS_FILE_NAME not in tree.members:+ return False+ return any(tree_path.endswith(PYTHON_SUFFIX) for tree_path in tree.tree_paths)+++class JudgedMethod(NamedTuple):+ """A method tree whose requirements are judged, with how a refusal names it and its file."""++ tree: MethodTree+ method_name: str+ requirements_source: str+++def _judged_method(*, tree: MethodTree, names_host_paths: bool) -> JudgedMethod:+ """A tree as a refusal names it: by its root on this host when it may name host paths, and as "The method" otherwise.++ The file is named by its path in the tree either way, never by its path on the host.+ """+ method_name = f"The method rooted at '{tree.directory}'" if names_host_paths and tree.directory is not None else "The method"+ return JudgedMethod(tree=tree, method_name=method_name, requirements_source=REQUIREMENTS_FILE_NAME)+++def _read_requirements_text(*, judged: JudgedMethod) -> str:+ """The text of a tree's `requirements.txt`.++ Raises:+ MethodRequirementsFileError: The file cannot be read, or is not UTF-8 text.+ """+ location = judged.tree.file_location(REQUIREMENTS_FILE_NAME)+ source = judged.requirements_source+ try:+ return location.read_bytes().decode("utf-8")+ except UnicodeDecodeError as exc:+ msg = f"{judged.method_name} declares its packages in a {REQUIREMENTS_FILE_NAME} that is not UTF-8 text, so they cannot be read."+ raise MethodRequirementsFileError(msg, items=[PipelexBundleBlueprintValidationErrorData(source=source, message=msg)]) from exc+ except OSError as exc:+ msg = f"{judged.method_name} declares its packages in a {REQUIREMENTS_FILE_NAME} that cannot be read: {exc.strerror or type(exc).__name__}."+ raise MethodRequirementsFileError(msg, items=[PipelexBundleBlueprintValidationErrorData(source=source, message=msg)]) from exc+++def _raise_refused_lines(*, judged: JudgedMethod, refused_lines: list[RefusedRequirementsLine]) -> None:+ """Refuse the file for its lines the standard refuses, naming each.++ Raises:+ MethodRequirementsFileError: Always.+ """+ items: list[PipelexBundleBlueprintValidationErrorData] = []+ line_texts: list[str] = []+ for refused_line in refused_lines:+ line_text = f"line {refused_line.line_number} '{refused_line.text}': {refused_line.reason}"+ line_texts.append(line_text)+ items.append(+ PipelexBundleBlueprintValidationErrorData(+ source=judged.requirements_source,+ line=refused_line.line_number,+ message=f"{REQUIREMENTS_FILE_NAME} {line_text}.",+ )+ )+ msg = (+ f"{judged.method_name} declares its packages in a {REQUIREMENTS_FILE_NAME} holding lines the MTHDS standard "+ f"refuses, where each line is blank, a comment, or exactly one PEP 508 requirement: {'; '.join(line_texts)}."+ )+ raise MethodRequirementsFileError(msg, items=items)+++def _raise_unmet(*, judged: JudgedMethod, unmet: list[UnmetRequirement], environment: PythonPackageEnvironment) -> None:+ """Refuse the method for the requirements the environment does not meet, naming each and what its author can do.++ Raises:+ MethodRequirementsNotMetError: Always.+ """+ items: list[PipelexBundleBlueprintValidationErrorData] = []+ line_texts: list[str] = []+ for unmet_requirement in unmet:+ requirement_line = unmet_requirement.requirement_line+ line_text = f"line {requirement_line.line_number} '{requirement_line.text}' ({unmet_requirement.reason})"+ line_texts.append(line_text)+ items.append(+ PipelexBundleBlueprintValidationErrorData(+ source=judged.requirements_source,+ line=requirement_line.line_number,+ message=f"The requirement '{requirement_line.text}' is not met by {environment.description}: {unmet_requirement.reason}.",+ )+ )+ advice = environment.advice(requirement_texts=[unmet_requirement.requirement_line.text for unmet_requirement in unmet])+ msg = (+ f"{judged.method_name} needs Python packages that {environment.description} "+ f"does not have, as its {REQUIREMENTS_FILE_NAME} declares: {'; '.join(line_texts)}. {advice}"+ )+ raise MethodRequirementsNotMetError(msg, items=items)+++def _ensure_judged_methods_met(*, judged_methods: Iterable[JudgedMethod], names_host_paths: bool) -> None:+ """Refuse the first of the methods whose `requirements.txt` the standard refuses or its PipeFunc environment does not meet.++ Each tree is judged once however often it is given, in the order given. The environment is resolved only once a tree+ declares a requirement to judge, so methods declaring none never read it.++ Raises:+ MethodRequirementsFileError: A tree's `requirements.txt` holds a line the standard refuses, or cannot be read.+ MethodRequirementsNotMetError: The environment does not meet a tree's requirements.+ PipeFuncPackageSetUnstatedError: PipeFunc steps run out of this process and the deployment states no package+ set for them.+ """+ environment: PythonPackageEnvironment | None = None+ judged_tree_keys: set[str] = set()+ for judged in judged_methods:+ tree_key = tree_key_of(tree=judged.tree)+ if tree_key in judged_tree_keys:+ continue+ judged_tree_keys.add(tree_key)+ if not holds_requirements_to_check(tree=judged.tree):+ continue+ parsed = parse_requirements_text(text=_read_requirements_text(judged=judged))+ if parsed.refused_lines:+ _raise_refused_lines(judged=judged, refused_lines=parsed.refused_lines)+ if not parsed.requirements:+ continue+ if environment is None:+ environment = resolve_pipe_func_package_environment(names_host_paths=names_host_paths)+ unmet = unmet_requirements(requirements=parsed.requirements, environment=environment)+ if unmet:+ _raise_unmet(judged=judged, unmet=unmet, environment=environment)+++def ensure_method_requirements_met(*, trees: Iterable[MethodTree], names_host_paths: bool) -> None:+ """Refuse a method among the trees whose `requirements.txt` the standard refuses or its PipeFunc environment does not meet.++ Each tree is judged once however often it is given, in the order given, and the first refused ends the check.++ Args:+ trees: The method trees a load reads, each checked when it lies on disk, declares requirements and holds Python.+ names_host_paths: Whether a refusal may name the method's root and this interpreter by their paths on this host,+ as a load's may: its refusal is the caller's own disk's, or the host's fault, withheld from the caller.++ Raises:+ MethodRequirementsFileError: A tree's `requirements.txt` holds a line the standard refuses, or cannot be read.+ MethodRequirementsNotMetError: The environment does not meet a tree's requirements.+ PipeFuncPackageSetUnstatedError: PipeFunc steps run out of this process and the deployment states no package+ set for them.+ """+ _ensure_judged_methods_met(+ judged_methods=(_judged_method(tree=tree, names_host_paths=names_host_paths) for tree in trees),+ names_host_paths=names_host_paths,+ )+++def ensure_package_requirements_met(*, tree: MethodTree, package_address: str) -> None:+ """Refuse a dependency package whose `requirements.txt` the standard refuses or its PipeFunc environment does not meet.++ The package is named by its address and its file by `<address>/requirements.txt`, never by the directory it was+ installed or fetched into, as its bundles are, since the package is the caller's dependency wherever it loads.++ Raises:+ MethodRequirementsFileError: The package's `requirements.txt` holds a line the standard refuses, or cannot be read.+ MethodRequirementsNotMetError: The environment does not meet the package's requirements.+ PipeFuncPackageSetUnstatedError: PipeFunc steps run out of this process and the deployment states no package+ set for them.+ """+ judged = JudgedMethod(+ tree=tree,+ method_name=f"The method package '{package_address}'",+ requirements_source=f"{package_address}/{REQUIREMENTS_FILE_NAME}",+ )+ _ensure_judged_methods_met(judged_methods=[judged], names_host_paths=False)+++def ensure_served_method_requirements_met(*, library_dirs: Sequence[Path], limits: TreeLimits | None = None) -> None:+ """Check the requirements of a caller's method a server holds in directories of its own, as loading them would, without loading them.++ For a server that hands the directories to a loader elsewhere, such as a validator on a worker that never reads+ them, or that loads them as its own library directories, so the refusal is the caller's and comes from the place+ that knows the method's files. The directories are the server's: a tree of them that cannot be listed is the+ host's fault, as a load of a host's library directories reports it, and a refusal names no path on the host, the+ directories' and this interpreter's included.++ Raises:+ HostLibraryTreeError: A tree of the directories could not be listed.+ MethodRequirementsFileError: A tree's `requirements.txt` holds a line the standard refuses, or cannot be read.+ MethodRequirementsNotMetError: The environment does not meet a tree's requirements.+ PipeFuncPackageSetUnstatedError: PipeFunc steps run out of this process and the deployment states no package+ set for them.+ """+ with reporting_host_library_tree_faults():+ library_trees = list_library_dir_trees(library_dirs=library_dirs, limits=limits)+ ensure_method_requirements_met(trees=[library_tree.tree for library_tree in library_trees], names_host_paths=False)diff --git a/pipelex/libraries/method_trees.py b/pipelex/libraries/method_trees.pyindex 8d39177e8..43a6540cd 100644--- a/pipelex/libraries/method_trees.py+++ b/pipelex/libraries/method_trees.py@@ -46,7 +46,7 @@ from mthds.method_tree.vocabulary import ExclusionRule, TreeRefusalCodefrom pipelex import logfrom pipelex.core.method_trees.tree_bundle import TreeBundlefrom pipelex.hosted.execution import get_or_load_pipelex_config-from pipelex.libraries.exceptions import HostLibraryTreeError, MethodTreeLoadError+from pipelex.libraries.exceptions import HostLibraryRequirementsError, HostLibraryTreeError, MethodRequirementsError, MethodTreeLoadErrorLIMITS_ADVICE = ("Hand over the method's own folder rather than a directory above it, or raise the limits of "@@ -121,20 +121,25 @@ def list_package_method_tree(*, root: Path, package_address: str, limits: TreeLi@contextmanagerdef reporting_host_library_tree_faults() -> Generator[None, None, None]:- """Report a method tree of this host's own library directories that cannot be listed as the host's fault.+ """Report a method tree of this host's own library directories that cannot be listed, or whose requirements refuse it, as the host's fault.Around a load of library directories that are not the caller's, such as the ones a server loads beside a request's- bundles, a `MethodTreeLoadError` leaves as a `HostLibraryTreeError`, which is a configuration fault and whose- message, naming paths on the host, never reaches the caller.+ bundles, a `MethodTreeLoadError` leaves as a `HostLibraryTreeError`, and a `MethodRequirementsError` as a+ `HostLibraryRequirementsError`: both are configuration faults, whose messages, naming paths on the host, never reach+ the caller.Raises:HostLibraryTreeError: A tree of the directories could not be listed.+ HostLibraryRequirementsError: A method of the directories declares requirements its PipeFunc environment refuses."""try:yieldexcept MethodTreeLoadError as exc:msg = f"A library directory of this host cannot be read as a method tree: {exc.message}"raise HostLibraryTreeError(msg) from exc+ except MethodRequirementsError as exc:+ msg = f"A library directory of this host holds a method whose requirements.txt refuses it: {exc.message}"+ raise HostLibraryRequirementsError(msg) from excdef tree_bundles_of(*, tree: MethodTree) -> list[TreeBundle]:@@ -389,11 +394,18 @@ def _spelled_from(*, library_dir: Path, location: Path) -> Path:return location-def list_library_bundle_files(*, library_dirs: Iterable[Path], limits: TreeLimits | None = None) -> list[LibraryBundleFile]:- """The bundle files of the library directories, each as a member of its method tree, each file once.+class LibraryTree(NamedTuple):+ """A method tree a library directory holds, with the directory as the caller gave it."""- The directories are read in the order given, and a file two of their trees hold, such as a directory and one of- its subdirectories both given, is the member of the first tree that holds it.+ library_dir: Path+ tree: MethodTree+++def list_library_dir_trees(*, library_dirs: Iterable[Path], limits: TreeLimits | None = None) -> list[LibraryTree]:+ """The method trees of the library directories, in the order the directories are given, each directory's own first.++ A tree holding no bundle is listed too, such as one holding only the Python a server received beside a method's+ bundles: a load reads its `requirements.txt` before it reads any of its Python.Args:library_dirs: The library directories.@@ -403,19 +415,48 @@ def list_library_bundle_files(*, library_dirs: Iterable[Path], limits: TreeLimitMethodTreeLoadError: A tree could not be listed."""active_limits = limits or local_tree_limits()- seen_locations: set[Path] = set()- bundle_files: list[LibraryBundleFile] = []+ library_trees: list[LibraryTree] = []for library_dir in library_dirs:for tree in list_library_trees(library_dir=library_dir, limits=active_limits):- for tree_bundle in tree_bundles_of(tree=tree):- location = tree_bundle.location- if location in seen_locations:- continue- seen_locations.add(location)- bundle_files.append(LibraryBundleFile(tree_bundle=tree_bundle, path=_spelled_from(library_dir=library_dir, location=location)))+ library_trees.append(LibraryTree(library_dir=library_dir, tree=tree))+ return library_trees+++def library_bundle_files_of(*, library_trees: Iterable[LibraryTree]) -> list[LibraryBundleFile]:+ """The bundle files of listed library trees, each as a member of its method tree, each file once.++ A file two of the trees hold, such as a directory and one of its subdirectories both given, is the member of the+ first tree that holds it, spelled from that tree's library directory.+ """+ seen_locations: set[Path] = set()+ bundle_files: list[LibraryBundleFile] = []+ for library_tree in library_trees:+ for tree_bundle in tree_bundles_of(tree=library_tree.tree):+ location = tree_bundle.location+ if location in seen_locations:+ continue+ seen_locations.add(location)+ spelled_path = _spelled_from(library_dir=library_tree.library_dir, location=location)+ bundle_files.append(LibraryBundleFile(tree_bundle=tree_bundle, path=spelled_path))return bundle_files+def list_library_bundle_files(*, library_dirs: Iterable[Path], limits: TreeLimits | None = None) -> list[LibraryBundleFile]:+ """The bundle files of the library directories, each as a member of its method tree, each file once.++ The directories are read in the order given, and a file two of their trees hold, such as a directory and one of+ its subdirectories both given, is the member of the first tree that holds it.++ Args:+ library_dirs: The library directories.+ limits: The limits to list each tree under, read from the configuration once when not given.++ Raises:+ MethodTreeLoadError: A tree could not be listed.+ """+ return library_bundle_files_of(library_trees=list_library_dir_trees(library_dirs=library_dirs, limits=limits))++def bundle_files_under(*, library_dirs: Iterable[Path]) -> list[Path]:"""Every `.mthds` file under the library directories and the trees they are rooted in, whatever the membership rule.diff --git a/pipelex/libraries/package_environments.py b/pipelex/libraries/package_environments.pynew file mode 100644index 000000000..d6cb320b8--- /dev/null+++ b/pipelex/libraries/package_environments.py@@ -0,0 +1,163 @@+"""The Python environment a method's PipeFunc steps run in, as the check of its `requirements.txt` reads it.++Where a PipeFunc runs decides what its requirements are checked against:++- In the `direct` execution mode it runs in this interpreter, so the check reads the distributions installed here and+ this interpreter's own marker environment, and can follow a requirement's extras through their metadata.+- In any other mode it runs in a sandbox this process never sees, such as the Daytona sandbox, whose packages are the+ whole offer. The plugin providing the mode states them (`pipelex/plugins/pipe_func_package_offer_registry.py`), and+ the check reads that statement alone: a marker variable it does not state cannot be judged, and an extra's needs,+ which it does not state, cannot be followed. A mode with no statement fails closed.++How an environment names itself in a refusal depends on who reads it: this interpreter is named by its path on the+host, and its advice gives the exact install command, only where the refusal may name paths on the host.+"""++import shlex+import sys+from collections.abc import Mapping+from importlib.metadata import PackageNotFoundError, distribution+from typing import Protocol++from packaging.markers import default_environment+from packaging.utils import canonicalize_name+from typing_extensions import override++from pipelex.config import get_config, get_pipe_func_execution_mode, is_pipe_func_sandbox_hosted+from pipelex.interpreter_hub import get_pipe_func_package_offer_registry+from pipelex.libraries.exceptions import PipeFuncPackageSetUnstatedError+from pipelex.plugins.pipe_func_package_offer_registry import PipeFuncPackageOffer+++class PythonPackageEnvironment(Protocol):+ """The Python packages and the marker environment a method's requirements are judged against."""++ @property+ def description(self) -> str:+ """The environment as a refusal names it, a noun phrase such as "this Python interpreter (/usr/bin/python3)"."""+ ...++ @property+ def marker_environment(self) -> Mapping[str, str]:+ """The PEP 508 marker values the environment states; a variable missing here cannot be judged."""+ ...++ def version_of(self, *, distribution_name: str) -> str | None:+ """The version of the distribution the environment holds under this name, or None when it holds none."""+ ...++ def dependencies_of(self, *, distribution_name: str) -> list[str] | None:+ """The requirements the distribution's metadata declares (its Requires-Dist), or None when the environment does not state them."""+ ...++ def advice(self, *, requirement_texts: list[str]) -> str:+ """What the method's author can do about the requirements given, as written in the file, that the environment does not meet."""+ ...+++class InterpreterPackageEnvironment(PythonPackageEnvironment):+ """This interpreter, where a PipeFunc runs in the `direct` execution mode: its installed distributions, read as needed.++ Its path on the host is named, in its description and in the install command its advice gives, only when+ `names_host_paths` says the refusal may name paths on the host.+ """++ def __init__(self, *, names_host_paths: bool) -> None:+ self._executable = sys.executable+ self._names_host_paths = names_host_paths+ self._marker_environment: dict[str, str] = {variable: str(value) for variable, value in default_environment().items()}++ @property+ @override+ def description(self) -> str:+ if self._names_host_paths:+ return f"this Python interpreter ({self._executable})"+ return "the Python interpreter Pipelex runs in"++ @property+ @override+ def marker_environment(self) -> Mapping[str, str]:+ return self._marker_environment++ @override+ def version_of(self, *, distribution_name: str) -> str | None:+ try:+ return distribution(distribution_name).version+ except PackageNotFoundError:+ return None++ @override+ def dependencies_of(self, *, distribution_name: str) -> list[str] | None:+ try:+ return distribution(distribution_name).requires or []+ except PackageNotFoundError:+ return []++ @override+ def advice(self, *, requirement_texts: list[str]) -> str:+ quoted = " ".join(shlex.quote(requirement_text) for requirement_text in requirement_texts)+ if self._names_host_paths:+ return (+ f"Install them into this interpreter with: uv pip install --python {shlex.quote(self._executable)} {quoted}. "+ "Pipelex never installs a method's requirements itself."+ )+ return (+ f"Install them into the Python interpreter Pipelex runs in, for instance with: uv pip install {quoted}. "+ "Pipelex never installs a method's requirements itself."+ )+++class OfferedPackageEnvironment(PythonPackageEnvironment):+ """The package set a plugin states for the sandbox an out-of-process execution mode runs a PipeFunc in."""++ def __init__(self, *, offer: PipeFuncPackageOffer):+ self._offer = offer+ self._versions: dict[str, str] = {canonicalize_name(name): version for name, version in offer.distributions.items()}++ @property+ @override+ def description(self) -> str:+ return self._offer.environment++ @property+ @override+ def marker_environment(self) -> Mapping[str, str]:+ return self._offer.marker_environment++ @override+ def version_of(self, *, distribution_name: str) -> str | None:+ return self._versions.get(canonicalize_name(distribution_name))++ @override+ def dependencies_of(self, *, distribution_name: str) -> list[str] | None:+ return None++ @override+ def advice(self, *, requirement_texts: list[str]) -> str:+ return (+ f"PipeFunc steps on this deployment run with the packages of {self._offer.environment} and no others, "+ "so require versions it holds, or remove the requirements it cannot meet."+ )+++def resolve_pipe_func_package_environment(*, names_host_paths: bool) -> PythonPackageEnvironment:+ """The environment this process's PipeFunc steps run in, as its execution mode decides.++ Args:+ names_host_paths: Whether a refusal naming the environment may name paths on this host, such as this+ interpreter's.++ Raises:+ PipeFuncPackageSetUnstatedError: The mode runs PipeFunc steps out of this process and no plugin states, or the+ stating plugin could not state, the packages found there.+ """+ if not is_pipe_func_sandbox_hosted():+ return InterpreterPackageEnvironment(names_host_paths=names_host_paths)+ execution_mode = get_pipe_func_execution_mode()+ offer_factory = get_pipe_func_package_offer_registry().get_optional(mode=execution_mode)+ if offer_factory is None:+ raise PipeFuncPackageSetUnstatedError(execution_mode=execution_mode, offer_registered=False)+ offer = offer_factory(get_config().interpreter.pipe_func)+ if offer is None:+ raise PipeFuncPackageSetUnstatedError(execution_mode=execution_mode, offer_registered=True)+ return OfferedPackageEnvironment(offer=offer)diff --git a/pipelex/pipelex.py b/pipelex/pipelex.pyindex dd7d83a82..dc9545c3f 100644--- a/pipelex/pipelex.py+++ b/pipelex/pipelex.py@@ -4,7 +4,7 @@interpreter half of the boot: it subclasses :class:`pipelex.runtime_boot.RuntimeBoot` — importing itdownward, which the interpreter layer may do — and appends the constructions that only a processwhich will actually load and run a method needs: the ``InterpreterHub``, the composed plugin-manifests, the ``PipeFuncExecutorRegistry`` and its executor, the ``LibraryManager``, the+manifests, the ``PipeFuncExecutorRegistry`` and its executor, the ``PipeFuncPackageOfferRegistry``, the ``LibraryManager``, the``PipelineManager``, the pipe-kind class registrations, the ``PipeRouter`` and the ``PipeRun``.Inheritance rather than composition is deliberate: every attribute address is preserved@@ -41,6 +41,7 @@ from pipelex.pipe_run.pipe_run import PipeRunfrom pipelex.pipeline.pipeline_manager import PipelineManagerfrom pipelex.pipeline.pipeline_manager_abstract import PipelineManagerAbstractfrom pipelex.plugins.pipe_func_executor_registry import PipeFuncExecutorRegistry+from pipelex.plugins.pipe_func_package_offer_registry import PipeFuncPackageOfferRegistryfrom pipelex.plugins.registrar import HubSlotfrom pipelex.reporting.reporting_protocol import ReportingProtocolfrom pipelex.runtime_boot import PACKAGE_VERSION, RuntimeBoot@@ -141,6 +142,11 @@ class Pipelex(RuntimeBoot):pipe_func_executor_registry = PipeFuncExecutorRegistry(plugin_registrar.pipe_func_executors)self.interpreter_hub.set_pipe_func_executor_registry(pipe_func_executor_registry)+ # The packages each out-of-process mode offers a PipeFunc, which a method's requirements.txt is checked against+ # when it loads; `direct` needs no entry, its packages being this interpreter's own.+ self.interpreter_hub.set_pipe_func_package_offer_registry(+ pipe_func_package_offer_registry=PipeFuncPackageOfferRegistry(plugin_registrar.pipe_func_package_offers)+ )# Injection precedence: explicit setup() param > plugin slot-claim thunk > config-selected# registry factory. The PipeFunc execution axis is orthogonal to orchestration: a Temporaldiff --git a/pipelex/plugins/exceptions.py b/pipelex/plugins/exceptions.pyindex f9a937e0b..a4fec94bc 100644--- a/pipelex/plugins/exceptions.py+++ b/pipelex/plugins/exceptions.py@@ -212,6 +212,20 @@ class DuplicatePipeFuncExecutorError(PluginError):super().__init__(message)+class DuplicatePipeFuncPackageOfferError(PluginError):+ """Two plugins stated the package offer of the same PipeFunc execution mode."""++ def __init__(self, *, mode: str, first_plugin: str, second_plugin: str):+ self.mode = mode+ self.first_plugin = first_plugin+ self.second_plugin = second_plugin+ message = (+ f"The PipeFunc package offer of execution mode '{mode}' is stated by both plugin "+ f"'{first_plugin}' and plugin '{second_plugin}'. Each mode must have a single package offer."+ )+ super().__init__(message)++class DuplicateHttpErrorMapperError(PluginError):"""Two plugins registered an HTTP-error mapper for the same exception type."""diff --git a/pipelex/plugins/pipe_func_package_offer_registry.py b/pipelex/plugins/pipe_func_package_offer_registry.pynew file mode 100644index 000000000..1071a7010--- /dev/null+++ b/pipelex/plugins/pipe_func_package_offer_registry.py@@ -0,0 +1,66 @@+"""The seam through which a deployment states the Python packages its PipeFunc steps run with, out of this process.++A method declares the packages its Python needs in the `requirements.txt` at the root of its method tree, and pipelex+checks them when the method loads (`pipelex/libraries/method_requirements.py`). In the `direct` execution mode a+PipeFunc runs in this interpreter, so the check reads this interpreter's installed distributions. In any other mode it+runs in a sandbox this process never sees, such as the Daytona sandbox, whose packages are the whole offer: pipelex+cannot read them, so the plugin that contributes the mode's executor states them here, with+``PluginRegistrar.add_pipe_func_package_offer``, keyed by the same execution mode token.++A mode for which no plugin states a package set, or whose factory answers ``None``, meets no requirement at all: the+check fails closed rather than letting a method load whose Python would then fail inside the sandbox.+"""++from collections.abc import Callable, Mapping+from typing import TypeAlias++from pydantic import BaseModel, ConfigDict, Field++from pipelex.system.configuration.pipe_func_config import PipeFuncConfig+++class PipeFuncPackageOffer(BaseModel):+ """The Python packages a PipeFunc step finds where an execution mode runs it, and the marker environment there."""++ model_config = ConfigDict(frozen=True, extra="forbid")++ environment: str = Field(+ description=(+ "The place the packages are offered, as a refusal names it to the method's author, such as "+ "\"the Daytona sandbox snapshot 'pipelex-base-dev'\"."+ )+ )+ distributions: dict[str, str] = Field(+ description="Each distribution the place holds, by its name (normalized or not), with its exact installed version."+ )+ marker_environment: dict[str, str] = Field(+ description=(+ "The PEP 508 environment marker values of the place's interpreter, such as python_version and sys_platform. "+ "A requirement whose marker reads a variable missing here cannot be judged, and is refused."+ )+ )+++# A plugin's factory for the package offer of one PipeFunc execution mode: the pipe_func config in, the offer out, or+# ``None`` when the deployment cannot state it. Called each time a method declaring requirements loads, never at+# registration, so a factory may read its own configuration and should cache what it computes.+PipeFuncPackageOfferFactoryFn: TypeAlias = Callable[[PipeFuncConfig], PipeFuncPackageOffer | None]+++class PipeFuncPackageOfferRegistry:+ """Read view over the package-offer factories contributed by discovered plugins, keyed by execution mode.++ Built once at boot from the registrar's accumulated ``pipe_func_package_offers`` and stored on the interpreter hub,+ beside the ``PipeFuncExecutorRegistry`` whose modes it mirrors. The ``direct`` mode needs no entry: its packages+ are this interpreter's own.+ """++ def __init__(self, pipe_func_package_offers: Mapping[str, PipeFuncPackageOfferFactoryFn]):+ self._pipe_func_package_offers: dict[str, PipeFuncPackageOfferFactoryFn] = dict(pipe_func_package_offers)++ def get_optional(self, *, mode: str) -> PipeFuncPackageOfferFactoryFn | None:+ return self._pipe_func_package_offers.get(mode)++ @property+ def modes(self) -> list[str]:+ return list(self._pipe_func_package_offers)diff --git a/pipelex/plugins/registrar.py b/pipelex/plugins/registrar.pyindex a434703b2..11f4c54c0 100644--- a/pipelex/plugins/registrar.py+++ b/pipelex/plugins/registrar.py@@ -20,6 +20,7 @@ from pipelex.plugins.exceptions import (DuplicateModelListerError,DuplicateOrchestratorError,DuplicatePipeFuncExecutorError,+ DuplicatePipeFuncPackageOfferError,DuplicateSecretsProviderError,DuplicateServiceErrorCodeError,DuplicateStorageProviderError,@@ -32,6 +33,7 @@ from pipelex.plugins.log_sink_registry import LogSinkFactoryFnfrom pipelex.plugins.model_lister_registry import ListModelsFnfrom pipelex.plugins.orchestrator_registry import OrchestratorProtocolfrom pipelex.plugins.pipe_func_executor_registry import PipeFuncExecutorFactoryFn+from pipelex.plugins.pipe_func_package_offer_registry import PipeFuncPackageOfferFactoryFnfrom pipelex.plugins.plugin_group import PluginGroupfrom pipelex.plugins.plugin_model_declarations import PluginDocGenDefault, PluginInternalModel, PluginModelDeclarationsfrom pipelex.plugins.secrets_provider_registry import SecretsProviderFactoryFn@@ -175,6 +177,7 @@ class PluginRegistrar:self.secrets_providers: dict[str, SecretsProviderFactoryFn] = {}self.log_sinks: dict[str, LogSinkFactoryFn] = {}self.pipe_func_executors: dict[str, PipeFuncExecutorFactoryFn] = {}+ self.pipe_func_package_offers: dict[str, PipeFuncPackageOfferFactoryFn] = {}# Plain data, read once by ``make_model_declarations`` for the model manager to merge at boot.# Keyed by the model's name and its model type as written, since a handle names one model per model type.self.internal_models: dict[tuple[str, str], dict[str, Any]] = {}@@ -195,6 +198,7 @@ class PluginRegistrar:self._secrets_provider_sources: dict[str, str] = {}self._log_sink_sources: dict[str, str] = {}self._pipe_func_executor_sources: dict[str, str] = {}+ self._pipe_func_package_offer_sources: dict[str, str] = {}self._internal_model_sources: dict[tuple[str, str], str] = {}self._doc_gen_default_sources: dict[tuple[DocGenFormat, DocGenSource], str] = {}self._service_error_code_sources: dict[str, str] = {}@@ -446,6 +450,28 @@ class PluginRegistrar:),)+ def add_pipe_func_package_offer(self, *, mode: str, factory: PipeFuncPackageOfferFactoryFn) -> None:+ """State the Python packages a PipeFunc step finds in one execution mode, keyed by the same ``mode`` token as its executor.++ A method's ``requirements.txt`` is checked when it loads against the packages its PipeFunc steps will run+ with. In ``direct`` mode those are this interpreter's own; in any other mode they are the sandbox's, which this+ process cannot read, so the plugin contributing that mode's executor states them here (e.g. our Daytona plugin,+ from the lock of the Daytona sandbox snapshot). ``factory`` is called with the pipe_func config each time a+ method declaring requirements loads, never here, and answers ``None`` when it cannot state the offer; a mode+ with no offer, or a ``None`` answer, meets no requirement. Fail-loud on a duplicate mode, naming both plugins.+ """+ self._require_interpreter_layer(capability=f"pipe_func package offer {mode}")+ self._add(+ store=self.pipe_func_package_offers,+ sources=self._pipe_func_package_offer_sources,+ key=mode,+ value=factory,+ contribution=f"pipe_func package offer {mode}",+ on_duplicate=lambda first_plugin, second_plugin: DuplicatePipeFuncPackageOfferError(+ mode=mode, first_plugin=first_plugin, second_plugin=second_plugin+ ),+ )+def add_http_error_mapper(self, *, exc_type_provider: HttpErrorTypeProviderFn, to_error_report: HttpErrorMapperFn) -> None:"""Contribute a mapping from a transport/runtime exception to a structured ``ErrorReport``.diff --git a/pyproject.toml b/pyproject.tomlindex ae0fe81a2..907f45dc8 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -34,6 +34,7 @@ dependencies = ["mthds==0.22.0","networkx>=3.4.2","openai>=2.0.0",+ "packaging>=24.0", # reads a method's requirements.txt: PEP 508 requirements, markers and versions"opentelemetry-api>=1.39.0","opentelemetry-exporter-otlp-proto-http>=1.38.0","opentelemetry-semantic-conventions>=0.59b0",diff --git a/tests/data/errors/error_identity.txt b/tests/data/errors/error_identity.txtindex 0cb6ea2a5..54b3561a6 100644--- a/tests/data/errors/error_identity.txt+++ b/tests/data/errors/error_identity.txt@@ -84,6 +84,7 @@ DuplicateLogSinkError | Duplicate log sink | https://docs.pipelex.com/latest/errDuplicateModelListerError | Duplicate model lister | https://docs.pipelex.com/latest/errors/duplicate-model-lister-error/DuplicateOrchestratorError | Duplicate orchestrator | https://docs.pipelex.com/latest/errors/duplicate-orchestrator-error/DuplicatePipeFuncExecutorError | Duplicate pipe func executor | https://docs.pipelex.com/latest/errors/duplicate-pipe-func-executor-error/+DuplicatePipeFuncPackageOfferError | Duplicate pipe func package offer | https://docs.pipelex.com/latest/errors/duplicate-pipe-func-package-offer-error/DuplicateSecretsProviderError | Duplicate secrets provider | https://docs.pipelex.com/latest/errors/duplicate-secrets-provider-error/DuplicateServiceErrorCodeError | Duplicate service error code | https://docs.pipelex.com/latest/errors/duplicate-service-error-code-error/DuplicateStorageProviderError | Duplicate storage provider | https://docs.pipelex.com/latest/errors/duplicate-storage-provider-error/@@ -115,6 +116,7 @@ GoogleImgGenWorkerError | Google img gen worker | https://docs.pipelex.com/latesGoogleLLMWorkerError | Google LLM worker | https://docs.pipelex.com/latest/errors/google-llm-worker-error/GraphSpecError | Graph spec | https://docs.pipelex.com/latest/errors/graph-spec-error/GraphSpecValidationError | Graph spec validation | https://docs.pipelex.com/latest/errors/graph-spec-validation-error/+HostLibraryRequirementsError | Host library requirements | https://docs.pipelex.com/latest/errors/host-library-requirements-error/HostLibraryTreeError | Host library tree | https://docs.pipelex.com/latest/errors/host-library-tree-error/HostedBaseUrlError | Hosted base url | https://docs.pipelex.com/latest/errors/hosted-base-url-error/HostedLocalFileUploadUnavailableError | Hosted local file upload unavailable | https://docs.pipelex.com/latest/errors/hosted-local-file-upload-unavailable-error/@@ -195,6 +197,9 @@ MethodPackageSymlinkError | Method package symlink | https://docs.pipelex.com/laMethodPackageTooLargeError | Method package too large | https://docs.pipelex.com/latest/errors/method-package-too-large-error/MethodRefError | Method reference error | https://docs.pipelex.com/latest/errors/method-ref-error/MethodRefParseError | Method ref parse | https://docs.pipelex.com/latest/errors/method-ref-parse-error/+MethodRequirementsError | Method requirements | https://docs.pipelex.com/latest/errors/method-requirements-error/+MethodRequirementsFileError | Method requirements file | https://docs.pipelex.com/latest/errors/method-requirements-file-error/+MethodRequirementsNotMetError | Method requirements not met | https://docs.pipelex.com/latest/errors/method-requirements-not-met-error/MethodStructuresRefusedError | Method structures refused | https://docs.pipelex.com/latest/errors/method-structures-refused-error/MethodTreeLoadError | Method tree load | https://docs.pipelex.com/latest/errors/method-tree-load-error/MigrationError | Migration error | https://docs.pipelex.com/latest/errors/migration-error/@@ -243,6 +248,7 @@ PipeExecutionError | Pipe execution | https://docs.pipelex.com/latest/errors/pipPipeExtractFactoryError | Pipe extract factory | https://docs.pipelex.com/latest/errors/pipe-extract-factory-error/PipeFactoryError | Pipe factory | https://docs.pipelex.com/latest/errors/pipe-factory-error/PipeFuncExecutionError | Pipe func execution | https://docs.pipelex.com/latest/errors/pipe-func-execution-error/+PipeFuncPackageSetUnstatedError | Pipe func package set unstated | https://docs.pipelex.com/latest/errors/pipe-func-package-set-unstated-error/PipeFuncTransportError | Pipe func transport | https://docs.pipelex.com/latest/errors/pipe-func-transport-error/PipeIOContractError | Pipe IO contract | https://docs.pipelex.com/latest/errors/pipe-io-contract-error/PipeImgGenFactoryError | Pipe img gen factory | https://docs.pipelex.com/latest/errors/pipe-img-gen-factory-error/diff --git a/tests/integration/pipelex/libraries/test_method_requirements_load.py b/tests/integration/pipelex/libraries/test_method_requirements_load.pynew file mode 100644index 000000000..d699366b7--- /dev/null+++ b/tests/integration/pipelex/libraries/test_method_requirements_load.py@@ -0,0 +1,176 @@+from pathlib import Path++import pytest+from pytest_mock import MockerFixture++from pipelex.interpreter_hub import get_library_manager, scoped_current_library+from pipelex.libraries.exceptions import MethodRequirementsNotMetError, PipeFuncPackageSetUnstatedError+from pipelex.mthds_parsing.parser import MthdsParser+from pipelex.plugins.pipe_func_package_offer_registry import PipeFuncPackageOffer, PipeFuncPackageOfferFactoryFn, PipeFuncPackageOfferRegistry+from pipelex.system.configuration.pipe_func_config import PipeFuncConfig+from pipelex.system.registries.func_registry import func_registry+from tests.integration.pipelex.libraries.installed_packages import isolate_installed_methods, vendor_probe+from tests.integration.pipelex.libraries.test_data import ProbePackageTestData++_ABSENT_PACKAGE = "pipelex-requirements-test-absent"+_FUNCTION_NAME = "requirements_test_crunch"+_OFFER = PipeFuncPackageOffer(+ environment="the Daytona sandbox snapshot 'pipelex-base-test'",+ distributions={"pandas": "2.2.3", "openpyxl": "3.1.5"},+ marker_environment={"python_version": "3.12", "sys_platform": "linux"},+)+_MANIFEST = '[package]\naddress = "github.com/pipelex-tests/requirements-load"\nversion = "1.0.0"\ndescription = "A method with requirements"\n'++_MTHDS = f"""\+domain = "requirements_load"+main_pipe = "crunch"++[concept]+Note = "A note produced by the method's own code"++[pipe.crunch]+type = "PipeFunc"+description = "Crunch numbers in the method's own code"+inputs = {{}}+output = "Note"+function_name = "{_FUNCTION_NAME}"+"""+++def _funcs_py(*, sentinel: Path) -> str:+ """A PipeFunc module whose import, were it ever imported, would write the sentinel file."""+ return f"""\+from pathlib import Path++from pipelex.core.memory.working_memory import WorkingMemory+from pipelex.core.stuffs.text_content import TextContent+from pipelex.system.registries.func_registry import pipe_func++Path({str(sentinel)!r}).write_text("imported", encoding="utf-8")+++@pipe_func(name="{_FUNCTION_NAME}")+async def {_FUNCTION_NAME}(working_memory: WorkingMemory) -> TextContent:+ return TextContent(text="crunched")+"""+++def _method(*, directory: Path, requirements: str, sentinel: Path) -> Path:+ directory.mkdir(parents=True, exist_ok=True)+ (directory / "main.mthds").write_text(_MTHDS, encoding="utf-8")+ (directory / "funcs.py").write_text(_funcs_py(sentinel=sentinel), encoding="utf-8")+ (directory / "requirements.txt").write_text(requirements, encoding="utf-8")+ return directory+++def _offering(mocker: MockerFixture, *, offer: PipeFuncPackageOffer | None) -> None:+ """Register what the `sandbox` execution mode's plugin would state, or nothing at all when `offer` is None."""++ def stated_offer(_config: PipeFuncConfig) -> PipeFuncPackageOffer | None:+ return offer++ offers: dict[str, PipeFuncPackageOfferFactoryFn] = {} if offer is None else {"sandbox": stated_offer}+ mocker.patch("pipelex.libraries.package_environments.get_pipe_func_package_offer_registry", return_value=PipeFuncPackageOfferRegistry(offers))+++class TestMethodRequirementsLoad:+ def test_locally_an_unmet_requirement_refuses_the_method_before_its_python_is_imported(self, tmp_path: Path) -> None:+ sentinel = tmp_path / "funcs-module-ran"+ library_dir = _method(directory=tmp_path / "method", requirements=f"pydantic>=2\n{_ABSENT_PACKAGE}>=1\n", sentinel=sentinel)+ library_manager = get_library_manager()+ library_id, _ = library_manager.open_library()+ try:+ with scoped_current_library(library_id=library_id), pytest.raises(MethodRequirementsNotMetError) as exc_info:+ library_manager.load_libraries(library_id=library_id, library_dirs=[library_dir])+ finally:+ library_manager.teardown(library_id=library_id)++ assert f"The method rooted at '{library_dir.resolve()}'" in exc_info.value.message+ assert f"line 2 '{_ABSENT_PACKAGE}>=1'" in exc_info.value.message+ assert "uv pip install --python" in exc_info.value.message+ assert not sentinel.exists(), "the method's Python was imported before its requirements were checked"+ assert func_registry.get_function(_FUNCTION_NAME) is None++ def test_a_bundle_file_named_alone_loads_under_its_trees_requirements_on_every_load(self, tmp_path: Path) -> None:+ """A retry of the same blueprint, placed in its tree by the refused load, is refused again rather than taken as judged."""+ sentinel = tmp_path / "funcs-module-ran"+ method_dir = _method(directory=tmp_path / "method", requirements=f"{_ABSENT_PACKAGE}\n", sentinel=sentinel)+ (method_dir / "METHODS.toml").write_text(_MANIFEST, encoding="utf-8")+ blueprint = MthdsParser.make_pipelex_bundle_blueprint(bundle_path=method_dir / "main.mthds")+ library_manager = get_library_manager()+ library_id, _ = library_manager.open_library()+ try:+ with scoped_current_library(library_id=library_id):+ with pytest.raises(MethodRequirementsNotMetError, match=_ABSENT_PACKAGE):+ library_manager.load_from_blueprints(library_id=library_id, blueprints=[blueprint])+ assert blueprint.tree_bundle is not None+ with pytest.raises(MethodRequirementsNotMetError, match=_ABSENT_PACKAGE):+ library_manager.load_from_blueprints(library_id=library_id, blueprints=[blueprint])+ finally:+ library_manager.teardown(library_id=library_id)++ def test_a_dependency_package_whose_requirements_are_not_met_refuses_the_load(self, mocker: MockerFixture, tmp_path: Path) -> None:+ isolate_installed_methods(mocker=mocker, root=tmp_path)+ methods_dir = vendor_probe(methods_dir=tmp_path / "request-methods")+ package_dir = methods_dir / ProbePackageTestData.METHOD_NAME+ sentinel = tmp_path / "funcs-module-ran"+ (package_dir / "funcs.py").write_text(_funcs_py(sentinel=sentinel), encoding="utf-8")+ (package_dir / "requirements.txt").write_text(f"pydantic>=2\n{_ABSENT_PACKAGE}>=1\n", encoding="utf-8")+ blueprint = MthdsParser.make_pipelex_bundle_blueprint(mthds_content=ProbePackageTestData.CONSUMER_BUNDLE)+ library_manager = get_library_manager()+ library_id, _ = library_manager.open_library()+ try:+ with scoped_current_library(library_id=library_id), pytest.raises(MethodRequirementsNotMetError) as exc_info:+ library_manager.load_from_blueprints(library_id=library_id, blueprints=[blueprint], methods_dirs=[methods_dir])+ finally:+ library_manager.teardown(library_id=library_id)++ message = exc_info.value.message+ assert message.startswith(f"The method package '{ProbePackageTestData.DEP_ALIAS}' needs Python packages")+ assert f"line 2 '{_ABSENT_PACKAGE}>=1' ({_ABSENT_PACKAGE} is missing)" in message+ assert str(tmp_path) not in message+ items = exc_info.value.blueprint_validation_errors+ assert [(item.source, item.line) for item in items] == [(f"{ProbePackageTestData.DEP_ALIAS}/requirements.txt", 2)]+ assert not sentinel.exists()++ @pytest.mark.usefixtures("sandbox_hosted_mode")+ def test_on_a_sandbox_hosted_deployment_the_stated_package_set_is_the_whole_offer(self, mocker: MockerFixture, tmp_path: Path) -> None:+ _offering(mocker, offer=_OFFER)+ sentinel = tmp_path / "funcs-module-ran"+ refused_dir = _method(directory=tmp_path / "refused", requirements="pandas>=2\npandas>=3\n", sentinel=sentinel)+ accepted_dir = _method(directory=tmp_path / "accepted", requirements="pandas>=2\nopenpyxl==3.1.5\n", sentinel=sentinel)+ library_manager = get_library_manager()+ library_id, _ = library_manager.open_library()+ try:+ with scoped_current_library(library_id=library_id):+ with pytest.raises(MethodRequirementsNotMetError) as exc_info:+ library_manager.load_libraries(library_id=library_id, library_dirs=[refused_dir])+ assert library_manager.get_crate(library_id=library_id) is None++ pipes = library_manager.load_libraries(library_id=library_id, library_dirs=[accepted_dir])+ assert [pipe.code for pipe in pipes] == ["crunch"]+ crate = library_manager.get_crate(library_id=library_id)+ assert crate is not None+ assert "funcs.py" in crate.python_sources+ finally:+ library_manager.teardown(library_id=library_id)++ # A load's refusal names the method's root: it is the caller's own disk's, or the host's fault, withheld from the caller.+ message = exc_info.value.message+ assert message.startswith(f"The method rooted at '{refused_dir.resolve()}' needs Python packages that the Daytona sandbox snapshot")+ assert "line 2 'pandas>=3' (pandas 2.2.3 is there)" in message+ assert not sentinel.exists()++ @pytest.mark.usefixtures("sandbox_hosted_mode")+ def test_a_sandbox_hosted_deployment_stating_no_package_set_refuses_a_method_with_requirements(+ self, mocker: MockerFixture, tmp_path: Path+ ) -> None:+ _offering(mocker, offer=None)+ library_dir = _method(directory=tmp_path / "method", requirements="pandas\n", sentinel=tmp_path / "funcs-module-ran")+ library_manager = get_library_manager()+ library_id, _ = library_manager.open_library()+ try:+ with scoped_current_library(library_id=library_id), pytest.raises(PipeFuncPackageSetUnstatedError, match="'sandbox' execution mode"):+ library_manager.load_libraries(library_id=library_id, library_dirs=[library_dir])+ finally:+ library_manager.teardown(library_id=library_id)diff --git a/tests/unit/pipelex/libraries/test_data.py b/tests/unit/pipelex/libraries/test_data.pyindex 5374fd063..4e87a96c2 100644--- a/tests/unit/pipelex/libraries/test_data.py+++ b/tests/unit/pipelex/libraries/test_data.py@@ -1,3 +1,5 @@+from typing import ClassVar+from pipelex.core.concepts.concept_blueprint import ConceptBlueprintfrom pipelex.core.concepts.concept_structure_blueprint import ConceptStructureBlueprint, ConceptStructureBlueprintFieldTypefrom pipelex.core.domains.domain_blueprint import DomainBlueprint@@ -8,6 +10,7 @@ from pipelex.pipe_controllers.parallel.pipe_parallel_blueprint import PipeParallfrom pipelex.pipe_controllers.sequence.pipe_sequence_blueprint import PipeSequenceBlueprintfrom pipelex.pipe_controllers.sub_pipe_blueprint import SubPipeBlueprintfrom pipelex.pipe_operators.llm.pipe_llm_blueprint import PipeLLMBlueprint+from pipelex.plugins.pipe_func_package_offer_registry import PipeFuncPackageOfferclass CrateQualificationTestData:@@ -63,3 +66,56 @@ class CrateQualificationTestData:},domains={"alpha": DomainBlueprint(code="alpha", description="alpha domain")},)+++class MethodRequirementsTestData:+ """Lines of a method's requirements.txt, as the MTHDS standard reads them, and a package set to judge them against."""++ # (topic, line, the requirement's name when the line declares one)+ ACCEPTED_LINES: ClassVar[list[tuple[str, str, str]]] = [+ ("bare name", "pandas", "pandas"),+ ("range", "pandas>=2.2,<3", "pandas"),+ ("pinned", "openpyxl==3.1.5", "openpyxl"),+ ("wildcard", "pandas==2.*", "pandas"),+ ("extras", "pandas[excel]>=2", "pandas"),+ ("marker", 'tzdata; python_version >= "3.12"', "tzdata"),+ ("spaced marker", "pandas >= 2.0 ; sys_platform == 'linux'", "pandas"),+ ("indented", " numpy", "numpy"),+ ("dotted name", "zope.interface>=6", "zope.interface"),+ ]++ # (topic, line, a fragment of the reason the line is refused)+ REFUSED_LINES: ClassVar[list[tuple[str, str, str]]] = [+ ("direct reference", "name @ https://example.com/name-1.0.whl", "naming a URL"),+ ("bare URL", "https://example.com/name-1.0.whl", "naming a URL"),+ ("requirement file option", "-r base.txt", "option line"),+ ("editable option", "-e .", "option line"),+ ("constraint option", "-c constraints.txt", "option line"),+ ("index option", "--index-url https://example.com/simple", "option line"),+ ("trailing comment", "openpyxl==3.1.5 # spreadsheets", "comment after a requirement"),+ ("trailing comment mentioning a URL", "foo # see https://x.y", "comment after a requirement"),+ ("direct reference with a hash fragment", "name @ https://example.com/name-1.0.whl#sha256=abc", "naming a URL"),+ ("direct reference with a trailing comment", "name @ https://example.com/name-1.0.whl # pinned", "naming a URL"),+ ("continued line", "pandas>=2 \\", "continued with a backslash"),+ ("local path", "./vendor/package", "not a PEP 508 requirement"),+ ("version without operator", "pandas 2.2", "not a PEP 508 requirement"),+ ]++ SANDBOX_ENVIRONMENT = "the Daytona sandbox snapshot 'pipelex-base-test'"++ @classmethod+ def sandbox_offer(cls) -> PipeFuncPackageOffer:+ """A package set as the Daytona plugin would state it for its snapshot: canonical names, a Linux CPython 3.12."""+ return PipeFuncPackageOffer(+ environment=cls.SANDBOX_ENVIRONMENT,+ distributions={"pandas": "2.2.3", "openpyxl": "3.1.5", "numpy": "2.1.0rc1", "Zope.Interface": "7.0"},+ marker_environment={+ "implementation_name": "cpython",+ "os_name": "posix",+ "platform_machine": "x86_64",+ "platform_python_implementation": "CPython",+ "platform_system": "Linux",+ "python_version": "3.12",+ "sys_platform": "linux",+ },+ )diff --git a/tests/unit/pipelex/libraries/test_method_requirements_check.py b/tests/unit/pipelex/libraries/test_method_requirements_check.pynew file mode 100644index 000000000..a7e07618f--- /dev/null+++ b/tests/unit/pipelex/libraries/test_method_requirements_check.py@@ -0,0 +1,221 @@+import shlex+import sys+from pathlib import Path++import pytest+from mthds.method_tree.lister import list_method_tree+from pytest_mock import MockerFixture++from pipelex.base_exceptions import INTERNAL_ERROR_PLACEHOLDER, DisclosureMode, ErrorDomain, ValidationErrorCategory+from pipelex.libraries.exceptions import (+ HostLibraryRequirementsError,+ HostLibraryTreeError,+ MethodRequirementsFileError,+ MethodRequirementsNotMetError,+)+from pipelex.libraries.method_requirements import (+ ensure_method_requirements_met,+ ensure_package_requirements_met,+ ensure_served_method_requirements_met,+)+from pipelex.libraries.method_trees import reporting_host_library_tree_faults+from pipelex.libraries.package_environments import OfferedPackageEnvironment+from pipelex.pipeline.exceptions import ValidateBundleError+from pipelex.pipeline.validate_bundle_translation import translate_to_validate_bundle_error+from tests.unit.pipelex.libraries.test_data import MethodRequirementsTestData++_MODULE = "pipelex.libraries.method_requirements"+_ABSENT_PACKAGE = "pipelex-requirements-test-absent"+_MANIFEST = '[package]\naddress = "github.com/pipelex-tests/requirements"\nversion = "1.0.0"\ndescription = "A nested package"\n'+++def _write(*, directory: Path, tree_path: str, content: str | bytes) -> None:+ file_path = directory / tree_path+ file_path.parent.mkdir(parents=True, exist_ok=True)+ if isinstance(content, bytes):+ file_path.write_bytes(content)+ else:+ file_path.write_text(content, encoding="utf-8")+++def _method(*, directory: Path, requirements: str | bytes | None, with_python: bool = True) -> Path:+ _write(directory=directory, tree_path="main.mthds", content="domain = 'requirements_tests'\n")+ if with_python:+ _write(directory=directory, tree_path="funcs/crunch.py", content="def crunch() -> int:\n return 1\n")+ if requirements is not None:+ _write(directory=directory, tree_path="requirements.txt", content=requirements)+ return directory+++class TestMethodRequirementsCheck:+ @pytest.mark.parametrize(+ ("topic", "requirements", "with_python", "nested_requirements"),+ [+ ("no Python to import a package", "-r base.txt\n", False, None),+ ("no requirements.txt", None, True, None),+ ("only the root's file declares anything", None, True, "-r base.txt\n"),+ ],+ )+ def test_a_tree_without_requirements_to_check_is_never_read(+ self, mocker: MockerFixture, tmp_path: Path, topic: str, requirements: str | None, with_python: bool, nested_requirements: str | None+ ) -> None:+ resolve = mocker.patch(f"{_MODULE}.resolve_pipe_func_package_environment")+ _method(directory=tmp_path, requirements=requirements, with_python=with_python)+ if nested_requirements is not None:+ _write(directory=tmp_path, tree_path="funcs/requirements.txt", content=nested_requirements)++ ensure_method_requirements_met(trees=[list_method_tree(tmp_path)], names_host_paths=True)++ assert resolve.call_count == 0, topic++ def test_refused_lines_are_named_each_before_any_environment_is_read(self, mocker: MockerFixture, tmp_path: Path) -> None:+ resolve = mocker.patch(f"{_MODULE}.resolve_pipe_func_package_environment")+ _method(directory=tmp_path, requirements="# data\n-r base.txt\npandas>=2\nname @ https://example.com/x.whl\n")++ with pytest.raises(MethodRequirementsFileError) as exc_info:+ ensure_method_requirements_met(trees=[list_method_tree(tmp_path)], names_host_paths=True)++ resolve.assert_not_called()+ message = exc_info.value.message+ assert f"The method rooted at '{tmp_path}'" in message+ assert "line 2 '-r base.txt'" in message+ assert "line 4 'name @ https://example.com/x.whl'" in message+ items = exc_info.value.blueprint_validation_errors+ assert [(item.source, item.line) for item in items] == [("requirements.txt", 2), ("requirements.txt", 4)]+ assert exc_info.value.to_error_report().error_domain == ErrorDomain.INPUT++ def test_a_file_that_is_not_utf8_is_refused(self, tmp_path: Path) -> None:+ _method(directory=tmp_path, requirements=b"pandas\xff\n")++ with pytest.raises(MethodRequirementsFileError, match="not UTF-8 text"):+ ensure_method_requirements_met(trees=[list_method_tree(tmp_path)], names_host_paths=True)++ def test_comments_alone_need_no_environment(self, mocker: MockerFixture, tmp_path: Path) -> None:+ resolve = mocker.patch(f"{_MODULE}.resolve_pipe_func_package_environment")+ _method(directory=tmp_path, requirements="# nothing yet\n\n")++ ensure_method_requirements_met(trees=[list_method_tree(tmp_path)], names_host_paths=True)++ resolve.assert_not_called()++ def test_locally_each_unmet_requirement_is_named_with_the_command_that_installs_it(self, tmp_path: Path) -> None:+ """This interpreter, read for real: the refusal names the method's root, each line and the uv command."""+ _method(directory=tmp_path, requirements=f"pydantic>=2\n{_ABSENT_PACKAGE}>=1\npydantic>=999\n")++ with pytest.raises(MethodRequirementsNotMetError) as exc_info:+ ensure_method_requirements_met(trees=[list_method_tree(tmp_path)], names_host_paths=True)++ message = exc_info.value.message+ assert message.startswith(f"The method rooted at '{tmp_path}' needs Python packages that this Python interpreter")+ assert f"line 2 '{_ABSENT_PACKAGE}>=1' ({_ABSENT_PACKAGE} is missing)" in message+ assert "line 3 'pydantic>=999' (pydantic " in message+ assert f"uv pip install --python {shlex.quote(sys.executable)} '{_ABSENT_PACKAGE}>=1' 'pydantic>=999'" in message+ items = exc_info.value.blueprint_validation_errors+ assert [item.line for item in items] == [2, 3]+ assert all(item.source == "requirements.txt" for item in items)++ def test_where_host_paths_may_not_be_named_the_refusal_names_none(self, mocker: MockerFixture, tmp_path: Path) -> None:+ mocker.patch(+ f"{_MODULE}.resolve_pipe_func_package_environment",+ return_value=OfferedPackageEnvironment(offer=MethodRequirementsTestData.sandbox_offer()),+ )+ _method(directory=tmp_path, requirements="pandas>=2.2\npandas>=3\npolars\n")++ with pytest.raises(MethodRequirementsNotMetError) as exc_info:+ ensure_method_requirements_met(trees=[list_method_tree(tmp_path)], names_host_paths=False)++ message = exc_info.value.message+ assert str(tmp_path) not in message+ assert message.startswith(f"The method needs Python packages that {MethodRequirementsTestData.SANDBOX_ENVIRONMENT} does not have")+ assert "line 2 'pandas>=3' (pandas 2.2.3 is there); line 3 'polars' (polars is missing)" in message+ assert [(item.source, item.line) for item in exc_info.value.blueprint_validation_errors] == [("requirements.txt", 2), ("requirements.txt", 3)]++ def test_the_environment_is_resolved_once_and_each_tree_judged_once(self, mocker: MockerFixture, tmp_path: Path) -> None:+ resolve = mocker.patch(+ f"{_MODULE}.resolve_pipe_func_package_environment",+ return_value=OfferedPackageEnvironment(offer=MethodRequirementsTestData.sandbox_offer()),+ )+ first = list_method_tree(_method(directory=tmp_path / "first", requirements="pandas\n"))+ second = list_method_tree(_method(directory=tmp_path / "second", requirements="openpyxl\n"))++ ensure_method_requirements_met(trees=[first, second, first], names_host_paths=True)++ resolve.assert_called_once_with(names_host_paths=True)++ def test_a_served_method_checks_the_packages_nested_in_its_directory(self, tmp_path: Path) -> None:+ _write(directory=tmp_path, tree_path="top.mthds", content="domain = 'requirements_top'\n")+ nested = tmp_path / "packages" / "invoicing"+ _write(directory=nested, tree_path="METHODS.toml", content=_MANIFEST)+ _method(directory=nested, requirements=f"{_ABSENT_PACKAGE}\n")++ with pytest.raises(MethodRequirementsNotMetError, match=f"line 1 '{_ABSENT_PACKAGE}'"):+ ensure_served_method_requirements_met(library_dirs=[tmp_path])++ def test_a_validation_reports_the_refusal_as_located_items(self, tmp_path: Path) -> None:+ _method(directory=tmp_path, requirements=f"{_ABSENT_PACKAGE}\n")++ with pytest.raises(ValidateBundleError) as exc_info, translate_to_validate_bundle_error():+ ensure_served_method_requirements_met(library_dirs=[tmp_path])++ report = exc_info.value.to_error_report()+ assert report.validation_errors is not None+ (item,) = report.validation_errors+ assert item.category == ValidationErrorCategory.BLUEPRINT_VALIDATION+ assert item.source == "requirements.txt"+ assert item.line == 1+ assert f"The requirement '{_ABSENT_PACKAGE}' is not met" in item.message++ def test_under_strict_disclosure_a_served_method_refused_in_direct_mode_names_no_host_path(self, tmp_path: Path) -> None:+ """This interpreter, read for real in the direct mode: the caller reads the method, its lines and the packages, and no path on the host."""+ method_dir = _method(directory=tmp_path / "request-files", requirements=f"pydantic>=2\n{_ABSENT_PACKAGE}>=1\n")++ with pytest.raises(MethodRequirementsNotMetError) as refusal_info:+ ensure_served_method_requirements_met(library_dirs=[method_dir])+ with pytest.raises(ValidateBundleError) as verdict_info, translate_to_validate_bundle_error():+ ensure_served_method_requirements_met(library_dirs=[method_dir])++ refusal = refusal_info.value.to_error_report().to_dict(disclosure_mode=DisclosureMode.STRICT)+ verdict = verdict_info.value.to_error_report().to_dict(disclosure_mode=DisclosureMode.STRICT)+ message = refusal["message"]+ assert message.startswith("The method needs Python packages that the Python interpreter Pipelex runs in does not have")+ assert f"line 2 '{_ABSENT_PACKAGE}>=1' ({_ABSENT_PACKAGE} is missing)" in message+ assert f"uv pip install '{_ABSENT_PACKAGE}>=1'" in message+ assert [(item["source"], item["line"]) for item in verdict["validation_errors"]] == [("requirements.txt", 2)]+ for disclosed in (str(refusal), str(verdict)):+ for host_path in (str(tmp_path), str(tmp_path.resolve()), sys.executable):+ assert host_path not in disclosed, host_path++ def test_a_served_method_whose_directory_cannot_be_listed_is_the_hosts_fault(self, tmp_path: Path) -> None:+ method_dir = _method(directory=tmp_path / "request-files", requirements=f"{_ABSENT_PACKAGE}\n")+ (method_dir / "linked.py").symlink_to(method_dir / "funcs" / "crunch.py")++ with pytest.raises(HostLibraryTreeError) as exc_info:+ ensure_served_method_requirements_met(library_dirs=[method_dir])++ report = exc_info.value.to_error_report()+ assert report.error_domain == ErrorDomain.CONFIG+ assert report.to_dict(disclosure_mode=DisclosureMode.STRICT)["message"] == INTERNAL_ERROR_PLACEHOLDER++ def test_around_a_hosts_library_directories_a_refusal_is_the_hosts_fault(self, tmp_path: Path) -> None:+ _method(directory=tmp_path, requirements=f"{_ABSENT_PACKAGE}\n")++ with pytest.raises(HostLibraryRequirementsError) as exc_info, reporting_host_library_tree_faults():+ ensure_method_requirements_met(trees=[list_method_tree(tmp_path)], names_host_paths=True)++ assert isinstance(exc_info.value.__cause__, MethodRequirementsNotMetError)+ report = exc_info.value.to_error_report()+ assert report.error_domain == ErrorDomain.CONFIG+ assert report.to_dict(disclosure_mode=DisclosureMode.STRICT)["message"] == INTERNAL_ERROR_PLACEHOLDER++ def test_a_dependency_package_is_named_by_its_address(self, tmp_path: Path) -> None:+ package_dir = _method(directory=tmp_path / "installed" / "invoicing", requirements=f"-r base.txt\n{_ABSENT_PACKAGE}\n")++ with pytest.raises(MethodRequirementsFileError) as exc_info:+ ensure_package_requirements_met(tree=list_method_tree(package_dir), package_address="github.com/pipelex-tests/invoicing")++ message = exc_info.value.message+ assert message.startswith("The method package 'github.com/pipelex-tests/invoicing' declares its packages")+ assert str(tmp_path) not in message+ assert [(item.source, item.line) for item in exc_info.value.blueprint_validation_errors] == [+ ("github.com/pipelex-tests/invoicing/requirements.txt", 1)+ ]diff --git a/tests/unit/pipelex/libraries/test_method_requirements_judgment.py b/tests/unit/pipelex/libraries/test_method_requirements_judgment.pynew file mode 100644index 000000000..226f90b20--- /dev/null+++ b/tests/unit/pipelex/libraries/test_method_requirements_judgment.py@@ -0,0 +1,141 @@+import shlex+import sys+from importlib.metadata import PackageNotFoundError++import pytest+from packaging.markers import Marker+from packaging.version import InvalidVersion+from pytest_mock import MockerFixture++from pipelex.libraries.method_requirements import marker_variables_of, parse_requirements_text, unmet_requirements+from pipelex.libraries.package_environments import InterpreterPackageEnvironment, OfferedPackageEnvironment+from tests.unit.pipelex.libraries.test_data import MethodRequirementsTestData++_DISTRIBUTION_TARGET = "pipelex.libraries.package_environments.distribution"+++def _unmet_reasons(*, text: str, environment: InterpreterPackageEnvironment | OfferedPackageEnvironment) -> list[str]:+ parsed = parse_requirements_text(text=text)+ assert parsed.refused_lines == []+ return [unmet_requirement.reason for unmet_requirement in unmet_requirements(requirements=parsed.requirements, environment=environment)]+++def _install(mocker: MockerFixture, *, installed: dict[str, tuple[str, list[str]]]) -> None:+ """Make this interpreter appear to hold exactly `installed`: each name with its version and its Requires-Dist."""++ def fake_distribution(distribution_name: str) -> object:+ normalized = distribution_name.lower().replace("_", "-")+ if normalized not in installed:+ raise PackageNotFoundError(distribution_name)+ version, requires = installed[normalized]+ return mocker.Mock(version=version, requires=requires)++ mocker.patch(_DISTRIBUTION_TARGET, side_effect=fake_distribution)+++class TestMethodRequirementsJudgment:+ @pytest.mark.parametrize(+ ("topic", "line", "expected_reason"),+ [+ ("range met", "pandas>=2.2,<3", None),+ ("pin met", "openpyxl==3.1.5", None),+ ("names compared normalized", "zope-interface>=7\nOpenPyXL==3.1.5", None),+ ("a pre-release held counts", "numpy>=2", None),+ ("version not accepted", "pandas>=3", "pandas 2.2.3 is there"),+ ("package not offered", "polars>=1", "polars is missing"),+ ("marker false means not needed", 'pywin32; sys_platform == "win32"', None),+ ("marker true is judged", 'polars; sys_platform == "linux"', "polars is missing"),+ ("a line names no extra", 'polars; extra == "fast"', None),+ ("marker variable not stated", 'pandas; platform_release >= "6"', "its marker reads platform_release, which the Daytona sandbox"),+ ("extras are not stated", "pandas[excel]>=2", "does not state what the extra 'excel' of pandas needs"),+ ],+ )+ def test_a_stated_package_set_judges_each_requirement(self, topic: str, line: str, expected_reason: str | None) -> None:+ """The offer is the whole environment: its distributions, its marker values, and nothing it does not state."""+ environment = OfferedPackageEnvironment(offer=MethodRequirementsTestData.sandbox_offer())++ reasons = _unmet_reasons(text=line, environment=environment)++ if expected_reason is None:+ assert reasons == [], topic+ else:+ (reason,) = reasons+ assert expected_reason in reason, topic++ def test_a_marker_value_spelled_like_a_variable_is_not_read_as_one(self) -> None:+ """Only the marker's variables are checked against what an environment states, never its quoted values."""+ parsed = parse_requirements_text(text='pandas; os_name == "python_full_version"\n')++ (requirement_line,) = parsed.requirements+ assert requirement_line.requirement.marker is not None+ assert marker_variables_of(marker=requirement_line.requirement.marker) == {"os_name"}+ environment = OfferedPackageEnvironment(offer=MethodRequirementsTestData.sandbox_offer())+ assert unmet_requirements(requirements=parsed.requirements, environment=environment) == []++ def test_this_interpreter_meets_what_it_holds(self) -> None:+ """Read for real: pipelex's own dependencies are installed here, and a package no index holds is not."""+ reasons = _unmet_reasons(+ text="pydantic>=2\nPyYAML\npipelex-requirements-test-absent>=1\n", environment=InterpreterPackageEnvironment(names_host_paths=True)+ )++ assert reasons == ["pipelex-requirements-test-absent is missing"]++ def test_an_extra_is_followed_through_the_metadata_of_this_interpreter(self, mocker: MockerFixture) -> None:+ """An extra needs what the distribution's metadata declares under it, and only that; extras naming each other end."""+ _install(+ mocker,+ installed={+ "fastlib": ("1.4", ["basedep>=1", 'turbo>=2; extra == "speed"', 'unrelated; extra == "other"', 'fastlib[cycle]; extra == "speed"']),+ "turbo": ("1.5", []),+ "basedep": ("0.1", []),+ },+ )++ reasons = _unmet_reasons(text="fastlib[speed]>=1\nfastlib[other]\n", environment=InterpreterPackageEnvironment(names_host_paths=True))++ assert reasons == [+ "the extra 'speed' of fastlib needs 'turbo>=2; extra == \"speed\"', and turbo 1.5 is there",+ "the extra 'other' of fastlib needs 'unrelated; extra == \"other\"', and unrelated is missing",+ ]++ def test_extras_naming_each_other_are_followed_once(self, mocker: MockerFixture) -> None:+ _install(mocker, installed={"loopy": ("2.0", ['loopy[beta]; extra == "alpha"', 'loopy[alpha]; extra == "beta"'])})++ assert _unmet_reasons(text="loopy[alpha]\n", environment=InterpreterPackageEnvironment(names_host_paths=True)) == []++ def test_an_installed_version_no_specifier_compares_is_not_met(self, mocker: MockerFixture) -> None:+ _install(mocker, installed={"oddball": ("not-a-version", [])})++ (reason,) = _unmet_reasons(text="oddball>=1\n", environment=InterpreterPackageEnvironment(names_host_paths=True))++ assert reason == "oddball not-a-version is there, a version no specifier can be compared with"++ def test_each_environment_advises_its_own_remedy(self) -> None:+ """Locally the advice is the uv command for this interpreter; on a sandbox-hosted deployment it is the offer's limits."""+ local_advice = InterpreterPackageEnvironment(names_host_paths=True).advice(+ requirement_texts=["pandas>=2.2", 'tzdata; python_version >= "3.12"']+ )+ sandbox_advice = OfferedPackageEnvironment(offer=MethodRequirementsTestData.sandbox_offer()).advice(requirement_texts=["pandas>=3"])++ assert f"uv pip install --python {shlex.quote(sys.executable)} 'pandas>=2.2' 'tzdata; python_version >= \"3.12\"'" in local_advice+ assert "never installs" in local_advice+ assert MethodRequirementsTestData.SANDBOX_ENVIRONMENT in sandbox_advice+ assert "uv" not in sandbox_advice++ def test_this_interpreter_is_named_by_its_path_only_where_host_paths_may_be_named(self) -> None:+ environment = InterpreterPackageEnvironment(names_host_paths=False)++ advice = environment.advice(requirement_texts=["pandas>=2.2"])++ assert sys.executable not in environment.description+ assert sys.executable not in advice+ assert environment.description == "the Python interpreter Pipelex runs in"+ assert "uv pip install 'pandas>=2.2'" in advice++ def test_a_marker_comparing_a_value_that_is_not_a_version_refuses_the_requirement(self, mocker: MockerFixture) -> None:+ """Releases of packaging before 22 raise InvalidVersion where later ones compare as strings; the requirement is refused, not the load."""+ mocker.patch.object(Marker, "evaluate", side_effect=InvalidVersion("Invalid version: '5.15.0-generic'"))++ (reason,) = _unmet_reasons(text='pandas; platform_release >= "5"\n', environment=InterpreterPackageEnvironment(names_host_paths=True))++ assert reason == "its marker cannot be evaluated: Invalid version: '5.15.0-generic'"diff --git a/tests/unit/pipelex/libraries/test_method_requirements_parsing.py b/tests/unit/pipelex/libraries/test_method_requirements_parsing.pynew file mode 100644index 000000000..68ab9480f--- /dev/null+++ b/tests/unit/pipelex/libraries/test_method_requirements_parsing.py@@ -0,0 +1,47 @@+import pytest++from pipelex.libraries.method_requirements import parse_requirements_text+from tests.unit.pipelex.libraries.test_data import MethodRequirementsTestData+++class TestMethodRequirementsParsing:+ @pytest.mark.parametrize(("topic", "line", "name"), MethodRequirementsTestData.ACCEPTED_LINES)+ def test_a_pep_508_requirement_is_declared(self, topic: str, line: str, name: str) -> None:+ """A line holding exactly one PEP 508 requirement, markers and extras included, declares it, stripped."""+ parsed = parse_requirements_text(text=f"{line}\n")++ assert parsed.refused_lines == [], topic+ (requirement_line,) = parsed.requirements+ assert requirement_line.line_number == 1+ assert requirement_line.text == line.strip()+ assert requirement_line.requirement.name == name++ @pytest.mark.parametrize(("topic", "line", "reason_fragment"), MethodRequirementsTestData.REFUSED_LINES)+ def test_a_line_the_standard_refuses_is_named_with_why(self, topic: str, line: str, reason_fragment: str) -> None:+ """A URL, an option, a comment after a requirement, a continued line and anything else not PEP 508 are refused."""+ parsed = parse_requirements_text(text=f"pandas>=2\n{line}\n")++ assert [requirement_line.text for requirement_line in parsed.requirements] == ["pandas>=2"], topic+ (refused_line,) = parsed.refused_lines+ assert refused_line.line_number == 2+ assert refused_line.text == line.strip()+ assert reason_fragment in refused_line.reason++ def test_blank_lines_and_comments_declare_nothing_and_lines_keep_their_numbers(self) -> None:+ """A byte order mark is skipped, a carriage return ends a line, and a comment may be indented by spaces or tabs."""+ text = "# data handling\r\n\r\n # indented\n\t# tabbed\n \npandas>=2.2\r\nopenpyxl\n"++ parsed = parse_requirements_text(text=text)++ assert parsed.refused_lines == []+ assert [(requirement_line.line_number, requirement_line.text) for requirement_line in parsed.requirements] == [+ (6, "pandas>=2.2"),+ (7, "openpyxl"),+ ]++ def test_every_refused_line_is_kept_not_only_the_first(self) -> None:+ """The author fixes the whole file in one pass, so each refused line is reported, in file order."""+ parsed = parse_requirements_text(text="-r base.txt\npandas\nname @ https://example.com/x.whl\n-e .\n")++ assert [refused_line.line_number for refused_line in parsed.refused_lines] == [1, 3, 4]+ assert [requirement_line.text for requirement_line in parsed.requirements] == ["pandas"]diff --git a/tests/unit/pipelex/libraries/test_package_environments.py b/tests/unit/pipelex/libraries/test_package_environments.pynew file mode 100644index 000000000..e7112fdf9--- /dev/null+++ b/tests/unit/pipelex/libraries/test_package_environments.py@@ -0,0 +1,73 @@+from types import SimpleNamespace++import pytest+from pytest_mock import MockerFixture++from pipelex.base_exceptions import ErrorDomain+from pipelex.libraries.exceptions import PipeFuncPackageSetUnstatedError+from pipelex.libraries.package_environments import InterpreterPackageEnvironment, OfferedPackageEnvironment, resolve_pipe_func_package_environment+from pipelex.plugins.pipe_func_package_offer_registry import PipeFuncPackageOffer, PipeFuncPackageOfferFactoryFn, PipeFuncPackageOfferRegistry+from tests.unit.pipelex.libraries.test_data import MethodRequirementsTestData++_MODULE = "pipelex.libraries.package_environments"+++def _sandbox_hosted(mocker: MockerFixture, *, offers: dict[str, PipeFuncPackageOfferFactoryFn]) -> object:+ """Make this process run PipeFunc steps in the `daytona` mode, with the given offers registered; return its pipe_func config."""+ pipe_func_config = object()+ mocker.patch(f"{_MODULE}.is_pipe_func_sandbox_hosted", return_value=True)+ mocker.patch(f"{_MODULE}.get_pipe_func_execution_mode", return_value="daytona")+ mocker.patch(f"{_MODULE}.get_pipe_func_package_offer_registry", return_value=PipeFuncPackageOfferRegistry(offers))+ mocker.patch(f"{_MODULE}.get_config", return_value=SimpleNamespace(interpreter=SimpleNamespace(pipe_func=pipe_func_config)))+ return pipe_func_config+++class TestPackageEnvironments:+ def test_in_direct_mode_the_environment_is_this_interpreter(self, mocker: MockerFixture) -> None:+ registry = mocker.patch(f"{_MODULE}.get_pipe_func_package_offer_registry")+ mocker.patch(f"{_MODULE}.is_pipe_func_sandbox_hosted", return_value=False)++ environment = resolve_pipe_func_package_environment(names_host_paths=True)++ assert isinstance(environment, InterpreterPackageEnvironment)+ registry.assert_not_called()++ def test_in_a_sandbox_mode_the_environment_is_the_offer_its_plugin_states(self, mocker: MockerFixture) -> None:+ factory = mocker.Mock(return_value=MethodRequirementsTestData.sandbox_offer())+ pipe_func_config = _sandbox_hosted(mocker, offers={"daytona": factory})++ environment = resolve_pipe_func_package_environment(names_host_paths=True)++ assert isinstance(environment, OfferedPackageEnvironment)+ factory.assert_called_once_with(pipe_func_config)+ assert environment.description == MethodRequirementsTestData.SANDBOX_ENVIRONMENT+ assert environment.version_of(distribution_name="zope-interface") == "7.0"+ assert environment.version_of(distribution_name="polars") is None+ assert environment.dependencies_of(distribution_name="pandas") is None++ @pytest.mark.parametrize(+ ("topic", "offer_registered", "remedy"),+ [+ ("no plugin registers the mode's packages", False, "must register its package offer with PluginRegistrar.add_pipe_func_package_offer"),+ ("the registered plugin cannot state them", True, "Check the plugin's configuration"),+ ],+ )+ def test_a_sandbox_mode_without_a_stated_package_set_fails_closed_naming_its_remedy(+ self, mocker: MockerFixture, topic: str, offer_registered: bool, remedy: str+ ) -> None:+ """No requirement can be judged, so a method declaring one is refused, as the deployment's fault rather than the method's."""++ def unstated(_config: object) -> PipeFuncPackageOffer | None:+ return None++ offers: dict[str, PipeFuncPackageOfferFactoryFn] = {"daytona": unstated} if offer_registered else {"elsewhere": unstated}+ _sandbox_hosted(mocker, offers=offers)++ with pytest.raises(PipeFuncPackageSetUnstatedError) as exc_info:+ resolve_pipe_func_package_environment(names_host_paths=True)++ assert exc_info.value.execution_mode == "daytona"+ assert exc_info.value.offer_registered is offer_registered, topic+ assert remedy in exc_info.value.message, topic+ assert ("registers a package offer that could not state" in exc_info.value.message) is offer_registered, topic+ assert exc_info.value.to_error_report().error_domain == ErrorDomain.CONFIGdiff --git a/tests/unit/pipelex/plugins/test_pipe_func_package_offer_registry.py b/tests/unit/pipelex/plugins/test_pipe_func_package_offer_registry.pynew file mode 100644index 000000000..794eb5b5f--- /dev/null+++ b/tests/unit/pipelex/plugins/test_pipe_func_package_offer_registry.py@@ -0,0 +1,61 @@+from __future__ import annotations++from types import SimpleNamespace+from typing import TYPE_CHECKING, cast++import pytest++from pipelex.plugins.contract import PLUGIN_API_VERSION+from pipelex.plugins.exceptions import DuplicatePipeFuncPackageOfferError+from pipelex.plugins.pipe_func_package_offer_registry import PipeFuncPackageOffer, PipeFuncPackageOfferRegistry+from pipelex.plugins.registrar import PluginOrigin, PluginRegistrar++if TYPE_CHECKING:+ from pipelex.system.configuration.configs import PipelexConfig+ from pipelex.system.configuration.pipe_func_config import PipeFuncConfig++_OFFER = PipeFuncPackageOffer(+ environment="the Daytona sandbox snapshot 'pipelex-base-test'",+ distributions={"pandas": "2.2.3"},+ marker_environment={"python_version": "3.12"},+)+++def _make_registrar() -> PluginRegistrar:+ return PluginRegistrar(config=cast("PipelexConfig", SimpleNamespace(runtime=SimpleNamespace(plugins=SimpleNamespace(disabled=[])))))+++def _offer_factory(_config: PipeFuncConfig) -> PipeFuncPackageOffer | None:+ return _OFFER+++class TestPipeFuncPackageOfferRegistry:+ def test_an_offer_registered_for_a_mode_is_found_by_that_mode_alone(self) -> None:+ registrar = _make_registrar()+ discovery = registrar.begin_plugin(name="daytona", origin=PluginOrigin.EXTERNAL, targets_api=PLUGIN_API_VERSION, group=None)++ registrar.add_pipe_func_package_offer(mode="daytona", factory=_offer_factory)++ registry = PipeFuncPackageOfferRegistry(registrar.pipe_func_package_offers)+ assert registry.get_optional(mode="daytona") is _offer_factory+ assert registry.get_optional(mode="direct") is None+ assert registry.modes == ["daytona"]+ assert "pipe_func package offer daytona" in discovery.contributions++ def test_two_plugins_stating_one_mode_fail_loud_naming_both(self) -> None:+ registrar = _make_registrar()+ registrar.begin_plugin(name="alpha", origin=PluginOrigin.EXTERNAL, targets_api=PLUGIN_API_VERSION, group=None)+ registrar.add_pipe_func_package_offer(mode="daytona", factory=_offer_factory)+ registrar.begin_plugin(name="beta", origin=PluginOrigin.EXTERNAL, targets_api=PLUGIN_API_VERSION, group=None)++ with pytest.raises(DuplicatePipeFuncPackageOfferError) as exc_info:+ registrar.add_pipe_func_package_offer(mode="daytona", factory=_offer_factory)++ assert (exc_info.value.mode, exc_info.value.first_plugin, exc_info.value.second_plugin) == ("daytona", "alpha", "beta")+ assert "'alpha'" in exc_info.value.message+ assert "'beta'" in exc_info.value.message++ def test_an_offer_refuses_a_field_it_does_not_define(self) -> None:+ """A plugin's statement is closed plain data, so a misspelled field fails at the plugin rather than reading as unstated."""+ with pytest.raises(ValueError, match="extra"):+ PipeFuncPackageOffer.model_validate({**_OFFER.model_dump(), "marker_environments": {}})diff --git a/tests/unit/pipelex/plugins/test_plugin_group_split.py b/tests/unit/pipelex/plugins/test_plugin_group_split.pyindex 9bcc34a65..2fe6cbece 100644--- a/tests/unit/pipelex/plugins/test_plugin_group_split.py+++ b/tests/unit/pipelex/plugins/test_plugin_group_split.py@@ -26,6 +26,7 @@ if TYPE_CHECKING:from pipelex.plugins.contract import PipelexPluginfrom pipelex.plugins.orchestrator_registry import OrchestratorProtocolfrom pipelex.plugins.pipe_func_executor_registry import PipeFuncExecutorFactoryFn+ from pipelex.plugins.pipe_func_package_offer_registry import PipeFuncPackageOfferFactoryFnfrom pipelex.plugins.registrar import PluginRegistrarfrom pipelex.system.configuration.configs import PipelexConfig@@ -36,6 +37,9 @@ INTERPRETER_TIER_CALLS: dict[str, Callable[[PluginRegistrar], None]] = {"orchestrator": lambda registrar: registrar.add_orchestrator(mode="temporal", orchestrator=cast("OrchestratorProtocol", object())),"bundle validator": lambda registrar: registrar.add_bundle_validator(mode="temporal", validator=cast("BundleValidatorProtocol", object())),"pipe_func executor": lambda registrar: registrar.add_pipe_func_executor(mode="daytona", factory=cast("PipeFuncExecutorFactoryFn", object)),+ "pipe_func package offer": lambda registrar: registrar.add_pipe_func_package_offer(+ mode="daytona", factory=cast("PipeFuncPackageOfferFactoryFn", object)+ ),"hub slot pipe_router": lambda registrar: registrar.claim_pipe_router(object),"hub slot pipe_run": lambda registrar: registrar.claim_pipe_run(object),"hub slot pipe_func_executor": lambda registrar: registrar.claim_pipe_func_executor(object),diff --git a/uv.lock b/uv.lockindex 328db3291..a9e13e41f 100644--- a/uv.lock+++ b/uv.lock@@ -3578,6 +3578,7 @@ dependencies = [{ name = "opentelemetry-exporter-otlp-proto-http" },{ name = "opentelemetry-sdk" },{ name = "opentelemetry-semantic-conventions" },+ { name = "packaging" },{ name = "pillow" },{ name = "pipelex-sdk" },{ name = "pipelex-tools-py" },@@ -3743,6 +3744,7 @@ requires-dist = [{ name = "opentelemetry-exporter-otlp-proto-http", specifier = ">=1.38.0" },{ name = "opentelemetry-sdk", specifier = ">=1.39.0" },{ name = "opentelemetry-semantic-conventions", specifier = ">=0.59b0" },+ { name = "packaging", specifier = ">=24.0" },{ name = "pillow", specifier = ">=12.1.1" },{ name = "pipelex-sdk", specifier = "==0.45.0" },{ name = "pipelex-tools", marker = "extra == 'dev'", specifier = ">=0.7.2" },