Build AI methods with your coding agent, then run them anywhere: as an MCP for chatbots, as a webapp for people, or via API for your software.
"""Resolve a `method_ref` into a method tree: fetch, read from the commit's git objects, list, refuse.
The wire field `method_ref` carries a globally resolvable address —
`github.com/<owner>/<repo>[/<selector>][@<tag>]` — and THIS runner is its resolver (the
layered extension policy's Rule 3: an address needs no catalog, so it is a layer-2 concept;
the hosted-only `method_id` never reaches this server). The grammar, the fetch-at-tag, the
manifest-identity package location and the bounds all live in `pipelex.methods`; this module
composes them behind the SHA-keyed clone cache (`pipelex_api.method_cache`), so a package is
cached by its commit and a second request for the same commit fetches nothing.
The package is a method tree (`mthds/docs/spec/method-tree.md`): the files of the package's directory in the resolved
commit, read from the commit's git objects as stored, with no checkout filter, line-ending conversion or
`export-ignore`, so its digest is the same on every machine, and filtered by the membership rule read with that
commit's own `.gitignore` files and the package's own `exclude` patterns. A submodule contributes no file, and a
symbolic link committed in the package is refused unless the rule excludes it. The tree's digest is recorded in the
run's provenance (`method_provenance.tree_digest`), beside the commit and the `display_name` the package's manifest
declares, which the fetched package's own provenance already carries.
The packages a package ships under its own `.mthds/methods/<name>/`, the layout of the local method cache, are out of
its tree (a dot directory), and each becomes a dependency tree of its own, keyed by the address its manifest declares,
as a dependency a request sends in `dependencies` is.
Every route taking a `method_ref` — the run routes, `/validate`, and the crate routes `/resolve`, `/codegen` and
`/pipe-io` — uses :func:`fetched_method`: the tree and its dependency trees, read in full, which the routes write out
and load as a request's `files` are (`pipelex_api.method_tree_request`), so a fetched package and a sent method run the
same path, the files a step reads, such as a `PipeDocGen` template, included.
The security gate (packaging invariant 7 — execution locus decides) is the request tree's
(`pipelex_api.method_tree_request.ensure_python_allowed`): on a deployment that is not sandbox-hosted, a package
carrying any `.py` is refused with a 501 `pipelex.deployment.custom_code_requires_sandbox`; on a sandbox-hosted one, PipeFunc `.py` is
acceptable but a package declaring `StructuredContent` subclasses is refused (`MethodStructuresRefusedError`, a 422
`pipelex.deployment.python_structures_refused`).
Everything a run needs is read OUT of the cached clone before this module returns — the tree's bytes into memory —
while the clone is pinned against eviction (`MethodCloneCache.pinned_package`), so cache eviction never races a read or
a running pipeline.
"""
import os
import shutil
import subprocess
import tempfile
from collections.abc import Generator
from contextlib import contextmanager
from pathlib import Path
from typing import NamedTuple
from mthds.protocol.problem.registry import CoreCode
from pipelex import log
from pipelex.errors.registry.pipelex_codes import PipelexErrorCode
from pipelex.libraries.method_trees import list_package_method_tree
from pipelex.methods.exceptions import MethodFetchError
from pipelex.methods.fetching import FetchedMethodPackage, MethodProvenance
from pipelex.methods.method_ref import parse_method_ref
from pipelex.methods.package_reason import MethodPackageReason
from pipelex_api.method_cache import get_method_clone_cache
from pipelex_api.method_tree_request import (
RequestMethod,
RequestTree,
add_dependency,
dependency_address_of,
ensure_method_within_limits,
ensure_tree_within_limits,
is_package_dir_name,
)
GIT_TIMEOUT_SECONDS = 60
# The git modes of a tree entry this reader treats apart from a file: an executable file, a symbolic link, and a submodule's commit.
_GIT_MODE_EXECUTABLE = "100755"
_GIT_MODE_SYMLINK = "120000"
_GIT_MODE_SUBMODULE = "160000"
# Where a package ships the method packages it calls, the layout of the local method cache.
_VENDORED_METHODS_DIR = Path(".mthds") / "methods"
class FetchedMethod(NamedTuple):
"""A fetched package as a method: its tree with the dependency trees it ships, and what the run records."""
request_method: RequestMethod
package: FetchedMethodPackage
provenance: MethodProvenance
@property
def label(self) -> str:
"""How messages name the package: its address, quoted."""
return f"'{self.package.full_address}'"
class _GitTreeEntry(NamedTuple):
mode: str
object_sha: str
path: str
def _run_git(*, clone_dir: Path, args: list[str], stdin: bytes | None, package_address: str) -> bytes:
"""Run one git command on a cached clone, its output as bytes.
Raises:
MethodFetchError: git is not installed, the command fails, or it times out.
"""
try:
result = subprocess.run( # noqa: S603 — fixed argv, no shell; every argument is ours or an object id git printed
["git", "-C", str(clone_dir), *args], # noqa: S607
input=stdin,
capture_output=True,
check=True,
timeout=GIT_TIMEOUT_SECONDS,
)
except FileNotFoundError as exc:
msg = "git is not installed or not found on PATH"
raise MethodFetchError(msg).with_error_code(PipelexErrorCode.CONFIG_DEPENDENCY_MISSING) from exc
except subprocess.CalledProcessError as exc:
msg = f"Could not read method package '{package_address}' from its commit: {exc.stderr.decode('utf-8', errors='replace').strip()}"
raise MethodFetchError(msg).with_error_code(CoreCode.METHOD_FETCH_FAILED).for_method_ref(method_ref=package_address) from exc
except subprocess.TimeoutExpired as exc:
msg = f"Timed out reading method package '{package_address}' from its commit"
raise MethodFetchError(msg).with_error_code(CoreCode.METHOD_FETCH_FAILED).for_method_ref(method_ref=package_address) from exc
return result.stdout
def _package_tree_entries(*, package: FetchedMethodPackage) -> list[_GitTreeEntry]:
"""Every entry of the package's directory in the fetched commit, by its path relative to the package's directory.
Raises:
MethodFetchError: git cannot list the commit's tree, or a path in it is not UTF-8 text.
"""
package_rel = package.package_dir.relative_to(package.clone_dir).as_posix()
# The clone the cache keys by the resolved commit is checked out at that commit, so its HEAD is the commit.
args = ["ls-tree", "-r", "-z", "--full-tree", "HEAD"]
prefix = ""
if package_rel != ".":
args += ["--", f":(literal){package_rel}"]
prefix = f"{package_rel}/"
output = _run_git(clone_dir=package.clone_dir, args=args, stdin=None, package_address=package.full_address)
entries: list[_GitTreeEntry] = []
for record in output.split(b"\0"):
if not record:
continue
header, _, raw_path = record.partition(b"\t")
mode, _, rest = header.decode("ascii").partition(" ")
_, _, object_sha = rest.partition(" ")
try:
path = raw_path.decode("utf-8")
except UnicodeDecodeError as exc:
msg = f"Method package '{package.full_address}' holds a file whose name is not UTF-8 text, which no method tree can hold."
raise (
MethodFetchError(msg).with_error_code(CoreCode.METHOD_PACKAGE_INVALID).with_params(reason=MethodPackageReason.FILE_NAME_NOT_UTF8)
) from exc
if not path.startswith(prefix):
continue
entries.append(_GitTreeEntry(mode=mode, object_sha=object_sha, path=path.removeprefix(prefix)))
return entries
def _read_git_objects(*, package: FetchedMethodPackage, object_shas: list[str]) -> dict[str, bytes]:
"""The contents of these blobs of the clone, read in one `git cat-file --batch`, by object id.
Raises:
MethodFetchError: git cannot read an object, or its answer is not the one asked for.
"""
if not object_shas:
return {}
unique_shas = sorted(set(object_shas))
output = _run_git(
clone_dir=package.clone_dir,
args=["cat-file", "--batch"],
stdin="".join(f"{object_sha}\n" for object_sha in unique_shas).encode("ascii"),
package_address=package.full_address,
)
contents: dict[str, bytes] = {}
cursor = 0
for object_sha in unique_shas:
line_end = output.index(b"\n", cursor)
header = output[cursor:line_end].decode("ascii").split(" ")
if len(header) != 3 or header[0] != object_sha:
msg = f"Could not read the object {object_sha} of method package '{package.full_address}' from its commit."
raise MethodFetchError(msg).with_error_code(CoreCode.METHOD_FETCH_FAILED).for_method_ref(method_ref=package.full_address)
size = int(header[2])
start = line_end + 1
contents[object_sha] = output[start : start + size]
cursor = start + size + 1
return contents
def _stage_package(*, package: FetchedMethodPackage, staging_dir: Path) -> None:
"""Write the package's directory as its commit stores it into `staging_dir`, from the clone's git objects.
A symbolic link is written as one, for the tree's lister to refuse unless the membership rule excludes it, and a
submodule contributes no file.
"""
entries = _package_tree_entries(package=package)
blobs = _read_git_objects(package=package, object_shas=[entry.object_sha for entry in entries if entry.mode != _GIT_MODE_SUBMODULE])
for entry in entries:
if entry.mode == _GIT_MODE_SUBMODULE:
continue
target = staging_dir.joinpath(*entry.path.split("/"))
target.parent.mkdir(parents=True, exist_ok=True)
data = blobs[entry.object_sha]
if entry.mode == _GIT_MODE_SYMLINK:
target.symlink_to(os.fsdecode(data))
continue
target.write_bytes(data)
if entry.mode == _GIT_MODE_EXECUTABLE:
target.chmod(0o755)
def _read_listed_tree(*, root: Path, package_address: str, label: str) -> RequestTree:
"""List a staged package directory as a method tree and read every file of it into memory.
Raises:
MethodTreeLoadError: The standard refused the tree, it passed a limit, or it could not be read.
"""
tree = list_package_method_tree(root=root, package_address=package_address)
contents = {tree_path: tree.file_location(tree_path).read_bytes() for tree_path in tree.tree_paths}
request_tree = RequestTree.make_from_contents(contents=contents)
ensure_tree_within_limits(tree=request_tree, label=label)
return request_tree
def _read_package_method(*, package: FetchedMethodPackage) -> RequestMethod:
"""The package's tree and the dependency trees it ships, read from the commit's git objects.
The staging directory is read into memory and removed here, and a route loading the method writes the tree out a
second time (`written_method`), rather than loading the staged directory in place. The staged copy is the commit
as git holds it: files the membership rule leaves out of the tree, symlinks, and entries under `.mthds/methods/`
that are no package all stay in it, so loading it would load more than the tree. The tree is needed in memory
anyway, for the Python policy, the limits, the `include_files` echo and the blob store, and every method a route
loads, sent or fetched, is then written by the one `written_method`, which owns its teardown.
"""
staging_dir = Path(tempfile.mkdtemp(prefix="pipelex-package-"))
try:
package_root = staging_dir / "package"
package_root.mkdir()
_stage_package(package=package, staging_dir=package_root)
label = f"Method package '{package.full_address}'"
tree = _read_listed_tree(root=package_root, package_address=package.full_address, label=label)
dependencies: dict[str, RequestTree] = {}
vendored_root = package_root / _VENDORED_METHODS_DIR
if vendored_root.is_dir():
for vendored_dir in sorted(vendored_root.iterdir()):
# A shipped package is a directory with a name a package's directory can take, as a `bundle_b64`
# archive's are (`read_archive_method`); anything else under `.mthds/methods/` is no package.
if not vendored_dir.is_dir() or vendored_dir.is_symlink() or not is_package_dir_name(vendored_dir.name):
continue
vendored_label = f"The package '.mthds/methods/{vendored_dir.name}/' of method package '{package.full_address}'"
dependency_tree = _read_listed_tree(root=vendored_dir, package_address=vendored_label, label=vendored_label)
root = f".mthds/methods/{vendored_dir.name}/"
address = dependency_address_of(tree=dependency_tree, fallback_name=vendored_dir.name, label=vendored_label, root=root)
add_dependency(dependencies=dependencies, address=address, tree=dependency_tree, label=vendored_label, root=root)
request_method = RequestMethod(tree=tree, dependencies=dependencies)
ensure_method_within_limits(request_method=request_method, label=label)
return request_method
finally:
shutil.rmtree(staging_dir, ignore_errors=True)
@contextmanager
def _fetched_package(method_ref: str) -> Generator[FetchedMethodPackage, None, None]:
"""Parse the reference and fetch its package through the SHA-keyed clone cache, its clone pinned in the block.
Every failure mode is a distinct pipelex `MethodRefError` subclass, rendered by the
global handler as the `problem+json` document of its code, with the status its code declares.
"""
ref = parse_method_ref(method_ref)
with get_method_clone_cache().pinned_package(ref=ref) as package:
# The tag and the commit say which version of the package a hosted run executed; the tag is `None` when the
# reference named none and the default branch was fetched.
log.info(
"A method reference was resolved to a package",
fields={
"method_ref": ref.ref_str,
"package_address": package.provenance.address,
"fetched_tag": package.provenance.tag,
"commit_sha": package.provenance.commit_sha,
},
)
yield package
def fetched_method(method_ref: str) -> FetchedMethod:
"""Fetch a `method_ref`'s package and read it as a method tree, with the dependency trees it ships.
Args:
method_ref: The raw `method_ref` string from the request.
Raises:
MethodRefError subclasses: parse, fetch, location and bounds failures, each rendered as `problem+json` by the
global handler.
MethodTreeLoadError: The standard refused the package's tree (a 422 naming the package by its address).
RequestMethodError: A tree past this runner's limits, or a shipped package without a manifest (422s).
"""
with _fetched_package(method_ref) as package:
request_method = _read_package_method(package=package)
# The package's own provenance already holds the address, the tag, the commit and the manifest's display name;
# only the digest of the tree this runner read is added here.
provenance = package.provenance.model_copy(update={"tree_digest": request_method.tree.tree.digest})
return FetchedMethod(request_method=request_method, package=package, provenance=provenance)