Skip to content

Build AI methods with your coding agent, then run them anywhere: as an MCP for chatbots, as a webapp for people, or via API for your software.

15.9 KB
Raw Blame
"""The blob routes of the MTHDS Protocol: `POST /v1/blobs/missing` and `PUT /v1/blobs/{sha256}`.

A sender asks which of its blobs the runner lacks, uploads each to the signed target the answer names, then sends its
method by hash. These pin the wire: the targets, the write-once upload and its refusals, the scopes that keep one
caller's blobs apart from another's, the tree digest the runner answers in one lookup once it has read that tree in
full, and the `blob_store` flag `/v1/version` declares. The blob store is the in-memory one the conftest installs.
"""

from typing import Any
from urllib.parse import parse_qs, urlsplit

import pytest
from fastapi import FastAPI
from fastapi.testclient import TestClient
from mthds.method_tree.digest import digest_of_contents, sha256_hex, tree_files_from_contents, tree_listing
from pipelex.tools.storage.in_memory_storage_provider import InMemoryStorageProvider
from pytest_mock import MockerFixture

from pipelex_api.exception_handlers import register_exception_handlers
from pipelex_api.routes import router as api_router
from pipelex_api.routes.blob_upload import router as blob_upload_router
from pipelex_api.routes.version import router as version_router
from tests.unit._constants import VALID_MTHDS
from tests.unit._problems import assert_problem

_TEMPLATE = b"<h1>A template</h1>"
_NOTES = b"Some notes"


def _build_client() -> TestClient:
    app = FastAPI()
    app.include_router(api_router, prefix="/v1")
    app.include_router(version_router, prefix="/v1")
    app.include_router(blob_upload_router, prefix="/v1")
    register_exception_handlers(app)
    return TestClient(app)


def _blob_ref(data: bytes) -> dict[str, Any]:
    return {"sha256": sha256_hex(data), "size": len(data)}


def _missing(client: TestClient, *blobs: bytes, digest: str | None = None, read_scope: str | None = None) -> list[dict[str, Any]]:
    body: dict[str, Any] = {"blobs": [_blob_ref(data) for data in blobs]}
    if digest is not None:
        body["digest"] = digest
    params = {"read_scope": read_scope} if read_scope is not None else None
    response = client.post("/v1/blobs/missing", json=body, params=params)
    assert response.status_code == 200, response.text
    missing: list[dict[str, Any]] = response.json()["missing"]
    return missing


def _upload(client: TestClient, *, target: dict[str, Any], data: bytes) -> Any:
    upload = target["upload"]
    return client.request(upload["method"], upload["url"], content=data, headers=upload["headers"])


def _assert_not_authorized(response: Any) -> None:
    """The refusal of an upload whose target this runner did not sign: what it lacks is a signed upload target."""
    problem = assert_problem(response, status=403, code="auth.forbidden")
    assert problem["params"] == {"requirement": "signed_upload_target"}


class TestBlobRoutes:
    def test_the_version_declares_a_blob_store(self) -> None:
        response = _build_client().get("/v1/version")

        assert response.status_code == 200, response.text
        assert response.json()["blob_store"] is True

    def test_each_blob_the_runner_lacks_comes_with_a_signed_upload_target(self) -> None:
        missing = _missing(_build_client(), _TEMPLATE, _NOTES)

        assert [(blob["sha256"], blob["size"]) for blob in missing] == [(sha256_hex(_TEMPLATE), len(_TEMPLATE)), (sha256_hex(_NOTES), len(_NOTES))]
        for blob in missing:
            upload = blob["upload"]
            assert upload["method"] == "PUT"
            assert upload["headers"] == {"content-type": "application/octet-stream"}
            url_text: str = upload["url"]
            url = urlsplit(url_text)
            assert url.path == f"/v1/blobs/{blob['sha256']}"
            assert set(parse_qs(url.query)) == {"scope", "expires", "signature"}

    def test_an_uploaded_blob_is_held_and_uploading_it_again_changes_nothing(self) -> None:
        client = _build_client()
        (target,) = _missing(client, _TEMPLATE)

        first = _upload(client, target=target, data=_TEMPLATE)
        again = _upload(client, target=target, data=_TEMPLATE)

        assert first.status_code == 204, first.text
        assert again.status_code == 204, again.text
        assert [blob["sha256"] for blob in _missing(client, _TEMPLATE, _NOTES)] == [sha256_hex(_NOTES)]

    def test_a_body_whose_hash_is_not_the_paths_is_refused_and_not_stored(self) -> None:
        client = _build_client()
        (target,) = _missing(client, _TEMPLATE)

        assert_problem(_upload(client, target=target, data=_NOTES), status=422, code="request.invalid", field="body")

        assert [blob["sha256"] for blob in _missing(client, _TEMPLATE, _NOTES)] == [sha256_hex(_TEMPLATE), sha256_hex(_NOTES)]

    @pytest.mark.parametrize("tampering", ["no_query", "forged_signature", "another_scope", "another_hash"])
    def test_an_upload_without_a_valid_target_is_refused(self, tampering: str) -> None:
        client = _build_client()
        (target,) = _missing(client, _TEMPLATE)
        url_text: str = target["upload"]["url"]
        url = urlsplit(url_text)
        query: dict[str, str] = {name: values[0] for name, values in parse_qs(url.query).items()}
        path: str = url.path
        match tampering:
            case "no_query":
                query = {}
            case "forged_signature":
                query["signature"] = "0" * 64
            case "another_scope":
                query["scope"] = "someone-else"
            case "another_hash":
                path = f"/v1/blobs/{sha256_hex(_NOTES)}"
            case _:
                pytest.fail(f"unknown tampering {tampering}")
        data = _NOTES if tampering == "another_hash" else _TEMPLATE

        response = client.put(path, params=query, content=data, headers={"content-type": "application/octet-stream"})

        _assert_not_authorized(response)
        assert len(_missing(client, _TEMPLATE, _NOTES)) == 2

    def test_an_expired_target_is_refused(self, mocker: MockerFixture) -> None:
        mocker.patch("pipelex_api.blob_upload.BLOB_UPLOAD_URL_TTL_SECONDS", -60)
        client = _build_client()
        (target,) = _missing(client, _TEMPLATE)

        _assert_not_authorized(_upload(client, target=target, data=_TEMPLATE))

    def test_an_upload_past_the_per_file_limit_is_refused(self, mocker: MockerFixture) -> None:
        client = _build_client()
        (target,) = _missing(client, _TEMPLATE)
        mocker.patch("pipelex_api.routes.blob_upload.MAX_METHOD_TREE_FILE_BYTES", len(_TEMPLATE) - 1)

        problem = assert_problem(_upload(client, target=target, data=_TEMPLATE), status=413, code="request.too_large")
        assert problem["params"] == {"limit": len(_TEMPLATE) - 1, "size": len(_TEMPLATE)}

    def test_one_hash_listed_with_two_sizes_is_refused(self) -> None:
        body = {"blobs": [_blob_ref(_TEMPLATE), {"sha256": sha256_hex(_TEMPLATE), "size": len(_TEMPLATE) + 1}]}

        assert_problem(_build_client().post("/v1/blobs/missing", json=body), status=422, code="request.invalid", field="blobs")

    @pytest.mark.parametrize("size", [12.0, 12.5, "12", True])
    def test_a_size_that_is_not_a_json_integer_is_refused(self, size: Any) -> None:
        body = {"blobs": [{"sha256": sha256_hex(_TEMPLATE), "size": size}]}

        response = _build_client().post("/v1/blobs/missing", json=body)

        assert response.status_code == 422, response.text

    def test_blobs_past_the_tree_limits_are_refused_before_any_upload(self, mocker: MockerFixture) -> None:
        mocker.patch("pipelex_api.method_tree_request.MAX_METHOD_TREE_FILES", 1)

        response = _build_client().post("/v1/blobs/missing", json={"blobs": [_blob_ref(_TEMPLATE), _blob_ref(_NOTES)]})

        problem = assert_problem(response, status=422, code="method.package_invalid")
        assert problem["params"] == {"reason": "too_many_files", "limit": 1}

    def test_one_scopes_blobs_are_not_held_for_another(self) -> None:
        client = _build_client()
        (target,) = _missing(client, _TEMPLATE, read_scope="org-a")
        assert _upload(client, target=target, data=_TEMPLATE).status_code == 204

        assert _missing(client, _TEMPLATE, read_scope="org-a") == []
        assert [blob["sha256"] for blob in _missing(client, _TEMPLATE, read_scope="org-b")] == [sha256_hex(_TEMPLATE)]
        assert [blob["sha256"] for blob in _missing(client, _TEMPLATE)] == [sha256_hex(_TEMPLATE)]

    def test_a_read_scope_that_is_not_a_scope_is_refused(self) -> None:
        response = _build_client().post("/v1/blobs/missing", json={"blobs": []}, params={"read_scope": "../escape"})

        assert_problem(response, status=422, code="request.invalid", field="read_scope")

    def test_a_tree_read_in_full_is_answered_by_its_digest(self) -> None:
        """Once a request sent the tree, the runner holds its listing, and asking with the tree's digest is one lookup."""
        client = _build_client()
        contents = {"main.mthds": VALID_MTHDS.encode("utf-8"), "notes.txt": _NOTES}
        digest = digest_of_contents(contents)
        assert len(_missing(client, *contents.values(), digest=digest, read_scope="org-a")) == 2

        validated = client.post(
            "/v1/validate", json={"files": {"main.mthds": VALID_MTHDS, "notes.txt": _NOTES.decode("utf-8")}, "read_scope": "org-a"}
        )

        assert validated.status_code == 200, validated.text
        assert validated.json()["is_valid"] is True, validated.text
        assert _missing(client, *contents.values(), digest=digest, read_scope="org-a") == []
        assert _missing(client, *contents.values(), read_scope="org-a") == []

    def test_a_validation_records_its_tree_only_under_a_read_scope_it_was_sent(self, blob_storage: InMemoryStorageProvider) -> None:
        """A dry run without a read scope leaves nothing stored, since its fallback scope, the caller's id, no host erases."""
        client = _build_client()
        files = {"main.mthds": VALID_MTHDS, "notes.txt": _NOTES.decode("utf-8")}
        contents = {"main.mthds": VALID_MTHDS.encode("utf-8"), "notes.txt": _NOTES}

        unscoped = client.post("/v1/validate", json={"files": files})

        assert unscoped.status_code == 200, unscoped.text
        assert unscoped.json()["is_valid"] is True, unscoped.text
        assert blob_storage.root == {}

        scoped = client.post("/v1/validate", json={"files": files, "read_scope": "org-a"})

        assert scoped.status_code == 200, scoped.text
        digest_hex = digest_of_contents(contents).removeprefix("sha256:")
        assert sorted(blob_storage.root) == sorted([f"org-a/blobs/{sha256_hex(data)}" for data in contents.values()] + [f"org-a/trees/{digest_hex}"])
        assert blob_storage.root[f"org-a/trees/{digest_hex}"] == tree_listing(tree_files_from_contents(contents))

    def test_a_validation_without_a_read_scope_still_reads_the_caller_s_blobs(self, blob_storage: InMemoryStorageProvider) -> None:
        """Only the recording waits on a read scope: hash entries resolve in the caller's own scope, as before."""
        client = _build_client()
        (target,) = _missing(client, _NOTES)
        assert _upload(client, target=target, data=_NOTES).status_code == 204
        held_before = dict(blob_storage.root)

        validated = client.post("/v1/validate", json={"files": {"main.mthds": VALID_MTHDS, "notes.txt": _blob_ref(_NOTES)}})

        assert validated.status_code == 200, validated.text
        assert validated.json()["is_valid"] is True, validated.text
        assert blob_storage.root == held_before

    def test_an_uploaded_listing_never_records_its_tree(self) -> None:
        """Anyone can upload bytes that happen to form a listing, so an upload never stands for a tree held in full."""
        client = _build_client()
        contents = {"main.mthds": VALID_MTHDS.encode("utf-8"), "notes.txt": _NOTES}
        listing = tree_listing(tree_files_from_contents(contents))
        (target,) = _missing(client, listing)
        assert _upload(client, target=target, data=listing).status_code == 204

        missing = _missing(client, *contents.values(), digest=digest_of_contents(contents))

        assert [blob["sha256"] for blob in missing] == [sha256_hex(data) for data in contents.values()]

    @pytest.mark.parametrize(
        ("name", "value"),
        [
            pytest.param("signature", "é" * 64, id="signature_outside_ascii"),
            pytest.param("expires", "soon", id="expires_not_a_number"),
            pytest.param("expires", "١٢٣", id="expires_in_other_digits"),
            pytest.param("expires", "-5", id="expires_signed"),
        ],
    )
    def test_a_malformed_target_is_refused_as_no_valid_target(self, name: str, value: str) -> None:
        """Every part of the query is the caller's text: one no target of this runner holds is a 403, never a server error."""
        client = _build_client()
        (target,) = _missing(client, _TEMPLATE)
        url = urlsplit(str(target["upload"]["url"]))
        query: dict[str, str] = {key: values[0] for key, values in parse_qs(url.query).items()}
        query[name] = value

        response = client.put(url.path, params=query, content=_TEMPLATE, headers={"content-type": "application/octet-stream"})

        _assert_not_authorized(response)

    def test_upload_targets_start_with_the_configured_public_base_url(self, mocker: MockerFixture) -> None:
        """Behind a proxy terminating TLS, the URL a request arrived at is not the one a sender reaches."""
        mocker.patch("pipelex_api.blob_upload.get_optional_env", return_value="https://runner.example.com/api/")
        client = _build_client()

        (target,) = _missing(client, _TEMPLATE)

        url_text: str = target["upload"]["url"]
        assert url_text.startswith(f"https://runner.example.com/api/v1/blobs/{sha256_hex(_TEMPLATE)}?")
        url = urlsplit(url_text)
        query: dict[str, str] = {key: values[0] for key, values in parse_qs(url.query).items()}
        uploaded = client.put(f"/v1/blobs/{sha256_hex(_TEMPLATE)}", params=query, content=_TEMPLATE, headers=target["upload"]["headers"])
        assert uploaded.status_code == 204, uploaded.text

    def test_without_a_public_base_url_targets_start_with_the_request_s_url(self) -> None:
        (target,) = _missing(_build_client(), _TEMPLATE)

        assert str(target["upload"]["url"]).startswith(f"http://testserver/v1/blobs/{sha256_hex(_TEMPLATE)}?")

    def test_a_public_base_url_that_is_not_an_http_url_is_a_configuration_error(self, mocker: MockerFixture) -> None:
        mocker.patch("pipelex_api.blob_upload.get_optional_env", return_value="runner.example.com")

        response = _build_client().post("/v1/blobs/missing", json={"blobs": [_blob_ref(_TEMPLATE)]})

        assert_problem(response, status=500, code="runtime.misconfigured")

    def test_a_blob_held_at_another_size_is_listed_and_its_upload_never_writes_over_it(self, blob_storage: InMemoryStorageProvider) -> None:
        """Bytes held short are damaged storage: they are asked for, and their upload is a 500 that leaves the object as it is."""
        client = _build_client()
        blob_storage.root[f"single-tenant/blobs/{sha256_hex(_TEMPLATE)}"] = _TEMPLATE[:5]

        (target,) = _missing(client, _TEMPLATE)
        uploaded = _upload(client, target=target, data=_TEMPLATE)

        assert_problem(uploaded, status=500, code="pipelex.storage.damaged")
        assert blob_storage.root[f"single-tenant/blobs/{sha256_hex(_TEMPLATE)}"] == _TEMPLATE[:5]

    def test_a_held_digest_answers_alone_only_for_blobs_of_its_tree(self) -> None:
        """A digest the runner holds says nothing of blobs outside its tree, which are looked up one by one."""
        client = _build_client()
        contents = {"main.mthds": VALID_MTHDS.encode("utf-8"), "notes.txt": _NOTES}
        validated = client.post(
            "/v1/validate", json={"files": {"main.mthds": VALID_MTHDS, "notes.txt": _NOTES.decode("utf-8")}, "read_scope": "org-a"}
        )
        assert validated.status_code == 200, validated.text

        missing = _missing(client, *contents.values(), _TEMPLATE, digest=digest_of_contents(contents), read_scope="org-a")

        assert [blob["sha256"] for blob in missing] == [sha256_hex(_TEMPLATE)]