Skip to content

Build AI methods with your coding agent, then run them anywhere: as an MCP for chatbots, as a webapp for people, or via API for your software.

24.0 KB
Raw Blame
"""A security refusal answered by the run routes fails under its security code and logs one security event naming the request.

`/execute` runs through the base runner, whose own boundary reports a refusal before the run; `/start` builds its job
itself, so a refusal before its run reaches the global handler unreported, which reports it there. A refusal the server
authors itself, a reserved object key, a callback URL aimed at a private address or a path leaving the method's tree,
is reported by the handler of its problem response. Either way the event is logged once, with the run's extras the body
carried and the request id, never a run id the caller chose, and never what was refused.
"""

import base64
import io
import json
import logging
import zipfile
from collections.abc import Generator
from contextlib import contextmanager
from typing import Any, NamedTuple

import pytest
from fastapi import FastAPI
from fastapi.testclient import TestClient
from mthds.protocol.problem.registry import CoreCode
from pipelex.errors.error_codes import ErrorCode
from pipelex.errors.registry.pipelex_codes import PipelexErrorCode
from pipelex.errors.security_event import SECURITY_EVENT_MESSAGE, SECURITY_EVENT_NAME
from pipelex.pipe_run.delivery_assignment import DeliveryAssignment
from pipelex.pipe_run.pipe_job import PipeJob
from pipelex.plugins.orchestrator_registry import OrchestratorRegistry
from pipelex.runtime_bridge.payloads import PipelexPipeDispatchAck, PipelexPipeRunOutput
from pipelex.tools.log.json_log_sink import JsonLogSink
from pytest_mock import MockerFixture

from pipelex_api.api_config import ApiConfig
from pipelex_api.exception_handlers import register_exception_handlers
from pipelex_api.middleware import REQUEST_ID_HEADER, RequestIdMiddleware
from pipelex_api.routes import router as api_router
from tests.unit._problems import assert_problem

_PIPELINE_NS = "pipelex_api.routes.pipelex.pipeline"
_MODE = "temporal"
_SECURITY_EVENT_LOGGER = "pipelex.errors.security_event"
_LOCAL_PATH = "/etc/passwd"
_REQUEST_ID = "req-security-event-1"
_METHODS_TOML = """\
[package]
address = "github.com/pipelex-tests/security-event"
version = "1.0.0"
description = "Prints a letter from a template file"
"""
# A method whose step names, as its template file, a path that climbs out of its tree to a file of the server's disk.
_TEMPLATE_OUTSIDE_THE_TREE_MTHDS = """\
domain = "security_event"
main_pipe = "print_letter"

[concept.Letter]
description = "A letter to print"

[concept.Letter.structure]
body = { type = "text", description = "The letter's text", required = true }

[concept.LetterFile]
description = "The letter as a printable file"
refines = "Document"

[pipe.print_letter]
type = "PipeDocGen"
description = "Print the letter"
inputs = { letter = "Letter" }
output = "LetterFile"
format = "docx"
template_file = "../../etc/passwd.docx"
"""
# Paths that leave the tree in a spelling other than a leading slash or a `..` between slashes, each with what of it the
# event must never carry: a backslash traversal, a drive and a network share.
_ESCAPING_PATHS: list[tuple[str, str, str]] = [
    ("..\\secrets\\key.pem", "key.pem", "backslash_dot_dot"),
    ("C:/Windows/win.ini", "win.ini", "drive"),
    ("\\\\server\\share\\key.pem", "key.pem", "network_share"),
]
_DESCRIBE_IMAGE_MTHDS = """\
domain = "security_event"
main_pipe = "describe_image"

[pipe.describe_image]
type = "PipeLLM"
description = "Describe an image"
inputs = { photo = "Image" }
output = "Text"
prompt = "Describe this image. $photo"
"""


def _bundle_b64(entries: dict[str, str]) -> str:
    """A `bundle_b64` archive holding each entry under its name as written, a backslash included."""
    buffer = io.BytesIO()
    with zipfile.ZipFile(buffer, "w", zipfile.ZIP_DEFLATED) as archive:
        for name, text in entries.items():
            archive.writestr(name, text)
    return base64.b64encode(buffer.getvalue()).decode("ascii")


def _b64(text: str) -> str:
    return base64.b64encode(text.encode("utf-8")).decode("ascii")


class _ServerRefusal(NamedTuple):
    """A refusal of a security rule the server authors itself, and the security event it logs."""

    route: str
    body: dict[str, Any]
    code: ErrorCode
    event_error_type: str
    target: str
    target_class: str | None
    refused_content: str
    """Something the request carried that the event must never hold."""


class _UnreachedOrchestrator:
    """An async-capable orchestrator no refused request reaches, so a route refusing before the run dispatches nothing."""

    supports_fire_and_forget = True

    async def execute(self, *, pipe_job: PipeJob, delivery_assignment: DeliveryAssignment | None) -> PipelexPipeRunOutput:  # noqa: ARG002
        msg = "a refused request must not reach the orchestrator"
        raise AssertionError(msg)

    async def start(self, *, pipe_job: PipeJob, delivery_assignment: DeliveryAssignment | None) -> PipelexPipeDispatchAck:  # noqa: ARG002
        msg = "a refused request must not reach the orchestrator"
        raise AssertionError(msg)


def _build_client(mocker: MockerFixture) -> TestClient:
    config = ApiConfig(orchestration_mode=_MODE, allow_request_orchestration_mode_override=False)
    mocker.patch(f"{_PIPELINE_NS}.get_api_config", return_value=config)
    mocker.patch(f"{_PIPELINE_NS}.get_orchestrator_registry", return_value=OrchestratorRegistry({_MODE: _UnreachedOrchestrator()}))
    app = FastAPI()
    app.include_router(api_router, prefix="/v1")
    register_exception_handlers(app)
    return TestClient(RequestIdMiddleware(app))


def _run_body() -> dict[str, Any]:
    """A run body that passes every check but the one a test breaks, naming the organization in its extras."""
    return {
        "mthds_contents": [_DESCRIBE_IMAGE_MTHDS],
        "inputs": {"photo": "https://example.com/photo.png"},
        "storage_scope": "org_abc/mt_1/run_1",
        "read_scope": "org_abc",
        "analytics_groups": {"org_id": "org_abc"},
    }


@contextmanager
def _security_event_lines() -> Generator[list[dict[str, Any]]]:
    """The security events logged in the block, as the `json` sink the server selects serializes them."""
    buffer = io.StringIO()
    handler = JsonLogSink(stream=buffer).handler
    logger = logging.getLogger(_SECURITY_EVENT_LOGGER)
    previous_level = logger.level
    logger.setLevel(logging.WARNING)
    logger.addHandler(handler)
    lines: list[dict[str, Any]] = []
    try:
        yield lines
    finally:
        logger.removeHandler(handler)
        logger.setLevel(previous_level)
        handler.close()
        lines.extend(json.loads(line) for line in buffer.getvalue().splitlines() if line)


class TestSecurityEventRoutes:
    @pytest.mark.parametrize(
        ("route", "pipeline_run_id"),
        [
            pytest.param("/v1/execute", None, id="execute"),
            pytest.param("/v1/start", "run-security-event-1", id="start"),
        ],
    )
    def test_a_refused_input_answers_its_security_code_and_logs_one_event(
        self, mocker: MockerFixture, route: str, pipeline_run_id: str | None
    ) -> None:
        client = _build_client(mocker)
        body: dict[str, Any] = {
            "mthds_contents": [_DESCRIBE_IMAGE_MTHDS],
            "inputs": {"photo": _LOCAL_PATH},
            "storage_scope": "org_abc/mt_1/run_1",
            "read_scope": "org_abc",
            "analytics_groups": {"org_id": "org_abc"},
        }
        if pipeline_run_id is not None:
            body["pipeline_run_id"] = pipeline_run_id

        with _security_event_lines() as lines:
            response = client.post(route, json=body, headers={REQUEST_ID_HEADER: _REQUEST_ID})

        assert response.status_code == 422, response.text
        assert response.json()["code"] == CoreCode.SECURITY_INPUT_URL_REFUSED
        (line,) = lines
        assert line["message"] == SECURITY_EVENT_MESSAGE
        assert line["event.name"] == SECURITY_EVENT_NAME
        assert line["error.code"] == CoreCode.SECURITY_INPUT_URL_REFUSED
        assert line["error.domain"] == "input"
        assert line["security.target"] == "url"
        assert line["security.target_class"] == "scheme"
        assert line["run.extras.org_id"] == "org_abc"
        assert line["request_id"] == _REQUEST_ID
        # A run id the body names is the caller's claim, not a run of this runtime, before any run exists.
        assert "pipeline_run_id" not in line
        assert _LOCAL_PATH not in json.dumps(line)

    @pytest.mark.parametrize(
        "case",
        [
            pytest.param(
                _ServerRefusal(
                    route="/v1/execute",
                    body={"inputs": {"photo": {"__class__": "subprocess.Popen", "args": ["id"]}}},
                    code=PipelexErrorCode.SECURITY_RESERVED_KEY,
                    event_error_type="ReservedObjectKeyError",
                    target="object_key",
                    target_class=None,
                    refused_content="__class__",
                ),
                id="reserved_object_key",
            ),
            pytest.param(
                _ServerRefusal(
                    route="/v1/start",
                    body={"callback_urls": ["http://169.254.169.254/latest/meta-data/"]},
                    code=CoreCode.SECURITY_INPUT_URL_REFUSED,
                    event_error_type="CallbackUrlRefusedError",
                    target="url",
                    target_class="metadata",
                    refused_content="169.254.169.254",
                ),
                id="callback_metadata_endpoint",
            ),
            pytest.param(
                _ServerRefusal(
                    route="/v1/start",
                    body={"callback_urls": ["https://10.0.0.5/hooks/done"]},
                    code=CoreCode.SECURITY_INPUT_URL_REFUSED,
                    event_error_type="CallbackUrlRefusedError",
                    target="url",
                    target_class="private",
                    refused_content="10.0.0.5",
                ),
                id="callback_private_address",
            ),
            pytest.param(
                _ServerRefusal(
                    route="/v1/start",
                    body={"callback_urls": ["http://METADATA.google.internal./computeMetadata/v1/"]},
                    code=CoreCode.SECURITY_INPUT_URL_REFUSED,
                    event_error_type="CallbackUrlRefusedError",
                    target="url",
                    target_class="metadata",
                    refused_content="computeMetadata",
                ),
                id="callback_metadata_alias_spelled_otherwise",
            ),
            pytest.param(
                _ServerRefusal(
                    route="/v1/start",
                    body={"callback_urls": ["file:///etc/passwd"]},
                    code=CoreCode.SECURITY_INPUT_URL_REFUSED,
                    event_error_type="CallbackUrlRefusedError",
                    target="url",
                    target_class="scheme",
                    refused_content="/etc/passwd",
                ),
                id="callback_scheme",
            ),
            pytest.param(
                _ServerRefusal(
                    route="/v1/start",
                    body={"callback_urls": ["//169.254.169.254/latest/meta-data"]},
                    code=CoreCode.SECURITY_INPUT_URL_REFUSED,
                    event_error_type="CallbackUrlRefusedError",
                    target="url",
                    target_class="metadata",
                    refused_content="169.254.169.254",
                ),
                id="callback_scheme_relative_metadata_endpoint",
            ),
            pytest.param(
                _ServerRefusal(
                    route="/v1/start",
                    body={"callback_urls": ["//127.0.0.1:8080/cb"]},
                    code=CoreCode.SECURITY_INPUT_URL_REFUSED,
                    event_error_type="CallbackUrlRefusedError",
                    target="url",
                    target_class="loopback",
                    refused_content="127.0.0.1",
                ),
                id="callback_scheme_relative_loopback",
            ),
            pytest.param(
                _ServerRefusal(
                    route="/v1/execute",
                    body={"resources": {"../secrets/key.pem": base64.b64encode(b"x").decode("ascii")}},
                    code=CoreCode.SECURITY_INPUT_PATH_REFUSED,
                    event_error_type="BundleResourcePathError",
                    target="path",
                    target_class=None,
                    refused_content="key.pem",
                ),
                id="resource_path_leaving_the_bundle",
            ),
            pytest.param(
                _ServerRefusal(
                    route="/v1/execute",
                    body={"mthds_contents": None, "files": {"main.mthds": _DESCRIBE_IMAGE_MTHDS, "../secrets/key.pem": "x"}},
                    code=CoreCode.SECURITY_INPUT_PATH_REFUSED,
                    event_error_type="RequestMethodError",
                    target="path",
                    target_class=None,
                    refused_content="key.pem",
                ),
                id="file_leaving_the_method_tree",
            ),
            pytest.param(
                _ServerRefusal(
                    route="/v1/execute",
                    body={"mthds_contents": None, "files": {"main.mthds": _DESCRIBE_IMAGE_MTHDS, "/etc/key.pem": "x"}},
                    code=CoreCode.SECURITY_INPUT_PATH_REFUSED,
                    event_error_type="RequestMethodError",
                    target="path",
                    target_class=None,
                    refused_content="key.pem",
                ),
                id="absolute_file_path",
            ),
            *[
                pytest.param(
                    _ServerRefusal(
                        route="/v1/execute",
                        body={"mthds_contents": None, "files": {"main.mthds": _DESCRIBE_IMAGE_MTHDS, path: "x"}},
                        code=CoreCode.SECURITY_INPUT_PATH_REFUSED,
                        event_error_type="RequestMethodError",
                        target="path",
                        target_class=None,
                        refused_content=refused_content,
                    ),
                    id=f"file_{case_id}",
                )
                for path, refused_content, case_id in _ESCAPING_PATHS
            ],
            *[
                pytest.param(
                    _ServerRefusal(
                        route="/v1/execute",
                        body={"mthds_contents": None, "bundle_b64": _bundle_b64({"main.mthds": _DESCRIBE_IMAGE_MTHDS, path: "x"})},
                        code=CoreCode.SECURITY_INPUT_PATH_REFUSED,
                        event_error_type="RequestMethodError",
                        target="path",
                        target_class=None,
                        refused_content=refused_content,
                    ),
                    id=f"archive_member_{case_id}",
                )
                for path, refused_content, case_id in _ESCAPING_PATHS
            ],
            *[
                pytest.param(
                    _ServerRefusal(
                        route="/v1/execute",
                        body={"resources": {path: _b64("x")}},
                        code=CoreCode.SECURITY_INPUT_PATH_REFUSED,
                        event_error_type="BundleResourcePathError",
                        target="path",
                        target_class=None,
                        refused_content=refused_content,
                    ),
                    id=f"resource_{case_id}",
                )
                for path, refused_content, case_id in _ESCAPING_PATHS
            ],
        ],
    )
    def test_a_refusal_the_server_authors_logs_one_event_without_its_content(self, mocker: MockerFixture, case: _ServerRefusal) -> None:
        """The server's own refusal of a security rule keeps its response and logs the event its code calls for."""
        client = _build_client(mocker)
        request_body = {**_run_body(), "pipeline_run_id": "run-chosen-by-the-caller", **case.body}
        request_body = {key: value for key, value in request_body.items() if value is not None}

        with _security_event_lines() as lines:
            response = client.post(case.route, json=request_body, headers={REQUEST_ID_HEADER: _REQUEST_ID})

        assert_problem(response, status=422, code=case.code)
        (line,) = lines
        assert line["message"] == SECURITY_EVENT_MESSAGE
        assert line["event.name"] == SECURITY_EVENT_NAME
        assert line["error.code"] == case.code
        assert line["error.domain"] == "input"
        assert line["error.type"] == case.event_error_type
        assert line["security.target"] == case.target
        assert line.get("security.target_class") == case.target_class
        assert line["run.extras.org_id"] == "org_abc"
        assert line["request_id"] == _REQUEST_ID
        assert "pipeline_run_id" not in line
        assert case.refused_content not in json.dumps(line)

    def test_a_method_naming_a_file_outside_its_tree_fails_under_its_security_code_and_logs_one_event(self, mocker: MockerFixture) -> None:
        """A path the method names that climbs out of its tree is the method's security refusal, never a verdict about the method."""
        client = _build_client(mocker)
        body = {
            **_run_body(),
            "mthds_contents": None,
            "pipe_code": "print_letter",
            "inputs": {"letter": {"concept": "security_event.Letter", "content": {"body": "Dear reader"}}},
            "files": {"METHODS.toml": _METHODS_TOML, "main.mthds": _TEMPLATE_OUTSIDE_THE_TREE_MTHDS},
        }

        with _security_event_lines() as lines:
            response = client.post("/v1/execute", json=body, headers={REQUEST_ID_HEADER: _REQUEST_ID})

        assert response.status_code == 422, response.text
        assert response.json()["code"] == CoreCode.SECURITY_METHOD_PATH_REFUSED, response.text
        (line,) = lines
        assert line["event.name"] == SECURITY_EVENT_NAME
        assert line["error.code"] == CoreCode.SECURITY_METHOD_PATH_REFUSED
        assert line["error.domain"] == "method"
        assert line["error.type"] == "PipeDocGenFactoryError"
        assert line["security.target"] == "path"
        assert line["run.extras.org_id"] == "org_abc"
        assert line["request_id"] == _REQUEST_ID
        assert "passwd" not in json.dumps(line)

    @pytest.mark.parametrize(
        ("route", "body", "event_error_type"),
        [
            pytest.param(
                "/v1/validate",
                {"mthds_contents": [_DESCRIBE_IMAGE_MTHDS], "resources": {"../private": "YQ=="}},
                "BundleResourcePathError",
                id="validate_resource",
            ),
            pytest.param(
                "/v1/validate",
                {"files": {"main.mthds": _DESCRIBE_IMAGE_MTHDS, "../secrets/key.pem": "x"}},
                "RequestMethodError",
                id="validate_tree",
            ),
            pytest.param(
                "/v1/resolve", {"files": {"main.mthds": _DESCRIBE_IMAGE_MTHDS, "../secrets/key.pem": "x"}}, "RequestMethodError", id="resolve"
            ),
            pytest.param(
                "/v1/codegen",
                {"kind": "types", "target": "ts-zod", "files": {"main.mthds": _DESCRIBE_IMAGE_MTHDS, "../secrets/key.pem": "x"}},
                "RequestMethodError",
                id="codegen",
            ),
            pytest.param(
                "/v1/pipe-io", {"files": {"main.mthds": _DESCRIBE_IMAGE_MTHDS, "../secrets/key.pem": "x"}}, "RequestMethodError", id="pipe_io"
            ),
        ],
    )
    def test_a_diagnostic_route_s_refusal_names_whose_request_it_was(
        self, mocker: MockerFixture, route: str, body: dict[str, Any], event_error_type: str
    ) -> None:
        """A route that runs nothing still binds the labels its body names before refusing it, so its event carries them."""
        client = _build_client(mocker)

        with _security_event_lines() as lines:
            response = client.post(route, json={**body, "analytics_groups": {"org_id": "org_abc"}}, headers={REQUEST_ID_HEADER: _REQUEST_ID})

        assert_problem(response, status=422, code=CoreCode.SECURITY_INPUT_PATH_REFUSED)
        (line,) = lines
        assert line["error.type"] == event_error_type
        assert line["run.extras.org_id"] == "org_abc"
        assert line["request_id"] == _REQUEST_ID
        assert "private" not in json.dumps(line)
        assert "key.pem" not in json.dumps(line)

    def test_labels_the_runtime_refuses_ride_no_event(self, mocker: MockerFixture) -> None:
        """Only labels that pass the runtime's own rules are bound: a refused value never reaches a security event."""
        client = _build_client(mocker)
        body = {"mthds_contents": [_DESCRIBE_IMAGE_MTHDS], "resources": {"../private": "YQ=="}, "analytics_groups": {"org_id": "org abc\nforged"}}

        with _security_event_lines() as lines:
            response = client.post("/v1/validate", json=body)

        # The labels are the request's own mistake, which its model refuses before the route reads any resource.
        assert_problem(response, status=422, code="request.invalid", field="analytics_groups")
        assert lines == []

    @pytest.mark.parametrize(
        ("route", "body", "code"),
        [
            pytest.param("/v1/start", {"callback_urls": [""]}, "request.invalid", id="empty_callback_url"),
            pytest.param("/v1/start", {"callback_urls": ["https://hooks.example.com/" + "a" * 5000]}, "request.invalid", id="long_callback_url"),
            pytest.param("/v1/start", {"callback_urls": ["hooks.example.com/done"]}, "request.invalid", id="callback_url_without_scheme"),
            pytest.param("/v1/start", {"callback_urls": ["http:///done"]}, "request.invalid", id="callback_url_without_host"),
            # A host and a port with no scheme open no authority, though `urlparse` takes the host for a scheme.
            pytest.param("/v1/start", {"callback_urls": ["example.com:8080/hook"]}, "request.invalid", id="callback_host_and_port_without_scheme"),
            pytest.param("/v1/start", {"callback_urls": ["localhost:3000/cb"]}, "request.invalid", id="callback_localhost_and_port_without_scheme"),
            # A scheme-relative URL naming a host the SSRF rule allows still names no scheme.
            pytest.param("/v1/start", {"callback_urls": ["//hooks.example.com/done"]}, "request.invalid", id="callback_scheme_relative_public_host"),
            pytest.param(
                "/v1/execute",
                {"mthds_contents": None, "files": {"main.mthds": _DESCRIBE_IMAGE_MTHDS, "notes//a.txt": "x"}},
                "method.package_invalid",
                id="empty_segment_in_a_file_path",
            ),
            pytest.param(
                "/v1/execute",
                {"mthds_contents": None, "files": {"main.mthds": _DESCRIBE_IMAGE_MTHDS, "notes\\a.txt": "x"}},
                "method.package_invalid",
                id="backslash_in_a_file_path",
            ),
            pytest.param(
                "/v1/execute",
                {"mthds_contents": None, "files": {"main.mthds": _DESCRIBE_IMAGE_MTHDS, "notes:v2.txt": "x"}},
                "method.package_invalid",
                id="colon_in_a_file_path",
            ),
            pytest.param("/v1/execute", {"resources": {"notes\\a.txt": _b64("x")}}, "request.invalid", id="backslash_in_a_resource_path"),
            pytest.param("/v1/execute", {"resources": {"notes:v2.txt": _b64("x")}}, "request.invalid", id="colon_in_a_resource_path"),
            pytest.param("/v1/execute", {"resources": {"": _b64("x")}}, "request.invalid", id="empty_resource_path"),
        ],
    )
    def test_a_refusal_of_the_request_shape_logs_no_event(self, mocker: MockerFixture, route: str, body: dict[str, Any], code: str) -> None:
        """A malformed field is the caller's mistake, not an attempt to reach what a security rule forbids."""
        client = _build_client(mocker)
        request_body = {key: value for key, value in {**_run_body(), **body}.items() if value is not None}

        with _security_event_lines() as lines:
            response = client.post(route, json=request_body, headers={REQUEST_ID_HEADER: _REQUEST_ID})

        assert_problem(response, status=422, code=code)
        assert lines == []