Skip to content

Build AI methods with your coding agent, then run them anywhere: as an MCP for chatbots, as a webapp for people, or via API for your software.

8.3 KB
Raw Blame
"""Smoke + validation tests for /validate and /models."""

import pytest
from fastapi import FastAPI
from fastapi.testclient import TestClient
from mthds.protocol.models import ModelCategory as MthdsModelCategory
from mthds.protocol.problem.registry import CoreCode
from pipelex.cogt.models.model_listing import ModelCategory
from pipelex.errors.error_codes import error_code_spec
from pipelex.interpreter_hub import get_library_manager
from pytest_mock import MockerFixture

from pipelex_api.exception_handlers import register_exception_handlers
from pipelex_api.routes import router as api_router
from tests.unit._constants import VALID_MTHDS
from tests.unit._problems import assert_problem


def _build_client() -> TestClient:
    app = FastAPI()
    app.include_router(api_router, prefix="/v1")
    register_exception_handlers(app)
    return TestClient(app)


class TestValidateAndModelsRoutes:
    def test_validate_rejects_oversized_mthds(self):
        client = _build_client()
        oversized = "a" * (2 * 1024 * 1024)  # 2 MiB > 1 MiB cap
        response = client.post(
            "/v1/validate",
            json={"mthds_contents": [oversized]},
        )
        assert_problem(response, status=422, code="request.invalid", field="mthds_contents")

    def test_validate_rejects_too_many_files(self):
        client = _build_client()
        response = client.post(
            "/v1/validate",
            json={"mthds_contents": [VALID_MTHDS] * 32},
        )
        assert_problem(response, status=422, code="request.invalid", field="mthds_contents")

    def test_validate_invalid_mthds_returns_200_invalid_report(self):
        # `/validate` is a diagnostic endpoint: an invalid bundle is a produced verdict, so it rides
        # a 200 `InvalidReport` (discriminated on `is_valid: false`) — NOT a 422 problem document,
        # and NOT the older `{success, mthds_contents, message}` 422 envelope. Invalid TOML reliably
        # triggers a `ValidateBundleError` from the interpreter, which the route converts to the
        # invalid arm. The verdict is carried by `is_valid`/`message`, not the status code.
        client = _build_client()
        response = client.post(
            "/v1/validate",
            json={"mthds_contents": ["this is not valid TOML !!!"]},
        )
        assert response.status_code == 200, response.text
        assert response.headers["content-type"].startswith("application/json")
        body = response.json()
        assert body["is_valid"] is False
        assert body["is_runnable"] is False
        # The verdict's message is the sentence its code renders, `method.invalid`'s; the interpreter's
        # actual complaint rides the item, so the client still reads what is wrong and where.
        assert body["message"] == error_code_spec(CoreCode.METHOD_INVALID).detail
        # The item list is non-empty on every invalid arm — the structured-info invariant is total. A raw
        # TOML-syntax error is the method's syntax finding, located where the parser stopped; a categorized
        # failure (e.g. an invalid `main_pipe`) yields its own codes, pinned in test_validate_errors.py.
        (item,) = body["errors"]
        assert item["code"] == "method.syntax"
        assert item["location"]["line"] == 1
        assert item["params"]["finding"].startswith("TOML syntax error")
        # The valid arm's structural artifacts + the retired `success` extra are absent.
        assert "bundle_blueprint" not in body
        assert "success" not in body

    # NOTE: the former `main_pipe` precondition on /validate is deleted (protocol alignment
    # D2) — a bundle without `main_pipe` now answers 200 with `graph_spec=null`. The
    # regression pin for that behavior (both backends) lives in `test_validate_envelope.py`.

    @pytest.mark.parametrize("bad_type", ["not-a-real-category", ""])
    def test_models_rejects_invalid_category(self, bad_type: str):
        # The empty string is an explicitly-supplied (invalid) filter value, not an absent
        # param — it must fail loudly like any other unknown category, not silently return
        # the unfiltered deck.
        client = _build_client()
        response = client.get(f"/v1/models?type={bad_type}")
        assert_problem(response, status=422, code="request.invalid", field="type")

    def test_models_rejects_repeated_type_param(self):
        # D11: the protocol's `type` filter is a single plain value. The old route accepted
        # repeated `?type=` values (list[str] Query) — that extension is dropped, and FastAPI
        # would otherwise silently keep one of the values, so the rejection is explicit.
        # Both values are valid categories — what's wrong is the arity, still a refusal of the field.
        client = _build_client()
        response = client.get("/v1/models?type=llm&type=extract")
        assert_problem(response, status=422, code="request.invalid", field="type")

    def test_models_returns_protocol_deck(self):
        # The deck is the protocol `ModelDeck` shape produced by `PipelexMTHDSProtocol.models`:
        # a non-empty flat `models` list (regression for the silently-empty deck the old raw
        # per-category payload caused in the SDK — F2) plus this implementation's routing
        # extensions, keyed by category (the same alias name exists in several categories —
        # a flat map would silently drop entries on collision). The old raw keys (`presets`
        # by category, `success`) are gone. Every entry of the flat list is typed by one of the
        # protocol's categories, `judgment` among them, and never by a value the protocol does not
        # define; the extensions are keyed by the categories this runtime serves.
        client = _build_client()
        response = client.get("/v1/models")
        assert response.status_code == 200, response.text
        body = response.json()
        assert body["models"], "the protocol deck must carry a non-empty models list"
        first_model = body["models"][0]
        assert first_model["name"]
        assert first_model["type"]
        assert {model["type"] for model in body["models"]} <= {category.value for category in MthdsModelCategory}
        valid_categories = {category.value for category in ModelCategory}
        assert set(body["aliases"]) <= valid_categories
        assert all(isinstance(category_aliases, dict) for category_aliases in body["aliases"].values())
        assert set(body["waterfalls"]) <= valid_categories
        assert "presets" not in body
        assert "success" not in body

    def test_models_single_type_filter(self):
        client = _build_client()
        response = client.get("/v1/models?type=llm")
        assert response.status_code == 200, response.text
        body = response.json()
        assert body["models"]
        assert {model["type"] for model in body["models"]} == {"llm"}
        assert set(body["aliases"]) <= {"llm"}

    def test_empty_mthds_contents_rejected(self):
        # `/validate` is the last route on the `mthds_contents` envelope (it is protocol-owned; the
        # crate routes take `files[]`, whose empty-selector rejection is pinned in the pipe-io suite).
        client = _build_client()
        response = client.post("/v1/validate", json={"mthds_contents": []})
        assert_problem(response, status=422, code="request.invalid", field="mthds_contents")

    def test_validate_tears_down_validate_bundle_library_without_leaking(self, mocker: MockerFixture):
        # validate_bundle opens a library and leaves it loaded + current on success (the D6 contract);
        # /validate must own that teardown. Before the fix the route never tore it down, orphaning it on
        # every successful call (the best-effort graph dry-run opens + tears down its OWN library via
        # PipelexRunner, so that one was balanced). Assert the conservation property: every library opened
        # during the request is torn down — open count == teardown count. A leak shows up as opens > teardowns.
        library_manager = get_library_manager()
        open_spy = mocker.spy(library_manager, "open_library")
        teardown_spy = mocker.spy(library_manager, "teardown")

        client = _build_client()
        response = client.post("/v1/validate", json={"mthds_contents": [VALID_MTHDS]})

        assert response.status_code == 200, response.text
        assert open_spy.call_count >= 1
        assert open_spy.call_count == teardown_spy.call_count