Build AI methods with your coding agent, then run them anywhere: as an MCP for chatbots, as a webapp for people, or via API for your software.
"""The model reference check route, `GET /v1/models/check`, on the server's own deck."""
from typing import Any
import pytest
from fastapi import FastAPI
from fastapi.testclient import TestClient
from pipelex_api.exception_handlers import register_exception_handlers
from pipelex_api.routes import router as api_router
from pipelex_api.routes.pipelex.agent.models import MAX_MODEL_REFERENCE_LENGTH
from tests.unit._problems import assert_problem
_CHECK_PATH = "/v1/models/check"
_VERDICT_FIELDS = {"reference", "kind", "name", "category", "resolution", "matches", "suggestions", "other_kinds", "other_categories"}
def _build_client() -> TestClient:
app = FastAPI()
app.include_router(api_router, prefix="/v1")
register_exception_handlers(app)
return TestClient(app)
def _verdict(client: TestClient, params: list[tuple[str, str]]) -> dict[str, Any]:
response = client.get(_CHECK_PATH, params=tuple(params))
assert response.status_code == 200, response.text
assert response.headers["content-type"].startswith("application/json")
verdict: dict[str, Any] = response.json()
assert set(verdict) == _VERDICT_FIELDS
return verdict
def _img_gen_bundle(*, model: str) -> str:
return f"""
domain = "harbour_board"
description = "Draw the board announcing the tide times"
main_pipe = "draw_tide_board"
[pipe.draw_tide_board]
type = "PipeImgGen"
description = "Draw the painted board announcing the tide times"
output = "Image"
model = "{model}"
prompt = "A painted wooden harbour board announcing the tide times, morning light"
"""
class TestModelsCheckRoute:
def test_a_preset_of_the_deck_resolves_in_its_category(self):
client = _build_client()
preset = client.get("/v1/models").json()["models"][0]
verdict = _verdict(client, [("reference", f"${preset['name']}"), ("type", preset["type"])])
assert verdict["resolution"] == "resolved"
assert verdict["kind"] == "preset"
assert verdict["name"] == preset["name"]
assert verdict["category"] == preset["type"]
(match,) = verdict["matches"]
assert set(match) == {"category", "resolves_to", "target", "description"}
assert match["category"] == preset["type"]
assert verdict["suggestions"] == verdict["other_kinds"] == verdict["other_categories"] == []
def test_an_alias_and_its_target_resolve_with_the_alias_in_via(self):
client = _build_client()
alias_name, alias_target = next(iter(client.get("/v1/models", params={"type": "llm"}).json()["aliases"]["llm"].items()))
alias_verdict = _verdict(client, [("reference", f" alias:{alias_name} "), ("type", "llm")])
handle_verdict = _verdict(client, [("reference", alias_target), ("type", "llm")])
assert alias_verdict["reference"] == f"alias:{alias_name}"
assert alias_verdict["matches"] == [{"category": "llm", "resolves_to": alias_target, "target": alias_target}]
assert handle_verdict["resolution"] == "resolved"
assert handle_verdict["kind"] == "handle"
(match,) = handle_verdict["matches"]
assert match["resolves_to"] == alias_target
assert f"@{alias_name}" in match["via"]
def test_an_llm_asked_as_an_image_generation_model_is_not_found_and_fails_validation_too(self):
"""The check and a validation agree: an LLM the deck serves is neither checked nor validated as an image-generation model."""
client = _build_client()
llm_handle = next(iter(client.get("/v1/models", params={"type": "llm"}).json()["aliases"]["llm"].values()))
verdict = _verdict(client, [("reference", llm_handle), ("type", "img_gen")])
validation = client.post("/v1/validate", json={"mthds_contents": [_img_gen_bundle(model=llm_handle)]})
assert verdict["resolution"] == "not_found"
assert verdict["matches"] == []
assert verdict["other_categories"] == ["llm"]
assert validation.status_code == 200, validation.text
report = validation.json()
assert report["is_valid"] is False
(item,) = report["errors"]
assert item["code"] == "model.unknown"
assert item["params"] == {"model": llm_handle, "model_type": "img_gen", "suggestions": []}
assert item["location"]["field"] == "model"
def test_doc_gen_is_a_category_of_the_check(self):
client = _build_client()
verdict = _verdict(client, [("reference", "$no-preset-is-named-this"), ("type", "doc_gen")])
assert verdict["category"] == "doc_gen"
assert verdict["resolution"] == "not_found"
def test_the_longest_accepted_reference_is_a_verdict(self):
client = _build_client()
verdict = _verdict(client, [("reference", "@" + "x" * (MAX_MODEL_REFERENCE_LENGTH - 1))])
assert verdict["resolution"] == "not_found"
assert verdict["category"] is None
@pytest.mark.parametrize(
("params", "field"),
[
pytest.param([], "reference", id="reference-missing"),
pytest.param([("reference", "@a"), ("reference", "@b")], "reference", id="reference-repeated"),
pytest.param([("reference", "@a"), ("type", "llm"), ("type", "extract")], "type", id="type-repeated"),
pytest.param([("reference", "")], "reference", id="empty"),
pytest.param([("reference", " ")], "reference", id="blank"),
pytest.param([("reference", "~")], "reference", id="sigil-alone"),
pytest.param([("reference", "handle:")], "reference", id="namespace-alone"),
pytest.param([("reference", "@" + "x" * MAX_MODEL_REFERENCE_LENGTH)], "reference", id="past-the-bound"),
pytest.param([("reference", "@a"), ("type", "bogus_category")], "type", id="type-unknown"),
pytest.param([("reference", "@a"), ("type", "")], "type", id="type-empty"),
],
)
def test_a_request_with_no_verdict_is_an_input_422(self, params: list[tuple[str, str]], field: str):
client = _build_client()
response = client.get(_CHECK_PATH, params=tuple(params))
problem = assert_problem(response, status=422, code="request.invalid", field=field)
assert problem["domain"] == "input"
assert "resolution" not in problem