:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
from fastapi.testclient import TestClient
from llm_gateway.auth.service import InvalidProjectScopeError, UnauthorizedProjectScopeError
from llm_gateway.rate_limiting.model_cost_service import ModelCost, ModelCostService
from llm_gateway.services.model_registry import ModelRegistryService
MOCK_COST_DATA: dict[str, ModelCost] = {
"gpt-4o": {
"litellm_provider": "openai",
"max_input_tokens": 128000,
"supports_vision": True,
"mode": "chat",
},
"gpt-4o-mini": {
"litellm_provider": "openai",
"max_input_tokens": 128000,
"supports_vision": True,
"mode": "chat",
},
"gpt-5.2": {
"litellm_provider": "openai",
"max_input_tokens": 200000,
"supports_vision": True,
"supports_prompt_caching": True,
"input_cost_per_token": 2.5e-6,
"output_cost_per_token": 10e-6,
"cache_read_input_token_cost": 0.25e-6,
"mode": "chat",
},
"gpt-5-mini": {
"litellm_provider": "openai",
"max_input_tokens": 200000,
"supports_vision": True,
"mode": "chat",
},
"gpt-5.3-codex": {
"litellm_provider": "openai",
"max_input_tokens": 200000,
"supports_vision": False,
"mode": "chat",
},
"o1": {
"litellm_provider": "openai",
"max_input_tokens": 200000,
"supports_vision": True,
"mode": "chat",
},
"claude-3-5-sonnet-20241022": {
"litellm_provider": "anthropic",
"max_input_tokens": 200000,
"supports_vision": True,
"mode": "chat",
},
"claude-opus-4-5": {
"litellm_provider": "anthropic",
"max_input_tokens": 200000,
"supports_vision": True,
"supports_prompt_caching": True,
"input_cost_per_token": 5e-6,
"output_cost_per_token": 25e-6,
"cache_read_input_token_cost": 0.5e-6,
"cache_creation_input_token_cost": 6.25e-6,
"mode": "chat",
},
"claude-opus-4-6": {
"litellm_provider": "anthropic",
"max_input_tokens": 200000,
"supports_vision": True,
"mode": "chat",
},
"claude-sonnet-4-5": {
"litellm_provider": "anthropic",
"max_input_tokens": 200000,
"supports_vision": True,
"mode": "chat",
},
"claude-sonnet-4-6": {
"litellm_provider": "anthropic",
"max_input_tokens": 200000,
"supports_vision": True,
"mode": "chat",
},
"claude-haiku-4-5": {
"litellm_provider": "anthropic",
"max_input_tokens": 200000,
"supports_vision": True,
"mode": "chat",
},
"claude-sonnet-4-5-20260101": {
"litellm_provider": "anthropic",
"max_input_tokens": 200000,
"supports_vision": True,
"mode": "chat",
},
"openrouter/anthropic/claude-3.5-sonnet": {
"litellm_provider": "openrouter",
"max_input_tokens": 200000,
"supports_vision": True,
"mode": "chat",
},
"fireworks_ai/accounts/fireworks/models/llama-v3p1-70b-instruct": {
"litellm_provider": "fireworks_ai",
"max_input_tokens": 131072,
"supports_vision": False,
"mode": "chat",
},
"cloudflare/@cf/zai-org/glm-5.2": {
"litellm_provider": "cloudflare",
"max_input_tokens": 128000,
"supports_prompt_caching": True,
"input_cost_per_token": 1.4e-6,
"output_cost_per_token": 4.4e-6,
"cache_read_input_token_cost": 0.14e-6,
"mode": "chat",
},
"baseten/deepseek-ai/deepseek-v4-flash-0731": {
"litellm_provider": "baseten",
"max_input_tokens": 1048000,
"supports_prompt_caching": True,
"input_cost_per_token": 0.13e-6,
"output_cost_per_token": 0.26e-6,
"cache_read_input_token_cost": 0.028e-6,
"mode": "chat",
},
}
def mock_get_costs(self: ModelCostService, model: str) -> ModelCost | None:
return MOCK_COST_DATA.get(model)
def mock_get_all_models(self: ModelCostService) -> dict[str, ModelCost]:
return MOCK_COST_DATA
def create_mock_settings() -> MagicMock:
settings = MagicMock()
settings.openai_api_key = "sk-test"
settings.anthropic_api_key = "sk-ant-test"
settings.openrouter_api_key = "or-test"
settings.fireworks_api_key = "fw-test"
return settings
@pytest.fixture(autouse=True)
def reset_services():
ModelRegistryService.reset_instance()
ModelCostService.reset_instance()
yield
ModelRegistryService.reset_instance()
ModelCostService.reset_instance()
@pytest.fixture(autouse=True)
def mock_cost_service():
with (
patch.object(ModelCostService, "get_costs", mock_get_costs),
patch.object(ModelCostService, "get_all_models", mock_get_all_models),
):
yield
@pytest.fixture(autouse=True)
def mock_settings():
with patch(
"llm_gateway.services.model_registry.get_settings",
return_value=create_mock_settings(),
):
yield
class TestListModelsEndpoint:
def test_returns_models_list(self, client: TestClient):
response = client.get("/v1/models")
assert response.status_code == 200
data = response.json()
assert data["object"] == "list"
assert isinstance(data["data"], list)
assert len(data["data"]) > 0
# codex-acp compatibility: `models` mirrors `data`
assert data["models"] == data["data"]
def test_model_object_has_required_fields(self, client: TestClient):
response = client.get("/v1/models")
data = response.json()
model = data["data"][0]
assert "id" in model
assert "object" in model
assert model["object"] == "model"
assert "created" in model
assert "owned_by" in model
assert "context_window" in model
assert "supports_streaming" in model
assert "supports_vision" in model
def test_truncation_policy_limit_is_non_zero(self, client: TestClient):
# Ensure truncation_policy.limit is always > 0 (prevents tool output breakage).
response = client.get("/v1/models")
for model in response.json()["data"]:
policy = model["truncation_policy"]
assert policy["mode"] in {"bytes", "tokens"}
assert policy["limit"] > 0, f"{model['id']} would truncate tool output to zero"
class TestListModelsForProductEndpoint:
def test_returns_models_for_llm_gateway(self, client: TestClient):
response = client.get("/llm_gateway/v1/models")
assert response.status_code == 200
data = response.json()
model_ids = {m["id"] for m in data["data"]}
assert "gpt-4o" in model_ids
assert "o1" in model_ids
assert "claude-sonnet-4-5" in model_ids
assert "claude-3-5-sonnet-20241022" in model_ids
def test_posthog_code_filters_models_by_allowed_list(self, client: TestClient):
response = client.get("/posthog_code/v1/models")
assert response.status_code == 200
data = response.json()
model_ids = {m["id"] for m in data["data"]}
assert "claude-sonnet-4-5" in model_ids
assert "claude-sonnet-4-5-20260101" not in model_ids
assert "gpt-4o" not in model_ids
assert "o1" not in model_ids
def test_posthog_code_returns_effective_pricing(self, client: TestClient):
response = client.get("/posthog_code/v1/models")
assert response.status_code == 200
models = {model["id"]: model for model in response.json()["data"]}
assert models["gpt-5.2"]["pricing"] == {
"prompt": "0.0000025",
"completion": "0.00001",
"input_cache_read": "0.00000025",
"input_cache_write": "0.0000025",
}
assert models["claude-opus-4-5"]["pricing"] == {
"prompt": "0.000005",
"completion": "0.000025",
"input_cache_read": "0.0000005",
"input_cache_write": "0.00000625",
}
assert models["@cf/zai-org/glm-5.2"]["pricing"] == {
"prompt": "0.0000014",
"completion": "0.0000044",
"input_cache_read": "0.00000014",
"input_cache_write": "0.0000014",
}
assert models["deepseek-ai/deepseek-v4-flash-0731"]["pricing"] == {
"prompt": "0.00000013",
"completion": "0.00000026",
"input_cache_read": "0.000000028",
"input_cache_write": "0.00000013",
}
@pytest.mark.parametrize("alias", ["twig", "array"])
def test_legacy_alias_routes_to_posthog_code(self, client: TestClient, alias: str):
response = client.get(f"/{alias}/v1/models")
assert response.status_code == 200
data = response.json()
model_ids = {m["id"] for m in data["data"]}
posthog_code_response = client.get("/posthog_code/v1/models")
posthog_code_model_ids = {m["id"] for m in posthog_code_response.json()["data"]}
assert model_ids == posthog_code_model_ids
def test_returns_error_for_invalid_product(self, client: TestClient):
response = client.get("/invalid_product/v1/models")
assert response.status_code == 400
assert "Invalid product" in response.json()["detail"]
class TestResponsesModeModels:
@pytest.mark.parametrize(
"endpoint", ["/v1/models", "/posthog_code/v1/models", "/array/v1/models", "/twig/v1/models"]
)
def test_models_endpoint_includes_responses_mode_models(self, client: TestClient, endpoint: str):
cost_data_with_responses = dict(MOCK_COST_DATA)
cost_data_with_responses["gpt-5.3-codex"] = {
**cost_data_with_responses["gpt-5.3-codex"],
"mode": "responses",
}
def get_costs_with_responses(self: ModelCostService, model: str) -> ModelCost | None:
return cost_data_with_responses.get(model)
def get_all_models_with_responses(self: ModelCostService) -> dict[str, ModelCost]:
return cost_data_with_responses
ModelRegistryService.reset_instance()
ModelCostService.reset_instance()
with (
patch.object(ModelCostService, "get_costs", get_costs_with_responses),
patch.object(ModelCostService, "get_all_models", get_all_models_with_responses),
):
response = client.get(endpoint)
assert response.status_code == 200
model_ids = {m["id"] for m in response.json()["data"]}
assert "gpt-5.3-codex" in model_ids
def _wire_authenticated_user(mock_db_pool, distinct_id: str):
from unittest.mock import AsyncMock
conn = AsyncMock()
conn.fetchrow = AsyncMock(
return_value={
"id": "key_id",
"user_id": 1,
"scopes": ["llm_gateway:read"],
"current_team_id": 1,
"distinct_id": distinct_id,
"is_staff": False,
}
)
mock_db_pool.acquire = AsyncMock(return_value=conn)
mock_db_pool.release = AsyncMock()
class TestFreeTierModelListing:
def test_anonymous_caller_gets_full_unrestricted_list(self, client: TestClient):
# an unidentifiable caller may be a billed org; never mark for those
response = client.get("/posthog_code/v1/models")
assert response.status_code == 200
models = response.json()["data"]
assert "claude-opus-4-5" in {m["id"] for m in models}
assert all(m["allowed"] for m in models)
@pytest.mark.parametrize(
"error,expected_status",
[
pytest.param(InvalidProjectScopeError(), 400, id="invalid_project"),
pytest.param(UnauthorizedProjectScopeError(), 403, id="unauthorized_project"),
],
)
def test_project_scope_errors_are_returned(
self,
app,
monkeypatch: pytest.MonkeyPatch,
error: Exception,
expected_status: int,
):
auth_service = MagicMock()
auth_service.authenticate_request = AsyncMock(side_effect=error)
with patch("llm_gateway.api.models.get_auth_service", return_value=auth_service), TestClient(app) as client:
response = client.get(
"/posthog_code/v1/models",
headers={"Authorization": "Bearer pha_scoped_models", "X-PostHog-Project-Id": "123"},
)
assert response.status_code == expected_status
def test_project_scope_errors_are_returned_on_bare_models_route(self, app):
auth_service = MagicMock()
auth_service.authenticate_request = AsyncMock(side_effect=UnauthorizedProjectScopeError())
with patch("llm_gateway.api.models.get_auth_service", return_value=auth_service), TestClient(app) as client:
response = client.get(
"/v1/models",
headers={"Authorization": "Bearer pha_scoped_models", "X-PostHog-Project-Id": "123"},
)
assert response.status_code == 403
def test_unbilled_org_gets_full_list_with_premium_models_marked(self, app, mock_db_pool):
from unittest.mock import AsyncMock
from llm_gateway.services.quota_resolver import QuotaResourceStatus
_wire_authenticated_user(mock_db_pool, "unbilled-user")
with TestClient(app) as c:
c.app.state.quota_resolver.get_resource_status = AsyncMock(
return_value=QuotaResourceStatus(limited=False, code_usage_billing_active=False)
)
response = c.get("/posthog_code/v1/models", headers={"Authorization": "Bearer phx_unbilled_org_models"})
assert response.status_code == 200
body = response.json()
by_id = {m["id"]: m for m in body["data"]}
premium = by_id["claude-opus-4-5"]
assert premium["allowed"] is False
assert premium["restriction_reason"] == "paid_plan_required"
# exact, not subset: every free-tier model must survive the allowlist
assert {m["id"] for m in body["data"] if m["allowed"]} == {
"@cf/zai-org/glm-5.2",
"deepseek-ai/deepseek-v4-flash-0731",
"zai-org/glm-5.3-flash",
}
# codex reads the `models` mirror; the marks must be there too
assert body["models"] == body["data"]
@pytest.mark.parametrize(
"flag_enabled,listed",
[(True, True), (False, False), (None, False)],
ids=["flag_on", "flag_off", "flag_unavailable"],
)
def test_flag_gated_model_is_listed_only_when_its_flag_clears(
self, app, mock_db_pool, flag_enabled: bool | None, listed: bool
):
# No model is behind a rollout flag now, so one is gated here for the listing to have
# something to drop.
_wire_authenticated_user(mock_db_pool, "gated-user")
evaluate = AsyncMock(side_effect=lambda flag_keys, _: dict.fromkeys(flag_keys, flag_enabled))
with (
patch(
"llm_gateway.api.models.get_required_model_flag",
side_effect=lambda model: "acme-gated-model" if model == "claude-opus-4-5" else None,
),
patch("llm_gateway.api.models.evaluate_flags", evaluate),
TestClient(app) as c,
):
response = c.get("/posthog_code/v1/models", headers={"Authorization": "Bearer phx_gated_models"})
assert response.status_code == 200
evaluate.assert_awaited_once()
body = response.json()
assert ("claude-opus-4-5" in {m["id"] for m in body["data"]}) is listed
assert body["models"] == body["data"]
# The listing drops a model whose flag cannot be evaluated, so a flag left behind after a
# rollout finished is what took the open-weights models off non-staff pickers.
def test_open_weights_models_are_listed_without_a_flag_evaluation(self, app, mock_db_pool):
_wire_authenticated_user(mock_db_pool, "open-weights-user")
evaluate = AsyncMock(side_effect=lambda flag_keys, _: dict.fromkeys(flag_keys, False))
with (
patch("llm_gateway.api.models.evaluate_flags", evaluate),
TestClient(app) as c,
):
response = c.get("/posthog_code/v1/models", headers={"Authorization": "Bearer phx_open_weights_models"})
assert response.status_code == 200
evaluate.assert_not_awaited()
listed = {m["id"] for m in response.json()["data"]}
# Kimi K3 is Modal-served, and Modal is unconfigured here, so it is out of scope for
# a listing test.
assert {
"deepseek-ai/deepseek-v4-flash-0731",
"zai-org/glm-5.3",
"zai-org/glm-5.3-flash",
} <= listed
def test_billed_org_sees_full_list(self, app, mock_db_pool):
from unittest.mock import AsyncMock
from llm_gateway.services.quota_resolver import QuotaResourceStatus
_wire_authenticated_user(mock_db_pool, "billed-user")
with TestClient(app) as c:
c.app.state.quota_resolver.get_resource_status = AsyncMock(
return_value=QuotaResourceStatus(limited=False, code_usage_billing_active=True)
)
response = c.get("/posthog_code/v1/models", headers={"Authorization": "Bearer phx_billed_org_models"})
assert response.status_code == 200
models = response.json()["data"]
assert "claude-opus-4-5" in {m["id"] for m in models}
assert all(m["allowed"] for m in models)
def test_auth_resolution_failure_serves_full_unrestricted_list(self, app, mock_db_pool):
from unittest.mock import AsyncMock
# an auth outage must not mark a possibly-billed caller's models
mock_db_pool.acquire = AsyncMock(side_effect=RuntimeError("db down"))
with TestClient(app) as c:
response = c.get("/posthog_code/v1/models", headers={"Authorization": "Bearer phx_auth_outage_models"})
assert response.status_code == 200
models = response.json()["data"]
assert "claude-opus-4-5" in {m["id"] for m in models}
assert all(m["allowed"] for m in models)