π OpenHands: AI-Driven Development
Deprecate /api/options/models, add /api/v1/config/models/search endpoint (#13799)
Tim O'Farrell2eefa5edfd76
diff --git a/openhands/app_server/config_api/config_models.py b/openhands/app_server/config_api/config_models.pynew file mode 100644index 0000000000..bccd654989--- /dev/null+++ b/openhands/app_server/config_api/config_models.py@@ -0,0 +1,32 @@+"""Config-related models for OpenHands App Server V1 API."""++from pydantic import BaseModel, Field+++class LLMModel(BaseModel):+ """LLM Model object for API responses.++ Attributes:+ name: The model name.+ verified: Whether the model is verified by OpenHands.+ """++ provider: str | None = Field(+ default=None, description='The name of the provider for this model'+ )+ name: str = Field(description='The name of this model')+ verified: bool = Field(+ default=False, description='Whether the model is verified by OpenHands'+ )+++class LLMModelPage(BaseModel):+ """Paginated response for LLM models.++ Attributes:+ items: List of LLM models in the current page.+ next_page_id: ID for the next page, or None if there are no more pages.+ """++ items: list[LLMModel]+ next_page_id: str | None = Nonediff --git a/openhands/app_server/config_api/config_router.py b/openhands/app_server/config_api/config_router.pynew file mode 100644index 0000000000..29df9e870c--- /dev/null+++ b/openhands/app_server/config_api/config_router.py@@ -0,0 +1,94 @@+"""Config router for OpenHands App Server V1 API.++This module provides V1 API endpoints for configuration, including model search+with pagination support.+"""++from typing import Annotated++from fastapi import APIRouter, Depends, Query++from openhands.app_server.config_api.config_models import LLMModel, LLMModelPage+from openhands.app_server.utils.dependencies import get_dependencies+from openhands.app_server.utils.paging_utils import (+ paginate_results,+)+from openhands.sdk.llm.utils.verified_models import VERIFIED_MODELS+from openhands.server.routes.public import get_llm_models_dependency+from openhands.utils.llm import ModelsResponse++# We use the get_dependencies method here to signal to the OpenAPI docs that this endpoint+# is protected. The actual protection is provided by SetAuthCookieMiddleware+router = APIRouter(+ prefix='/config',+ tags=['Config'],+ dependencies=get_dependencies(),+)++[email protected]('/models/search')+async def search_models(+ page_id: Annotated[+ str | None,+ Query(title='Optional next_page_id from the previously returned page'),+ ] = None,+ limit: Annotated[+ int,+ Query(title='The max number of results in the page', gt=0, le=100),+ ] = 50,+ query: Annotated[+ str | None,+ Query(title='Filter models by name (case-insensitive substring match)'),+ ] = None,+ verified__eq: Annotated[+ bool | None,+ Query(title='Filter by verified status (true/false, omit for all)'),+ ] = None,+ models: ModelsResponse = Depends(get_llm_models_dependency),+) -> LLMModelPage:+ """Search for LLM models with pagination and filtering.++ Returns a paginated list of models that can be filtered by name+ (contains) and verified status.+ """+ filtered_models = _get_all_models_with_verified(models)++ if query is not None:+ query_lower = query.lower()+ filtered_models = [m for m in filtered_models if query_lower in m.name.lower()]++ if verified__eq is not None:+ filtered_models = [m for m in filtered_models if m.verified == verified__eq]++ # Apply pagination+ items, next_page_id = paginate_results(filtered_models, page_id, limit)++ return LLMModelPage(items=items, next_page_id=next_page_id)+++def _get_verified_models() -> set[str]:+ verified_models = set()+ for provider, models in VERIFIED_MODELS.items():+ for name in models:+ verified_models.add(f'{provider}/{name}')+ return verified_models+++def _get_all_models_with_verified(models: ModelsResponse) -> list[LLMModel]:+ verified_models = _get_verified_models()+ results = []+ for model_name in models.models:+ verified = model_name in verified_models+ parts = model_name.split('/', 1)+ if len(parts) == 2:+ provider, name = parts+ else:+ provider = None+ name = parts[0]+ result = LLMModel(+ provider=provider,+ name=name,+ verified=verified,+ )+ results.append(result)+ return resultsdiff --git a/openhands/app_server/v1_router.py b/openhands/app_server/v1_router.pyindex 292b77b2e1..eae32aa933 100644--- a/openhands/app_server/v1_router.py+++ b/openhands/app_server/v1_router.py@@ -1,6 +1,7 @@from fastapi import APIRouterfrom openhands.app_server.app_conversation import app_conversation_router+from openhands.app_server.config_api.config_router import router as config_routerfrom openhands.app_server.event import event_routerfrom openhands.app_server.event_callback import (webhook_router,@@ -33,3 +34,4 @@ router.include_router(skills_router.router)router.include_router(webhook_router.router)router.include_router(web_client_router.router)router.include_router(git_router)+router.include_router(config_router)diff --git a/openhands/server/routes/public.py b/openhands/server/routes/public.pyindex 6c8067f87c..3ceb3f0bbd 100644--- a/openhands/server/routes/public.py+++ b/openhands/server/routes/public.py@@ -29,10 +29,18 @@ async def get_llm_models_dependency(request: Request) -> ModelsResponse:return get_supported_llm_models(config)[email protected]('/models')[email protected]('/models', deprecated=True)async def get_litellm_models(models: ModelsResponse = Depends(get_llm_models_dependency),) -> ModelsResponse:+ """Get all supported LLM models.++ .. deprecated::+ This endpoint is deprecated. Use `/api/v1/config/models/search` instead.++ Returns:+ ModelsResponse: Response containing models, verified_models, verified_providers, and default_model.+ """return modelsdiff --git a/tests/unit/app_server/test_config_router.py b/tests/unit/app_server/test_config_router.pynew file mode 100644index 0000000000..8bd5eeb850--- /dev/null+++ b/tests/unit/app_server/test_config_router.py@@ -0,0 +1,206 @@+"""Unit tests for the config_models and config_router.++This module tests the config router endpoints,+focusing on the search_models endpoint for LLM models.+"""++import pytest+from fastapi import FastAPI, status+from fastapi.testclient import TestClient++from openhands.app_server.config_api.config_models import LLMModel+from openhands.app_server.config_api.config_router import (+ _get_all_models_with_verified,+ router,+)+from openhands.app_server.utils.dependencies import check_session_api_key+from openhands.app_server.utils.paging_utils import encode_page_id, paginate_results+from openhands.server.shared import config+from openhands.utils.llm import get_supported_llm_models+++class TestLLMModel:+ """Test suite for LLMModel."""++ def test_create_model_with_name_and_verified(self):+ """Test that LLMModel can be created with name and verified."""+ model = LLMModel(provider='openai', name='gpt-4', verified=True)++ assert model.provider == 'openai'+ assert model.name == 'gpt-4'+ assert model.verified is True++ def test_create_model_with_default_verified_false(self):+ """Test that verified defaults to False."""+ model = LLMModel(provider='openai', name='gpt-4')++ assert model.provider == 'openai'+ assert model.name == 'gpt-4'+ assert model.verified is False+++class TestPagination:+ """Test suite for pagination helper function."""++ def test_returns_first_page_when_no_page_id(self):+ """Test that first page is returned when no page_id is provided."""+ models = [+ LLMModel(provider='openai', name='gpt-4', verified=True),+ LLMModel(provider='anthropic', name='claude-3', verified=True),+ LLMModel(provider='openai', name='gpt-3.5', verified=False),+ ]++ result, next_page_id = paginate_results(models, None, 2)++ assert len(result) == 2+ assert next_page_id == encode_page_id(2)++ def test_returns_second_page_when_page_id_provided(self):+ """Test that correct page is returned when page_id is provided."""+ models = [+ LLMModel(provider='openai', name='gpt-4', verified=True),+ LLMModel(provider='anthropic', name='claude-3', verified=True),+ LLMModel(provider='openai', name='gpt-3.5', verified=False),+ ]+ encoded_page_id = encode_page_id(2)++ result, next_page_id = paginate_results(models, encoded_page_id, 2)++ assert len(result) == 1+ assert result[0].provider == 'openai'+ assert result[0].name == 'gpt-3.5'+ assert next_page_id is None+++class TestGetAllModelsWithVerified:+ """Test suite for _get_all_models_with_verified function."""++ def test_returns_list_of_llm_models(self):+ """Test that function returns list of LLMModel objects."""+ models = _get_all_models_with_verified(get_supported_llm_models(config))++ assert isinstance(models, list)+ assert all(isinstance(m, LLMModel) for m in models)++ def test_models_verified_mix(self):+ """Test that models contains a mix of verified and unverified."""+ models = _get_all_models_with_verified(get_supported_llm_models(config))++ assert any(m.verified is True for m in models)+ assert any(m.verified is False for m in models)+++def test_client():+ """Create a test client with the actual config router and mocked dependencies.++ We override check_session_api_key to bypass auth checks.+ This allows us to test the actual Query parameter validation in the router.+ """+ app = FastAPI()+ app.include_router(router)++ # Override the auth dependency to always pass+ app.dependency_overrides[check_session_api_key] = lambda: None++ client = TestClient(app, raise_server_exceptions=False)+ yield client++ # Clean up+ app.dependency_overrides.clear()+++class TestSearchModelsEndpoint:+ """Test suite for /models/search endpoint."""++ def test_returns_200_with_paginated_results(self, test_client):+ """Test that endpoint returns 200 with paginated results."""+ response = test_client.get('/config/models/search')++ assert response.status_code == status.HTTP_200_OK+ data = response.json()+ assert 'items' in data+ assert 'next_page_id' in data++ def test_respects_limit_parameter(self, test_client):+ """Test that limit parameter is respected."""+ response = test_client.get('/config/models/search', params={'limit': 2})++ assert response.status_code == status.HTTP_200_OK+ data = response.json()+ assert len(data['items']) <= 2++ def test_filters_by_query_name_contains(self, test_client):+ """Test that query parameter filters by name (case-insensitive)."""+ response = test_client.get('/config/models/search', params={'query': 'gpt'})++ assert response.status_code == status.HTTP_200_OK+ data = response.json()+ for item in data['items']:+ assert 'gpt' in item['name'].lower()++ def test_filters_by_verified_eq_true(self, test_client):+ """Test that verified__eq=true filters to verified models only."""+ response = test_client.get(+ '/config/models/search', params={'verified__eq': True}+ )++ assert response.status_code == status.HTTP_200_OK+ data = response.json()+ for item in data['items']:+ assert item['verified'] is True++ def test_filters_by_verified_eq_false(self, test_client):+ """Test that verified__eq=false filters to non-verified models only."""+ # Since all models from _SDK_VERIFIED_MODELS are verified,+ # we expect empty results when filtering for non-verified+ response = test_client.get(+ '/config/models/search', params={'verified__eq': False}+ )++ assert response.status_code == status.HTTP_200_OK+ data = response.json()+ for item in data['items']:+ assert item['verified'] is False++ def test_combines_query_and_verified_filters(self, test_client):+ """Test that query and verified filters are combined."""+ response = test_client.get(+ '/config/models/search', params={'query': 'gpt', 'verified__eq': True}+ )++ assert response.status_code == status.HTTP_200_OK+ data = response.json()+ for item in data['items']:+ assert 'gpt' in item['name'].lower()+ assert item['verified'] is True++ def test_pagination_with_page_id(self, test_client):+ """Test that pagination works with page_id."""+ # First request - get first page+ response1 = test_client.get('/config/models/search', params={'limit': 1})+ data1 = response1.json()++ # If there's a next page, test it+ if data1.get('next_page_id'):+ response2 = test_client.get(+ '/config/models/search',+ params={'limit': 1, 'page_id': data1['next_page_id']},+ )+ data2 = response2.json()++ assert response2.status_code == status.HTTP_200_OK+ # The items should be different+ assert data1['items'][0]['name'] != data2['items'][0]['name']++ def test_invalid_limit_parameter_returns_422(self, test_client):+ """Test that invalid limit parameter returns 422."""+ response = test_client.get('/config/models/search', params={'limit': 0})++ assert response.status_code == status.HTTP_422_UNPROCESSABLE_ENTITY++ def test_limit_exceeds_max_returns_422(self, test_client):+ """Test that limit exceeding max returns 422."""+ response = test_client.get('/config/models/search', params={'limit': 101})++ assert response.status_code == status.HTTP_422_UNPROCESSABLE_ENTITY