The world's first agentic reverse engineer.
fix: probe uses OpenAI SDK to validate custom endpoints
Amrutha6079c5d1cbb
diff --git a/kong/llm/probe.py b/kong/llm/probe.pyindex 1bb5e96..781a253 100644--- a/kong/llm/probe.py+++ b/kong/llm/probe.py@@ -9,13 +9,14 @@ from __future__ import annotationsimport loggingimport anthropic-import httpximport openaifrom kong.config import LLMConfig, LLMProviderlogger = logging.getLogger(__name__)+_PROBE_DUMMY_KEY = "not-needed"+def probe_endpoint(config: LLMConfig) -> bool:if config.provider is LLMProvider.CUSTOM:@@ -26,18 +27,22 @@ def probe_endpoint(config: LLMConfig) -> bool:def _probe_custom(config: LLMConfig) -> bool:- url = f"{config.base_url}/models"+ api_key = config.api_key if config.api_key else _PROBE_DUMMY_KEYtry:- response = httpx.get(url, timeout=10.0)- if response.status_code == 200:- return True- logger.warning("Custom endpoint returned status %d", response.status_code)+ client = openai.OpenAI(api_key=api_key, base_url=config.base_url)+ client.models.list()+ return True+ except openai.AuthenticationError:+ logger.warning("Custom endpoint rejected API key")return False- except (httpx.ConnectError, httpx.TimeoutException):+ except openai.APIConnectionError:logger.warning("Could not connect to %s", config.base_url)return False- except httpx.HTTPError as e:- logger.warning("HTTP error probing %s: %s", config.base_url, e)+ except openai.APIError as e:+ logger.warning("Custom endpoint error: %s", e.message)+ return False+ except Exception as e:+ logger.warning("Could not validate %s: %s", config.base_url, e)return False@@ -50,7 +55,7 @@ def _probe_openai(config: LLMConfig) -> bool:logger.warning("OpenAI API key is invalid")return Falseexcept openai.APIError as e:- logger.warning("OpenAI API error: %s", e)+ logger.warning("OpenAI API error: %s", e.message)return False@@ -63,5 +68,5 @@ def _probe_anthropic(config: LLMConfig) -> bool:logger.warning("Anthropic API key is invalid")return Falseexcept anthropic.APIError as e:- logger.warning("Anthropic API error: %s", e)+ logger.warning("Anthropic API error: %s", e.message)return Falsediff --git a/tests/test_cli.py b/tests/test_cli.pyindex 5d0c4ba..4a28c01 100644--- a/tests/test_cli.py+++ b/tests/test_cli.py@@ -228,7 +228,7 @@ class TestSetupWizardCustom:class TestCustomProviderIntegration:- @patch("kong.llm.probe.httpx.get")+ @patch("kong.llm.probe.probe_endpoint", return_value=True)@patch("kong.llm.openai_client.openai.OpenAI")@patch("kong.config.find_ghidra_install", return_value=None)def test_analyze_with_base_url_flag(@@ -236,7 +236,6 @@ class TestCustomProviderIntegration:):monkeypatch.setenv("KONG_CONFIG_DIR", str(tmp_path))save_setup(enabled=[LLMProvider.CUSTOM], default=LLMProvider.CUSTOM)- mock_probe.return_value = MagicMock(status_code=200, json=lambda: {"data": []})binary = tmp_path / "test_binary"binary.write_bytes(b"\x00" * 16)diff --git a/tests/test_probe.py b/tests/test_probe.pyindex 7a40bad..7901bb8 100644--- a/tests/test_probe.py+++ b/tests/test_probe.py@@ -10,47 +10,78 @@ from kong.llm.probe import probe_endpointclass TestProbeCustom:- @patch("kong.llm.probe.httpx.get")- def test_custom_probe_success(self, mock_get):- mock_get.return_value = MagicMock(status_code=200, json=lambda: {"data": []})+ @patch("kong.llm.probe.openai.OpenAI")+ def test_custom_probe_success(self, mock_openai_cls):+ mock_client = MagicMock()+ mock_openai_cls.return_value = mock_client+ mock_client.models.list.return_value = MagicMock()config = LLMConfig(provider=LLMProvider.CUSTOM,base_url="http://localhost:11434/v1",)assert probe_endpoint(config) is True- mock_get.assert_called_once_with(- "http://localhost:11434/v1/models",- timeout=10.0,+ mock_openai_cls.assert_called_once_with(+ api_key="not-needed",+ base_url="http://localhost:11434/v1",)- @patch("kong.llm.probe.httpx.get")- def test_custom_probe_connection_error(self, mock_get):- import httpx+ @patch("kong.llm.probe.openai.OpenAI")+ def test_custom_probe_with_api_key(self, mock_openai_cls):+ mock_client = MagicMock()+ mock_openai_cls.return_value = mock_client+ mock_client.models.list.return_value = MagicMock()+ config = LLMConfig(+ provider=LLMProvider.CUSTOM,+ base_url="https://openrouter.ai/api/v1",+ api_key="sk-or-test",+ )+ assert probe_endpoint(config) is True+ mock_openai_cls.assert_called_once_with(+ api_key="sk-or-test",+ base_url="https://openrouter.ai/api/v1",+ )- mock_get.side_effect = httpx.ConnectError("Connection refused")+ @patch("kong.llm.probe.openai.OpenAI")+ def test_custom_probe_connection_error(self, mock_openai_cls):+ mock_client = MagicMock()+ mock_openai_cls.return_value = mock_client+ mock_client.models.list.side_effect = openai.APIConnectionError(+ request=MagicMock(),+ )config = LLMConfig(provider=LLMProvider.CUSTOM,base_url="http://localhost:11434/v1",)assert probe_endpoint(config) is False- @patch("kong.llm.probe.httpx.get")- def test_custom_probe_timeout(self, mock_get):- import httpx-- mock_get.side_effect = httpx.ReadTimeout("timed out")+ @patch("kong.llm.probe.openai.OpenAI")+ def test_custom_probe_non_openai_endpoint(self, mock_openai_cls):+ mock_client = MagicMock()+ mock_openai_cls.return_value = mock_client+ mock_client.models.list.side_effect = openai.APIStatusError(+ message="not found",+ response=MagicMock(status_code=404, reason_phrase="Not Found"),+ body=None,+ )config = LLMConfig(provider=LLMProvider.CUSTOM,- base_url="http://localhost:11434/v1",+ base_url="https://example.com",)assert probe_endpoint(config) is False- @patch("kong.llm.probe.httpx.get")- def test_custom_probe_non_200(self, mock_get):- mock_get.return_value = MagicMock(status_code=500)+ @patch("kong.llm.probe.openai.OpenAI")+ def test_custom_probe_auth_failure(self, mock_openai_cls):+ mock_client = MagicMock()+ mock_openai_cls.return_value = mock_client+ mock_client.models.list.side_effect = openai.AuthenticationError(+ message="invalid key",+ response=MagicMock(status_code=401),+ body=None,+ )config = LLMConfig(provider=LLMProvider.CUSTOM,- base_url="http://localhost:11434/v1",+ base_url="https://openrouter.ai/api/v1",+ api_key="bad-key",)assert probe_endpoint(config) is False