Skip to content

armgit / kong

Public Imported from GitHub
0

The world's first agentic reverse engineer.

fix: probe uses OpenAI SDK to validate custom endpoints

Amrutha6079c5d1cbb
diff --git a/kong/llm/probe.py b/kong/llm/probe.py
index 1bb5e96..781a253 100644
--- a/kong/llm/probe.py
+++ b/kong/llm/probe.py
@@ -9,13 +9,14 @@ from __future__ import annotations
import logging
import anthropic
-import httpx
import openai
from kong.config import LLMConfig, LLMProvider
logger = logging.getLogger(__name__)
+_PROBE_DUMMY_KEY = "not-needed"
+
def probe_endpoint(config: LLMConfig) -> bool:
if config.provider is LLMProvider.CUSTOM:
@@ -26,18 +27,22 @@ def probe_endpoint(config: LLMConfig) -> bool:
def _probe_custom(config: LLMConfig) -> bool:
- url = f"{config.base_url}/models"
+ api_key = config.api_key if config.api_key else _PROBE_DUMMY_KEY
try:
- response = httpx.get(url, timeout=10.0)
- if response.status_code == 200:
- return True
- logger.warning("Custom endpoint returned status %d", response.status_code)
+ client = openai.OpenAI(api_key=api_key, base_url=config.base_url)
+ client.models.list()
+ return True
+ except openai.AuthenticationError:
+ logger.warning("Custom endpoint rejected API key")
return False
- except (httpx.ConnectError, httpx.TimeoutException):
+ except openai.APIConnectionError:
logger.warning("Could not connect to %s", config.base_url)
return False
- except httpx.HTTPError as e:
- logger.warning("HTTP error probing %s: %s", config.base_url, e)
+ except openai.APIError as e:
+ logger.warning("Custom endpoint error: %s", e.message)
+ return False
+ except Exception as e:
+ logger.warning("Could not validate %s: %s", config.base_url, e)
return False
@@ -50,7 +55,7 @@ def _probe_openai(config: LLMConfig) -> bool:
logger.warning("OpenAI API key is invalid")
return False
except openai.APIError as e:
- logger.warning("OpenAI API error: %s", e)
+ logger.warning("OpenAI API error: %s", e.message)
return False
@@ -63,5 +68,5 @@ def _probe_anthropic(config: LLMConfig) -> bool:
logger.warning("Anthropic API key is invalid")
return False
except anthropic.APIError as e:
- logger.warning("Anthropic API error: %s", e)
+ logger.warning("Anthropic API error: %s", e.message)
return False
diff --git a/tests/test_cli.py b/tests/test_cli.py
index 5d0c4ba..4a28c01 100644
--- a/tests/test_cli.py
+++ b/tests/test_cli.py
@@ -228,7 +228,7 @@ class TestSetupWizardCustom:
class TestCustomProviderIntegration:
- @patch("kong.llm.probe.httpx.get")
+ @patch("kong.llm.probe.probe_endpoint", return_value=True)
@patch("kong.llm.openai_client.openai.OpenAI")
@patch("kong.config.find_ghidra_install", return_value=None)
def test_analyze_with_base_url_flag(
@@ -236,7 +236,6 @@ class TestCustomProviderIntegration:
):
monkeypatch.setenv("KONG_CONFIG_DIR", str(tmp_path))
save_setup(enabled=[LLMProvider.CUSTOM], default=LLMProvider.CUSTOM)
- mock_probe.return_value = MagicMock(status_code=200, json=lambda: {"data": []})
binary = tmp_path / "test_binary"
binary.write_bytes(b"\x00" * 16)
diff --git a/tests/test_probe.py b/tests/test_probe.py
index 7a40bad..7901bb8 100644
--- a/tests/test_probe.py
+++ b/tests/test_probe.py
@@ -10,47 +10,78 @@ from kong.llm.probe import probe_endpoint
class TestProbeCustom:
- @patch("kong.llm.probe.httpx.get")
- def test_custom_probe_success(self, mock_get):
- mock_get.return_value = MagicMock(status_code=200, json=lambda: {"data": []})
+ @patch("kong.llm.probe.openai.OpenAI")
+ def test_custom_probe_success(self, mock_openai_cls):
+ mock_client = MagicMock()
+ mock_openai_cls.return_value = mock_client
+ mock_client.models.list.return_value = MagicMock()
config = LLMConfig(
provider=LLMProvider.CUSTOM,
base_url="http://localhost:11434/v1",
)
assert probe_endpoint(config) is True
- mock_get.assert_called_once_with(
- "http://localhost:11434/v1/models",
- timeout=10.0,
+ mock_openai_cls.assert_called_once_with(
+ api_key="not-needed",
+ base_url="http://localhost:11434/v1",
)
- @patch("kong.llm.probe.httpx.get")
- def test_custom_probe_connection_error(self, mock_get):
- import httpx
+ @patch("kong.llm.probe.openai.OpenAI")
+ def test_custom_probe_with_api_key(self, mock_openai_cls):
+ mock_client = MagicMock()
+ mock_openai_cls.return_value = mock_client
+ mock_client.models.list.return_value = MagicMock()
+ config = LLMConfig(
+ provider=LLMProvider.CUSTOM,
+ base_url="https://openrouter.ai/api/v1",
+ api_key="sk-or-test",
+ )
+ assert probe_endpoint(config) is True
+ mock_openai_cls.assert_called_once_with(
+ api_key="sk-or-test",
+ base_url="https://openrouter.ai/api/v1",
+ )
- mock_get.side_effect = httpx.ConnectError("Connection refused")
+ @patch("kong.llm.probe.openai.OpenAI")
+ def test_custom_probe_connection_error(self, mock_openai_cls):
+ mock_client = MagicMock()
+ mock_openai_cls.return_value = mock_client
+ mock_client.models.list.side_effect = openai.APIConnectionError(
+ request=MagicMock(),
+ )
config = LLMConfig(
provider=LLMProvider.CUSTOM,
base_url="http://localhost:11434/v1",
)
assert probe_endpoint(config) is False
- @patch("kong.llm.probe.httpx.get")
- def test_custom_probe_timeout(self, mock_get):
- import httpx
-
- mock_get.side_effect = httpx.ReadTimeout("timed out")
+ @patch("kong.llm.probe.openai.OpenAI")
+ def test_custom_probe_non_openai_endpoint(self, mock_openai_cls):
+ mock_client = MagicMock()
+ mock_openai_cls.return_value = mock_client
+ mock_client.models.list.side_effect = openai.APIStatusError(
+ message="not found",
+ response=MagicMock(status_code=404, reason_phrase="Not Found"),
+ body=None,
+ )
config = LLMConfig(
provider=LLMProvider.CUSTOM,
- base_url="http://localhost:11434/v1",
+ base_url="https://example.com",
)
assert probe_endpoint(config) is False
- @patch("kong.llm.probe.httpx.get")
- def test_custom_probe_non_200(self, mock_get):
- mock_get.return_value = MagicMock(status_code=500)
+ @patch("kong.llm.probe.openai.OpenAI")
+ def test_custom_probe_auth_failure(self, mock_openai_cls):
+ mock_client = MagicMock()
+ mock_openai_cls.return_value = mock_client
+ mock_client.models.list.side_effect = openai.AuthenticationError(
+ message="invalid key",
+ response=MagicMock(status_code=401),
+ body=None,
+ )
config = LLMConfig(
provider=LLMProvider.CUSTOM,
- base_url="http://localhost:11434/v1",
+ base_url="https://openrouter.ai/api/v1",
+ api_key="bad-key",
)
assert probe_endpoint(config) is False