Skip to content

Your phone, on your command. Gotcha is world's first fully on-device AI copilot for Android that turns natural language into real device actions — safely.

12.9 KB
Raw Blame
package com.gotcha.llm

import com.gotcha.i18n.Language
import kotlinx.coroutines.test.runTest
import kotlinx.serialization.json.Json
import kotlinx.serialization.json.JsonPrimitive
import kotlinx.serialization.json.buildJsonObject
import kotlinx.serialization.json.put
import okhttp3.mockwebserver.MockResponse
import okhttp3.mockwebserver.MockWebServer
import okhttp3.mockwebserver.SocketPolicy
import org.junit.After
import org.junit.Assert.assertEquals
import org.junit.Assert.assertTrue
import org.junit.Assert.fail
import org.junit.Before
import org.junit.Test
import java.net.SocketTimeoutException
import java.util.concurrent.TimeUnit

class LLMClientTest {

    private lateinit var server: MockWebServer
    private lateinit var client: LLMClient

    private val dummyTool = ToolDefinition(
        function = FunctionDefinition(
            name = "dial_number",
            description = "Dial a phone number",
            parameters = buildJsonObject {
                put("type", "object")
            }
        )
    )

    @Before
    fun setUp() {
        server = MockWebServer()
        server.start()
        client = LLMClient(
            apiKey = "test-key",
            baseUrl = server.url("/").toString()
        )
    }

    @After
    fun tearDown() {
        server.shutdown()
    }

    @Test
    fun `chat parses tool_calls response`() = runTest {
        server.enqueue(
            MockResponse().setBody(
                """
                {
                  "choices": [
                    {
                      "message": {
                        "role": "assistant",
                        "tool_calls": [
                          {
                            "id": "call_1",
                            "type": "function",
                            "function": {
                              "name": "dial_number",
                              "arguments": "{\"number\":\"555-1234\"}"
                            }
                          }
                        ]
                      },
                      "finish_reason": "tool_calls"
                    }
                  ]
                }
                """.trimIndent()
            ).setHeader("Content-Type", "application/json")
        )

        val response = client.chat(
            messages = listOf(ChatMessage(role = "user", content = JsonPrimitive("Call mom"))),
            tools = listOf(dummyTool)
        )

        val message = response.choices.single().message
        val toolCall = message.toolCalls?.single()
        assertEquals("dial_number", toolCall?.function?.name)
        assertEquals("{\"number\":\"555-1234\"}", toolCall?.function?.arguments)
    }

    @Test
    fun `chat parses direct content response`() = runTest {
        server.enqueue(
            MockResponse().setBody(
                """{"choices":[{"message":{"role":"assistant","content":"Hi there"}}]}"""
            ).setHeader("Content-Type", "application/json")
        )

        val response = client.chat(
            messages = listOf(ChatMessage(role = "user", content = JsonPrimitive("Hello")))
        )

        assertEquals("Hi there", response.choices.single().message.textContent)
    }

    @Test
    fun `identical deterministic request is served from cache`() = runTest {
        server.enqueue(
            MockResponse().setBody(
                """{"choices":[{"message":{"role":"assistant","content":"cached answer"}}]}"""
            ).setHeader("Content-Type", "application/json")
        )

        val messages = listOf(ChatMessage(role = "user", content = JsonPrimitive("What is 2+2?")))

        val first = client.chat(messages = messages)
        val second = client.chat(messages = messages)

        assertEquals("cached answer", first.choices.single().message.textContent)
        assertEquals(first.choices.single().message.textContent, second.choices.single().message.textContent)
        assertEquals(1, server.requestCount)
    }

    @Test
    fun `response containing tool_calls is never cached`() = runTest {
        val toolCallBody = """
            {
              "choices": [
                {
                  "message": {
                    "role": "assistant",
                    "tool_calls": [
                      {
                        "id": "call_1",
                        "type": "function",
                        "function": { "name": "dial_number", "arguments": "{}" }
                      }
                    ]
                  }
                }
              ]
            }
        """.trimIndent()
        server.enqueue(MockResponse().setBody(toolCallBody).setHeader("Content-Type", "application/json"))
        server.enqueue(MockResponse().setBody(toolCallBody).setHeader("Content-Type", "application/json"))

        val messages = listOf(ChatMessage(role = "user", content = JsonPrimitive("Call mom")))

        client.chat(messages = messages, tools = listOf(dummyTool))
        client.chat(messages = messages, tools = listOf(dummyTool))

        assertTrue(server.requestCount == 2)
    }

    @Test
    fun `cleanText request body contains the language name and no-translate clause`() = runTest {
        server.enqueue(
            MockResponse().setBody(
                """{"choices":[{"message":{"role":"assistant","content":"नमस्ते, आप कैसे हैं?"}}]}"""
            ).setHeader("Content-Type", "application/json")
        )

        client.cleanText("namaste aap kaise hain", language = Language.HINDI)

        val request = server.takeRequest()
        val body = request.body.readUtf8()
        assertTrue(body.contains(Language.HINDI.label))
        assertTrue(body.contains("Never translate"))
        assertTrue(body.contains("<dictation>"))
    }

    @Test
    fun `listModels filters out audio models by provider_type`() = runTest {
        server.enqueue(
            MockResponse().setBody(
                """
                {
                  "data": [
                    {"id": "gpt-4o", "provider_type": "llm"},
                    {"id": "kokoro-82m", "provider_type": "tts"},
                    {"id": "whisper-1", "provider_type": "stt"},
                    {"id": "claude-opus-4", "provider_type": "llm"}
                  ]
                }
                """.trimIndent()
            ).setHeader("Content-Type", "application/json")
        )

        val result = client.listModels().getOrThrow()

        assertEquals(listOf("claude-opus-4", "gpt-4o"), result)
    }

    @Test
    fun `listModels filters out audio models by task field`() = runTest {
        server.enqueue(
            MockResponse().setBody(
                """
                {
                  "data": [
                    {"id": "gpt-4o-mini"},
                    {"id": "tts-1", "task": "text-to-speech"},
                    {"id": "whisper-large-v3", "task": "automatic-speech-recognition"}
                  ]
                }
                """.trimIndent()
            ).setHeader("Content-Type", "application/json")
        )

        val result = client.listModels().getOrThrow()

        assertEquals(listOf("gpt-4o-mini"), result)
    }

    @Test
    fun `listModels falls back to name heuristics when no hint field is present`() = runTest {
        // OpenAI's /v1/models omits both `provider_type` and `task`; the name
        // heuristic is the last line of defence.
        server.enqueue(
            MockResponse().setBody(
                """
                {
                  "data": [
                    {"id": "gpt-4o"},
                    {"id": "tts-1"},
                    {"id": "whisper-1"},
                    {"id": "llama-3-8b-instruct"}
                  ]
                }
                """.trimIndent()
            ).setHeader("Content-Type", "application/json")
        )

        val result = client.listModels().getOrThrow()

        assertEquals(listOf("gpt-4o", "llama-3-8b-instruct"), result)
    }

    @Test
    fun `listModels keeps explicitly tagged LLM models alongside UNKNOWN ones`() = runTest {
        server.enqueue(
            MockResponse().setBody(
                """
                {
                  "data": [
                    {"id": "samosa-llm-pro", "provider_type": "llm"},
                    {"id": "gpt-4o"},
                    {"id": "kokoro-82m", "provider_type": "tts"}
                  ]
                }
                """.trimIndent()
            ).setHeader("Content-Type", "application/json")
        )

        val result = client.listModels().getOrThrow()

        assertEquals(listOf("gpt-4o", "samosa-llm-pro"), result)
    }

    @Test
    fun `listModels keeps UNKNOWN-categorized models with no hint and no name match`() = runTest {
        // Custom / private model with neither provider_type nor task field, and
        // an id that triggers no audio name heuristic — must survive filtering.
        server.enqueue(
            MockResponse().setBody(
                """
                {
                  "data": [
                    {"id": "my-private-llm"},
                    {"id": "tts-1"},
                    {"id": "whisper-large-v3"}
                  ]
                }
                """.trimIndent()
            ).setHeader("Content-Type", "application/json")
        )

        val result = client.listModels().getOrThrow()

        assertEquals(listOf("my-private-llm"), result)
    }

    @Test
    fun `malformed tool-call arguments are sanitized before they leave the client`() = runTest {
        server.enqueue(
            MockResponse().setBody(
                """{"choices":[{"message":{"role":"assistant","content":"ok"}}]}"""
            ).setHeader("Content-Type", "application/json")
        )

        // The exact shape that bricked issue #13: the model cut the summary short
        // and closed with `</summary>`/`</function>` tags inside the JSON string.
        val poisoned = ChatMessage(
            role = "assistant",
            toolCalls = listOf(
                ToolCall(
                    id = "call_poisoned",
                    function = FunctionCall(
                        "finish_task",
                        """{"summary": "रिशभ…कर सकते हैं:</summary>\n</function>\n</tool_call>"""
                    )
                ),
                ToolCall(
                    id = "call_fine",
                    function = FunctionCall("websearch", """{"query":"hi","numResults":5}""")
                )
            )
        )

        client.chat(
            messages = listOf(
                ChatMessage(role = "user", content = JsonPrimitive("continue")),
                poisoned
            )
        )

        val body = server.takeRequest().body.readUtf8()
        val sent = Json { ignoreUnknownKeys = true }.decodeFromString(ChatRequest.serializer(), body)

        val calls = sent.messages.flatMap { it.toolCalls.orEmpty() }
        assertEquals("call_poisoned", calls[0].id)
        assertEquals("{}", calls[0].function.arguments)
        assertEquals("call_fine", calls[1].id)
        assertEquals("""{"query":"hi","numResults":5}""", calls[1].function.arguments)
        assertTrue(
            "the malformed fragment must never leave the client:\n$body",
            !body.contains("</summary>")
        )
    }

    /** A server that accepts the request and never answers must end the call, not hang it (#104). */
    @Test
    fun `a server that never responds times out`() = runTest {
        server.enqueue(MockResponse().setSocketPolicy(SocketPolicy.NO_RESPONSE))
        val shortTimeout = LLMClient(
            apiKey = "test-key",
            baseUrl = server.url("/").toString(),
            apiTimeoutSeconds = 1L
        )
        try {
            shortTimeout.chat(listOf(ChatMessage(role = "user", content = JsonPrimitive("Hello?"))))
            fail("expected the request to time out")
        } catch (_: SocketTimeoutException) {
            // expected
        }
    }

    /**
     * The timeout is a gap between bytes, not a cap on the whole request: a reply
     * that takes longer than the timeout overall but keeps arriving is not cut off.
     */
    @Test
    fun `a slow reply that keeps arriving is not cut off`() = runTest {
        server.enqueue(
            MockResponse()
                .setBody("""{"choices":[{"message":{"role":"assistant","content":"slow but steady"}}]}""")
                .throttleBody(32, 400, TimeUnit.MILLISECONDS)
        )
        val shortTimeout = LLMClient(
            apiKey = "test-key",
            baseUrl = server.url("/").toString(),
            apiTimeoutSeconds = 1L
        )
        val response = shortTimeout.chat(listOf(ChatMessage(role = "user", content = JsonPrimitive("Take your time"))))
        assertEquals("slow but steady", response.choices.first().message.textContent)
    }

    @Test
    fun `the API timeout is split into OkHttp timeouts`() {
        assertEquals(LlmTimeouts(15L, 180L, 180L), llmTimeouts(180L))
        assertEquals(LlmTimeouts(5L, 5L, 5L), llmTimeouts(5L))
        assertEquals("0 is the user asking for no timeout", LlmTimeouts(0L, 0L, 0L), llmTimeouts(0L))
    }
}