Skip to content

Your phone, on your command. Gotcha is world's first fully on-device AI copilot for Android that turns natural language into real device actions — safely.

13.8 KB
Raw Blame
package com.gotcha.agent

import android.content.Context
import androidx.test.core.app.ApplicationProvider
import com.gotcha.data.ChatHistoryRepository
import com.gotcha.data.Settings
import com.gotcha.llm.LLMClient
import com.gotcha.testsupport.FakeAndroidKeyStore
import com.gotcha.testsupport.ShadowExternalStorageManager
import com.gotcha.tools.AgentMode
import com.gotcha.tools.FileResolver
import kotlinx.coroutines.test.runTest
import okhttp3.mockwebserver.MockResponse
import okhttp3.mockwebserver.MockWebServer
import org.junit.After
import org.junit.Assert.assertEquals
import org.junit.Assert.assertTrue
import org.junit.Before
import org.junit.Test
import org.junit.runner.RunWith
import org.robolectric.RobolectricTestRunner
import org.robolectric.annotation.Config
import java.io.File

/**
 * Context-window accounting in [AgentEngine]:
 *  - Post-compaction estimate includes the static prompt overhead so the meter
 *    doesn't collapse (Bug 1).
 *  - When the API omits `usage`, the meter still moves (Bug 3).
 *  - Same fallback lets `checkAndCompactHistory()` fire so history doesn't
 *    grow unbounded against the real model context (Bug 3 cont.).
 */
@RunWith(RobolectricTestRunner::class)
@Config(sdk = [34], shadows = [ShadowExternalStorageManager::class])
class AgentEngineOverheadTest {

    private val context: Context = ApplicationProvider.getApplicationContext()

    private lateinit var server: MockWebServer
    private lateinit var engine: AgentEngine
    private lateinit var events: TokenRecordingAgentEvents
    private lateinit var workDir: File
    private lateinit var savedWorkingDir: String

    @Before
    fun setUp() {
        FakeAndroidKeyStore.setUp()
        server = MockWebServer()
        server.start()

        workDir = File(context.filesDir, "agent-overhead-test").apply {
            deleteRecursively()
            mkdirs()
        }
        savedWorkingDir = FileResolver.WORKING_DIR_BASE

        events = TokenRecordingAgentEvents()
        engine = AgentEngine(
            appContext = context,
            events = events,
            historyRepository = ChatHistoryRepository(context, "agent-overhead-test-chats"),
            settingsProvider = {
                Settings(
                    apiKey = "test-key",
                    baseUrl = server.url("/").toString(),
                    // Keep the loop short and put the threshold in reach so the
                    // compaction path is exercised on demand.
                    maxContextTokens = 800,
                    maxToolRounds = 4
                )
            },
            clientProvider = {
                LLMClient(apiKey = "test-key", baseUrl = server.url("/").toString())
            },
            workingDirRoot = workDir.absolutePath
        )
        engine.sessionId = "agent-overhead-test"
        FileResolver.WORKING_DIR_BASE = workDir.absolutePath
    }

    @After
    fun tearDown() {
        server.shutdown()
        FileResolver.WORKING_DIR_BASE = savedWorkingDir
        workDir.deleteRecursively()
        ShadowExternalStorageManager.resetGranted()
    }

    // ---- scripting helpers ----

    /**
     * Enqueue a plain assistant reply. The response payload omits `usage` so
     * the engine's missing-usage fallback path is exercised end-to-end.
     */
    private fun enqueueTextReplyWithoutUsage(text: String) {
        server.enqueue(
            MockResponse().setBody(
                """{"choices":[{"message":{"role":"assistant","content":"$text"}}]}"""
            )
        )
    }

    /**
     * Enqueue a compaction summary that the engine produces when shrinking
     * history. The body deliberately omits `usage`.
     */
    private fun enqueueCompactionSummary(summary: String) {
        server.enqueue(
            MockResponse().setBody(
                """{"choices":[{"message":{"role":"assistant","content":${summary.toJsonStringLiteral()}}}]}"""
            )
        )
    }

    private fun String.toJsonStringLiteral(): String =
        "\"" + replace("\\", "\\\\").replace("\"", "\\\"") + "\""

    // ---- tests ----

    /**
     * Bug 1 — Post-compaction estimate previously measured history only
     * (`(summary.length + preserveLast.length) / 4`), so the meter visibly
     * collapsed. With the fix, it includes [AgentEngine.PROMPT_OVERHEAD_TOKENS]
     * so the value is in the same units as the next round's API-reported count.
     */
    @Test
    fun `post compaction estimate includes the prompt overhead constant`() = runTest {
        // First reply: a long, chatty turn that pushes us past the 80 %
        // threshold (800 * 0.8 = 640). The body is over 2 560 characters
        // long, so even with no `usage` reported the fallback estimate lands
        // above the threshold.
        val firstReply = "x".repeat(3_000)
        enqueueTextReplyWithoutUsage(firstReply)

        // Second reply (post-compaction): a short summary.
        enqueueCompactionSummary("Compacted summary.")

        // Seed the chat with enough history that compaction is meaningful.
        engine.history += com.gotcha.llm.ChatMessage(
            role = "user",
            content = kotlinx.serialization.json.JsonPrimitive("do the thing")
        )

        engine.run(AgentMode.OPERATOR)

        // We expect at least one compaction-trigger onTokenCount emission.
        val compactionEvents = events.tokenCounts.filter { it > 0 }
        assertTrue(
            "expected onTokenCount to fire with the fallback estimate, got: ${events.tokenCounts}",
            compactionEvents.isNotEmpty()
        )

        // The fallback path also covers history-only messages that are shorter
        // than the overhead, so the bar can never collapse below the static
        // prefix: every post-compaction onTokenCount call must be at least
        // PROMPT_OVERHEAD_TOKENS.
        compactionEvents.forEach {
            assertTrue(
                "post-compaction onTokenCount must include the overhead floor ($it < ${AgentEngine.PROMPT_OVERHEAD_TOKENS})",
                it >= AgentEngine.PROMPT_OVERHEAD_TOKENS
            )
        }

        // The persisted engine tokenCount must also be at least the overhead.
        assertTrue(
            "engine.tokenCount must include the overhead floor (${engine.tokenCount} < ${AgentEngine.PROMPT_OVERHEAD_TOKENS})",
            engine.tokenCount >= AgentEngine.PROMPT_OVERHEAD_TOKENS
        )
    }

    /**
     * Bug 3 — If the server omits `usage`, the meter used to stay at 0
     * forever and compaction never fired. The fix falls back to a local
     * `(history.length / 4) + PROMPT_OVERHEAD_TOKENS` estimate so onTokenCount
     * still fires and `checkAndCompactHistory` can still trigger.
     */
    @Test
    fun `missing usage falls back to a local estimate so the meter moves`() = runTest {
        // Pre-populate with enough history that the fallback estimate will be
        // visibly non-zero.
        engine.history += com.gotcha.llm.ChatMessage(
            role = "user",
            content = kotlinx.serialization.json.JsonPrimitive(
                "tell me a long story about a robot that wanted to learn to paint"
            )
        )
        enqueueTextReplyWithoutUsage("ok")

        engine.run(AgentMode.OPERATOR)

        // The fallback path should have produced at least one non-zero
        // onTokenCount emission, and the engine's tokenCount should reflect it.
        assertTrue(
            "expected onTokenCount fallback to be non-zero, got: ${events.tokenCounts}",
            events.tokenCounts.any { it > 0 }
        )
        assertTrue(
            "engine.tokenCount must reflect the fallback (${engine.tokenCount})",
            engine.tokenCount > 0
        )
        assertTrue(
            "engine.tokenCount must be at least the overhead floor (${engine.tokenCount} < ${AgentEngine.PROMPT_OVERHEAD_TOKENS})",
            engine.tokenCount >= AgentEngine.PROMPT_OVERHEAD_TOKENS
        )
    }

    /**
     * Companion to the test above: when the fallback estimate crosses the
     * 80% compaction threshold, the engine must actually call the compaction
     * LLM. Without this, history grows unbounded against the real model
     * context window.
     */
    @Test
    fun `missing usage still triggers compaction at the threshold`() = runTest {
        // Push the engine over the threshold directly so we don't have to
        // script many long rounds.
        engine.tokenCount = (Settings().maxContextTokens * 0.85).toInt()

        // Two enqueued responses: one for the compaction summarization call,
        // then a normal text reply for the resumed loop.
        enqueueCompactionSummary("Compacted.")
        enqueueTextReplyWithoutUsage("done")

        engine.history += com.gotcha.llm.ChatMessage(
            role = "user",
            content = kotlinx.serialization.json.JsonPrimitive("hi")
        )

        engine.run(AgentMode.OPERATOR)

        // Compaction fires before the loop's first round and replaces history
        // with `[assistant: summary, user: preserved]`. The resumed loop then
        // appends one more assistant reply, so we end up at size 3 — what
        // matters is that compaction happened at all, which is observable
        // through the summary bubble it injects.
        assertEquals(
            "compaction summary must be in history",
            true,
            engine.history.any { it.textContent.contains("Compacted.") }
        )
        assertTrue(
            "compaction must have fired — history should contain the user message that survived it",
            engine.history.any { it.role == "user" && it.textContent == "hi" }
        )
    }

    /**
     * Issue #156 review — successful compaction must not drop the memory
     * context for the request that triggered it. The outgoing model request
     * after compaction carries exactly one [memory_startup] message.
     */
    @Test
    fun `compaction preserves the memory startup message`() = runTest {
        // Point memory at an isolated dir so the test never touches real files.
        val memoryDir = File(workDir, "memory-test").apply { mkdirs() }
        val previousRoot = runCatching { com.gotcha.data.GotchaStorage.rootPath }.getOrNull()
        com.gotcha.data.GotchaStorage.rootPath = File(memoryDir, "Gotcha").absolutePath
        try {
            com.gotcha.data.MemoryStore.ensureDefaults(Settings())
            engine.tokenCount = (Settings().maxContextTokens * 0.85).toInt()
            enqueueCompactionSummary("Compacted.")
            enqueueTextReplyWithoutUsage("done")
            engine.history += com.gotcha.llm.ChatMessage(
                role = "user",
                content = kotlinx.serialization.json.JsonPrimitive("hi")
            )
            engine.run(AgentMode.OPERATOR)
            // Two server hits: the compaction summary call, then the real turn.
            assertEquals(2, server.requestCount)
            server.takeRequest()
            val mainRequest = server.takeRequest().body.readUtf8()
            val histDump = engine.history.map { m -> m.role + ":" + m.textContent.contains(com.gotcha.data.MemoryStore.STARTUP_MARKER) }
            // Count real startup messages (content starting with the marker),
            // not the pointer block merely naming it.
            val startupHits = "\"content\":\"\\[memory_startup\\]".toRegex()
                .findAll(mainRequest).count()
            assertEquals(
                "post-compaction request must carry exactly one memory startup message; history=$histDump",
                1,
                startupHits
            )
            // The startup system message now sits ahead of the assistant
            // summary, so the user-first guard must look past system messages:
            // providers reject a request whose first non-system turn is the
            // assistant's (PR #174 review).
            val roles = kotlinx.serialization.json.Json.parseToJsonElement(mainRequest)
                .let { it as kotlinx.serialization.json.JsonObject }["messages"]
                .let { it as kotlinx.serialization.json.JsonArray }
                .map { (it as kotlinx.serialization.json.JsonObject)["role"].toString().trim('"') }
            assertEquals(
                "first non-system message after compaction must be the user's; roles=$roles",
                "user",
                roles.first { it != "system" }
            )
        } finally {
            if (previousRoot != null) {
                com.gotcha.data.GotchaStorage.rootPath = previousRoot
            } else {
                com.gotcha.data.GotchaStorage.resetRootPathForTests()
            }
        }
    }
}

/**
 * A small [AgentEvents] variant that captures token-count emissions so the
 * overhead-aware tests can assert on the fallback path. Mirrors the recording
 * shape of [RecordingAgentEvents] but keeps a separate `tokenCounts` list.
 */
private class TokenRecordingAgentEvents : AgentEvents {

    val tokenCounts = mutableListOf<Int>()
    val assistantReplies = mutableListOf<String>()
    val uiMessages = mutableListOf<String>()
    var historyResets = 0
        private set

    override fun onUi(
        kind: MessageKind,
        text: String,
        imageBase64: String?,
        subAgentSteps: List<String>,
        reasoningContent: String?
    ) {
        uiMessages += text
    }

    override fun onActivity(activity: String?) = Unit

    override fun onTokenCount(totalTokens: Int) {
        tokenCounts += totalTokens
    }

    override fun onAssistantReply(text: String) {
        assistantReplies += text
    }

    override fun onSubAgentUpdate(running: String?, currentAction: String?) = Unit

    override fun onPermissionRequest(marker: String) = Unit

    override fun onHistoryReset() {
        historyResets++
    }

    override suspend fun awaitQuestionAnswer(question: PendingQuestion): String = ""

    override suspend fun awaitConfirmation(toolNames: List<String>, description: String): Boolean = true
}