Skip to content

armgit / andee

Public Imported from GitHub
0

Andee lives on your phone or Mac and uses any app like you do—no integrations, no root, any model; memory and passwords never leave your device | Andee 住进你的手机或 Mac,像你一样操作任何 App——无需集成、无需 Root、支持任意模型,记忆和密码绝不离开设备。

feat(andee): 接入 CodeFlying 市场工具,顺带修两个挡路的 bug

kuafu-limanxiang1b3deaaa6a4e
diff --git a/app/src/main/java/net/kuafuai/andee/ScreenBodyService.kt b/app/src/main/java/net/kuafuai/andee/ScreenBodyService.kt
index c86210a..8de162c 100644
--- a/app/src/main/java/net/kuafuai/andee/ScreenBodyService.kt
+++ b/app/src/main/java/net/kuafuai/andee/ScreenBodyService.kt
@@ -108,6 +108,15 @@ class ScreenBodyService : AccessibilityService() {
*/
private val callExtensions = net.kuafuai.andee.net.CallExtensionRegistry()
+ /**
+ * The CodeFlying market as a second external tool source, or null before
+ * [startBrain] has run — and permanently null in the hub branch, which never
+ * builds one. Read inside the `forwardExternal` lambda rather than captured
+ * by value, because the dispatcher is built well before the brain is.
+ */
+ @Volatile
+ private var marketTools: net.kuafuai.andee.market.MarketTools? = null
+
// Closed by [stopEverything], reopened when the user starts the next turn.
// Cancelling the brain's task is asynchronous, so progress / message frames
// already in flight still arrive after the stop; without this gate the
@@ -429,14 +438,24 @@ class ScreenBodyService : AccessibilityService() {
dogMotion,
this,
forwardExternal = { method, params ->
- // Unknown to the dispatcher → try the call extension. The
- // server enforces the timeout and turns a missing extension
- // into a proper error; the throw here becomes brain-visible.
- val srv = wsServer
- ?: throw IllegalStateException(
- "no local server running — cannot reach call extension"
- )
- srv.forwardToExtension(method, params)
+ // Unknown to the dispatcher → the market, then the call
+ // extension. Both are "tools that live somewhere else", and the
+ // market goes first because it is the cheap check — an array
+ // walk — while the extension path throws when nothing is
+ // plugged in, which would mask a market hit.
+ val market = marketTools
+ if (market != null && market.hasMethod(method)) {
+ market.call(method, params)
+ } else {
+ // The server enforces the timeout and turns a missing
+ // extension into a proper error; the throw here becomes
+ // brain-visible.
+ val srv = wsServer
+ ?: throw IllegalStateException(
+ "no local server running — cannot reach call extension"
+ )
+ srv.forwardToExtension(method, params)
+ }
},
// The page no longer blocks the tool call, so this callback is the
// only way the brain learns the user is done with it. See
@@ -1877,11 +1896,21 @@ class ScreenBodyService : AccessibilityService() {
if (bc.apiKey.isEmpty()) {
reportNoBrain(AppLocale.str(this, R.string.svc_no_brain_local), announce)
}
+ // CodeFlying is the only backend with a market behind it. Everywhere
+ // else this object is inert — empty tool list, `hasMethod` always false
+ // — so a local or hub brain's tool surface is exactly what it was.
+ val market = net.kuafuai.andee.market.MarketTools(
+ baseUrl = net.kuafuai.andee.config.VoiceConfig.codeFlyingMarketBaseUrl(),
+ apiKey = bc.apiKey,
+ enabled = bc.mode == net.kuafuai.andee.config.VoiceConfig.BRAIN_CODEFLYING,
+ )
+ marketTools = market
localBrain = net.kuafuai.andee.brain.LocalBrain(
cfg = bc,
appContext = this,
dispatcher = dispatcher,
registry = callExtensions,
+ market = market,
onProgress = ::onBrainProgress,
onFinal = ::onBrainFinal,
onTurnEnd = ::onTurnEnd,
diff --git a/app/src/main/java/net/kuafuai/andee/brain/LlmClient.kt b/app/src/main/java/net/kuafuai/andee/brain/LlmClient.kt
index 6f79dc7..cf91351 100644
--- a/app/src/main/java/net/kuafuai/andee/brain/LlmClient.kt
+++ b/app/src/main/java/net/kuafuai/andee/brain/LlmClient.kt
@@ -169,7 +169,19 @@ class LlmClient(
assistantMessage = sanitizeAssistant(msg),
promptTokens = usage?.optInt("prompt_tokens") ?: 0,
completionTokens = usage?.optInt("completion_tokens") ?: 0,
- cachedTokens = usage?.optInt("prompt_cache_hit_tokens") ?: 0,
+ // 命中缓存的 prompt 有多少。火山方舟(现在的上游)把它放在
+ // usage.prompt_tokens_details.cached_tokens 里,扁平的
+ // prompt_cache_hit_tokens 是 DeepSeek 自家 API 的字段 —— 方舟不发。
+ // optInt 取不到就是 0,所以之前每一行日志都写 cached 0,
+ // 看着像缓存没生效,其实是根本没读对地方。
+ // 两个都认,以后换上游不用再动这里。
+ cachedTokens = usage?.let { u ->
+ // optInt 对「key 不存在」和「值是 0」返回的一样,所以先 has 再取 ——
+ // 否则嵌套块在但缺 cached_tokens 时不会回落到扁平字段。
+ val details = u.optJSONObject("prompt_tokens_details")
+ if (details != null && details.has("cached_tokens")) details.optInt("cached_tokens")
+ else u.optInt("prompt_cache_hit_tokens")
+ } ?: 0,
)
}
diff --git a/app/src/main/java/net/kuafuai/andee/brain/LocalBrain.kt b/app/src/main/java/net/kuafuai/andee/brain/LocalBrain.kt
index b5ddaba..3163363 100644
--- a/app/src/main/java/net/kuafuai/andee/brain/LocalBrain.kt
+++ b/app/src/main/java/net/kuafuai/andee/brain/LocalBrain.kt
@@ -5,6 +5,7 @@ import net.kuafuai.andee.R
import net.kuafuai.andee.config.Notebook
import net.kuafuai.andee.config.VoiceConfig
import net.kuafuai.andee.i18n.AppLocale
+import net.kuafuai.andee.market.MarketTools
import net.kuafuai.andee.net.CallExtensionRegistry
import net.kuafuai.andee.net.CommandDispatcher
import net.kuafuai.andee.net.ToolSchemas
@@ -68,6 +69,14 @@ class LocalBrain(
private val appContext: android.content.Context,
private val dispatcher: CommandDispatcher,
private val registry: CallExtensionRegistry,
+ /**
+ * The CodeFlying market, as a second external tool source beside the call
+ * extension. Empty outside CodeFlying mode, so a local or hub brain sees the
+ * same tool list it always did — the gate lives in [MarketTools] rather than
+ * here, because "which backend is this" is the service's question, not the
+ * brain's.
+ */
+ private val market: MarketTools,
private val onProgress: (String) -> Unit,
private val onFinal: (String) -> Unit,
/**
@@ -346,6 +355,7 @@ class LocalBrain(
// make the *next* turn refuse every device tool.
currentKind = TurnKind.USER
busy.set(false)
+ dropUnfinishedHarvest(kind)
if (kind == TurnKind.SWEEP) onSweepSpent(turnTokens)
if (gen == generation.get()) {
drainWakes()
@@ -498,7 +508,7 @@ class LocalBrain(
if (!internal && images.isNotEmpty()) attachUserPhotos(images)
- val tools = LlmClient.toolSpecs(ToolSchemas.all(), registry.tools())
+ val tools = LlmClient.toolSpecs(ToolSchemas.all(), registry.tools(), market.tools())
var lastSignature = ""
var repeats = 0
@@ -644,6 +654,7 @@ class LocalBrain(
val builtin = ToolSchemas.methodOf(tc.name)
val method = builtin
?: tc.name.takeIf { registry.hasMethod(it) }
+ ?: tc.name.takeIf { market.hasMethod(it) }
?: return toolMessage(tc.id, "No such tool: ${tc.name}")
val result = dispatchWithTimeout(method, args, silent, untrusted)
@@ -942,6 +953,34 @@ class LocalBrain(
launch(HARVEST_INSTRUCTION, TurnKind.HARVEST)
}
+ /**
+ * Throw away a review turn's own messages when the review did not finish.
+ *
+ * The harvest appends its prompt to [history] before calling the model — see
+ * the `internal` branch in [runTurn]. On success [applyHarvest] removes that
+ * scaffolding along with the folded conversation. On failure nothing did, so
+ * the history kept one extra message per attempt.
+ *
+ * That growth is exactly what made a failed review re-fire forever:
+ * [maybeHarvest] skips a repeat by comparing `history.size` against the size
+ * it last fired at, and one extra message per attempt deflected that check
+ * every single time. Measured on a device: 218 attempts about 5 s apart,
+ * until the app was reinstalled. Dropping the scaffolding puts the size back
+ * where it was, the guard holds, and the next turn is an ordinary one.
+ *
+ * Only [TurnKind.HARVEST]: [applyHarvest] is the only consumer of
+ * [internalFromIndex], and the sweep never calls it — clearing a *successful*
+ * sweep's messages is not this change's business.
+ */
+ private fun dropUnfinishedHarvest(kind: TurnKind) {
+ if (kind != TurnKind.HARVEST) return
+ val from = internalFromIndex
+ if (from < 0 || from >= history.size) return
+ Log.i(TAG, "harvest did not finish; dropping ${history.size - from} scaffolding message(s)")
+ history.subList(from, history.size).clear()
+ internalFromIndex = -1
+ }
+
/**
* Swap the reviewed part of the conversation for the summary.
*
diff --git a/app/src/main/java/net/kuafuai/andee/config/VoiceConfig.kt b/app/src/main/java/net/kuafuai/andee/config/VoiceConfig.kt
index 2564587..905e3ed 100644
--- a/app/src/main/java/net/kuafuai/andee/config/VoiceConfig.kt
+++ b/app/src/main/java/net/kuafuai/andee/config/VoiceConfig.kt
@@ -94,17 +94,40 @@ data class VoiceConfig(
private fun codeFlyingHttpScheme(): String =
if (BuildConfig.CODEFLYING_SSL_ENABLED) "https" else "http"
+ /** Bare host, no scheme and no trailing slash. One place, so the three URLs below cannot drift. */
+ private fun codeFlyingHost(): String =
+ BuildConfig.CODEFLYING_DOMAIN.trim().trimEnd('/')
+
+ /** Bare CodeFlying origin, e.g. `http://124.71.176.202`. */
+ fun codeFlyingBaseUrl(): String = "${codeFlyingHttpScheme()}://${codeFlyingHost()}"
+
+ /**
+ * Origin for the market tool endpoints, e.g. `http://host/baas-api`.
+ *
+ * `/baas-api/` exists because the bare `/api/` on that host is the
+ * front-end's static directory — nginx only forwards that tree to baas
+ * under this prefix. The voice endpoints need no such prefix (`/voice/`
+ * has its own location), which is why this is a separate accessor rather
+ * than folded into [codeFlyingBaseUrl]: pointing the LLM and ASR/TTS at
+ * `/baas-api/` as well would put the WebSocket upgrade on a location that
+ * may not carry the `Upgrade` headers.
+ *
+ * The `/baas-api` segment is a deployment detail, like the `/voice/llm`
+ * suffix above it — it belongs to whichever nginx the build points at.
+ */
+ fun codeFlyingMarketBaseUrl(): String = "${codeFlyingBaseUrl()}/baas-api"
+
+ /** WebSocket origin, same host on the `ws`/`wss` scheme. */
+ private fun codeFlyingWsBaseUrl(): String = "${codeFlyingWsScheme()}://${codeFlyingHost()}"
+
/** Full LLM REST base URL for CodeFlying, e.g. `https://foo.net/voice/llm`. */
- fun codeFlyingLlmBaseUrl(): String =
- "${codeFlyingHttpScheme()}://${BuildConfig.CODEFLYING_DOMAIN.trim().trimEnd('/')}/voice/llm"
+ fun codeFlyingLlmBaseUrl(): String = "${codeFlyingBaseUrl()}/voice/llm"
/** Full ASR WebSocket URL for CodeFlying. */
- fun codeFlyingAsrEndpoint(): String =
- "${codeFlyingWsScheme()}://${BuildConfig.CODEFLYING_DOMAIN.trim().trimEnd('/')}/voice/asr/bigmodel_async"
+ fun codeFlyingAsrEndpoint(): String = "${codeFlyingWsBaseUrl()}/voice/asr/bigmodel_async"
/** Full TTS WebSocket URL for CodeFlying. */
- fun codeFlyingTtsEndpoint(): String =
- "${codeFlyingWsScheme()}://${BuildConfig.CODEFLYING_DOMAIN.trim().trimEnd('/')}/voice/tts/bidirection"
+ fun codeFlyingTtsEndpoint(): String = "${codeFlyingWsBaseUrl()}/voice/tts/bidirection"
/** Whether to show the 云端 (hub) tab in the backend picker. Default off. */
fun showHub(context: Context): Boolean =
diff --git a/app/src/main/java/net/kuafuai/andee/market/MarketTools.kt b/app/src/main/java/net/kuafuai/andee/market/MarketTools.kt
new file mode 100644
index 0000000..2cc869a
--- /dev/null
+++ b/app/src/main/java/net/kuafuai/andee/market/MarketTools.kt
@@ -0,0 +1,246 @@
+package net.kuafuai.andee.market
+
+import android.util.Log
+import okhttp3.MediaType.Companion.toMediaType
+import okhttp3.OkHttpClient
+import okhttp3.Request
+import okhttp3.RequestBody.Companion.toRequestBody
+import org.json.JSONArray
+import org.json.JSONObject
+import java.util.concurrent.TimeUnit
+
+/**
+ * The CodeFlying app market, lent to the brain as two tools.
+ *
+ * **Only exists in CodeFlying mode.** [enabled] is wired from `BrainConfig.mode`;
+ * when it is false [tools] returns an empty array and [hasMethod] says no, so a
+ * local or hub brain is byte-for-byte what it was before this class existed.
+ * That is deliberate: the market is a CodeFlying capability, and the other two
+ * backends answer to a different server (or none).
+ *
+ * **Two tools, not the whole market.** The market holds hundreds of HTTP APIs and
+ * grows. Handing the model every one of them is the context-blowout that tool
+ * search exists to avoid — Claude Code measured 93 tools at ~46K tokens before
+ * it started deferring them. Two specs cost nothing per turn, and the round-trip
+ * is only paid on the turns that actually reach outside the device:
+ *
+ * ```
+ * market_search {query, limit} → the handful of tools that match
+ * market_call {name, arguments} → run one of them
+ * ```
+ *
+ * This is the same shape as Claude Code's `tool_search` → specific call, and the
+ * server side of it is `POST /api/tools/search` + `POST /api/tools/call` in baas.
+ *
+ * **Why [call] is blocking.** The brain dispatches external tools on its timeout
+ * pool ([net.kuafuai.andee.brain.LocalBrain.dispatchWithTimeout]), so a call that
+ * takes seconds is exactly what that pool is for. Returning a future would buy
+ * nothing and cost the deadline.
+ */
+class MarketTools(
+ private val baseUrl: String,
+ private val apiKey: String,
+ private val enabled: Boolean,
+) {
+
+ private val http = OkHttpClient.Builder()
+ .connectTimeout(15, TimeUnit.SECONDS)
+ // A market call is a third party's API on the far side of baas — image
+ // generation and the like are slow by nature, and the model may declare a
+ // budget of its own via `timeout_ms`. This has to sit **above** the
+ // largest budget the model is told to use, or the socket would give up
+ // before the brain does and the model would see a transport error instead
+ // of the answer. The brain abandons the *wait* independently; a call that
+ // outlives it keeps running on the dispatch pool.
+ .readTimeout(300, TimeUnit.SECONDS)
+ .writeTimeout(30, TimeUnit.SECONDS)
+ .build()
+
+ /** Built once — the specs are static, unlike the market itself. */
+ private val specs: JSONArray = if (enabled) buildSpecs() else JSONArray()
+
+ fun tools(): JSONArray = specs
+
+ /**
+ * Whether this source owns the tool called [name]. Consulted by the brain
+ * before dispatch.
+ *
+ * [name] is the model-facing tool name (`market_search`), **not** a
+ * `ToolSchemas` internal method (`screen.tap_id`). The two look alike enough
+ * to be worth saying out loud: these tools have no separate internal name —
+ * the brain hands the tool name straight to the dispatcher, the same way it
+ * does for the call extension, and this is where that string is claimed.
+ */
+ fun hasMethod(name: String): Boolean = name == NAME_SEARCH || name == NAME_CALL
+
+ /**
+ * Run one of the two tools. Never throws on a tool-level failure — a bad
+ * payload comes back as a readable string, because "the model gets told what
+ * went wrong" beats "the turn dies".
+ */
+ fun call(tool: String, params: JSONObject?): Any = when (tool) {
+ NAME_SEARCH -> search(params)
+ NAME_CALL -> execute(params)
+ else -> throw IllegalArgumentException("unknown tool: $tool")
+ }
+
+ private fun search(params: JSONObject?): JSONObject {
+ val query = params?.optString("query").orEmpty().trim()
+ if (query.isEmpty()) return failure("market_search needs a 'query'.")
+
+ val limit = (params?.optInt("limit", DEFAULT_LIMIT) ?: DEFAULT_LIMIT).coerceIn(1, MAX_LIMIT)
+ val body = JSONObject().put("query", query).put("limit", limit)
+ val envelope = post("/api/tools/search", body)
+ ?: return failure("market_search could not reach the market.")
+ refusal(envelope)?.let { return it }
+
+ val tools = envelope.optJSONArray("data") ?: JSONArray()
+ if (tools.length() == 0) {
+ return JSONObject()
+ .put("tools", tools)
+ .put("note", "Nothing in the market matches '$query'. Tell the user that " +
+ "rather than inventing a tool or a tool name.")
+ }
+ return JSONObject().put("tools", tools)
+ }
+
+ private fun execute(params: JSONObject?): JSONObject {
+ val name = params?.optString("name").orEmpty().trim()
+ if (name.isEmpty()) return failure("market_call needs the 'name' from a market_search result.")
+ val arguments = params?.optJSONObject("arguments") ?: JSONObject()
+
+ val body = JSONObject().put("name", name).put("arguments", arguments)
+ val envelope = post("/api/tools/call", body)
+ ?: return failure("market_call '$name' could not reach the market.")
+ refusal(envelope)?.let { return it }
+
+ return JSONObject().put("result", envelope.opt("data") ?: JSONObject.NULL)
+ }
+
+ /**
+ * A `code != 0` envelope, turned into the reply the model should see — or
+ * null when the call was fine.
+ *
+ * **The server's own message is the payload here.** baas validates arguments
+ * against the market tool's schema before spending the upstream call, and
+ * says so in the same words a person could act on ("参数 speaker 取值不合法:
+ * zh_female_9,可选值: [...]"). Swallowing that into a generic "the call
+ * failed" would throw away the one sentence that lets the model retry
+ * correctly — and a market call that reaches the upstream and fails is
+ * charged for anyway.
+ */
+ private fun refusal(envelope: JSONObject): JSONObject? {
+ if (envelope.optInt("code", -1) == 0) return null
+ val message = envelope.optString("message").ifBlank { "the market refused the call" }
+ Log.w(TAG, "refused: ${message.take(200)}")
+ return failure(message)
+ }
+
+ /**
+ * POST [body] to a baas path and hand back the parsed envelope, or null when
+ * the request never produced one. A non-2xx or unparseable body is a
+ * transport failure here; a well-formed envelope carrying `code != 0` is not
+ * — that is the server answering, and [refusal] decides what to do with it.
+ */
+ private fun post(path: String, body: JSONObject): JSONObject? {
+ val req = Request.Builder()
+ .url("$baseUrl$path")
+ // Same credential and same shape as the LLM calls: the kft_ token in
+ // Authorization. baas resolves the appId from it, which is also what
+ // decides whose balance a billed tool is charged against.
+ .addHeader("Authorization", "Bearer $apiKey")
+ .addHeader("Content-Type", "application/json")
+ .post(body.toString().toRequestBody(JSON))
+ .build()
+ return try {
+ http.newCall(req).execute().use { resp ->
+ val text = resp.body?.string().orEmpty()
+ val envelope = runCatching { JSONObject(text) }.getOrNull()
+ if (envelope == null) {
+ Log.w(TAG, "$path http ${resp.code}, unparseable: ${text.take(200)}")
+ }
+ envelope
+ }
+ } catch (t: Throwable) {
+ Log.w(TAG, "$path failed: ${t.message}")
+ null
+ }
+ }
+
+ private fun failure(text: String): JSONObject = JSONObject().put("error", text)
+
+ private fun buildSpecs(): JSONArray = JSONArray().apply {
+ put(
+ JSONObject()
+ .put("name", NAME_SEARCH)
+ .put("description", DESC_SEARCH)
+ .put(
+ "parameters", JSONObject()
+ .put("type", "object")
+ .put(
+ "properties", JSONObject()
+ .put("query", prop("string", "What the user wants, in their own words — Chinese, English, keywords, all fine. e.g. \"查快递\", \"generate an image of a cat\", \"translate to Japanese\"."))
+ .put("limit", prop("integer", "How many tools to return. Default $DEFAULT_LIMIT."))
+ )
+ .put("required", JSONArray().put("query"))
+ )
+ )
+ put(
+ JSONObject()
+ .put("name", NAME_CALL)
+ .put("description", DESC_CALL)
+ .put(
+ "parameters", JSONObject()
+ .put("type", "object")
+ .put(
+ "properties", JSONObject()
+ .put("name", prop("string", "The exact 'name' from a market_search result."))
+ .put("arguments", prop("object", ARG_DESC))
+ .put("timeout_ms", prop("integer", TIMEOUT_DESC))
+ )
+ .put("required", JSONArray().put("name").put("arguments"))
+ )
+ )
+ }
+
+ private fun prop(type: String, description: String): JSONObject =
+ JSONObject().put("type", type).put("description", description)
+
+ private companion object {
+ const val TAG = "BodyMarket"
+
+ const val NAME_SEARCH = "market_search"
+ const val NAME_CALL = "market_call"
+
+ const val DEFAULT_LIMIT = 8
+ const val MAX_LIMIT = 30
+
+ val JSON = "application/json; charset=utf-8".toMediaType()
+
+ val DESC_SEARCH =
+ "Search the app market for a third-party capability this device does not have — " +
+ "express delivery tracking, weather, news, translation, image and music generation, " +
+ "SMS, stock quotes, and several hundred more. Call this FIRST whenever the user asks " +
+ "for something none of the device.* or screen.* tools cover; do not tell them the " +
+ "phone cannot do it before you have looked here. Returns each match's name, what it " +
+ "does, and the parameters it takes — read those before calling market_call."
+
+ val DESC_CALL =
+ "Run a market tool. Pass the exact 'name' and the 'arguments' object from a " +
+ "market_search result — the names are per-tool and cannot be guessed. Some tools " +
+ "cost the user money and some take a while; search first. If the tool needs a " +
+ "task id or a follow-up query (video and music generation do), it says so in its " +
+ "result and there is a second tool in the market for that."
+
+ val ARG_DESC =
+ "The parameters object, built from the 'parameters' schema the search result showed " +
+ "for this tool."
+
+ val TIMEOUT_DESC =
+ "How long to wait for this call before giving up on it. Default 60000. Most market " +
+ "tools answer in a second or two, but image and video generation do not — pass " +
+ "180000 for those rather than assuming the call failed. Giving up early only " +
+ "abandons the wait: the tool may still be running, and it has been paid for " +
+ "either way."
+ }
+}