Skip to content

The agent that grows with you

fix(desktop): tool media producer reads decoded results, earliest in the transcript

Tekniuma4d824fd896a
diff --git a/apps/desktop/electron/desktop-plugin-install.test.ts b/apps/desktop/electron/desktop-plugin-install.test.ts
index 8f85372edd8..c9e4db48185 100644
--- a/apps/desktop/electron/desktop-plugin-install.test.ts
+++ b/apps/desktop/electron/desktop-plugin-install.test.ts
@@ -164,7 +164,7 @@ describe('installDesktopPluginFromGit', () => {
/** A git repo whose root is one plugin: `desktop/plugin.js` plus, when
* `agentName` is given, the agent half that makes it a unified package. */
- function pluginRepo(agentName: null | string): string {
+ function pluginRepo(agentName: null | string, desktopFiles: Record<string, string> = {}): string {
const repo = mkdtemp('hermes-plugin-install-')
roots.push(repo)
const git = (...args: string[]) => execFileSync('git', args, { cwd: repo, stdio: 'pipe' })
@@ -172,6 +172,10 @@ describe('installDesktopPluginFromGit', () => {
fs.mkdirSync(path.join(repo, 'desktop'), { recursive: true })
fs.writeFileSync(path.join(repo, 'desktop', 'plugin.js'), 'export function register() {}\n')
+ for (const [name, text] of Object.entries(desktopFiles)) {
+ fs.writeFileSync(path.join(repo, 'desktop', name), text)
+ }
+
if (agentName) {
fs.writeFileSync(path.join(repo, 'plugin.yaml'), `name: ${agentName}\n`)
fs.writeFileSync(path.join(repo, '__init__.py'), 'def register(ctx): pass\n')
@@ -254,4 +258,19 @@ describe('installDesktopPluginFromGit', () => {
expect(fs.existsSync(path.join(appRoot, String(result.pluginName), 'plugin.js'))).toBe(true)
expect(fs.existsSync(path.join(appRoot, String(result.pluginName), PACKAGE_MARKER))).toBe(false)
})
+
+ it('drops a package marker a desktop-only repo ships, so it cannot pose as another package', async () => {
+ // The marker is what pairs a desktop half with an agent package (and lets
+ // it render that package's tool media); only this app may write it.
+ const forged = JSON.stringify({ package: 'meshy-ish', source: '/elsewhere' })
+ const repo = pluginRepo(null, { [PACKAGE_MARKER]: forged })
+ const appRoot = mkdtemp('hermes-plugin-root-')
+ roots.push(appRoot)
+
+ const result = await installDesktopPluginFromGit('git', pathToFileURL(repo).href, appRoot)
+
+ expect(result.ok).toBe(true)
+ expect(fs.existsSync(path.join(appRoot, String(result.pluginName), 'plugin.js'))).toBe(true)
+ expect(fs.existsSync(path.join(appRoot, String(result.pluginName), PACKAGE_MARKER))).toBe(false)
+ })
})
diff --git a/apps/desktop/electron/desktop-plugin-install.ts b/apps/desktop/electron/desktop-plugin-install.ts
index f091547a376..defdb3ec679 100644
--- a/apps/desktop/electron/desktop-plugin-install.ts
+++ b/apps/desktop/electron/desktop-plugin-install.ts
@@ -10,7 +10,7 @@ import fsp from 'node:fs/promises'
import os from 'node:os'
import path from 'node:path'
-import { publishDesktopTree, withDesktopPluginsRoot, writeDesktopHalfMarker } from './desktop-plugins-root'
+import { PACKAGE_MARKER, publishDesktopTree, withDesktopPluginsRoot, writeDesktopHalfMarker } from './desktop-plugins-root'
import { execGit, hiddenGitSpawnSpec } from './no-console-git'
const GITHUB_BROWSER_SEGMENTS = new Set(['tree', 'blob', 'commit'])
@@ -462,6 +462,12 @@ export async function installDesktopPluginFromGit(
// half loads default-enabled instead of opt-in.
await publishDesktopTree(sourceDir, targetDir, async staged => {
if (!packageName) {
+ // A desktop-only repo is a standalone plugin. A package marker it
+ // ships would name another package's agent half as its own (pairing
+ // it on the Plugins page and letting it render that package's tool
+ // media), so only this installer or the reconcile pass writes one.
+ await fsp.rm(path.join(staged, PACKAGE_MARKER), { force: true })
+
return
}
diff --git a/apps/desktop/src/lib/tool-media-renderers.test.ts b/apps/desktop/src/lib/tool-media-renderers.test.ts
index a64eb2de58b..0bcc2efc803 100644
--- a/apps/desktop/src/lib/tool-media-renderers.test.ts
+++ b/apps/desktop/src/lib/tool-media-renderers.test.ts
@@ -20,58 +20,100 @@ const call = (toolName: string, result: unknown, extra: Record<string, unknown>
...extra
})
+const turn = (...content: unknown[]) => [
+ { id: 'u', role: 'user', content: [{ type: 'text', text: 'make it' }] },
+ { id: 'a', role: 'assistant', content }
+]
+
describe('findProducingToolCall', () => {
const path = '/tmp/out/model.glb'
- it('picks the earliest call in the turn whose result names the path (a later ls only mentions it)', () => {
+ it('picks the earliest call whose result names the path (a later ls or a plugin reading it only mentions it)', () => {
const messages = [
{ id: 'u0', role: 'user', content: [] },
{ id: 'a0', role: 'assistant', content: [call('mcp__meshy__download', { path })] },
{ id: 'u1', role: 'user', content: [] },
- { id: 'a1', role: 'assistant', content: [call('mcp__meshy__download', `saved to ${path}`)] },
+ { id: 'a1', role: 'assistant', content: [call('sfx_analyze', `read ${path}`)] },
{ id: 'a2', role: 'assistant', content: [call('terminal', { output: path }), { type: 'text', text: 'MEDIA' }] }
]
- expect(findProducingToolCall(messages, 'a2', path)?.toolCallId).toBe('mcp__meshy__download-id')
- // The previous turn's call never counts for this turn.
- expect(findProducingToolCall(messages.slice(2), 'a2', path)?.toolName).toBe('mcp__meshy__download')
+ // Re-delivered in a later turn, the file still belongs to the call that made it.
+ expect(findProducingToolCall(messages, 'a2', path)?.toolName).toBe('mcp__meshy__download')
+ expect(findProducingToolCall(messages.slice(2), 'a2', path)?.toolName).toBe('sfx_analyze')
expect(findProducingToolCall([messages[2], messages[4]], 'a2', path)?.toolName).toBe('terminal')
+ // Nothing after the delivering message counts.
+ expect(findProducingToolCall(messages, 'u1', path)?.toolName).toBe('mcp__meshy__download')
})
it('matches whole paths only: a longer sibling file never claims this one', () => {
- const messages = [
- { id: 'u', role: 'user', content: [] },
- {
- id: 'a',
- role: 'assistant',
- content: [
- call('first', { path: `${path}.bak` }),
- call('nested', `copied /x${path}`),
- call('second', `saved to ${path}.`)
- ]
- }
- ]
+ const messages = turn(
+ call('first', { path: `${path}.bak` }),
+ call('nested', `copied /x${path}`),
+ call('url', `https://cdn.example${path}`),
+ call('second', `saved to ${path}.`)
+ )
expect(findProducingToolCall(messages, 'a', path)?.toolName).toBe('second')
expect(namesWholePath(`MEDIA:${path}`, path)).toBe(true)
+ expect(namesWholePath(`${path}: 3 MB`, path)).toBe(true)
expect(namesWholePath(`${path}.bak`, path)).toBe(false)
})
- it('ignores failed calls and finds Windows paths inside JSON results', () => {
- const win = 'C:\\Users\\me\\out\\clip.wav'
+ it('reads every file of a result that joins several MCP media blocks with newlines', () => {
+ // An MCP result is text inside a JSON string: the first path is followed by
+ // the escaped `\n`, which must still end the path.
+ const result = JSON.stringify({ result: 'Saved /out/a.glb\nSaved /out/b.glb\nSaved /out/c.png' })
+ const messages = turn(call('mcp__meshy__download', result))
+
+ for (const file of ['/out/a.glb', '/out/b.glb', '/out/c.png']) {
+ expect(findProducingToolCall(messages, 'a', file)?.toolName).toBe('mcp__meshy__download')
+ }
+
+ // Stored again as an object (a reloaded transcript), the same holds.
+ expect(findProducingToolCall(turn(call('mcp__meshy__download', JSON.parse(result))), 'a', '/out/a.glb')).not.toBeNull()
+ })
+
+ it('keeps a core tool the producer when its output names the file before a newline', () => {
+ const messages = turn(
+ call('terminal', JSON.stringify({ output: 'wrote /out/a.mp3\nDone' })),
+ call('sfx_analyze', { path: '/out/a.mp3' })
+ )
+
+ expect(findProducingToolCall(messages, 'a', '/out/a.mp3')?.toolName).toBe('terminal')
+ })
+
+ it('never credits a failed call, live or reloaded (a stored result is never marked isError)', () => {
+ const messages = turn(
+ call('sfx', { path }, { isError: true }),
+ call('meshy_x', JSON.stringify({ error: `exists: ${path}` })),
+ call('meshy_y', { success: false, message: `cannot write ${path}` }),
+ call('image_generate', { path })
+ )
+
+ expect(findProducingToolCall(messages, 'a', path)?.toolName).toBe('image_generate')
+ expect(findProducingToolCall(turn(call('ok', { error: null, path })), 'a', path)?.toolName).toBe('ok')
+ })
+
+ it('gives a file the user named first (an upload) no producer, even when a plugin echoes it', () => {
+ const upload = '/home/u/uploads/voice.ogg'
const messages = [
- { id: 'u', role: 'user', content: [] },
- {
- id: 'a',
- role: 'assistant',
- content: [call('sfx', { path }, { isError: true }), call('sfx', JSON.stringify({ win }))]
- }
+ { id: 'u', role: 'user', content: [{ type: 'text', text: `transcode ${upload}` }] },
+ { id: 'a', role: 'assistant', content: [call('sfx_transcode', { input: upload })] }
]
- expect(findProducingToolCall(messages, 'a', path)).toBeNull()
- expect(findProducingToolCall(messages, 'a', win)?.toolName).toBe('sfx')
- expect(findProducingToolCall(messages, 'missing', win)).toBeNull()
+ expect(findProducingToolCall(messages, 'a', upload)).toBeNull()
+ })
+
+ it('finds Windows, file URL and escaped non-ASCII spellings of the same path', () => {
+ const win = 'C:\\Users\\me\\out\\clip.wav'
+
+ expect(findProducingToolCall(turn(call('sfx', JSON.stringify({ win }))), 'a', win)?.toolName).toBe('sfx')
+ expect(findProducingToolCall(turn(call('sfx', 'saved c:/users/ME/out/clip.wav')), 'a', win)?.toolName).toBe('sfx')
+ expect(findProducingToolCall(turn(call('sfx', { win })), 'a', 'file:///C:/Users/me/out/clip.wav')).not.toBeNull()
+ // Python's json.dumps escapes non-ASCII by default.
+ expect(findProducingToolCall(turn(call('sfx', '{"path": "/out/caf\\u00e9.wav"}')), 'a', '/out/café.wav')).not.toBeNull()
+ expect(findProducingToolCall(turn(call('sfx', { win })), 'missing', win)).toBeNull()
})
})
@@ -84,6 +126,24 @@ describe('effectiveToolCall', () => {
expect(effectiveToolCall(batch).toolName).toBe('tool_call')
expect(effectiveToolCall(call('terminal', {})).toolName).toBe('terminal')
})
+
+ it('reads every shape the backend runs: legacy single, JSON-string calls, one object, string arguments', () => {
+ const shapes = [
+ { name: 'sfx_make', arguments: '{"prompt":"zap"}' },
+ { calls: JSON.stringify([{ name: 'sfx_make', arguments: { prompt: 'zap' } }]) },
+ { calls: { name: ' sfx_make ', arguments: { prompt: 'zap' } } }
+ ]
+
+ for (const args of shapes) {
+ expect(effectiveToolCall(call('tool_call', {}, { args }))).toMatchObject({
+ args: { prompt: 'zap' },
+ toolName: 'sfx_make'
+ })
+ }
+
+ // The bridge never names itself.
+ expect(effectiveToolCall(call('tool_call', {}, { args: { name: 'tool_call' } })).toolName).toBe('tool_call')
+ })
})
describe('ownership', () => {
diff --git a/apps/desktop/src/lib/tool-media-renderers.ts b/apps/desktop/src/lib/tool-media-renderers.ts
index 6a446db5b4c..d78ad79c45e 100644
--- a/apps/desktop/src/lib/tool-media-renderers.ts
+++ b/apps/desktop/src/lib/tool-media-renderers.ts
@@ -6,7 +6,7 @@ import type { MediaKind } from '@/lib/media'
* TOOL MEDIA RENDERERS — a plugin draws the media ITS OWN tools produced.
*
* When a transcript shows a `MEDIA:` attachment, the host finds the tool call
- * in that turn whose result produced the path. If the plugin that owns that
+ * whose result first named the path (earlier in the transcript). If the plugin that owns that
* tool registered a renderer for the tool's name, its component replaces the
* built-in player; every other file (core voice replies, other plugins' and
* core tools' output, user uploads, paths no tool in the turn produced) keeps
@@ -81,25 +81,63 @@ interface MessageLike {
role?: string
}
-function payloadText(value: unknown): string {
- if (typeof value === 'string') {
- return value
+// The texts a value carries, decoded: a JSON-string result is parsed and its
+// string leaves read, so a path next to an escaped `\n`, or a non-ASCII path
+// written as `\u00e9`, reads as the path itself. Bounded (depth, leaf count).
+function textLeaves(value: unknown, out: string[] = [], depth = 0): string[] {
+ if (out.length >= MAX_LEAVES || depth > 6) {
+ return out
}
- try {
- return JSON.stringify(value) ?? ''
- } catch {
- return ''
+ if (typeof value === 'string') {
+ out.push(value)
+ const head = value.trimStart()[0]
+
+ if ((head === '{' || head === '[' || head === '"') && value.length <= MAX_PARSE_CHARS) {
+ try {
+ const parsed: unknown = JSON.parse(value)
+
+ if (parsed !== value) {
+ textLeaves(parsed, out, depth + 1)
+ }
+ } catch {
+ // Plain text that happens to start with a bracket.
+ }
+ }
+ } else if (value && typeof value === 'object') {
+ for (const item of Array.isArray(value) ? value : Object.values(value)) {
+ textLeaves(item, out, depth + 1)
+ }
}
+
+ return out
}
-function pathNeedles(path: string): string[] {
- const bare = path.startsWith('file:') ? decodeURIComponentSafe(path.replace(/^file:\/\//, '')) : path
+const MAX_LEAVES = 2_000
+const MAX_PARSE_CHARS = 2_000_000
- // JSON-escaped forms too: a Windows path sits in a JSON result as `C:\\Users\\…`.
- return [...new Set([path, bare, JSON.stringify(path).slice(1, -1), JSON.stringify(bare).slice(1, -1)])].filter(
- Boolean
- )
+const WINDOWS_PATH = /^(?:[a-z]:[\\/]|\\\\|\/\/)/i
+
+/** One spelling for every form a path takes: `file://` URLs unwrapped,
+ * backslashes as `/`, and Windows paths (case-insensitive) lower-cased. */
+function canonicalText(text: string, windows: boolean): string {
+ const unwrapped = text.replace(/file:\/\/(?:\/(?=[a-z]:))?/gi, '')
+ const slashed = windows ? unwrapped.replace(/\\/g, '/') : unwrapped
+
+ return windows ? slashed.toLowerCase() : slashed
+}
+
+interface PathNeedle {
+ needles: string[]
+ windows: boolean
+}
+
+function pathNeedle(path: string): PathNeedle {
+ const bare = canonicalText(path, false)
+ const windows = WINDOWS_PATH.test(bare)
+ const decoded = decodeURIComponentSafe(bare)
+
+ return { needles: [...new Set([bare, decoded].map(p => canonicalText(p, windows)))].filter(Boolean), windows }
}
function decodeURIComponentSafe(value: string): string {
@@ -112,11 +150,19 @@ function decodeURIComponentSafe(value: string): string {
const escapeRegExp = (value: string) => value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')
+// A path continues through these; anything else (whitespace, quotes, `:`,
+// brackets, a sentence's final `.`) ends it.
+const PATH_CHAR = String.raw`[\p{L}\p{N}_\-~%+@/\\]`
+
/** Whether `text` names `needle` as a WHOLE path: `/out/a.glb` matches in
- * `{"path":"/out/a.glb"}`, `saved to /out/a.glb.` and `MEDIA:/out/a.glb`, but
- * never inside `/out/a.glb.bak` or `/x/out/a.glb`. */
+ * `{"path":"/out/a.glb"}`, `saved to /out/a.glb.`, `/out/a.glb: 3 MB` and
+ * `MEDIA:/out/a.glb`, but never inside `/out/a.glb.bak` or `/x/out/a.glb`. */
export function namesWholePath(text: string, needle: string): boolean {
- return new RegExp(`(?<=^|[\\s"'\`(=\\[{,:])${escapeRegExp(needle)}(?=$|[\\s"'\`),;\\]}>]|\\.(?:$|\\s))`).test(text)
+ if (!needle || !text.includes(needle)) {
+ return false
+ }
+
+ return new RegExp(`(?<!${PATH_CHAR}|\\.)${escapeRegExp(needle)}(?!${PATH_CHAR}|\\.${PATH_CHAR})`, 'u').test(text)
}
function toolCallsOf(message: MessageLike): ToolCallLike[] {
@@ -127,11 +173,101 @@ function toolCallsOf(message: MessageLike): ToolCallLike[] {
: []
}
+/** A call that failed produced nothing. A reloaded transcript marks every
+ * stored result as a success, so the payload's own verdict counts too. */
+function callFailed(call: ToolCallLike): boolean {
+ if (call.isError) {
+ return true
+ }
+
+ let payload = call.result
+
+ if (typeof payload === 'string' && payload.trimStart().startsWith('{') && payload.length <= MAX_PARSE_CHARS) {
+ try {
+ payload = JSON.parse(payload)
+ } catch {
+ return false
+ }
+ }
+
+ if (!payload || typeof payload !== 'object' || Array.isArray(payload)) {
+ return false
+ }
+
+ const { error, success } = payload as { error?: unknown; success?: unknown }
+
+ return success === false || (error !== undefined && error !== null && error !== false && error !== '')
+}
+
+// Decoded, canonical texts per part and spelling. Parts and messages are the
+// thread store's own objects, stable across streaming deltas, so a long
+// transcript is decoded once instead of on every update.
+const leavesCache = new WeakMap<object, Map<boolean, string[]>>()
+
+function canonicalLeaves(owner: object, value: unknown, windows: boolean): string[] {
+ let byForm = leavesCache.get(owner)
+
+ if (!byForm) {
+ byForm = new Map()
+ leavesCache.set(owner, byForm)
+ }
+
+ let leaves = byForm.get(windows)
+
+ if (!leaves) {
+ leaves = textLeaves(value).map(text => canonicalText(text, windows))
+ byForm.set(windows, leaves)
+ }
+
+ return leaves
+}
+
+type Mention = 'user' | ToolCallLike | null
+
+const mentionCache = new WeakMap<object, Map<string, Mention>>()
+
+/** The first thing in `message` that names the path: the user's own text
+ * (an upload, a file they pointed at) or a successful tool call's result. */
+function firstMention(message: MessageLike, target: PathNeedle): Mention {
+ const key = `${target.windows ? 'w' : 'p'}\u0000${target.needles.join('\u0000')}`
+ let cached = mentionCache.get(message)
+
+ if (cached?.has(key)) {
+ return cached.get(key) ?? null
+ }
+
+ const names = (leaves: string[]) => leaves.some(text => target.needles.some(needle => namesWholePath(text, needle)))
+ let found: Mention = null
+
+ if (message.role === 'user') {
+ found = names(canonicalLeaves(message, message.content, target.windows)) ? 'user' : null
+ } else {
+ found =
+ toolCallsOf(message).find(
+ call =>
+ call.result !== undefined &&
+ !callFailed(call) &&
+ names(canonicalLeaves(call as object, call.result, target.windows))
+ ) ?? null
+ }
+
+ if (!cached) {
+ cached = new Map()
+ mentionCache.set(message, cached)
+ }
+
+ cached.set(key, found)
+
+ return found
+}
+
/**
- * The tool call whose result produced `path`, from the turn that holds message
- * `messageId`: that message and the assistant messages back to the user's
- * prompt. The EARLIEST matching call wins, because a file is produced once and
- * later calls (`ls`, a read) only mention it. Failed calls produced nothing.
+ * The tool call whose result produced `path`, read from the transcript up to
+ * message `messageId`. The EARLIEST mention wins, because a file is produced
+ * once and later calls (`ls`, a read, a plugin taking it as input) only name
+ * it. A path the user named first (an upload) has no producer, and failed
+ * calls produced nothing. Rewriting a file in place later never moves it to
+ * another tool: ownership stays with the call that first produced it.
*/
export function findProducingToolCall(
messages: readonly MessageLike[],
@@ -144,51 +280,75 @@ export function findProducingToolCall(
return null
}
- let start = index
+ const target = pathNeedle(path)
+
+ for (let at = 0; at <= index; at += 1) {
+ const message = messages[at]
+ const mention = message ? firstMention(message, target) : null
- while (start > 0 && messages[start - 1]?.role !== 'user') {
- start -= 1
+ if (mention) {
+ return mention === 'user' ? null : mention
+ }
}
- const needles = pathNeedles(path)
+ return null
+}
- for (const message of messages.slice(start, index + 1)) {
- for (const call of toolCallsOf(message)) {
- if (call.isError || call.result === undefined) {
- continue
- }
+function bridgeEntries(args: Record<string, unknown>): unknown[] | null {
+ let calls: unknown = args.calls
- const text = payloadText(call.result)
+ if (calls === undefined || calls === null) {
+ calls = typeof args.name === 'string' && args.name.trim() ? [{ arguments: args.arguments, name: args.name }] : null
+ }
- if (needles.some(needle => namesWholePath(text, needle))) {
- return call
- }
+ if (typeof calls === 'string') {
+ try {
+ calls = JSON.parse(calls)
+ } catch {
+ return null
}
}
- return null
+ if (calls && typeof calls === 'object' && !Array.isArray(calls)) {
+ calls = [calls]
+ }
+
+ return Array.isArray(calls) ? calls : null
+}
+
+function bridgeArguments(raw: unknown): Record<string, unknown> {
+ if (typeof raw === 'string') {
+ try {
+ raw = raw.trim() ? JSON.parse(raw) : {}
+ } catch {
+ return {}
+ }
+ }
+
+ return raw && typeof raw === 'object' && !Array.isArray(raw) ? (raw as Record<string, unknown>) : {}
}
/** The tool a call really ran. The deferred-tool bridge (`tool_call`) runs one
* local tool per call (batches are split before they run), with the tool's
- * name and arguments in `args.calls[0]` and its own result as the result, so
- * a plugin tool loaded on demand is still recognised. A multi-entry batch
- * (connectors only) stays the bridge: no plugin owns it. */
+ * name and arguments in `args.calls[0]` (or the legacy `{name, arguments}`
+ * shape, a JSON-string `calls`, a single object) and its own result as the
+ * result, so a plugin tool loaded on demand is still recognised. Mirrors the
+ * backend's `normalize_tool_call_entries`. A multi-entry batch (connectors
+ * only) stays the bridge: no plugin owns it. */
export function effectiveToolCall(part: ToolCallLike): ToolCallLike {
- const calls =
+ const entries =
part.toolName === 'tool_call' && part.args && typeof part.args === 'object'
- ? (part.args as { calls?: unknown }).calls
- : undefined
+ ? bridgeEntries(part.args as Record<string, unknown>)
+ : null
- if (!Array.isArray(calls) || calls.length !== 1) {
+ if (!entries || entries.length !== 1) {
return part
}
- const entry = calls[0] as { arguments?: unknown; name?: unknown } | null
+ const entry = entries[0] as { arguments?: unknown; name?: unknown } | null
+ const name = typeof entry?.name === 'string' ? entry.name.trim() : ''
- return typeof entry?.name === 'string' && entry.name
- ? { ...part, args: entry.arguments ?? {}, toolName: entry.name }
- : part
+ return name && name !== 'tool_call' ? { ...part, args: bridgeArguments(entry?.arguments), toolName: name } : part
}
/** The package that owns `toolName`: the one package whose registered tools
diff --git a/apps/desktop/src/store/tool-media-owners.ts b/apps/desktop/src/store/tool-media-owners.ts
index 9fed8015807..98a1e81c82e 100644
--- a/apps/desktop/src/store/tool-media-owners.ts
+++ b/apps/desktop/src/store/tool-media-owners.ts
@@ -10,7 +10,7 @@ import {
activeGatewayProfileKey,
gatewayActivationEpoch
} from '@/store/gateway'
-import { requestForSessionProfile, type SessionOwnerScope } from '@/store/session-request-router'
+import { isSessionOwnerRoute, requestForSessionProfile, type SessionOwnerScope } from '@/store/session-request-router'
import { knownOwnerForSession } from '@/store/session-states'
/**
@@ -18,14 +18,23 @@ import { knownOwnerForSession } from '@/store/session-states'
* (connection + profile) — the authority behind tool media renderers. Read
* from the owning backend's `plugins.manage list` (`tools`), never from the
* renderer: a desktop half could otherwise name any tool. Loaded lazily the
- * first time a claimed attachment needs it, refetched when the agent plugin
- * list changes, retried with a bounded backoff after a failure. A backend that
- * predates the field reports nothing, so every claim keeps the built-in player.
+ * first time a claimed attachment needs it, retried with a bounded backoff
+ * after a failure. A plugin-list change re-reads every list while the previous
+ * answer keeps drawing (no widget remounts), and a claimed tool nobody owns
+ * yet (its MCP server still connecting) re-reads on a bounded backoff. A
+ * backend that predates the field reports nothing, so every claim keeps the
+ * built-in player.
*/
export type ToolMediaOwnersEntry =
| { status: 'loading' | 'error'; request: number }
- | { status: 'ready'; request: number; owners: ReadonlyMap<string, ToolOwnership> }
+ | {
+ status: 'ready'
+ request: number
+ owners: ReadonlyMap<string, ToolOwnership>
+ /** `due`: re-read on next use (keeps `owners` meanwhile); `pending`: re-reading. */
+ refresh?: 'due' | 'pending'
+ }
export const $toolMediaOwners = atom<Readonly<Record<string, ToolMediaOwnersEntry>>>({})
@@ -33,6 +42,12 @@ type OwnershipRow = AgentPluginRow & { tools?: string[] }
let requests = 0
const failures = new Map<string, number>()
+// Re-reads spent on a claimed-but-unowned tool, per key, and the pending timer.
+const revalidations = new Map<string, { attempts: number; timer?: ReturnType<typeof setTimeout> }>()
+
+/** How many times a claimed tool with no owner re-reads ownership (1 s, then
+ * 5 s doubling to 60 s: about two and a half minutes in all) before it stays built-in. */
+export const MAX_OWNERSHIP_REVALIDATIONS = 6
export function ownersFromPluginRows(rows: readonly OwnershipRow[]): Map<string, ToolOwnership> {
const owners = new Map<string, ToolOwnership>()
@@ -48,6 +63,22 @@ export function ownersFromPluginRows(rows: readonly OwnershipRow[]): Map<string,
return owners
}
+function sameOwners(a: ReadonlyMap<string, ToolOwnership>, b: ReadonlyMap<string, ToolOwnership>): boolean {
+ if (a.size !== b.size) {
+ return false
+ }
+
+ for (const [pkg, owned] of a) {
+ const other = b.get(pkg)
+
+ if (!other || other.tools.size !== owned.tools.size || [...owned.tools].some(tool => !other.tools.has(tool))) {
+ return false
+ }
+ }
+
+ return true
+}
+
/** The backend profile a session owner names (the profile `plugins.manage` must list). */
function ownerProfile(owner: SessionOwnerScope): null | string {
if (!owner) {
@@ -57,6 +88,8 @@ function ownerProfile(owner: SessionOwnerScope): null | string {
return typeof owner === 'string' ? owner : (owner.targetProfile ?? owner.profile)
}
+const ACTIVE_PREFIX = 'active:'
+
/** Cache key for the chat `sessionId` belongs to: its exact owner when known, else the active route. */
export function toolMediaOwnerKey(sessionId: null | string): string {
const owner = knownOwnerForSession(sessionId)
@@ -65,11 +98,26 @@ export function toolMediaOwnerKey(sessionId: null | string): string {
return typeof owner === 'string' ? `profile:${owner}` : `route:${owner.connectionId}|${ownerProfile(owner)}`
}
- return `active:${activeGatewayConnectionId() ?? ''}|${activeGatewayProfileKey()}|${gatewayActivationEpoch()}`
+ return `${ACTIVE_PREFIX}${activeGatewayConnectionId() ?? ''}|${activeGatewayProfileKey()}|${gatewayActivationEpoch()}`
+}
+
+function forgetKey(key: string) {
+ failures.delete(key)
+ clearTimeout(revalidations.get(key)?.timer)
+ revalidations.delete(key)
}
function setEntry(key: string, entry: null | ToolMediaOwnersEntry) {
const next = { ...$toolMediaOwners.get() }
+ // An `active:` key names one activation of the route; older ones can never be read again.
+ const epoch = `|${gatewayActivationEpoch()}`
+
+ for (const stale of Object.keys(next)) {
+ if (stale.startsWith(ACTIVE_PREFIX) && !stale.endsWith(epoch)) {
+ delete next[stale]
+ forgetKey(stale)
+ }
+ }
if (entry) {
next[key] = entry
@@ -96,34 +144,52 @@ function scheduleRetry(key: string, stillCurrent: () => boolean) {
)
}
-/** Fetch the ownership list for the chat `sessionId` belongs to, unless it is loaded, loading or backing off. */
+/** Fetch the ownership list for the chat `sessionId` belongs to, unless it is
+ * loaded (and not due a re-read), loading or backing off. */
export function ensureToolMediaOwners(sessionId: null | string): void {
const key = toolMediaOwnerKey(sessionId)
+ const existing = $toolMediaOwners.get()[key]
+ const owner = knownOwnerForSession(sessionId)
const gateway = activeGateway()
- if ($toolMediaOwners.get()[key] || !gateway) {
+ if (existing && !(existing.status === 'ready' && existing.refresh === 'due')) {
+ return
+ }
+
+ // A routed owner names its own connection; everything else needs the active one.
+ if (!gateway && !isSessionOwnerRoute(owner)) {
return
}
- const owner = knownOwnerForSession(sessionId)
const profile = ownerProfile(owner)
const request = ++requests
+ const previous = existing?.status === 'ready' ? existing.owners : null
- setEntry(key, { request, status: 'loading' })
+ // A re-read keeps drawing with the previous answer until the new one lands.
+ setEntry(key, previous ? { owners: previous, refresh: 'pending', request, status: 'ready' } : { request, status: 'loading' })
// Only the newest request for a key may write it: an invalidation (plugin
- // toggle) mid-flight clears the entry, and the older answer is dropped.
+ // toggle) mid-flight re-stamps the entry, and the older answer is dropped.
const stillCurrent = () => $toolMediaOwners.get()[key]?.request === request
requestForSessionProfile<{ plugins?: OwnershipRow[] }>(
owner,
- (method, params) => gateway.request(method, params),
+ (method, params) =>
+ gateway ? gateway.request(method, params) : Promise.reject(new Error('No active gateway')),
'plugins.manage',
profile ? { action: 'list', profile } : { action: 'list' }
)
.then(result => {
if (stillCurrent()) {
+ const owners = ownersFromPluginRows(result?.plugins ?? [])
+ const changed = !previous || !sameOwners(previous, owners)
+
failures.delete(key)
- setEntry(key, { owners: ownersFromPluginRows(result?.plugins ?? []), request, status: 'ready' })
+
+ if (changed) {
+ revalidations.delete(key)
+ }
+
+ setEntry(key, { owners: previous && !changed ? previous : owners, request, status: 'ready' })
}
})
.catch(() => {
@@ -134,12 +200,57 @@ export function ensureToolMediaOwners(sessionId: null | string): void {
})
}
-// A toggle/install/update in Capabilities reloads `$agentPlugins`: drop every
-// cached list (orphaning in-flight answers) so the next use re-reads ownership.
-$agentPlugins.listen(() => {
+/** A claimed tool has no owner in the chat's loaded answer: its package's MCP
+ * server may not be connected yet (or was lazy when the list was read). Mark
+ * the answer due a re-read after a bounded backoff; the next use fetches it
+ * while the current answer keeps drawing. */
+export function revalidateToolMediaOwners(sessionId: null | string): void {
+ const key = toolMediaOwnerKey(sessionId)
+ const entry = $toolMediaOwners.get()[key]
+ const state = revalidations.get(key) ?? { attempts: 0 }
+
+ if (entry?.status !== 'ready' || entry.refresh || state.timer || state.attempts >= MAX_OWNERSHIP_REVALIDATIONS) {
+ return
+ }
+
+ const delay = state.attempts === 0 ? 1_000 : Math.min(60_000, 5_000 * 2 ** (state.attempts - 1))
+ const request = entry.request
+
+ state.attempts += 1
+ state.timer = setTimeout(() => {
+ state.timer = undefined
+ const current = $toolMediaOwners.get()[key]
+
+ if (current?.status === 'ready' && current.request === request && !current.refresh) {
+ setEntry(key, { ...current, refresh: 'due' })
+ }
+ }, delay)
+ revalidations.set(key, state)
+}
+
+/** Every loaded list is due a re-read: it keeps drawing until the answer lands
+ * (widgets do not remount), and anything in flight is orphaned by a new stamp.
+ * Called when what a package registered can have changed: the plugin list
+ * reloads (toggle/install/update) or MCP reconnects (`reload.mcp`). */
+export function refreshToolMediaOwners(): void {
failures.clear()
- $toolMediaOwners.set({})
-})
+
+ for (const key of [...revalidations.keys()]) {
+ forgetKey(key)
+ }
+
+ const next: Record<string, ToolMediaOwnersEntry> = {}
+
+ for (const [key, entry] of Object.entries($toolMediaOwners.get())) {
+ if (entry.status === 'ready') {
+ next[key] = { owners: entry.owners, refresh: 'due', request: ++requests, status: 'ready' }
+ }
+ }
+
+ $toolMediaOwners.set(next)
+}
+
+$agentPlugins.listen(refreshToolMediaOwners)
/** The agent package a contribution's plugin is the desktop half of (`plugin:<id>` →
* its record's `packageName`); null for standalone desktop plugins and core. */
diff --git a/tests/tui_gateway/test_plugins_manage_tool_ownership.py b/tests/tui_gateway/test_plugins_manage_tool_ownership.py
index dd88030bc1c..83828143dd0 100644
--- a/tests/tui_gateway/test_plugins_manage_tool_ownership.py
+++ b/tests/tui_gateway/test_plugins_manage_tool_ownership.py
@@ -1,5 +1,4 @@
-"""``plugins.manage list`` names the tools each package's own code produces (``tools`` /
-``tool_prefixes``): Desktop lets a plugin's desktop half render media only from those, so the
+"""``plugins.manage list`` names the tools each package's own code produces (``tools``): Desktop lets a plugin's desktop half render media only from those, so the
answer must come from what the loader actually registered, never from a manifest claim.
Real discovery against the sandbox ``HERMES_HOME``: two native plugins and one portable MCP package.
@@ -79,3 +78,39 @@ def test_list_reports_only_tools_the_package_itself_registered():
assert rows["other"]["tools"] == ["other_tool"]
# A portable package owns exactly the tools its own MCP server registered.
assert rows["meshy-ish"]["tools"] == ["mcp__meshy_3d__download"]
+
+
+def _list_meshy_tools(live: list[str] | None = None, lazy: list[str] | None = None,
+ native: dict | None = None) -> list[str]:
+ root = Path(os.environ["HERMES_HOME"]) / "plugins"
+ if not (root / "meshy-ish").exists():
+ _portable_plugin(root)
+ cfg = load_config()
+ cfg["plugins"] = {"enabled": ["meshy-ish"]}
+ cfg["mcp_servers"] = native or {}
+ save_config(cfg)
+ from tools.mcp_tool_common import _core
+ from tools.mcp_tool_scope import _resolve_server_key
+ key = _resolve_server_key("meshy-3d")
+ servers = {key: SimpleNamespace(_registered_tool_names=live)} if live is not None else {}
+ with patch.dict(_core._servers, servers), \
+ patch.dict(_core._lazy_server_tool_names, {key: lazy} if lazy is not None else {}), \
+ patch("hermes_cli.plugins_cmd_catalog.catalog_rows_maps", return_value=({}, {}, {})):
+ resp = server.handle_request({"id": "1", "method": "plugins.manage", "params": {"action": "list"}})
+ return {row["name"]: row for row in resp["result"]["plugins"]}["meshy-ish"]["tools"]
+
+
+def test_a_lazily_cached_server_still_owns_its_tools():
+ # A cache-registered (lazy) server keeps its tool names outside the live connection table;
+ # they are still that server's tools, so its package owns them before it connects.
+ assert _list_meshy_tools(lazy=["mcp__meshy_3d__download"]) == ["mcp__meshy_3d__download"]
+ assert _list_meshy_tools() == []
+
+
+def test_a_native_server_of_the_same_name_credits_the_package_only_when_it_runs_the_same_thing():
+ live = ["mcp__meshy_3d__download"]
+ # The user's own, different server under the package's server name is the one that runs:
+ # its tools are not the package's.
+ assert _list_meshy_tools(live, native={"meshy-3d": {"command": "my-own-server"}}) == []
+ # The same server re-declared natively (the way a user widens its trust) stays the package's.
+ assert _list_meshy_tools(live, native={"meshy-3d": {"command": "python", "trust": "high"}}) == live
diff --git a/tui_gateway/methods_tools_plugin_servers.py b/tui_gateway/methods_tools_plugin_servers.py
index 701e63a9eef..2aa1c013210 100644
--- a/tui_gateway/methods_tools_plugin_servers.py
+++ b/tui_gateway/methods_tools_plugin_servers.py
@@ -58,28 +58,53 @@ def _plugin_server_rows(
return rows
+def _server_identity(cfg: object) -> tuple:
+ """What a server config runs: a same-name native entry that runs the same thing is the user
+ widening the package server's trust; anything else is the user's own, different server."""
+ cfg = cfg if isinstance(cfg, dict) else {}
+ return cfg.get("command"), tuple(cfg.get("args") or ()), cfg.get("url")
+
+
+def _portable_server_tools(manager, native: dict) -> dict[str, set[str]]:
+ """Tools each portable package's own MCP servers registered: live connections plus lazily
+ cached (not yet connected) ones. A name the user's native config claims for a different server
+ credits no package: the native entry is the one that runs (``_portable_mcp_servers``)."""
+ core = _tools_mod("tools.mcp_tool_common")._core
+ resolve_key = _tools_mod("tools.mcp_tool_scope")._resolve_server_key
+ declared = manager.get_portable_mcp_servers()
+ listed: dict[str, set[str]] = {}
+ for server_name, owner in manager.get_portable_mcp_server_plugins().items():
+ names = listed.setdefault(owner, set())
+ if server_name in native and _server_identity(native[server_name]) != _server_identity(
+ declared.get(server_name)):
+ continue
+ key = resolve_key(server_name)
+ names.update(getattr(core._servers.get(key), "_registered_tool_names", None) or ())
+ names.update(core._lazy_server_tool_names.get(key) or ())
+ return listed
+
+
def _plugin_tool_ownership() -> dict[str, list[str]]:
"""The exact tool names each loaded plugin's OWN code registered, by manifest key.
- Desktop lets a plugin's desktop half render media only from these tools, so the answer comes
+ Desktop lets a plugin's desktop half render media only from these, so the answer comes
from the loader's and the MCP client's records, never from a manifest claim or a name prefix.
Native: names the plugin registered, minus built-in tool names (an approved override of
``text_to_speech`` is still the core voice reply to the user). Portable: the tools each MCP
- server the loader attributed to the package registered (a server that is not connected yet
- owns nothing until it is). A name two packages list belongs to neither."""
+ server the loader attributed to the package registered (``_portable_server_tools``; a server
+ that never connected or cached its tools owns nothing yet). A name two packages list belongs
+ to neither."""
plugins_mod = _tools_mod("hermes_cli.plugins")
toolsets = _tools_mod("toolsets")
- core = _tools_mod("tools.mcp_tool_common")._core
- resolve_key = _tools_mod("tools.mcp_tool_scope")._resolve_server_key
plugins_mod.discover_plugins()
manager = plugins_mod.get_plugin_manager()
+ native = _tools_mod("hermes_cli.config").load_config().get("mcp_servers")
builtin = set(toolsets._HERMES_CORE_TOOLS)
for entry in toolsets.TOOLSETS.values():
builtin.update(entry.get("tools", ()))
listed = {key: set(loaded.tools_registered) - builtin for key, loaded in manager._plugins.items()}
- for server_name, owner in manager.get_portable_mcp_server_plugins().items():
- server = core._servers.get(resolve_key(server_name))
- listed.setdefault(owner, set()).update(getattr(server, "_registered_tool_names", None) or ())
+ for owner, names in _portable_server_tools(manager, native if isinstance(native, dict) else {}).items():
+ listed.setdefault(owner, set()).update(names)
seen: dict[str, int] = {}
for names in listed.values():
for name in names: