Skip to content

The agent that grows with you

feat(desktop): plugins render the media their own tools produce

Teknium6dc1c398e2ce
diff --git a/apps/desktop/src/app/capabilities/plugins/plugin-packages.ts b/apps/desktop/src/app/capabilities/plugins/plugin-packages.ts
index 1674135d1fb..dc64e7ed3fd 100644
--- a/apps/desktop/src/app/capabilities/plugins/plugin-packages.ts
+++ b/apps/desktop/src/app/capabilities/plugins/plugin-packages.ts
@@ -54,7 +54,7 @@ const DESKTOP_KIND_RANK: Record<PluginRecord['kind'], number> = { disk: 0, runti
* `plugins/<folder>/`, and a hand-copied folder need not match its manifest
* name). Nested rows (`image_gen/foo`) keep their unique canonical key; the
* manifest name is the fallback for backends that report no `install_dir`. */
-function agentPackageName(row: AgentPluginRow): string {
+export function agentPackageName(row: AgentPluginRow): string {
if (row.key?.includes('/')) {
return row.key
}
diff --git a/apps/desktop/src/components/assistant-ui/markdown-text.tsx b/apps/desktop/src/components/assistant-ui/markdown-text.tsx
index 01b4039bcbf..ca5cd1e543b 100644
--- a/apps/desktop/src/components/assistant-ui/markdown-text.tsx
+++ b/apps/desktop/src/components/assistant-ui/markdown-text.tsx
@@ -47,6 +47,7 @@ import { ArtifactCard } from './artifact-card'
import { SessionRefLink } from './directive-text'
import { detectEmbed, extractAlert, MarkdownAlert, RichCodeBlock, UrlEmbed } from './embeds'
import { ResizableMarkdownTable, ResizableMarkdownTh } from './markdown-table'
+import { ToolMediaSlot } from './tool-media-slot'
import { paragraphPlainText, TranscriptDirectiveLeaf, useResolvedParagraph } from './transcript-directive'
// Math rendering plugin (KaTeX). Configured once at module scope — the
@@ -286,21 +287,22 @@ function MarkdownLink({ children, className, href, ...props }: ComponentProps<'a
// download: route it to the preview rail (which renders .md with a
// rendered/source toggle) instead of the download-link fallback that
// `mediaKind() === 'file'` would produce. (#84951)
- if (isMarkdownDocumentPath(mediaPath)) {
- return <PreviewAttachment target={mediaPath} />
- }
-
+ //
// Non-media files (PDFs, data files, anything outside MEDIA_BY_EXT):
// MediaAttachment's kind==='file' branch is a degraded dead-end (bare
// "Open <name>" anchor). Route through the preview pipeline instead —
// the same file card + "Open preview" the bare-path markdown-link
// branch below produces — so MEDIA: uniformly delivers the richest
// rendering for every file type.
- if (mediaKind(mediaPath) === 'file') {
- return <PreviewAttachment target={mediaPath} />
- }
+ const builtIn =
+ isMarkdownDocumentPath(mediaPath) || mediaKind(mediaPath) === 'file' ? (
+ <PreviewAttachment target={mediaPath} />
+ ) : (
+ <MediaAttachment path={mediaPath} />
+ )
- return <MediaAttachment path={mediaPath} />
+ // The plugin that owns the tool which produced this file may draw it instead.
+ return <ToolMediaSlot path={mediaPath}>{builtIn}</ToolMediaSlot>
}
const previewTarget = previewTargetFromMarkdownHref(href)
@@ -330,10 +332,14 @@ function MarkdownLink({ children, className, href, ...props }: ComponentProps<'a
const fileHref = href && !href.startsWith('#') && isFileMediaPath(href) ? href : null
if (fileHref) {
- return mediaKind(fileHref) === 'file' ? (
- <PreviewAttachment target={fileHref} />
- ) : (
- <MediaAttachment path={fileHref} />
+ return (
+ <ToolMediaSlot path={fileHref}>
+ {mediaKind(fileHref) === 'file' ? (
+ <PreviewAttachment target={fileHref} />
+ ) : (
+ <MediaAttachment path={fileHref} />
+ )}
+ </ToolMediaSlot>
)
}
diff --git a/apps/desktop/src/components/assistant-ui/tool-media-slot.test.tsx b/apps/desktop/src/components/assistant-ui/tool-media-slot.test.tsx
new file mode 100644
index 00000000000..6ca2690e07d
--- /dev/null
+++ b/apps/desktop/src/components/assistant-ui/tool-media-slot.test.tsx
@@ -0,0 +1,184 @@
+// @vitest-environment jsdom
+import { MessagePrimitive, type ThreadMessage, ThreadPrimitive } from '@assistant-ui/react'
+import { cleanup, render, screen, waitFor } from '@testing-library/react'
+import { afterEach, beforeEach, describe, expect, it, type MockInstance, vi } from 'vitest'
+
+import { $pluginRecords } from '@/contrib/plugins-store'
+import { registry } from '@/contrib/registry'
+import { renderMediaTags } from '@/lib/chat-messages/parts'
+import {
+ TOOL_MEDIA_RENDERER_AREA,
+ type ToolMediaRendererContribution,
+ type ToolMediaRendererProps
+} from '@/lib/tool-media-renderers'
+import { $toolMediaOwners } from '@/store/tool-media-owners'
+
+import { MarkdownText, MessageTextContent } from './markdown-text'
+import { createdAt, stubThreadEnvironment, ThreadRuntime, userMessage } from './test-utils'
+
+// A real transcript: one turn where the probe plugin's tool and core TTS each
+// produced an audio file, and the reply delivers both with MEDIA: lines.
+const SFX_PATH = '/home/u/.hermes/cache/generated/audio/boom.mp3'
+const TTS_PATH = '/home/u/.hermes/cache/audio/tts_reply.ogg'
+const UPLOAD_PATH = '/home/u/Music/upload.mp3'
+
+function turn(): ThreadMessage[] {
+ return [
+ userMessage('u1', 'make a boom and say hi'),
+ {
+ id: 'a1',
+ role: 'assistant',
+ content: [
+ {
+ type: 'tool-call',
+ toolCallId: 'call-sfx',
+ toolName: 'sfx_make',
+ args: { prompt: 'boom' },
+ argsText: '{"prompt":"boom"}',
+ result: { file_path: SFX_PATH, media_tag: `MEDIA:${SFX_PATH}` }
+ },
+ {
+ type: 'tool-call',
+ toolCallId: 'call-tts',
+ toolName: 'text_to_speech',
+ args: { text: 'hi' },
+ argsText: '{"text":"hi"}',
+ result: JSON.stringify({ file_path: TTS_PATH })
+ },
+ {
+ type: 'text',
+ text: renderMediaTags(`Here:\n\nMEDIA:${SFX_PATH}\n\nMEDIA:${TTS_PATH}\n\nMEDIA:${UPLOAD_PATH}`)
+ }
+ ],
+ status: { type: 'complete', reason: 'stop' },
+ createdAt,
+ metadata: { unstable_state: null, unstable_annotations: [], unstable_data: [], steps: [], custom: {} }
+ } as unknown as ThreadMessage
+ ]
+}
+
+const Assistant = () => (
+ <MessagePrimitive.Root>
+ <MessagePrimitive.Parts components={{ Text: MarkdownText }} />
+ </MessagePrimitive.Root>
+)
+
+function renderTurn() {
+ return render(
+ <ThreadRuntime messages={turn()}>
+ <ThreadPrimitive.Root>
+ <ThreadPrimitive.Messages components={{ AssistantMessage: Assistant, UserMessage: () => null }} />
+ </ThreadPrimitive.Root>
+ </ThreadRuntime>
+ )
+}
+
+const seen: ToolMediaRendererProps[] = []
+
+function claim(pluginId: string, tools: string[], render?: ToolMediaRendererContribution['render']) {
+ return registry.register({
+ id: `${pluginId}:media`,
+ area: TOOL_MEDIA_RENDERER_AREA,
+ source: `plugin:${pluginId}`,
+ data: {
+ tools,
+ render:
+ render ??
+ (props => {
+ seen.push(props)
+
+ return <div data-testid={`card-${pluginId}`}>{props.file.name}</div>
+ })
+ } satisfies ToolMediaRendererContribution
+ })
+}
+
+describe('tool media renderers', () => {
+ const disposers: (() => void)[] = []
+ let warn: MockInstance<typeof console.warn>
+
+ beforeEach(() => {
+ stubThreadEnvironment()
+ seen.length = 0
+ warn = vi.spyOn(console, 'warn').mockImplementation(() => {})
+ $pluginRecords.set({
+ sfx: { id: 'sfx', kind: 'disk', name: 'SFX', packageName: 'sfx', status: 'loaded' },
+ thief: { id: 'thief', kind: 'disk', name: 'Thief', packageName: 'thief', status: 'loaded' }
+ })
+ // What the backend's plugins.manage list reported for the active route.
+ $toolMediaOwners.set({
+ key: '|default|0',
+ owners: new Map([
+ ['sfx', { prefixes: [], tools: new Set(['sfx_make']) }],
+ ['thief', { prefixes: [], tools: new Set(['thief_tool']) }]
+ ]),
+ status: 'ready'
+ })
+ })
+
+ afterEach(() => {
+ disposers.splice(0).forEach(dispose => dispose())
+ cleanup()
+ warn.mockRestore()
+ $pluginRecords.set({})
+ })
+
+ it('renders the owning plugin component for the file its tool produced, and the built-in player for the rest', async () => {
+ disposers.push(claim('sfx', ['sfx_make']))
+ const { container } = renderTurn()
+
+ await waitFor(() => expect(screen.getByTestId('card-sfx').textContent).toBe('boom.mp3'))
+ // Core TTS and a file no tool in the turn produced keep the built-in <audio> player.
+ await waitFor(() => expect(container.querySelectorAll('audio[controls]')).toHaveLength(2))
+ expect(screen.getAllByTestId('card-sfx')).toHaveLength(1)
+
+ const props = seen.at(-1)!
+ expect(props.toolName).toBe('sfx_make')
+ expect(props.toolCallId).toBe('call-sfx')
+ expect(props.args).toEqual({ prompt: 'boom' })
+ expect(props.result).toEqual({ file_path: SFX_PATH, media_tag: `MEDIA:${SFX_PATH}` })
+ expect(props.file).toMatchObject({ kind: 'audio', name: 'boom.mp3', path: SFX_PATH })
+ expect(typeof props.file.resolveUrl).toBe('function')
+ expect(props.fallback).toBeTruthy()
+ })
+
+ it('refuses a claim on a core tool or another plugin’s tool, with a console warning', async () => {
+ disposers.push(claim('thief', ['sfx_make', 'text_to_speech']))
+ const { container } = renderTurn()
+
+ await waitFor(() => expect(container.querySelectorAll('audio[controls]')).toHaveLength(3))
+ expect(screen.queryByTestId('card-thief')).toBeNull()
+ expect(warn.mock.calls.map(call => String(call[0])).join('\n')).toMatch(/plugin:thief.*refused.*"text_to_speech"/)
+ expect(warn.mock.calls.map(call => String(call[0])).join('\n')).toMatch(/plugin:thief.*refused.*"sfx_make"/)
+ })
+
+ it('falls back to the built-in player when the plugin renderer throws', async () => {
+ disposers.push(
+ claim('sfx', ['sfx_make'], () => {
+ throw new Error('boom')
+ })
+ )
+ vi.spyOn(console, 'error').mockImplementation(() => {})
+ const { container } = renderTurn()
+
+ await waitFor(() => expect(container.querySelectorAll('audio[controls]')).toHaveLength(3))
+ })
+
+ it('keeps the built-in player once the plugin is disabled (its contributions are gone)', async () => {
+ const dispose = claim('sfx', ['sfx_make'])
+ const { container } = renderTurn()
+
+ await waitFor(() => expect(screen.getByTestId('card-sfx')).toBeTruthy())
+ dispose()
+ await waitFor(() => expect(container.querySelectorAll('audio[controls]')).toHaveLength(3))
+ expect(screen.queryByTestId('card-sfx')).toBeNull()
+ })
+
+ it('keeps the built-in player outside a transcript turn (a Bot group room has no tool calls)', async () => {
+ disposers.push(claim('sfx', ['sfx_make']))
+ const { container } = render(<MessageTextContent text={`MEDIA:${SFX_PATH}`} />)
+
+ await waitFor(() => expect(container.querySelector('audio[controls]')).not.toBeNull())
+ expect(screen.queryByTestId('card-sfx')).toBeNull()
+ })
+})
diff --git a/apps/desktop/src/components/assistant-ui/tool-media-slot.tsx b/apps/desktop/src/components/assistant-ui/tool-media-slot.tsx
new file mode 100644
index 00000000000..16180becd4d
--- /dev/null
+++ b/apps/desktop/src/components/assistant-ui/tool-media-slot.tsx
@@ -0,0 +1,168 @@
+import { useAuiState } from '@assistant-ui/react'
+import { useStore } from '@nanostores/react'
+import { type FC, type ReactNode, useEffect, useMemo } from 'react'
+
+import { useSessionView } from '@/app/chat/session-view'
+import { ErrorBoundary } from '@/components/error-boundary'
+import { type Contribution, useContributions } from '@/contrib'
+import { $pluginRecords } from '@/contrib/plugins-store'
+import { mediaKind, mediaMime, mediaName, resolveMediaPlaybackSrc } from '@/lib/media'
+import {
+ findProducingToolCall,
+ refusedClaimPatterns,
+ resolveToolMediaClaim,
+ TOOL_MEDIA_RENDERER_AREA,
+ type ToolCallLike,
+ type ToolMediaFile,
+ type ToolMediaRendererContribution,
+ type ToolMediaRendererProps
+} from '@/lib/tool-media-renderers'
+import { $toolMediaOwners, ensureToolMediaOwners, packageOfContributionSource } from '@/store/tool-media-owners'
+
+/**
+ * Where a `MEDIA:` attachment meets the tool media renderer area. With no
+ * renderer registered this is a pass-through (no subscriptions); otherwise it
+ * finds the tool call in this turn that produced the path and, when the plugin
+ * that OWNS that tool claims it, mounts the plugin's component with the
+ * built-in player as both its `fallback` prop and its error fallback.
+ */
+
+const warned = new Set<string>()
+
+function warnOnce(key: string, message: string) {
+ if (!warned.has(key)) {
+ warned.add(key)
+ console.warn(message)
+ }
+}
+
+function warnRefused(claim: Pick<Contribution, 'id' | 'source'>, tool: string) {
+ warnOnce(
+ `${claim.id}\u0000${tool}`,
+ `[${claim.source ?? 'core'}] tool media renderer refused for "${tool}": it is not a tool this plugin's package registered`
+ )
+}
+
+// The turn's producer, read straight from the thread store so the selector
+// returns the store's own part object (a stable reference across deltas).
+// Outside a message (a Bot group room, a preview) there is no turn: no producer.
+function useProducingToolCall(path: string): ToolCallLike | null {
+ return useAuiState(state => {
+ try {
+ return findProducingToolCall(state.thread.messages, state.message.id, path)
+ } catch {
+ return null
+ }
+ })
+}
+
+export const ToolMediaSlot: FC<{ children: ReactNode; path: string }> = ({ children, path }) => {
+ const claims = useContributions(TOOL_MEDIA_RENDERER_AREA)
+
+ if (claims.length === 0) {
+ return <>{children}</>
+ }
+
+ return (
+ <ClaimedToolMediaSlot claims={claims} path={path}>
+ {children}
+ </ClaimedToolMediaSlot>
+ )
+}
+
+const ClaimedToolMediaSlot: FC<{ children: ReactNode; claims: readonly Contribution[]; path: string }> = ({
+ children,
+ claims,
+ path
+}) => {
+ const producer = useProducingToolCall(path)
+ const owners = useStore($toolMediaOwners)
+ // Re-resolve when a plugin record (enable state, package) changes.
+ useStore($pluginRecords)
+
+ useEffect(() => {
+ if (producer) {
+ ensureToolMediaOwners()
+ }
+ }, [producer, owners])
+
+ const ready = owners.status === 'ready' ? owners.owners : null
+
+ useEffect(() => {
+ if (!ready) {
+ return
+ }
+
+ for (const { claim, pattern } of refusedClaimPatterns(claims, ready, packageOfContributionSource)) {
+ warnRefused(claim, pattern)
+ }
+ }, [claims, ready])
+
+ const claim =
+ producer?.toolName && ready
+ ? resolveToolMediaClaim(claims, producer.toolName, {
+ onRefused: warnRefused,
+ owners: ready,
+ packageOf: packageOfContributionSource
+ })
+ : null
+
+ if (!claim || !producer?.toolName) {
+ return <>{children}</>
+ }
+
+ return (
+ <ErrorBoundary fallback={() => children} label={`contrib:${claim.id}`}>
+ <ToolMediaLeaf contribution={claim} fallback={children} path={path} producer={producer} />
+ </ErrorBoundary>
+ )
+}
+
+const ToolMediaLeaf: FC<{
+ contribution: Contribution
+ fallback: ReactNode
+ path: string
+ producer: ToolCallLike
+}> = ({ contribution, fallback, path, producer }) => {
+ const render = (contribution.data as ToolMediaRendererContribution).render
+ const view = useSessionView()
+ const storedId = useStore(view.$storedId)
+ const runtimeId = useStore(view.$runtimeId)
+
+ const file = useMemo<ToolMediaFile>(
+ () => ({
+ kind: mediaKind(path),
+ mime: mediaMime(path),
+ name: mediaName(path),
+ path,
+ resolveUrl: () => resolveMediaPlaybackSrc(path)
+ }),
+ [path]
+ )
+
+ const args = useMemo(
+ () => (producer.args && typeof producer.args === 'object' ? (producer.args as Record<string, unknown>) : {}),
+ [producer.args]
+ )
+
+ // Stable component identity per renderer, so a re-render never remounts the widget.
+ const Leaf = useMemo(
+ () =>
+ function ToolMediaRendererHost(props: ToolMediaRendererProps) {
+ return <>{render(props)}</>
+ },
+ [render]
+ )
+
+ return (
+ <Leaf
+ args={args}
+ fallback={fallback}
+ file={file}
+ result={producer.result}
+ sessionId={storedId || runtimeId || null}
+ toolCallId={producer.toolCallId ?? ''}
+ toolName={producer.toolName ?? ''}
+ />
+ )
+}
diff --git a/apps/desktop/src/lib/tool-media-renderers.test.ts b/apps/desktop/src/lib/tool-media-renderers.test.ts
new file mode 100644
index 00000000000..4987d85c9ec
--- /dev/null
+++ b/apps/desktop/src/lib/tool-media-renderers.test.ts
@@ -0,0 +1,103 @@
+import { describe, expect, it, vi } from 'vitest'
+
+import {
+ claimPatternOwned,
+ findProducingToolCall,
+ ownerOfTool,
+ refusedClaimPatterns,
+ resolveToolMediaClaim,
+ type ToolOwnership
+} from './tool-media-renderers'
+
+const call = (toolName: string, result: unknown, extra: Record<string, unknown> = {}) => ({
+ type: 'tool-call',
+ toolCallId: `${toolName}-id`,
+ toolName,
+ args: {},
+ result,
+ ...extra
+})
+
+describe('findProducingToolCall', () => {
+ const path = '/tmp/out/model.glb'
+
+ it('picks the earliest call in the turn whose result names the path (a later ls only mentions it)', () => {
+ const messages = [
+ { id: 'u0', role: 'user', content: [] },
+ { id: 'a0', role: 'assistant', content: [call('mcp__meshy__download', { path })] },
+ { id: 'u1', role: 'user', content: [] },
+ { id: 'a1', role: 'assistant', content: [call('mcp__meshy__download', `saved to ${path}`)] },
+ { id: 'a2', role: 'assistant', content: [call('terminal', { output: path }), { type: 'text', text: 'MEDIA' }] }
+ ]
+
+ expect(findProducingToolCall(messages, 'a2', path)?.toolCallId).toBe('mcp__meshy__download-id')
+ // The previous turn's call never counts for this turn.
+ expect(findProducingToolCall(messages.slice(2), 'a2', path)?.toolName).toBe('mcp__meshy__download')
+ expect(findProducingToolCall([messages[2], messages[4]], 'a2', path)?.toolName).toBe('terminal')
+ })
+
+ it('ignores failed calls and finds Windows paths inside JSON results', () => {
+ const win = 'C:\\Users\\me\\out\\clip.wav'
+
+ const messages = [
+ { id: 'u', role: 'user', content: [] },
+ {
+ id: 'a',
+ role: 'assistant',
+ content: [call('sfx', { path }, { isError: true }), call('sfx', JSON.stringify({ win }))]
+ }
+ ]
+
+ expect(findProducingToolCall(messages, 'a', path)).toBeNull()
+ expect(findProducingToolCall(messages, 'a', win)?.toolName).toBe('sfx')
+ expect(findProducingToolCall(messages, 'missing', win)).toBeNull()
+ })
+})
+
+describe('ownership', () => {
+ const owners = new Map<string, ToolOwnership>([
+ ['sfx', { prefixes: [], tools: new Set(['sfx_make']) }],
+ ['meshy', { prefixes: ['mcp__meshy__'], tools: new Set() }],
+ ['nested', { prefixes: ['mcp__meshy__x__'], tools: new Set() }]
+ ])
+
+ it('resolves a tool to exactly one package, longest MCP namespace first', () => {
+ expect(ownerOfTool('sfx_make', owners)).toBe('sfx')
+ expect(ownerOfTool('mcp__meshy__download', owners)).toBe('meshy')
+ expect(ownerOfTool('mcp__meshy__x__gen', owners)).toBe('nested')
+ expect(ownerOfTool('text_to_speech', owners)).toBeNull()
+ })
+
+ it('lets a wildcard claim only inside the package’s own namespace', () => {
+ expect(claimPatternOwned('mcp__meshy__*', owners.get('meshy'))).toBe(true)
+ expect(claimPatternOwned('mcp__*', owners.get('meshy'))).toBe(false)
+ expect(claimPatternOwned('sfx_*', owners.get('sfx'))).toBe(false)
+ expect(claimPatternOwned('text_to_speech', owners.get('sfx'))).toBe(false)
+ expect(claimPatternOwned('sfx_make', undefined)).toBe(false)
+ })
+
+ it('serves the first claim whose package owns the tool and reports every other match', () => {
+ const render = () => null
+
+ const packages: Record<string, string | null> = {
+ 'plugin:meshy': 'meshy',
+ 'plugin:thief': 'thief',
+ 'plugin:solo': null
+ }
+
+ const claims = [
+ { id: 'thief:x', source: 'plugin:thief', data: { tools: ['mcp__meshy__*'], render } },
+ { id: 'solo:x', source: 'plugin:solo', data: { tools: ['mcp__meshy__download'], render } },
+ { id: 'meshy:x', source: 'plugin:meshy', data: { tools: ['mcp__meshy__*'], render } }
+ ]
+
+ const onRefused = vi.fn()
+ const packageOf = (source?: string) => (source ? (packages[source] ?? null) : null)
+
+ expect(resolveToolMediaClaim(claims, 'mcp__meshy__download', { onRefused, owners, packageOf })?.id).toBe('meshy:x')
+ expect(onRefused.mock.calls.map(([c]) => c.id)).toEqual(['thief:x', 'solo:x'])
+ // The nested server's tools are not meshy's, even through meshy's own wildcard.
+ expect(resolveToolMediaClaim(claims, 'mcp__meshy__x__gen', { owners, packageOf })).toBeNull()
+ expect(refusedClaimPatterns(claims, owners, packageOf).map(r => r.claim.id)).toEqual(['thief:x', 'solo:x'])
+ })
+})
diff --git a/apps/desktop/src/lib/tool-media-renderers.ts b/apps/desktop/src/lib/tool-media-renderers.ts
new file mode 100644
index 00000000000..19781624448
--- /dev/null
+++ b/apps/desktop/src/lib/tool-media-renderers.ts
@@ -0,0 +1,270 @@
+import type { ReactNode } from 'react'
+
+import type { MediaKind } from '@/lib/media'
+
+/**
+ * TOOL MEDIA RENDERERS — a plugin draws the media ITS OWN tools produced.
+ *
+ * When a transcript shows a `MEDIA:` attachment, the host finds the tool call
+ * in that turn whose result produced the path. If the plugin that owns that
+ * tool registered a renderer for the tool's name, its component replaces the
+ * built-in player; every other file (core voice replies, other plugins' and
+ * core tools' output, user uploads, paths no tool in the turn produced) keeps
+ * the built-in player.
+ *
+ * Ownership is provenance, never file type or name: the backend reports which
+ * tools each installed package's own code registered (`plugins.manage list` →
+ * `tools` / `tool_prefixes`), and a claim on any other tool is refused.
+ */
+
+export const TOOL_MEDIA_RENDERER_AREA = 'transcript.toolMedia'
+
+/** The file the tool produced, as the transcript names it. */
+export interface ToolMediaFile {
+ /** Path on the BACKEND's machine, exactly as the `MEDIA:` line wrote it.
+ * Never open it directly (the backend may be remote): use `resolveUrl()`,
+ * or hand the path to your own backend route. */
+ path: string
+ name: string
+ kind: MediaKind
+ mime: string
+ /** A URL this window can load, local or remote backend alike: a range
+ * streaming route for audio/video, a data URL for anything else (size-capped
+ * by the host's file reader). */
+ resolveUrl: () => Promise<string>
+}
+
+/** Props handed to a tool media renderer's `render`. */
+export interface ToolMediaRendererProps {
+ file: ToolMediaFile
+ /** The producing tool call: registry name, call id, arguments and result. */
+ toolName: string
+ toolCallId: string
+ args: Readonly<Record<string, unknown>>
+ result: unknown
+ /** The chat whose transcript renders this (stored id, else runtime id), or null. */
+ sessionId: null | string
+ /** The built-in player for this file: return it for anything you don't draw. */
+ fallback: ReactNode
+}
+
+/** Payload of a `transcript.toolMedia` contribution's `data`. */
+export interface ToolMediaRendererContribution {
+ /** Tool names this renderer draws. A trailing `*` matches a namespace the
+ * package owns (`mcp__meshy__*` for its MCP server). Claims on tools the
+ * package did not register are refused with a console warning. */
+ tools: readonly string[]
+ /** Mounted inside an error boundary whose fallback is the built-in player. */
+ render: (props: ToolMediaRendererProps) => ReactNode
+}
+
+/** What one package's own code registered. */
+export interface ToolOwnership {
+ tools: ReadonlySet<string>
+ /** MCP server namespaces (`mcp__<server>__`). */
+ prefixes: readonly string[]
+}
+
+/** The tool-call fields a producer lookup reads (an assistant-ui part). */
+export interface ToolCallLike {
+ args?: unknown
+ isError?: boolean
+ result?: unknown
+ toolCallId?: string
+ toolName?: string
+ type?: string
+}
+
+interface MessageLike {
+ content?: readonly unknown[] | string
+ id?: string
+ role?: string
+}
+
+function payloadText(value: unknown): string {
+ if (typeof value === 'string') {
+ return value
+ }
+
+ try {
+ return JSON.stringify(value) ?? ''
+ } catch {
+ return ''
+ }
+}
+
+function pathNeedles(path: string): string[] {
+ const bare = path.startsWith('file:') ? decodeURIComponentSafe(path.replace(/^file:\/\//, '')) : path
+
+ // JSON-escaped forms too: a Windows path sits in a JSON result as `C:\\Users\\…`.
+ return [...new Set([path, bare, JSON.stringify(path).slice(1, -1), JSON.stringify(bare).slice(1, -1)])]
+}
+
+function decodeURIComponentSafe(value: string): string {
+ try {
+ return decodeURIComponent(value)
+ } catch {
+ return value
+ }
+}
+
+function toolCallsOf(message: MessageLike): ToolCallLike[] {
+ return Array.isArray(message.content)
+ ? (message.content as ToolCallLike[]).filter(
+ part => part?.type === 'tool-call' && typeof part.toolName === 'string'
+ )
+ : []
+}
+
+/**
+ * The tool call whose result produced `path`, from the turn that holds message
+ * `messageId`: that message and the assistant messages back to the user's
+ * prompt. The EARLIEST matching call wins, because a file is produced once and
+ * later calls (`ls`, a read) only mention it. Failed calls produced nothing.
+ */
+export function findProducingToolCall(
+ messages: readonly MessageLike[],
+ messageId: null | string | undefined,
+ path: string
+): ToolCallLike | null {
+ const index = messageId ? messages.findIndex(message => message.id === messageId) : -1
+
+ if (index < 0) {
+ return null
+ }
+
+ let start = index
+
+ while (start > 0 && messages[start - 1]?.role !== 'user') {
+ start -= 1
+ }
+
+ const needles = pathNeedles(path)
+
+ for (const message of messages.slice(start, index + 1)) {
+ for (const call of toolCallsOf(message)) {
+ if (call.isError || call.result === undefined) {
+ continue
+ }
+
+ const text = payloadText(call.result)
+
+ if (needles.some(needle => needle && text.includes(needle))) {
+ return call
+ }
+ }
+ }
+
+ return null
+}
+
+/** The package that owns `toolName`: an exact registration first, else the
+ * longest MCP namespace that covers it (so `mcp__a__` never takes a server
+ * named `a__b`'s tools). */
+export function ownerOfTool(toolName: string, owners: ReadonlyMap<string, ToolOwnership>): null | string {
+ let best: null | string = null
+ let bestLength = 0
+
+ for (const [pkg, owned] of owners) {
+ if (owned.tools.has(toolName)) {
+ return pkg
+ }
+
+ for (const prefix of owned.prefixes) {
+ if (toolName.startsWith(prefix) && toolName.length > prefix.length && prefix.length > bestLength) {
+ best = pkg
+ bestLength = prefix.length
+ }
+ }
+ }
+
+ return best
+}
+
+export function claimPatternMatches(pattern: string, toolName: string): boolean {
+ return pattern.endsWith('*') ? pattern.length > 1 && toolName.startsWith(pattern.slice(0, -1)) : toolName === pattern
+}
+
+/** Whether `pattern` names only tools `owned` registered (a wildcard must sit
+ * inside one of the package's MCP namespaces). */
+export function claimPatternOwned(pattern: string, owned: ToolOwnership | undefined): boolean {
+ if (!owned) {
+ return false
+ }
+
+ if (pattern.endsWith('*')) {
+ const stem = pattern.slice(0, -1)
+
+ return owned.prefixes.some(prefix => stem.startsWith(prefix))
+ }
+
+ return (
+ owned.tools.has(pattern) ||
+ owned.prefixes.some(prefix => pattern.startsWith(prefix) && pattern.length > prefix.length)
+ )
+}
+
+export interface ToolMediaClaim {
+ id: string
+ source?: string
+ data?: unknown
+}
+
+export interface ResolveToolMediaOptions {
+ owners: ReadonlyMap<string, ToolOwnership>
+ /** The package a contribution's plugin belongs to (null: none, e.g. a standalone desktop plugin). */
+ packageOf: (source: string | undefined) => null | string
+ onRefused?: (claim: ToolMediaClaim, toolName: string) => void
+}
+
+/** The first claim on `toolName` whose plugin's package owns that tool. A
+ * matching claim from any other plugin is refused (and reported). */
+export function resolveToolMediaClaim<C extends ToolMediaClaim>(
+ claims: readonly C[],
+ toolName: string,
+ { onRefused, owners, packageOf }: ResolveToolMediaOptions
+): C | null {
+ const owner = ownerOfTool(toolName, owners)
+
+ for (const claim of claims) {
+ const data = claim.data as Partial<ToolMediaRendererContribution> | undefined
+
+ if (typeof data?.render !== 'function' || !Array.isArray(data.tools)) {
+ continue
+ }
+
+ if (!data.tools.some(pattern => typeof pattern === 'string' && claimPatternMatches(pattern, toolName))) {
+ continue
+ }
+
+ const pkg = packageOf(claim.source)
+
+ if (owner !== null && pkg === owner) {
+ return claim
+ }
+
+ onRefused?.(claim, toolName)
+ }
+
+ return null
+}
+
+/** Every pattern in `claims` its plugin's package does not own — what the
+ * host warns about once the ownership list is known. */
+export function refusedClaimPatterns<C extends ToolMediaClaim>(
+ claims: readonly C[],
+ owners: ReadonlyMap<string, ToolOwnership>,
+ packageOf: (source: string | undefined) => null | string
+): { claim: C; pattern: string }[] {
+ return claims.flatMap(claim => {
+ const tools = (claim.data as Partial<ToolMediaRendererContribution> | undefined)?.tools
+ const pkg = packageOf(claim.source)
+ const owned = pkg ? owners.get(pkg) : undefined
+
+ return Array.isArray(tools)
+ ? tools
+ .filter(pattern => !claimPatternOwned(String(pattern), owned))
+ .map(pattern => ({ claim, pattern: String(pattern) }))
+ : []
+ })
+}
diff --git a/apps/desktop/src/sdk/areas.ts b/apps/desktop/src/sdk/areas.ts
index be025ae186f..b76717cd8d8 100644
--- a/apps/desktop/src/sdk/areas.ts
+++ b/apps/desktop/src/sdk/areas.ts
@@ -27,3 +27,11 @@ export {
type KanbanCardBadgeTone,
type KanbanCardBoard
} from '@/contrib/kanban-card-badges'
+/** Draw the `MEDIA:` files your package's OWN tools produced, in place of the
+ * built-in player (ownership is checked against the backend's tool registry). */
+export {
+ TOOL_MEDIA_RENDERER_AREA,
+ type ToolMediaFile,
+ type ToolMediaRendererContribution,
+ type ToolMediaRendererProps
+} from '@/lib/tool-media-renderers'
diff --git a/apps/desktop/src/store/tool-media-owners.ts b/apps/desktop/src/store/tool-media-owners.ts
new file mode 100644
index 00000000000..7c1f0a2cc0b
--- /dev/null
+++ b/apps/desktop/src/store/tool-media-owners.ts
@@ -0,0 +1,97 @@
+import { atom } from 'nanostores'
+
+import { agentPackageName, desktopPackageName } from '@/app/capabilities/plugins/plugin-packages'
+import { $pluginRecords } from '@/contrib/plugins-store'
+import type { ToolOwnership } from '@/lib/tool-media-renderers'
+import { $agentPlugins, type AgentPluginRow } from '@/store/agent-plugins'
+import {
+ activeGateway,
+ activeGatewayConnectionId,
+ activeGatewayProfileKey,
+ gatewayActivationEpoch
+} from '@/store/gateway'
+
+/**
+ * Which tools each installed package's own code registered, per active backend
+ * route — the authority behind tool media renderers. Read from the backend's
+ * `plugins.manage list` (`tools` / `tool_prefixes`), never from the renderer:
+ * a desktop half could otherwise name any tool. Loaded lazily the first time a
+ * claimed attachment needs it and refetched when the route or the agent plugin
+ * list changes. A backend that predates the fields reports nothing, so every
+ * claim falls back to the built-in player.
+ */
+
+export type ToolMediaOwnersState =
+ | { status: 'idle' | 'loading' | 'error'; key: string }
+ | { status: 'ready'; key: string; owners: ReadonlyMap<string, ToolOwnership> }
+
+export const $toolMediaOwners = atom<ToolMediaOwnersState>({ key: '', status: 'idle' })
+
+type OwnershipRow = AgentPluginRow & { tool_prefixes?: string[]; tools?: string[] }
+
+function routeKey(): string {
+ return `${activeGatewayConnectionId() ?? ''}|${activeGatewayProfileKey()}|${gatewayActivationEpoch()}`
+}
+
+export function ownersFromPluginRows(rows: readonly OwnershipRow[]): Map<string, ToolOwnership> {
+ const owners = new Map<string, ToolOwnership>()
+
+ for (const row of rows) {
+ const tools = Array.isArray(row.tools) ? row.tools.filter(name => typeof name === 'string') : []
+
+ const prefixes = Array.isArray(row.tool_prefixes)
+ ? row.tool_prefixes.filter(prefix => typeof prefix === 'string' && prefix.length > 0)
+ : []
+
+ if (tools.length > 0 || prefixes.length > 0) {
+ owners.set(agentPackageName(row), { prefixes, tools: new Set(tools) })
+ }
+ }
+
+ return owners
+}
+
+/** Fetch the ownership list for the active route unless it is loaded or loading. */
+export function ensureToolMediaOwners(): void {
+ const key = routeKey()
+ const current = $toolMediaOwners.get()
+
+ if (current.key === key && current.status !== 'idle') {
+ return
+ }
+
+ const gateway = activeGateway()
+
+ if (!gateway) {
+ return
+ }
+
+ $toolMediaOwners.set({ key, status: 'loading' })
+ void gateway
+ .request<{ plugins?: OwnershipRow[] }>('plugins.manage', { action: 'list' })
+ .then(result => {
+ if ($toolMediaOwners.get().key === key) {
+ $toolMediaOwners.set({ key, owners: ownersFromPluginRows(result?.plugins ?? []), status: 'ready' })
+ }
+ })
+ .catch(() => {
+ if ($toolMediaOwners.get().key === key) {
+ $toolMediaOwners.set({ key, status: 'error' })
+ }
+ })
+}
+
+// A toggle/install/update in Capabilities reloads `$agentPlugins`: re-read ownership on next use.
+$agentPlugins.listen(() => $toolMediaOwners.set({ key: '', status: 'idle' }))
+
+/** The agent package a contribution's plugin is the desktop half of (`plugin:<id>` →
+ * its record's `packageName`); null for standalone desktop plugins and core. */
+export function packageOfContributionSource(source: string | undefined): null | string {
+ if (!source?.startsWith('plugin:')) {
+ return null
+ }
+
+ const record = $pluginRecords.get()[source.slice('plugin:'.length)]
+
+ return record?.status === 'loaded' ? desktopPackageName(record) : null
+}
diff --git a/apps/shared/src/gateway-contract.generated.ts b/apps/shared/src/gateway-contract.generated.ts
index 35567c74c39..20315a911e3 100644
--- a/apps/shared/src/gateway-contract.generated.ts
+++ b/apps/shared/src/gateway-contract.generated.ts
@@ -4443,6 +4443,8 @@ export interface AgentPluginRow {
update_target?: string | null
pinned_sha?: string | null
settings_schema?: PluginSettingField[] | null
+ tools?: string[]
+ tool_prefixes?: string[]
}
export interface PluginServerRow {
name: string
diff --git a/apps/shared/src/gateway-contract.openrpc.json b/apps/shared/src/gateway-contract.openrpc.json
index 3d64ed20c0c..b6e36c12ce7 100644
--- a/apps/shared/src/gateway-contract.openrpc.json
+++ b/apps/shared/src/gateway-contract.openrpc.json
@@ -5076,6 +5076,20 @@
],
"default": null,
"title": "Settings Schema"
+ },
+ "tools": {
+ "items": {
+ "type": "string"
+ },
+ "title": "Tools",
+ "type": "array"
+ },
+ "tool_prefixes": {
+ "items": {
+ "type": "string"
+ },
+ "title": "Tool Prefixes",
+ "type": "array"
}
},
"required": [
diff --git a/skills/autonomous-ai-agents/hermes-agent/references/desktop-plugins.md b/skills/autonomous-ai-agents/hermes-agent/references/desktop-plugins.md
index cd6374ef5c6..31552801022 100644
--- a/skills/autonomous-ai-agents/hermes-agent/references/desktop-plugins.md
+++ b/skills/autonomous-ai-agents/hermes-agent/references/desktop-plugins.md
@@ -113,6 +113,13 @@ The ONLY import surface is `@hermes/plugin-sdk` (plus `react` /
back to plain text; core's own `::preview{file="…"}` is the reference.
After registering one, TELL the model it exists (a bundled skill or the
user's instructions) — it won't discover the name on its own.
+- TOOL media: a unified package's desktop half can draw the `MEDIA:` files
+ its OWN tools produced — `area: TOOL_MEDIA_RENDERER_AREA`,
+ `data: { tools: ['my_tool'] /* or 'mcp__myserver__*' */, render: ({ file, toolName, args, result, sessionId, fallback }) => … }`.
+ Ownership is checked against the backend's registry (claims on core or
+ other plugins' tools are refused); return `fallback` for anything you
+ don't draw; `file.resolveUrl()` works on remote backends. No skill
+ instruction needed — the tool keeps emitting plain `MEDIA:` lines.
- `ctx.storage.get/set/remove` — persistence namespaced to your plugin.
- `ctx.files` — the only file door: the plugin's own data folder (`'data'`) and
folders the user grants through `ctx.files.requestFolder()` (the app's
diff --git a/tests/tui_gateway/test_plugins_manage_tool_ownership.py b/tests/tui_gateway/test_plugins_manage_tool_ownership.py
new file mode 100644
index 00000000000..a5efe62e50f
--- /dev/null
+++ b/tests/tui_gateway/test_plugins_manage_tool_ownership.py
@@ -0,0 +1,73 @@
+"""``plugins.manage list`` names the tools each package's own code produces (``tools`` /
+``tool_prefixes``): Desktop lets a plugin's desktop half render media only from those, so the
+answer must come from what the loader actually registered, never from a manifest claim.
+
+Real discovery against the sandbox ``HERMES_HOME``: two native plugins and one portable MCP package.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+from pathlib import Path
+from unittest.mock import patch
+
+from hermes_cli.config import load_config, save_config
+from tui_gateway import server
+
+_NATIVE = """
+def _handler(args, **_kw):
+ return "{}"
+
+def register(ctx):
+ schema = {"name": "%(tool)s", "description": "d", "parameters": {"type": "object", "properties": {}}}
+ ctx.register_tool(name="%(tool)s", toolset="%(name)s", schema=schema, handler=_handler)
+ %(extra)s
+"""
+
+_OVERRIDE_TTS = (
+ 'ctx.register_tool(name="text_to_speech", toolset="%(name)s", override=True, handler=_handler, '
+ 'schema={"name": "text_to_speech", "description": "d", "parameters": {"type": "object", "properties": {}}})'
+)
+
+
+def _native_plugin(root: Path, name: str, tool: str, extra: str = "") -> None:
+ plugin = root / name
+ plugin.mkdir(parents=True)
+ (plugin / "plugin.yaml").write_text(f"name: {name}\nversion: '1.0'\ndescription: d\n")
+ (plugin / "__init__.py").write_text(_NATIVE % {"name": name, "tool": tool, "extra": extra % {"name": name}})
+
+
+def _portable_plugin(root: Path) -> None:
+ from hermes_cli.agent_plugins import MCP_SCHEMA_V1, PLUGIN_SCHEMA_V1
+ plugin = root / "meshy-ish"
+ plugin.mkdir(parents=True)
+ (plugin / "plugin.json").write_text(json.dumps({"$schema": PLUGIN_SCHEMA_V1, "name": "meshy-ish"}))
+ (plugin / "mcp.json").write_text(json.dumps({
+ "$schema": MCP_SCHEMA_V1, "mcpServers": {"meshy-3d": {"type": "stdio", "command": "python"}}}))
+
+
+def test_list_reports_only_tools_the_package_itself_registered():
+ root = Path(os.environ["HERMES_HOME"]) / "plugins"
+ _native_plugin(root, "sfx", "sfx_make", _OVERRIDE_TTS)
+ _native_plugin(root, "other", "other_tool")
+ _portable_plugin(root)
+ cfg = load_config()
+ cfg["plugins"] = {"enabled": ["sfx", "other", "meshy-ish"],
+ "entries": {"sfx": {"allow_tool_override": True}}}
+ save_config(cfg)
+
+ with patch("hermes_cli.plugins_cmd_catalog.catalog_rows_maps", return_value=({}, {}, {})):
+ resp = server.handle_request({"id": "1", "method": "plugins.manage", "params": {"action": "list"}})
+
+ rows = {row["name"]: row for row in resp["result"]["plugins"]}
+ from hermes_cli.plugins import get_plugin_manager
+ assert "text_to_speech" in get_plugin_manager()._plugins["sfx"].tools_registered # the override took
+ # The core voice-reply tool stays core even after an approved override; another plugin's tool
+ # is never listed on this row.
+ assert rows["sfx"]["tools"] == ["sfx_make"]
+ assert rows["other"]["tools"] == ["other_tool"]
+ assert rows["sfx"]["tool_prefixes"] == [] and rows["other"]["tool_prefixes"] == []
+ # A portable package owns its MCP server's tool namespace (the wire name sanitizes '-').
+ assert rows["meshy-ish"]["tools"] == []
+ assert rows["meshy-ish"]["tool_prefixes"] == ["mcp__meshy_3d__"]
diff --git a/tui_gateway/contracts/tools_mcp_plugins.py b/tui_gateway/contracts/tools_mcp_plugins.py
index d47f04c2a68..49b89e09bd0 100644
--- a/tui_gateway/contracts/tools_mcp_plugins.py
+++ b/tui_gateway/contracts/tools_mcp_plugins.py
@@ -713,6 +713,10 @@ class AgentPluginRow(Result):
update_target: str | None = None
pinned_sha: str | None = None
settings_schema: list[PluginSettingField] | None = None
+ # Tools this package's own code registered (native names; MCP ``mcp__<server>__`` namespaces):
+ # Desktop renders a plugin's tool-media renderer only for these.
+ tools: list[str] = Field(default_factory=list)
+ tool_prefixes: list[str] = Field(default_factory=list)
class PluginLiveServer(Result):
diff --git a/tui_gateway/methods_tools.py b/tui_gateway/methods_tools.py
index 82dd29cb8cf..0aeb6215e91 100644
--- a/tui_gateway/methods_tools.py
+++ b/tui_gateway/methods_tools.py
@@ -1669,6 +1669,7 @@ def _plugin_rows() -> list[dict]:
git_updates = _tools_mod("hermes_cli.plugins_cadence").cached_git_updates(ref_pins)
out = []
active = pc._category_active_names()
+ owned_tools, owned_prefixes = _plugin_tool_ownership()
for name, version, desc, source, _dir, key in sorted(pc._discover_all_plugins()):
# Bundled backends/platforms/providers and the live memory provider run without an explicit
# enable: _plugin_status reports the truthful default instead of "not enabled" (reads as OFF).
@@ -1686,6 +1687,8 @@ def _plugin_rows() -> list[dict]:
# Manifest ``config_schema`` + current values: the Plugins hub renders these as a form.
"settings_schema": _tools_mod("hermes_cli.plugins_settings").plugin_settings_fields(key, _dir_path),
"servers": _plugin_server_rows(_dir_path, key, portable=portable, catalog_titles=titles),
+ # Tools this package's own code produces: Desktop's tool-media renderer ownership check.
+ "tools": owned_tools.get(key, []), "tool_prefixes": owned_prefixes.get(key, []),
**cat.catalog_row_fields(_dir, pins, versions),
**({"update_available": True, "update_target": target}
if _dir_path and (target := git_updates.get(_dir_path.name)) else {}),
diff --git a/tui_gateway/methods_tools_plugin_servers.py b/tui_gateway/methods_tools_plugin_servers.py
index c70bd6c1064..ad6a968e940 100644
--- a/tui_gateway/methods_tools_plugin_servers.py
+++ b/tui_gateway/methods_tools_plugin_servers.py
@@ -58,5 +58,35 @@ def _plugin_server_rows(
return rows
+def _plugin_tool_ownership() -> tuple[dict[str, list[str]], dict[str, list[str]]]:
+ """Which tools each loaded plugin's OWN code produces, by manifest key: ``(tools, prefixes)``.
+
+ Desktop lets a plugin's desktop half render media only from these tools, so the answer comes
+ from the loader's records, never from a manifest claim. Native: names the plugin registered,
+ minus built-in tool names (an approved override of ``text_to_speech`` is still the core voice
+ reply to the user) and minus any name two plugins list. Portable: one ``mcp__<server>__``
+ prefix per MCP server the loader attributed to the package (duplicate server names are refused
+ at load, and a clamped long tool name keeps its prefix)."""
+ plugins_mod = _tools_mod("hermes_cli.plugins")
+ toolsets = _tools_mod("toolsets")
+ schema = _tools_mod("tools.mcp_tool_schema")
+ plugins_mod.discover_plugins()
+ manager = plugins_mod.get_plugin_manager()
+ builtin = set(toolsets._HERMES_CORE_TOOLS)
+ for entry in toolsets.TOOLSETS.values():
+ builtin.update(entry.get("tools", ()))
+ listed = {key: set(loaded.tools_registered) - builtin for key, loaded in manager._plugins.items()}
+ seen: dict[str, int] = {}
+ for names in listed.values():
+ for name in names:
+ seen[name] = seen.get(name, 0) + 1
+ tools = {key: sorted(n for n in names if seen[n] == 1) for key, names in listed.items()}
+ prefixes: dict[str, list[str]] = {}
+ for server, owner in manager.get_portable_mcp_server_plugins().items():
+ prefix = f"{schema.MCP_TOOL_NAME_PREFIX}{schema.sanitize_mcp_name_component(server)}__"
+ prefixes.setdefault(owner, []).append(prefix)
+ return tools, {key: sorted(values) for key, values in prefixes.items()}
+
+
def register(server) -> None:
bind_module(globals(), server)
diff --git a/website/docs/developer-guide/desktop-plugin-sdk.md b/website/docs/developer-guide/desktop-plugin-sdk.md
index 4e83feb369c..1f2d10fae44 100644
--- a/website/docs/developer-guide/desktop-plugin-sdk.md
+++ b/website/docs/developer-guide/desktop-plugin-sdk.md
@@ -251,6 +251,7 @@ Import the area constants from the SDK; each area has its own `data` payload.
| Model menu rows | `MODEL_MENU_ROW_AREA` | `data: ModelMenuRowContribution` — a leading icon / trailing badge per model |
| Kanban card badges | `KANBAN_CARD_AREAS.badges` | `data: KanbanCardBadgeContribution` — short text labels per card, painted by the board |
| Appearance settings | `APPEARANCE_AREAS.extra` | `render` — controls appended to Settings → Appearance |
+| Tool media | `TOOL_MEDIA_RENDERER_AREA` (`'transcript.toolMedia'`) | `data: { tools, render }` — draw the `MEDIA:` files your package's own tools produced ([Tool media renderers](#tool-media-renderers)) |
| Plugin settings page | `SETTINGS_PLUGINS_AREA` (`'settings.plugins'`) | Use `ctx.registerSettingsPage({ id, title, render, icon?, order?, children? })` — your own entry (with sub-pages) under Settings → Plugins |
### Panes
@@ -1160,6 +1161,90 @@ prompt reaches the chat's composer, or `{ ok: false, error }` where `error` is
or `undelivered` (no visible composer took it). Check it before showing a
widget as saved.
+### Tool media renderers — draw the files your own tools produce {#tool-media-renderers}
+
+A unified package (agent tools + a desktop half) can decide how Desktop shows
+the media **its own tools** produce. When an assistant reply delivers a file
+with a `MEDIA:` line, Desktop looks for the tool call in that turn whose result
+produced the path. If the plugin that owns that tool registered a renderer for
+the tool's name, your component replaces the built-in player:
+
+```javascript
+import * as sdk from '@hermes/plugin-sdk'
+import { useEffect, useState } from 'react'
+import { jsx, jsxs } from 'react/jsx-runtime'
+
+export default {
+ id: 'sound-fx', // the desktop half of the `sound-fx` package
+ register(ctx) {
+ // Older Desktops don't export the area: keep your old path there.
+ if (typeof sdk.TOOL_MEDIA_RENDERER_AREA !== 'string') return
+
+ ctx.register({
+ id: 'sound-card',
+ area: sdk.TOOL_MEDIA_RENDERER_AREA,
+ data: {
+ tools: ['sound_fx_generate'], // a tool this package's plugin.yaml registers
+ render: props => jsx(SoundCard, props)
+ }
+ })
+ }
+}
+
+function SoundCard({ file, args, result, sessionId, fallback }) {
+ const [src, setSrc] = useState(null)
+ useEffect(() => {
+ let live = true
+ file.resolveUrl().then(url => live && setSrc(url), () => live && setSrc(''))
+ return () => { live = false }
+ }, [file])
+
+ if (file.kind !== 'audio' || src === '') return fallback // the built-in player
+ return jsxs('figure', {
+ children: [
+ jsx('figcaption', { children: args.prompt ?? file.name }),
+ src && jsx('audio', { controls: true, src })
+ ]
+ })
+}
+```
+
+`render` receives:
+
+| Prop | What it is |
+|------|------------|
+| `file` | `{ path, name, kind, mime, resolveUrl() }`. `path` is the path on the **backend's** machine exactly as the `MEDIA:` line wrote it; never open it yourself. `resolveUrl()` returns a URL this window can load with a local or a remote backend: a range-streaming route for audio and video, a data URL for anything else (capped by the host's file reader). A large file (a 3D model) is better served by your own `plugin_api.py` route, called with `ctx.rest` and the `path`. |
+| `toolName`, `toolCallId` | The producing call. |
+| `args`, `result` | The call's arguments and result payload (the result as the tool returned it: an object, or a string). |
+| `sessionId` | The chat whose transcript shows the file (same meaning as a directive's `sessionId`), or `null`. |
+| `fallback` | The built-in player for this file. Return it for anything you don't draw. |
+
+What the host guarantees:
+
+- **Ownership is provenance, never file type.** A claim counts only for tools
+ your package's own code registered: native tools from `ctx.register_tool` in
+ the same package's `plugin.yaml` plugin, or the tools of the MCP servers your
+ portable package declares (claim those with a trailing `*`, e.g.
+ `mcp__meshy__*`). The backend reports this per package (`plugins.manage
+ list` → `tools` / `tool_prefixes`), so a claim on a core tool
+ (`text_to_speech`), another plugin's tool, or from a standalone desktop
+ plugin with no agent half is refused with a console warning.
+- **Everything else keeps the built-in player:** core voice replies, files
+ from other tools, user uploads, and any path no tool in that turn produced.
+ The earliest call whose result names the path is the producer, so a later
+ `ls` that merely prints it does not count.
+- **Fail-safe:** a renderer that throws, a plugin that is disabled, a backend
+ too old to report ownership, or a tool whose agent half is not loaded all
+ fall back to the built-in player.
+- **Other surfaces are unchanged.** Tools keep emitting plain `MEDIA:` lines;
+ the CLI, the TUI and messaging platforms deliver the file as before.
+
+Before this area existed, a plugin had to make the model emit its own
+`::directive` to get a custom card, which needs a skill instruction and leaves
+non-Desktop users with a raw directive line. With a renderer, the tool returns
+the ordinary `MEDIA:` tag every surface understands and Desktop alone upgrades
+it.
+
### Mount-scoped chrome (`Contribute`)
`ctx.register` is for **permanent** contributions. When chrome should live and