LLM inference in C/C++
C++ ggml
ui: add read_media tool (#25877)
parabelboi4dd127584b87
diff --git a/tools/server/server-tools.cpp b/tools/server/server-tools.cppindex a4c1059fff..fd0ff8ddd7 100644--- a/tools/server/server-tools.cpp+++ b/tools/server/server-tools.cpp@@ -1,6 +1,7 @@#include "server-tools.h"#include "subproc.h"+#include "base64.hpp"#include <filesystem>#include <fstream>@@ -864,6 +865,7 @@ static bool path_glob_match(const std::string & pattern, const std::string & rel//static constexpr size_t SERVER_TOOL_READ_FILE_MAX_SIZE = 16 * 1024; // 16 KB+static constexpr size_t SERVER_TOOL_READ_FILE_MAX_SIZE_BASE64 = 32 * 1024 * 1024; // 32 MBstruct server_tool_read_file : server_tool {server_tool_read_file() {@@ -899,6 +901,8 @@ struct server_tool_read_file : server_tool {int start_line = json_value(params, "start_line", 1);int end_line = json_value(params, "end_line", -1); // -1 = no limitbool append_loc = json_value(params, "append_loc", false);+ // comes from the x-resp-type header, the model cannot ask for it+ bool as_base64 = json_value(params, "resp_type", std::string()) == "base64";auto io = make_tools_io(params);@@ -906,6 +910,23 @@ struct server_tool_read_file : server_tool {if (!io->file_size(path, file_size)) {return {{"error", "cannot stat file: " + path}};}++ if (as_base64) {+ if (file_size > SERVER_TOOL_READ_FILE_MAX_SIZE_BASE64) {+ return {{"error", string_format(+ "file too large (%zu bytes, max %zu)",+ (size_t)file_size, SERVER_TOOL_READ_FILE_MAX_SIZE_BASE64)}};+ }+ std::string content;+ if (!io->read_file(path, content)) {+ return {{"error", "failed to open file: " + path}};+ }+ return {+ {"base64", base64::encode(content.data(), content.size())},+ {"size_bytes", (size_t) content.size()},+ };+ }+if (file_size > SERVER_TOOL_READ_FILE_MAX_SIZE && end_line == -1) {return {{"error", string_format("file too large (%zu bytes, max %zu). Use start_line/end_line to read a portion.",@@ -2135,6 +2156,15 @@ void server_tools::setup(const std::vector<std::string> & enabled_tools,params["runtime"] = runtime->spec();}+ // x-resp-type header is only used by read_file for now+ if (params.contains("resp_type")) {+ params.erase("resp_type");+ }+ auto resp_type = get_header(req.headers, "x-resp-type");+ if (!resp_type.empty()) {+ params["resp_type"] = resp_type;+ }+server_tool & tool = find_tool(tools, tool_name, stream);if (stream) {diff --git a/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/ChatMessageToolCallBlock.svelte b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/ChatMessageToolCallBlock.svelteindex 5bbaa4ea31..7a00f1a8ea 100644--- a/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/ChatMessageToolCallBlock.svelte+++ b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/ChatMessageToolCallBlock.svelte@@ -7,6 +7,7 @@import ChatMessageToolCallBlockGetInfo from './ChatMessageToolCallBlockGetInfo.svelte';import ChatMessageToolCallBlockGrepSearch from './ChatMessageToolCallBlockGrepSearch.svelte';import ChatMessageToolCallBlockReadFile from './ChatMessageToolCallBlockReadFile.svelte';+ import ChatMessageToolCallBlockReadMedia from './ChatMessageToolCallBlockReadMedia.svelte';import ChatMessageToolCallBlockRunJavascript from './ChatMessageToolCallBlockRunJavascript.svelte';import ChatMessageToolCallBlockSearchResults from './ChatMessageToolCallBlockSearchResults.svelte';import ChatMessageToolCallBlockWriteFile from './ChatMessageToolCallBlockWriteFile.svelte';@@ -45,6 +46,8 @@<ChatMessageToolCallBlockGetInfo {section} {isStreaming} />{:else if section.toolName === BuiltInTool.READ_FILE}<ChatMessageToolCallBlockReadFile {section} {open} {isStreaming} {onToggle} />+{:else if section.toolName === BuiltInTool.READ_MEDIA}+ <ChatMessageToolCallBlockReadMedia {section} {open} {isStreaming} {onToggle} />{:else if section.toolName === BuiltInTool.EDIT_FILE}<ChatMessageToolCallBlockEditFile {section} {open} {isStreaming} {onToggle} />{:else if section.toolName === BuiltInTool.WRITE_FILE}diff --git a/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/ChatMessageToolCallBlockDefault.svelte b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/ChatMessageToolCallBlockDefault.svelteindex 34dfde78b8..87208e2daa 100644--- a/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/ChatMessageToolCallBlockDefault.svelte+++ b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/ChatMessageToolCallBlockDefault.svelte@@ -8,14 +8,16 @@import { MarkdownContent, SyntaxHighlightedCode } from '$lib/components/app';import { MAX_HEIGHT_CODE_BLOCK } from '$lib/constants';import { getBuiltinToolUi } from '$lib/constants/built-in-tools';- import { FileTypeText, ToolResultKind } from '$lib/enums';+ import { AttachmentType, FileTypeText, MimeTypeAudio, ToolResultKind } from '$lib/enums';import type { DatabaseMessageExtra } from '$lib/types';import {type AgenticSection,classifyToolResult,formatJsonPretty,- parseToolResultWithImages+ parseToolResultWithMedia,+ type ToolResultLine} from '$lib/utils';+ import { createBase64DataUrl } from '$lib/utils/data-url';interface Props {section: AgenticSection;@@ -29,8 +31,8 @@const title = $derived(getBuiltinToolUi(section.toolName)?.label ?? section.toolName ?? '');const outputKind = $derived(classifyToolResult(section.toolResult));- const parsedLines = $derived(- section.toolResult ? parseToolResultWithImages(section.toolResult, attachments) : []+ const parsedLines: ToolResultLine[] = $derived(+ section.toolResult ? parseToolResultWithMedia(section.toolResult, attachments) : []);</script>@@ -103,13 +105,26 @@<div class="font-mono text-[11px] leading-relaxed whitespace-pre-wrap">{line.text}</div>- {#if line.image}- <img- src={line.image.base64Url}- alt={line.image.name}- class="mt-2 mb-2 h-auto max-w-full rounded-lg"- loading="lazy"- />+ {#if line.media}+ {#if line.media.type === AttachmentType.AUDIO}+ {@const audioMimeType = line.media.mimeType ?? MimeTypeAudio.MP3_MPEG}+ <div class="mt-2 mb-2">+ <audio controls class="w-full rounded-lg">+ <source+ src={createBase64DataUrl(audioMimeType, line.media.base64Data)}+ type={audioMimeType}+ />+ Your browser does not support the audio element.+ </audio>+ </div>+ {:else}+ <img+ src={line.media.base64Url}+ alt={line.media.name}+ class="mt-2 mb-2 h-auto max-w-full rounded-lg"+ loading="lazy"+ />+ {/if}{/if}{/each}</div>diff --git a/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/ChatMessageToolCallBlockExecShellCommand.svelte b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/ChatMessageToolCallBlockExecShellCommand.svelteindex b2fa8b331c..907a3cd12a 100644--- a/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/ChatMessageToolCallBlockExecShellCommand.svelte+++ b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/ChatMessageToolCallBlockExecShellCommand.svelte@@ -23,7 +23,7 @@isExitCodeSummaryLine,parseExecShellCommandError,parseExecShellCommandExitStatus,- parseToolResultWithImages,+ parseToolResultWithMedia,type ToolResultLine} from '$lib/utils';@@ -53,7 +53,7 @@);const parsedLines: ToolResultLine[] = $derived(- section.toolResult ? parseToolResultWithImages(section.toolResult, attachments) : []+ section.toolResult ? parseToolResultWithMedia(section.toolResult, attachments) : []);// Drop the trailing "[exit code: N]" line - rendered as a colored@@ -223,10 +223,10 @@>{#each outputLines as line, i (i)}<div class="font-mono text-[11px] leading-relaxed whitespace-pre-wrap">{line.text}</div>- {#if line.image}+ {#if line.media}<img- src={line.image.base64Url}- alt={line.image.name}+ src={line.media.base64Url}+ alt={line.media.name}class="mt-2 mb-2 h-auto max-w-full rounded-lg"loading="lazy"/>diff --git a/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/ChatMessageToolCallBlockReadMedia.svelte b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/ChatMessageToolCallBlockReadMedia.sveltenew file mode 100644index 0000000000..424c794d0b--- /dev/null+++ b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/ChatMessageToolCallBlockReadMedia.svelte@@ -0,0 +1,99 @@+<script lang="ts">+ import { parseReadMediaMeta } from './parsers/read-media';+ import ToolCallBlock from './ToolCallBlock.svelte';+ import { ATTACHMENT_SAVED_REGEX } from '$lib/constants/agentic';+ import { AttachmentType, MimeTypeAudio } from '$lib/enums';+ import type { DatabaseMessageExtraAudioFile, DatabaseMessageExtraImageFile } from '$lib/types';+ import { type AgenticSection } from '$lib/utils';+ import { createBase64DataUrl } from '$lib/utils/data-url';++ interface Props {+ section: AgenticSection;+ open: boolean;+ isStreaming: boolean;+ onToggle?: () => void;+ }++ let { isStreaming, onToggle, open, section }: Props = $props();++ const readMediaMeta = $derived(parseReadMediaMeta(section));++ // extractBase64Attachments swapped the data URI line for [Attachment saved: name]+ // and moved the bytes to the message extras, so the name is the only link back+ const mediaAttachment = $derived.by(() => {+ const extras = section.toolResultExtras;++ if (!extras || extras.length === 0) return null;++ const match = section.toolResult?.match(ATTACHMENT_SAVED_REGEX);++ if (!match) return null;++ const attachmentName = match[1];++ return (+ extras.find(+ (e): e is DatabaseMessageExtraImageFile | DatabaseMessageExtraAudioFile =>+ (e.type === AttachmentType.IMAGE || e.type === AttachmentType.AUDIO) &&+ e.name === attachmentName+ ) ?? null+ );+ });++ const audioMimeType = $derived(readMediaMeta?.mimeType ?? MimeTypeAudio.MP3_MPEG);+</script>++<ToolCallBlock {section} {open} {isStreaming} meta={readMediaMeta} {onToggle}>+ {#snippet titleSnippet()}+ <span class="text-muted-foreground">Read media </span>+ <span class="font-mono">{readMediaMeta?.fileName}</span>+ {/snippet}++ {#snippet children(_meta, _ctx)}+ {#if section.toolResult}+ {#if !mediaAttachment}+ <div class="rounded bg-muted/20 p-2 text-xs text-muted-foreground/70 italic">+ Media attachment not found in message extras+ </div>+ {:else if mediaAttachment.type === AttachmentType.AUDIO}+ <div class="mt-2">+ <audio controls class="w-full rounded-lg">+ <source+ src={createBase64DataUrl(audioMimeType, mediaAttachment.base64Data)}+ type={audioMimeType}+ />+ Your browser does not support the audio element.+ </audio>+ </div>+ {:else}+ <div class="mt-2">+ <img+ src={mediaAttachment.base64Url}+ alt={readMediaMeta?.fileName ?? 'media'}+ class="max-h-[60vh] max-w-full rounded-lg object-contain shadow-lg"+ loading="lazy"+ />+ </div>+ {/if}++ {#if readMediaMeta?.sizeBytes || readMediaMeta?.mimeType}+ <div class="mt-2 flex gap-4 text-xs text-muted-foreground">+ {#if readMediaMeta?.sizeBytes}+ <span>Size: {readMediaMeta.sizeBytes} bytes</span>+ {/if}+ {#if readMediaMeta?.mimeType}+ <span>MIME: {readMediaMeta.mimeType}</span>+ {/if}+ </div>+ {/if}++ {#if readMediaMeta?.path}+ <div class="mt-1 font-mono text-xs text-muted-foreground/60">{readMediaMeta.path}</div>+ {/if}+ {:else}+ <div class="rounded bg-muted/20 p-2 text-xs text-muted-foreground/70 italic">+ Waiting for media data...+ </div>+ {/if}+ {/snippet}+</ToolCallBlock>diff --git a/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/parsers/read-media.ts b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/parsers/read-media.tsnew file mode 100644index 0000000000..ab1a4d6aa0--- /dev/null+++ b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/parsers/read-media.ts@@ -0,0 +1,56 @@+import { FILE_PATH_SEPARATOR_REGEX, NEWLINE } from '$lib/constants/code';+import {+ PREFIX_FILE,+ PREFIX_MIME,+ PREFIX_SIZE,+ READ_MEDIA_SIZE_REGEX+} from '$lib/constants/read-media';+import type { AgenticSection } from '$lib/utils';++export interface ReadMediaMeta {+ fileName: string;+ path: string;+ sizeBytes?: number;+ mimeType?: string;+}++/**+ * Parse read_media tool result to extract metadata.+ * Expected format (after extractBase64Attachments processing):+ * File: /path/to/file.png+ * Size: 12345 bytes+ * MIME: image/png+ * [Attachment saved: mcp-attachment-xxx.png]+ *+ * The data URI line is replaced by the attachment marker by+ * agenticStore.extractBase64Attachments before storage.+ */+export function parseReadMediaMeta(section: AgenticSection): ReadMediaMeta | null {+ if (!section.toolResult) return null;++ const lines = section.toolResult.split(NEWLINE);++ let fileName = '';+ let path = '';+ let sizeBytes: number | undefined;+ let mimeType: string | undefined;++ for (const line of lines) {+ const trimmed = line.trim();++ if (trimmed.startsWith(PREFIX_FILE)) {+ path = trimmed.slice(PREFIX_FILE.length).trim();+ fileName = path.split(FILE_PATH_SEPARATOR_REGEX).pop() ?? path;+ } else if (trimmed.startsWith(PREFIX_SIZE)) {+ const match = trimmed.match(READ_MEDIA_SIZE_REGEX);++ if (match) sizeBytes = Number(match[1]);+ } else if (trimmed.startsWith(PREFIX_MIME)) {+ mimeType = trimmed.slice(PREFIX_MIME.length).trim();+ }+ }++ if (!path) return null;++ return { fileName, mimeType, path, sizeBytes };+}diff --git a/tools/ui/src/lib/constants/built-in-tools.ts b/tools/ui/src/lib/constants/built-in-tools.tsindex 89bcbecb8a..5bd24ffe83 100644--- a/tools/ui/src/lib/constants/built-in-tools.ts+++ b/tools/ui/src/lib/constants/built-in-tools.ts@@ -10,6 +10,7 @@import {Braces,Clock,+ Eye,FilePen,FilePlus,FileSearch,@@ -47,6 +48,7 @@ export const BUILTIN_TOOL_UI: Readonly<Record<BuiltInTool, BuiltinToolUiEntry>>source: ToolSource.BUILTIN},[BuiltInTool.READ_FILE]: { icon: FileText, label: 'Read file', source: ToolSource.BUILTIN },+ [BuiltInTool.READ_MEDIA]: { icon: Eye, label: 'Read media', source: ToolSource.FRONTEND },[BuiltInTool.RUN_JAVASCRIPT]: {icon: Braces,label: 'Run JavaScript',diff --git a/tools/ui/src/lib/constants/code.ts b/tools/ui/src/lib/constants/code.tsindex e57e1e6ec5..4b41142001 100644--- a/tools/ui/src/lib/constants/code.ts+++ b/tools/ui/src/lib/constants/code.ts@@ -18,6 +18,9 @@ export const TRIM_TRAILING_PADDING_REGEX = /(?:\n[ \t]*)+$/;// `C:\foo\bar.txt`. Used wherever a parameter accepts a user-supplied path.export const FILE_PATH_SEPARATOR_REGEX = /[\\/]/;+// Separates a file name from its extension, e.g. the '.' in `cover.png`.+export const FILE_EXTENSION_SEPARATOR = '.';+// Matches the `text:` prefix that file-type identifiers use to denote a// plain-text language (e.g. `text:typescript`). Used by tool-call renderers// to recover the underlying highlight.js language.diff --git a/tools/ui/src/lib/constants/index.ts b/tools/ui/src/lib/constants/index.tsindex 09006cf1e9..357a33a624 100644--- a/tools/ui/src/lib/constants/index.ts+++ b/tools/ui/src/lib/constants/index.ts@@ -49,6 +49,7 @@ export * from './sse';export * from './precision';export * from './processing-info';export * from './pwa';+export * from './read-media';export * from './routes';export * from './sandbox';export * from './settings-keys';diff --git a/tools/ui/src/lib/constants/mcp-resource.ts b/tools/ui/src/lib/constants/mcp-resource.tsindex 0ef8d9ec13..c2639daa12 100644--- a/tools/ui/src/lib/constants/mcp-resource.ts+++ b/tools/ui/src/lib/constants/mcp-resource.ts@@ -1,4 +1,4 @@-import { MimeTypeImage } from '$lib/enums';+import { MimeTypeAudio, MimeTypeImage } from '$lib/enums';// File extension patterns for resource type detectionexport const IMAGE_FILE_EXTENSION_REGEX = /\.(png|jpg|jpeg|gif|svg|webp)$/i;@@ -27,6 +27,9 @@ export const MCP_RESOURCE_ATTACHMENT_ID_PREFIX = 'res';// Default file extension for unknown image typesexport const DEFAULT_IMAGE_EXTENSION = 'img';+// Default file extension for unknown audio types+export const DEFAULT_AUDIO_EXTENSION = 'mp3';+// Default filename for resource content downloadsexport const DEFAULT_RESOURCE_FILENAME = 'resource.txt';@@ -53,3 +56,18 @@ export const IMAGE_MIME_TO_EXTENSION: Record<string, string> = {[MimeTypeImage.PNG]: 'png',[MimeTypeImage.WEBP]: 'webp'} as const;++/**+ * Mapping from audio MIME types to file extensions.+ * Used for generating attachment filenames from MIME types.+ */+export const AUDIO_MIME_TO_EXTENSION: Record<string, string> = {+ [MimeTypeAudio.MP3]: 'mp3',+ [MimeTypeAudio.MP3_MPEG]: 'mp3',+ [MimeTypeAudio.VND_WAVE]: 'wav',+ [MimeTypeAudio.WAV]: 'wav',+ [MimeTypeAudio.WAVE]: 'wav',+ [MimeTypeAudio.X_PN_WAV]: 'wav',+ [MimeTypeAudio.X_WAV]: 'wav',+ [MimeTypeAudio.X_WAVE]: 'wav'+} as const;diff --git a/tools/ui/src/lib/constants/read-media.ts b/tools/ui/src/lib/constants/read-media.tsnew file mode 100644index 0000000000..525c5e9029--- /dev/null+++ b/tools/ui/src/lib/constants/read-media.ts@@ -0,0 +1,66 @@+import {+ BuiltInTool,+ JsonSchemaType,+ MimeTypeAudio,+ MimeTypeImage,+ ToolCallType+} from '$lib/enums';+import type { OpenAIToolDefinition } from '$lib/types';++export const READ_MEDIA_TOOL_NAME = BuiltInTool.READ_MEDIA;++// header lines of the tool result, parsed back by the read_media renderer+export const PREFIX_FILE = 'File: ';+export const PREFIX_SIZE = 'Size: ';+export const PREFIX_MIME = 'MIME: ';++/** Byte count of the `Size: ` header line, e.g. `Size: 12345 bytes` -> capture group 1 is `12345`. */+export const READ_MEDIA_SIZE_REGEX = new RegExp(`^${PREFIX_SIZE}\\s*(\\d+)\\s*bytes`);++/** Image extensions the tool accepts. The server decodes images with stb_image, which has no webp or tiff. */+export const READ_MEDIA_IMAGE_MIME: Record<string, string> = {+ gif: MimeTypeImage.GIF,+ jpeg: MimeTypeImage.JPEG,+ jpg: MimeTypeImage.JPEG,+ png: MimeTypeImage.PNG+} as const;++/** Audio extensions the tool accepts. The `input_audio` API only takes wav and mp3. */+export const READ_MEDIA_AUDIO_MIME: Record<string, string> = {+ mp3: MimeTypeAudio.MP3_MPEG,+ wav: MimeTypeAudio.WAV+} as const;++/**+ * Build the read_media tool definition for the modalities the active model has.+ * At least one of the two flags must be true, otherwise the tool is not offered+ * at all - a model that cannot see or hear has nothing to do with the bytes.+ */+export function buildReadMediaToolDefinition(+ supportsVision: boolean,+ supportsAudio: boolean+): OpenAIToolDefinition {+ const kinds: string[] = [];++ if (supportsVision) kinds.push(`images (${Object.keys(READ_MEDIA_IMAGE_MIME).join(', ')})`);++ if (supportsAudio) kinds.push(`audio (${Object.keys(READ_MEDIA_AUDIO_MIME).join(', ')})`);++ return {+ function: {+ description: `Read a media file and attach it to the conversation so it can be perceived directly. Supports ${kinds.join(' and ')}.`,+ name: READ_MEDIA_TOOL_NAME,+ parameters: {+ properties: {+ path: {+ description: 'Path to the media file',+ type: JsonSchemaType.STRING+ }+ },+ required: ['path'],+ type: JsonSchemaType.OBJECT+ }+ },+ type: ToolCallType.FUNCTION+ };+}diff --git a/tools/ui/src/lib/constants/tools.ts b/tools/ui/src/lib/constants/tools.tsindex 65f4457c96..a467cda2b9 100644--- a/tools/ui/src/lib/constants/tools.ts+++ b/tools/ui/src/lib/constants/tools.ts@@ -3,6 +3,12 @@ import { ToolSource } from '$lib/enums/tools.enums';/** HTTP header carrying the working directory a tool call runs in. The server resolves relative paths against it; the model cannot override it. */export const X_TOOL_CWD_HEADER = 'x-tool-cwd';+/** HTTP header asking the server to encode a tool's output differently, e.g. read_file returning base64. Not a tool parameter, so it stays out of the definition the model sees. */+export const X_RESP_TYPE_HEADER = 'x-resp-type';++/** `X_RESP_TYPE_HEADER` value that makes read_file return the raw bytes as base64 instead of text. */+export const RESP_TYPE_BASE64 = 'base64';+export const TOOL_GROUP_LABELS = {[ToolSource.BUILTIN]: 'Built-in',[ToolSource.CUSTOM]: 'JSON Schema',diff --git a/tools/ui/src/lib/enums/files.enums.ts b/tools/ui/src/lib/enums/files.enums.tsindex eecb36c23e..5785428cf0 100644--- a/tools/ui/src/lib/enums/files.enums.ts+++ b/tools/ui/src/lib/enums/files.enums.ts@@ -163,6 +163,7 @@ export enum FileExtensionText {// MIME type prefixes and includes for content detectionexport enum MimeTypePrefix {IMAGE = 'image/',+ AUDIO = 'audio/',TEXT = 'text'}diff --git a/tools/ui/src/lib/enums/tools.enums.ts b/tools/ui/src/lib/enums/tools.enums.tsindex 7a9751ee79..31c992fef4 100644--- a/tools/ui/src/lib/enums/tools.enums.ts+++ b/tools/ui/src/lib/enums/tools.enums.ts@@ -37,6 +37,7 @@ export enum GlobSearchType {*/export enum BuiltInTool {READ_FILE = 'read_file',+ READ_MEDIA = 'read_media',EDIT_FILE = 'edit_file',WRITE_FILE = 'write_file',GET_DATETIME = 'get_datetime',diff --git a/tools/ui/src/lib/services/chat.service.ts b/tools/ui/src/lib/services/chat.service.tsindex 775a5d6e06..540d742393 100644--- a/tools/ui/src/lib/services/chat.service.ts+++ b/tools/ui/src/lib/services/chat.service.ts@@ -1,4 +1,5 @@import { settingsStore } from '../stores/settings.svelte';+import { getAudioInputFormat } from '../utils/audio-format';import { capImageDataURLSize } from '../utils/cap-img-size';import {API_CHAT,@@ -20,18 +21,12 @@ import {import {AttachmentType,ContentPartType,- FileTypeAudio,MessageRole,- MimeTypeAudio,ReasoningFormat,StreamConnectionState} from '$lib/enums';import { modelsStore } from '$lib/stores/models.svelte';-import type {- AudioInputFormat,- DatabaseMessageExtraMcpPrompt,- DatabaseMessageExtraMcpResource-} from '$lib/types';+import type { DatabaseMessageExtraMcpPrompt, DatabaseMessageExtraMcpResource } from '$lib/types';import type {ApiChatCompletionToolCall,ApiChatMessageContentPart,@@ -43,23 +38,6 @@ import { getAuthHeaders, getJsonHeaders } from '$lib/utils/api-headers';import { formatAttachmentText } from '$lib/utils/formatters';import { streamIdentity } from '$lib/utils/stream-identity';-function getAudioInputFormat(mimeType: string): AudioInputFormat {- const normalizedMimeType = mimeType.trim().toLowerCase();-- if (- normalizedMimeType === MimeTypeAudio.WAV ||- normalizedMimeType === MimeTypeAudio.WAVE ||- normalizedMimeType === MimeTypeAudio.X_WAV ||- normalizedMimeType === MimeTypeAudio.X_WAVE ||- normalizedMimeType === MimeTypeAudio.VND_WAVE ||- normalizedMimeType === MimeTypeAudio.X_PN_WAV- ) {- return FileTypeAudio.WAV;- }-- return FileTypeAudio.MP3;-}-interface ResumableStreamState {bytesReceived: number;updatedAt: number;diff --git a/tools/ui/src/lib/services/read-media.service.ts b/tools/ui/src/lib/services/read-media.service.tsnew file mode 100644index 0000000000..fd66350d00--- /dev/null+++ b/tools/ui/src/lib/services/read-media.service.ts@@ -0,0 +1,112 @@+import { ToolsService } from './tools.service';+import {+ FILE_EXTENSION_SEPARATOR,+ FILE_PATH_SEPARATOR_REGEX,+ NEWLINE,+ PREFIX_FILE,+ PREFIX_MIME,+ PREFIX_SIZE,+ READ_MEDIA_AUDIO_MIME,+ READ_MEDIA_IMAGE_MIME,+ RESP_TYPE_BASE64+} from '$lib/constants';+import { BuiltInTool, ToolResponseField } from '$lib/enums';+import type { ToolExecutionResult } from '$lib/types';++/** Modalities of the model the tool call runs for. */+export interface ReadMediaCapabilities {+ audio: boolean;+ vision: boolean;+}++/** Lowercase extension of a path, without the dot. Empty when the file name has none. */+function fileExtension(path: string): string {+ const name = path.split(FILE_PATH_SEPARATOR_REGEX).pop() ?? '';+ const dot = name.lastIndexOf(FILE_EXTENSION_SEPARATOR);++ return dot > 0 ? name.slice(dot + 1).toLowerCase() : '';+}++/**+ * **ReadMediaService** - frontend executor for the `read_media` tool+ *+ * The tool is synthetic: no such tool exists on the server. It reads the file+ * through the built-in `read_file` tool with the `base64` response type, then+ * turns the bytes into a data URI line. The agentic store lifts that line into+ * an image or audio attachment on the tool result message, which is what makes+ * the model perceive the file instead of reading a wall of base64.+ *+ * Living in the frontend is what lets it exist only for models that can+ * actually use the result - the server has no idea which model is selected.+ *+ * @see buildReadMediaToolDefinition in constants/read-media.ts - tool schema sent to the LLM+ * @see agenticStore in stores/agentic.svelte.ts - tool dispatch and attachment extraction+ */+export class ReadMediaService {+ static async executeTool(+ params: Record<string, unknown>,+ capabilities: ReadMediaCapabilities,+ signal?: AbortSignal,+ cwd?: string+ ): Promise<ToolExecutionResult> {+ const path = typeof params.path === 'string' ? params.path : '';++ if (!path) {+ return { content: 'Error: missing "path" argument.', isError: true };+ }++ const extension = fileExtension(path);+ const imageMime = READ_MEDIA_IMAGE_MIME[extension];+ const audioMime = READ_MEDIA_AUDIO_MIME[extension];++ let resolvedMime: string | undefined;++ if (imageMime && capabilities.vision) resolvedMime = imageMime;+ else if (audioMime && capabilities.audio) resolvedMime = audioMime;++ if (!resolvedMime) {+ const supported = [+ ...(capabilities.vision ? Object.keys(READ_MEDIA_IMAGE_MIME) : []),+ ...(capabilities.audio ? Object.keys(READ_MEDIA_AUDIO_MIME) : [])+ ];+ // an unreadable-by-this-model file is a dead end, so say why instead of failing silently+ const reason =+ imageMime || audioMime+ ? `the current model cannot perceive ".${extension}" files`+ : `".${extension}" is not a supported media type`;++ return {+ content: `Error: ${reason}. Supported: ${supported.join(', ')}.`,+ isError: true+ };+ }++ const raw = await ToolsService.executeToolRaw(+ BuiltInTool.READ_FILE,+ { path },+ signal,+ cwd,+ RESP_TYPE_BASE64+ );++ if (ToolResponseField.ERROR in raw) {+ return { content: String(raw[ToolResponseField.ERROR]), isError: true };+ }++ const base64 = typeof raw.base64 === 'string' ? raw.base64 : '';++ if (!base64) {+ return { content: `Error: no data returned for ${path}.`, isError: true };+ }++ const sizeBytes = typeof raw.size_bytes === 'number' ? raw.size_bytes : 0;+ const content = [+ `${PREFIX_FILE}${path}`,+ `${PREFIX_SIZE}${sizeBytes} bytes`,+ `${PREFIX_MIME}${resolvedMime}`,+ `data:${resolvedMime};base64,${base64}`+ ].join(NEWLINE);++ return { content, isError: false };+ }+}diff --git a/tools/ui/src/lib/services/tools.service.ts b/tools/ui/src/lib/services/tools.service.tsindex cd6b12cebe..9cb5ecb46d 100644--- a/tools/ui/src/lib/services/tools.service.ts+++ b/tools/ui/src/lib/services/tools.service.ts@@ -1,5 +1,5 @@import { base } from '$app/paths';-import { API_TOOLS, X_TOOL_CWD_HEADER } from '$lib/constants';+import { API_TOOLS, X_RESP_TYPE_HEADER, X_TOOL_CWD_HEADER } from '$lib/constants';import { ToolResponseField } from '$lib/enums';import type { ServerBuiltinToolInfo, ToolExecutionResult } from '$lib/types';import { apiFetch } from '$lib/utils';@@ -51,16 +51,26 @@ export class ToolsService {* Execute a built-in tool and return the raw JSON response. Unlike* executeTool, this preserves structured fields (e.g. file_glob_search's* `entries` and `base`) that the flattened ToolExecutionResult drops.+ *+ * @param respType - sent as the x-resp-type request header. Only read_file+ * honors it, with `base64` to get the raw bytes instead of decoded text.*/static async executeToolRaw(toolName: string,params: Record<string, unknown>,signal?: AbortSignal,- cwd?: string+ cwd?: string,+ respType?: string): Promise<Record<string, unknown>> {+ const headers: Record<string, string> = {};++ if (cwd) headers[X_TOOL_CWD_HEADER] = cwd;++ if (respType) headers[X_RESP_TYPE_HEADER] = respType;+return apiFetch<Record<string, unknown>>(API_TOOLS.EXECUTE, {body: JSON.stringify({ params, tool: toolName }),- headers: cwd ? { [X_TOOL_CWD_HEADER]: cwd } : undefined,+ headers: Object.keys(headers).length > 0 ? headers : undefined,method: 'POST',signal});diff --git a/tools/ui/src/lib/stores/agentic.svelte.ts b/tools/ui/src/lib/stores/agentic.svelte.tsindex 080ed7c89a..a84fa34dc2 100644--- a/tools/ui/src/lib/stores/agentic.svelte.ts+++ b/tools/ui/src/lib/stores/agentic.svelte.ts@@ -22,7 +22,9 @@import { DEFAULT_AGENTIC_CONFIG, NEWLINE } from '$lib/constants';import {+ AUDIO_MIME_TO_EXTENSION,DATA_URI_BASE64_REGEX,+ DEFAULT_AUDIO_EXTENSION,DEFAULT_IMAGE_EXTENSION,IMAGE_MIME_TO_EXTENSION,MCP_ATTACHMENT_NAME_PREFIX@@ -36,6 +38,7 @@ import {ToolCallType} from '$lib/enums';import { ChatService } from '$lib/services';+import { ReadMediaService } from '$lib/services/read-media.service';import { SandboxService } from '$lib/services/sandbox.service';import { ToolsService } from '$lib/services/tools.service';import { conversationsStore } from '$lib/stores/conversations.svelte';@@ -75,9 +78,10 @@ import type {import type {DatabaseMessage,DatabaseMessageExtra,+ DatabaseMessageExtraAudioFile,DatabaseMessageExtraImageFile} from '$lib/types/database';-import { isAbortError } from '$lib/utils';+import { getAudioInputFormat, isAbortError } from '$lib/utils';import { SvelteMap } from 'svelte/reactivity';function createDefaultSession(): AgenticSession {@@ -900,7 +904,18 @@ class AgenticStore {if (executionResult.isError) toolSuccess = false;} else if (toolSource === ToolSource.FRONTEND) {const args = this.parseToolArguments(toolCall.function.arguments);- const executionResult = await SandboxService.executeTool(toolName, args, signal);+ const executionResult =+ toolName === BuiltInTool.READ_MEDIA+ ? await ReadMediaService.executeTool(+ args,+ {+ audio: modelsStore.modelSupportsAudio(effectiveModel),+ vision: modelsStore.modelSupportsVision(effectiveModel)+ },+ signal,+ conversationsStore.activeConversation?.cwd+ )+ : await SandboxService.executeTool(toolName, args, signal);result = executionResult.content;@@ -990,7 +1005,19 @@ class AgenticStore {];for (const attachment of attachments) {- if (attachment.type === AttachmentType.IMAGE) {+ if (attachment.type === AttachmentType.AUDIO) {+ if (modelsStore.modelSupportsAudio(effectiveModel)) {+ contentParts.push({+ input_audio: {+ data: (attachment as DatabaseMessageExtraAudioFile).base64Data,+ format: getAudioInputFormat(+ (attachment as DatabaseMessageExtraAudioFile).mimeType+ )+ },+ type: ContentPartType.INPUT_AUDIO+ });+ }+ } else if (attachment.type === AttachmentType.IMAGE) {if (modelsStore.modelSupportsVision(effectiveModel)) {contentParts.push({image_url: {@@ -1101,6 +1128,18 @@ class AgenticStore {return `[Attachment saved: ${name}]`;}+ if (mimeType.startsWith(MimeTypePrefix.AUDIO)) {+ // audio extras hold the bare base64, the input_audio part has no room for a data URI+ attachments.push({+ base64Data,+ mimeType,+ name,+ type: AttachmentType.AUDIO+ });++ return `[Attachment saved: ${name}]`;+ }+return line;});@@ -1108,7 +1147,9 @@ class AgenticStore {}private buildAttachmentName(mimeType: string, index: number): string {- const extension = IMAGE_MIME_TO_EXTENSION[mimeType] ?? DEFAULT_IMAGE_EXTENSION;+ const extension = mimeType.startsWith(MimeTypePrefix.AUDIO)+ ? (AUDIO_MIME_TO_EXTENSION[mimeType] ?? DEFAULT_AUDIO_EXTENSION)+ : (IMAGE_MIME_TO_EXTENSION[mimeType] ?? DEFAULT_IMAGE_EXTENSION);return `${MCP_ATTACHMENT_NAME_PREFIX}-${Date.now()}-${index}.${extension}`;}diff --git a/tools/ui/src/lib/stores/models.svelte.ts b/tools/ui/src/lib/stores/models.svelte.tsindex 145de119ce..150fb3800f 100644--- a/tools/ui/src/lib/stores/models.svelte.ts+++ b/tools/ui/src/lib/stores/models.svelte.ts@@ -18,7 +18,10 @@ import { ModelsService } from '$lib/services/models.service';import { PropsService } from '$lib/services/props.service';import { conversationsStore } from '$lib/stores/conversations.svelte';import { isRouterMode, serverStore } from '$lib/stores/server.svelte';-import { getAuthHeaders, TTLCache } from '$lib/utils';+// deep imports, not the '$lib/utils' barrel: it re-exports modules that reach back+// into the stores, and going through it here would read a half-built module+import { getAuthHeaders } from '$lib/utils/api-headers';+import { TTLCache } from '$lib/utils/cache-ttl';import {detectThinkingSupport,detectThinkingSupportWithReasondiff --git a/tools/ui/src/lib/stores/tools.svelte.ts b/tools/ui/src/lib/stores/tools.svelte.tsindex b1a946391a..3984dd2bc8 100644--- a/tools/ui/src/lib/stores/tools.svelte.ts+++ b/tools/ui/src/lib/stores/tools.svelte.ts@@ -1,4 +1,5 @@import {+ buildReadMediaToolDefinition,buildSandboxToolDefinition,DISABLED_TOOL_KEYS_LOCALSTORAGE_KEY,HOME_TILDE,@@ -15,6 +16,7 @@ import {} from '$lib/enums';import { ToolsService } from '$lib/services/tools.service';import { mcpStore } from '$lib/stores/mcp.svelte';+import { modelsStore, selectedModelName } from '$lib/stores/models.svelte';import { config } from '$lib/stores/settings.svelte';import type { OpenAIToolDefinition, ToolEntry, ToolGroup } from '$lib/types';import { SvelteMap, SvelteSet } from 'svelte/reactivity';@@ -168,9 +170,42 @@ class ToolsStore {}get frontendTools(): OpenAIToolDefinition[] {- return config().jsSandboxEnabled- ? [buildSandboxToolDefinition(!!config().symbolicMathEnabled)]- : [];+ const tools: OpenAIToolDefinition[] = [];++ if (config().jsSandboxEnabled) {+ tools.push(buildSandboxToolDefinition(!!config().symbolicMathEnabled));+ }++ const readMedia = this.readMediaTool();++ if (readMedia) tools.push(readMedia);++ return tools;+ }++ /**+ * `read_media` runs in the frontend on top of the server's `read_file`, so it+ * exists only when that tool is served and the active model can perceive the+ * bytes. The server cannot make this call - it does not know which model the+ * conversation uses.+ */+ private readMediaTool(): OpenAIToolDefinition | null {+ const hasReadFile = this._builtinTools.some(+ (def) => def.function.name === BuiltInTool.READ_FILE+ );++ if (!hasReadFile) return null;++ const model = selectedModelName() ?? modelsStore.models[0]?.model ?? '';++ if (!model) return null;++ const vision = modelsStore.modelSupportsVision(model);+ const audio = modelsStore.modelSupportsAudio(model);++ if (!vision && !audio) return null;++ return buildReadMediaToolDefinition(vision, audio);}get customTools(): OpenAIToolDefinition[] {diff --git a/tools/ui/src/lib/utils/agentic.ts b/tools/ui/src/lib/utils/agentic.tsindex 9c92a42d40..b2dd2cd9ed 100644--- a/tools/ui/src/lib/utils/agentic.ts+++ b/tools/ui/src/lib/utils/agentic.ts@@ -50,11 +50,11 @@ export interface AgenticSection {}/**- * Represents a tool result line that may reference an image attachment+ * Represents a tool result line that may reference a media attachment (image or audio)*/export type ToolResultLine = {text: string;- image?: DatabaseMessageExtraImageFile;+ media?: DatabaseMessageExtraImageFile | DatabaseMessageExtraAudioFile;};/**@@ -301,16 +301,16 @@ export function splitSearchSummaryList(return { lines };}-/** Bounded cache for parseToolResultWithImages results. */+/** Bounded cache for parseToolResultWithMedia results. */const TOOL_RESULT_LINES_CACHE_MAX_SIZE = 32;const toolResultLinesCache = new Map<string, ToolResultLine[]>();/**- * Parse tool result text into lines, matching image attachments by name.+ * Parse tool result text into lines, matching media attachments (images and audio) by name.* Memoized: called per render during streaming on unchanged tool result* strings with unchanged extras.*/-export function parseToolResultWithImages(+export function parseToolResultWithMedia(toolResult: string,extras?: DatabaseMessageExtra[]): ToolResultLine[] {@@ -332,12 +332,13 @@ export function parseToolResultWithImages(if (!match || !extras) return { text: line };const attachmentName = match[1];- const image = extras.find(- (e): e is DatabaseMessageExtraImageFile =>- e.type === AttachmentType.IMAGE && e.name === attachmentName+ const media = extras.find(+ (e): e is DatabaseMessageExtraImageFile | DatabaseMessageExtraAudioFile =>+ (e.type === AttachmentType.IMAGE || e.type === AttachmentType.AUDIO) &&+ e.name === attachmentName);- return { image, text: line };+ return { media, text: line };});if (toolResultLinesCache.size >= TOOL_RESULT_LINES_CACHE_MAX_SIZE) {diff --git a/tools/ui/src/lib/utils/audio-format.ts b/tools/ui/src/lib/utils/audio-format.tsnew file mode 100644index 0000000000..4f597aad34--- /dev/null+++ b/tools/ui/src/lib/utils/audio-format.ts@@ -0,0 +1,22 @@+import { FileTypeAudio, MimeTypeAudio } from '$lib/enums';+import type { AudioInputFormat } from '$lib/types/api';++/**+ * Map a MIME type to the AudioInputFormat expected by the API.+ */+export function getAudioInputFormat(mimeType: string): AudioInputFormat {+ const normalizedMimeType = mimeType.trim().toLowerCase();++ if (+ normalizedMimeType === MimeTypeAudio.WAV ||+ normalizedMimeType === MimeTypeAudio.WAVE ||+ normalizedMimeType === MimeTypeAudio.X_WAV ||+ normalizedMimeType === MimeTypeAudio.X_WAVE ||+ normalizedMimeType === MimeTypeAudio.VND_WAVE ||+ normalizedMimeType === MimeTypeAudio.X_PN_WAV+ ) {+ return FileTypeAudio.WAV;+ }++ return FileTypeAudio.MP3;+}diff --git a/tools/ui/src/lib/utils/index.ts b/tools/ui/src/lib/utils/index.tsindex 5762e8e4bb..c3585bd5f8 100644--- a/tools/ui/src/lib/utils/index.ts+++ b/tools/ui/src/lib/utils/index.ts@@ -248,7 +248,7 @@ export {export {deriveAgenticSections,buildAssistantRawOutput,- parseToolResultWithImages,+ parseToolResultWithMedia,splitSearchSummaryList,hasAgenticContent,classifyToolResult,@@ -325,3 +325,6 @@ export { uuid } from './uuid';// CSS utilitiesexport { remToPx } from './css';++// Audio format helper (used by agentic store and chat service)+export { getAudioInputFormat } from './audio-format';diff --git a/tools/ui/tests/client/README-perf.md b/tools/ui/tests/client/README-perf.mdindex a4e3e4c644..198b219568 100644--- a/tools/ui/tests/client/README-perf.md+++ b/tools/ui/tests/client/README-perf.md@@ -33,12 +33,12 @@ npx vitest --project=client --run tests/client/agentic-stream.perf.svelte.test.tThe point of the harness is the _scaling curve_, not any single number.-| Knob | Reads on |-| --------------------------- | ---------------------------------------------------------------------------------------------------- |-| `priorToolCalls` (0/1/5/20) | the reactive fan-out. Flat => no fan-out. Linear => confirmed. |-| `toolResultBytes` | whole-blob string scans (`extractSearchResults`, `parseToolResultWithImages`, `classifyToolResult`). |-| `editFileEdits` | `computeLineDiff`, the O(m\*n) LCS. |-| `openCodeFence` | `hljs.highlightAuto` on partial code. |+| Knob | Reads on |+| --------------------------- | --------------------------------------------------------------------------------------------------- |+| `priorToolCalls` (0/1/5/20) | the reactive fan-out. Flat => no fan-out. Linear => confirmed. |+| `toolResultBytes` | whole-blob string scans (`extractSearchResults`, `parseToolResultWithMedia`, `classifyToolResult`). |+| `editFileEdits` | `computeLineDiff`, the O(m\*n) LCS. |+| `openCodeFence` | `hljs.highlightAuto` on partial code. |Deliberately no hard assertions: CI timing is noisy and the value here is thebefore/after delta, not a gate.diff --git a/tools/ui/tests/unit/agentic-hotpath.bench.ts b/tools/ui/tests/unit/agentic-hotpath.bench.tsindex 8bdaa095a8..24dcb48911 100644--- a/tools/ui/tests/unit/agentic-hotpath.bench.ts+++ b/tools/ui/tests/unit/agentic-hotpath.bench.ts@@ -6,7 +6,7 @@//// Run: npx vitest bench --project=unit tests/unit/agentic-hotpath.bench.ts-import { classifyToolResult, parseToolResultWithImages } from '$lib/utils/agentic';+import { classifyToolResult, parseToolResultWithMedia } from '$lib/utils/agentic';import { detectIncompleteCodeBlock, highlightCode } from '$lib/utils/code';import { computeLineDiff } from '$lib/utils/compute-line-diff';import { preprocessLaTeX } from '$lib/utils/latex-protection';@@ -200,17 +200,17 @@ describe('exit-code regex', () => {// --- per-line result parsers -----------------------------------------------describe('parseToolResultWithImages', () => {+describe('parseToolResultWithMedia', () => {bench('1KB', () => {- parseToolResultWithImages(SHELL_OUTPUT_1KB, []);+ parseToolResultWithMedia(SHELL_OUTPUT_1KB, []);});bench('200KB', () => {- parseToolResultWithImages(SHELL_OUTPUT_200KB, []);+ parseToolResultWithMedia(SHELL_OUTPUT_200KB, []);});bench('2MB', () => {- parseToolResultWithImages(SHELL_OUTPUT_2MB, []);+ parseToolResultWithMedia(SHELL_OUTPUT_2MB, []);});});