LLM inference in C/C++
C++ ggml
ui: add browser get_info tool (#27251)
Xuan-Son Nguyen7077abbe14c5
diff --git a/tools/ui/src/lib/constants/browser-info.ts b/tools/ui/src/lib/constants/browser-info.tsnew file mode 100644index 0000000000..86d999643a--- /dev/null+++ b/tools/ui/src/lib/constants/browser-info.ts@@ -0,0 +1,36 @@+import { CLI_FLAGS } from './cli-flags.constants';+import { BuiltInTool, JsonSchemaType, ToolCallType } from '$lib/enums';+import type { OpenAIToolDefinition } from '$lib/types';++export const BROWSER_INFO_TOOL_NAME = BuiltInTool.GET_INFO;++/** UA token to OS name, first match wins - Android and iOS UAs also carry the Linux / Mac OS X tokens */+export const BROWSER_INFO_OS_UA_PATTERNS: readonly [RegExp, string][] = [+ [/Windows NT/, 'Windows'],+ [/Android/, 'Android'],+ [/iPhone|iPad|iPod/, 'iOS'],+ [/CrOS/, 'ChromeOS'],+ [/Mac OS X/, 'macOS'],+ [/Linux/, 'Linux']+];++export const BROWSER_INFO_OS_UNKNOWN = 'unknown';++/** Sent to the model as the `note` field of the tool result, next to the OS name */+export const BROWSER_INFO_NOTE = `This environment is browser-only, it cannot read or modify local files, and it cannot run shell commands. To get local file access, tell user to launch llama-server with the ${CLI_FLAGS.AGENT} argument.`;++export function buildBrowserInfoToolDefinition(): OpenAIToolDefinition {+ return {+ function: {+ description:+ 'Get runtime info (OS name), may call when user asks about local files or shell commands',+ name: BROWSER_INFO_TOOL_NAME,+ parameters: {+ properties: {},+ required: [],+ type: JsonSchemaType.OBJECT+ }+ },+ type: ToolCallType.FUNCTION+ };+}diff --git a/tools/ui/src/lib/constants/cli-flags.constants.ts b/tools/ui/src/lib/constants/cli-flags.constants.tsindex 4fbee8a369..c4af2b6f46 100644--- a/tools/ui/src/lib/constants/cli-flags.constants.ts+++ b/tools/ui/src/lib/constants/cli-flags.constants.ts@@ -1,4 +1,5 @@export const CLI_FLAGS = {+ AGENT: '--agent',API_KEY: '--api-key',MCP_PROXY: '--ui-mcp-proxy',SLOTS: '--slots',diff --git a/tools/ui/src/lib/constants/index.ts b/tools/ui/src/lib/constants/index.tsindex 17e5f4d00a..cdd91a0738 100644--- a/tools/ui/src/lib/constants/index.ts+++ b/tools/ui/src/lib/constants/index.ts@@ -59,3 +59,4 @@ export * from './uri-template.constants';export * from './url.constants';export * from './working-directory.constants';export * from './read-media';+export * from './browser-info';diff --git a/tools/ui/src/lib/stores/agentic.svelte.ts b/tools/ui/src/lib/stores/agentic.svelte.tsindex 075c46c975..d6b57ffa80 100644--- a/tools/ui/src/lib/stores/agentic.svelte.ts+++ b/tools/ui/src/lib/stores/agentic.svelte.ts@@ -56,7 +56,8 @@ import type {AgenticSession,McpServerOverride,MCPToolCall,- SettingsConfigType+ SettingsConfigType,+ ToolExecutionResult} from '$lib/types';import type {AgenticFlowCallbacks,@@ -83,7 +84,7 @@ import type {DatabaseMessageExtraAudioFile,DatabaseMessageExtraImageFile} from '$lib/types/database';-import { getAudioInputFormat, isAbortError } from '$lib/utils';+import { executeBrowserInfoTool, getAudioInputFormat, isAbortError } from '$lib/utils';import { SvelteMap } from 'svelte/reactivity';function createDefaultSession(): AgenticSession {@@ -942,18 +943,24 @@ class AgenticStore {if (executionResult.isError) toolSuccess = false;} else if (toolSource === ToolSource.FRONTEND) {const args = this.parseToolArguments(toolCall.function.arguments);- const executionResult =- toolName === BuiltInTool.READ_MEDIA- ? await ReadMediaService.executeTool(- args,- {- audio: modelsStore.modelSupportsAudio(effectiveModel),- vision: modelsStore.modelSupportsVision(effectiveModel)- },- signal,- conversationsStore.activeConversation?.cwd- )- : await SandboxService.executeTool(toolName, args, signal);++ let executionResult: ToolExecutionResult;++ if (toolName === BuiltInTool.GET_INFO) {+ executionResult = executeBrowserInfoTool();+ } else if (toolName === BuiltInTool.READ_MEDIA) {+ executionResult = await ReadMediaService.executeTool(+ args,+ {+ audio: modelsStore.modelSupportsAudio(effectiveModel),+ vision: modelsStore.modelSupportsVision(effectiveModel)+ },+ signal,+ conversationsStore.activeConversation?.cwd+ );+ } else {+ executionResult = await SandboxService.executeTool(toolName, args, signal);+ }result = executionResult.content;diff --git a/tools/ui/src/lib/stores/tools.svelte.ts b/tools/ui/src/lib/stores/tools.svelte.tsindex 6426629288..fb3662025d 100644--- a/tools/ui/src/lib/stores/tools.svelte.ts+++ b/tools/ui/src/lib/stores/tools.svelte.ts@@ -1,5 +1,6 @@import { browser } from '$app/environment';import {+ buildBrowserInfoToolDefinition,buildReadMediaToolDefinition,DISABLED_TOOL_KEYS_LOCALSTORAGE_KEY,HOME_TILDE,@@ -185,9 +186,18 @@ class ToolsStore {if (readMedia) tools.push(readMedia);+ // provide browser's get_info tool if server doesn't provide one+ if (!this.hasBuiltinTool(BuiltInTool.GET_INFO)) {+ tools.push(buildBrowserInfoToolDefinition());+ }+return tools;}+ private hasBuiltinTool(name: BuiltInTool): boolean {+ return this._builtinTools.some((def) => def.function.name === name);+ }+/*** `read_media` runs in the frontend on top of the server's `read_file`, so it* exists only when that tool is served and the active model can perceive the@@ -195,11 +205,7 @@ class ToolsStore {* conversation uses.*/private readMediaTool(): OpenAIToolDefinition | null {- const hasReadFile = this._builtinTools.some(- (def) => def.function.name === BuiltInTool.READ_FILE- );-- if (!hasReadFile) return null;+ if (!this.hasBuiltinTool(BuiltInTool.READ_FILE)) return null;const model = modelsStore.selectedModelName ?? modelsStore.models[0]?.model ?? '';diff --git a/tools/ui/src/lib/utils/browser-info.ts b/tools/ui/src/lib/utils/browser-info.tsnew file mode 100644index 0000000000..c96abb01e0--- /dev/null+++ b/tools/ui/src/lib/utils/browser-info.ts@@ -0,0 +1,39 @@+/**+ * Browser fallback for the server's `get_info` tool, offered only when the+ * server does not serve one (llama-server without --agent). It tells the model+ * which OS the browser runs on and that there is no local file or shell access,+ * so it does not plan around tools that are not there.+ *+ * @see server_tool_get_info in tools/server/server-tools.cpp - the served variant+ * @see buildBrowserInfoToolDefinition in constants/browser-info.ts - tool schema sent to the LLM+ */++import { browser } from '$app/environment';+import {+ BROWSER_INFO_NOTE,+ BROWSER_INFO_OS_UA_PATTERNS,+ BROWSER_INFO_OS_UNKNOWN+} from '$lib/constants';+import type { ToolExecutionResult } from '$lib/types';++function detectOs(userAgent: string): string {+ for (const [pattern, os] of BROWSER_INFO_OS_UA_PATTERNS) {+ if (pattern.test(userAgent)) return os;+ }++ return BROWSER_INFO_OS_UNKNOWN;+}++/**+ * Result shape mirrors the server tool's JSON so the `get_info` renderer reads+ * `os` the same way, minus `cwd` - there is no working directory to report.+ */+export function executeBrowserInfoTool(): ToolExecutionResult {+ return {+ content: JSON.stringify({+ note: BROWSER_INFO_NOTE,+ os: browser ? detectOs(navigator.userAgent) : BROWSER_INFO_OS_UNKNOWN+ }),+ isError: false+ };+}diff --git a/tools/ui/src/lib/utils/index.ts b/tools/ui/src/lib/utils/index.tsindex 3fb701b323..d991363fc5 100644--- a/tools/ui/src/lib/utils/index.ts+++ b/tools/ui/src/lib/utils/index.ts@@ -331,6 +331,9 @@ export { getChatCommands } from './chat-commands';// SANDBOX_TOOL_DEFINITION is deprecated; kept for backward compatibility.export { buildSandboxToolDefinition, SANDBOX_TOOL_DEFINITION } from './sandbox-tool';+// Browser fallback for the server's get_info tool+export { executeBrowserInfoTool } from './browser-info';+// Cryptography utilitiesexport { uuid } from './uuid';