Skip to content

LLM inference in C/C++

C++ ggml

ui: add browser get_info tool (#27251)

Xuan-Son Nguyen7077abbe14c5
diff --git a/tools/ui/src/lib/constants/browser-info.ts b/tools/ui/src/lib/constants/browser-info.ts
new file mode 100644
index 0000000000..86d999643a
--- /dev/null
+++ b/tools/ui/src/lib/constants/browser-info.ts
@@ -0,0 +1,36 @@
+import { CLI_FLAGS } from './cli-flags.constants';
+import { BuiltInTool, JsonSchemaType, ToolCallType } from '$lib/enums';
+import type { OpenAIToolDefinition } from '$lib/types';
+
+export const BROWSER_INFO_TOOL_NAME = BuiltInTool.GET_INFO;
+
+/** UA token to OS name, first match wins - Android and iOS UAs also carry the Linux / Mac OS X tokens */
+export const BROWSER_INFO_OS_UA_PATTERNS: readonly [RegExp, string][] = [
+ [/Windows NT/, 'Windows'],
+ [/Android/, 'Android'],
+ [/iPhone|iPad|iPod/, 'iOS'],
+ [/CrOS/, 'ChromeOS'],
+ [/Mac OS X/, 'macOS'],
+ [/Linux/, 'Linux']
+];
+
+export const BROWSER_INFO_OS_UNKNOWN = 'unknown';
+
+/** Sent to the model as the `note` field of the tool result, next to the OS name */
+export const BROWSER_INFO_NOTE = `This environment is browser-only, it cannot read or modify local files, and it cannot run shell commands. To get local file access, tell user to launch llama-server with the ${CLI_FLAGS.AGENT} argument.`;
+
+export function buildBrowserInfoToolDefinition(): OpenAIToolDefinition {
+ return {
+ function: {
+ description:
+ 'Get runtime info (OS name), may call when user asks about local files or shell commands',
+ name: BROWSER_INFO_TOOL_NAME,
+ parameters: {
+ properties: {},
+ required: [],
+ type: JsonSchemaType.OBJECT
+ }
+ },
+ type: ToolCallType.FUNCTION
+ };
+}
diff --git a/tools/ui/src/lib/constants/cli-flags.constants.ts b/tools/ui/src/lib/constants/cli-flags.constants.ts
index 4fbee8a369..c4af2b6f46 100644
--- a/tools/ui/src/lib/constants/cli-flags.constants.ts
+++ b/tools/ui/src/lib/constants/cli-flags.constants.ts
@@ -1,4 +1,5 @@
export const CLI_FLAGS = {
+ AGENT: '--agent',
API_KEY: '--api-key',
MCP_PROXY: '--ui-mcp-proxy',
SLOTS: '--slots',
diff --git a/tools/ui/src/lib/constants/index.ts b/tools/ui/src/lib/constants/index.ts
index 17e5f4d00a..cdd91a0738 100644
--- a/tools/ui/src/lib/constants/index.ts
+++ b/tools/ui/src/lib/constants/index.ts
@@ -59,3 +59,4 @@ export * from './uri-template.constants';
export * from './url.constants';
export * from './working-directory.constants';
export * from './read-media';
+export * from './browser-info';
diff --git a/tools/ui/src/lib/stores/agentic.svelte.ts b/tools/ui/src/lib/stores/agentic.svelte.ts
index 075c46c975..d6b57ffa80 100644
--- a/tools/ui/src/lib/stores/agentic.svelte.ts
+++ b/tools/ui/src/lib/stores/agentic.svelte.ts
@@ -56,7 +56,8 @@ import type {
AgenticSession,
McpServerOverride,
MCPToolCall,
- SettingsConfigType
+ SettingsConfigType,
+ ToolExecutionResult
} from '$lib/types';
import type {
AgenticFlowCallbacks,
@@ -83,7 +84,7 @@ import type {
DatabaseMessageExtraAudioFile,
DatabaseMessageExtraImageFile
} from '$lib/types/database';
-import { getAudioInputFormat, isAbortError } from '$lib/utils';
+import { executeBrowserInfoTool, getAudioInputFormat, isAbortError } from '$lib/utils';
import { SvelteMap } from 'svelte/reactivity';
function createDefaultSession(): AgenticSession {
@@ -942,18 +943,24 @@ class AgenticStore {
if (executionResult.isError) toolSuccess = false;
} else if (toolSource === ToolSource.FRONTEND) {
const args = this.parseToolArguments(toolCall.function.arguments);
- const executionResult =
- toolName === BuiltInTool.READ_MEDIA
- ? await ReadMediaService.executeTool(
- args,
- {
- audio: modelsStore.modelSupportsAudio(effectiveModel),
- vision: modelsStore.modelSupportsVision(effectiveModel)
- },
- signal,
- conversationsStore.activeConversation?.cwd
- )
- : await SandboxService.executeTool(toolName, args, signal);
+
+ let executionResult: ToolExecutionResult;
+
+ if (toolName === BuiltInTool.GET_INFO) {
+ executionResult = executeBrowserInfoTool();
+ } else if (toolName === BuiltInTool.READ_MEDIA) {
+ executionResult = await ReadMediaService.executeTool(
+ args,
+ {
+ audio: modelsStore.modelSupportsAudio(effectiveModel),
+ vision: modelsStore.modelSupportsVision(effectiveModel)
+ },
+ signal,
+ conversationsStore.activeConversation?.cwd
+ );
+ } else {
+ executionResult = await SandboxService.executeTool(toolName, args, signal);
+ }
result = executionResult.content;
diff --git a/tools/ui/src/lib/stores/tools.svelte.ts b/tools/ui/src/lib/stores/tools.svelte.ts
index 6426629288..fb3662025d 100644
--- a/tools/ui/src/lib/stores/tools.svelte.ts
+++ b/tools/ui/src/lib/stores/tools.svelte.ts
@@ -1,5 +1,6 @@
import { browser } from '$app/environment';
import {
+ buildBrowserInfoToolDefinition,
buildReadMediaToolDefinition,
DISABLED_TOOL_KEYS_LOCALSTORAGE_KEY,
HOME_TILDE,
@@ -185,9 +186,18 @@ class ToolsStore {
if (readMedia) tools.push(readMedia);
+ // provide browser's get_info tool if server doesn't provide one
+ if (!this.hasBuiltinTool(BuiltInTool.GET_INFO)) {
+ tools.push(buildBrowserInfoToolDefinition());
+ }
+
return tools;
}
+ private hasBuiltinTool(name: BuiltInTool): boolean {
+ return this._builtinTools.some((def) => def.function.name === name);
+ }
+
/**
* `read_media` runs in the frontend on top of the server's `read_file`, so it
* exists only when that tool is served and the active model can perceive the
@@ -195,11 +205,7 @@ class ToolsStore {
* conversation uses.
*/
private readMediaTool(): OpenAIToolDefinition | null {
- const hasReadFile = this._builtinTools.some(
- (def) => def.function.name === BuiltInTool.READ_FILE
- );
-
- if (!hasReadFile) return null;
+ if (!this.hasBuiltinTool(BuiltInTool.READ_FILE)) return null;
const model = modelsStore.selectedModelName ?? modelsStore.models[0]?.model ?? '';
diff --git a/tools/ui/src/lib/utils/browser-info.ts b/tools/ui/src/lib/utils/browser-info.ts
new file mode 100644
index 0000000000..c96abb01e0
--- /dev/null
+++ b/tools/ui/src/lib/utils/browser-info.ts
@@ -0,0 +1,39 @@
+/**
+ * Browser fallback for the server's `get_info` tool, offered only when the
+ * server does not serve one (llama-server without --agent). It tells the model
+ * which OS the browser runs on and that there is no local file or shell access,
+ * so it does not plan around tools that are not there.
+ *
+ * @see server_tool_get_info in tools/server/server-tools.cpp - the served variant
+ * @see buildBrowserInfoToolDefinition in constants/browser-info.ts - tool schema sent to the LLM
+ */
+
+import { browser } from '$app/environment';
+import {
+ BROWSER_INFO_NOTE,
+ BROWSER_INFO_OS_UA_PATTERNS,
+ BROWSER_INFO_OS_UNKNOWN
+} from '$lib/constants';
+import type { ToolExecutionResult } from '$lib/types';
+
+function detectOs(userAgent: string): string {
+ for (const [pattern, os] of BROWSER_INFO_OS_UA_PATTERNS) {
+ if (pattern.test(userAgent)) return os;
+ }
+
+ return BROWSER_INFO_OS_UNKNOWN;
+}
+
+/**
+ * Result shape mirrors the server tool's JSON so the `get_info` renderer reads
+ * `os` the same way, minus `cwd` - there is no working directory to report.
+ */
+export function executeBrowserInfoTool(): ToolExecutionResult {
+ return {
+ content: JSON.stringify({
+ note: BROWSER_INFO_NOTE,
+ os: browser ? detectOs(navigator.userAgent) : BROWSER_INFO_OS_UNKNOWN
+ }),
+ isError: false
+ };
+}
diff --git a/tools/ui/src/lib/utils/index.ts b/tools/ui/src/lib/utils/index.ts
index 3fb701b323..d991363fc5 100644
--- a/tools/ui/src/lib/utils/index.ts
+++ b/tools/ui/src/lib/utils/index.ts
@@ -331,6 +331,9 @@ export { getChatCommands } from './chat-commands';
// SANDBOX_TOOL_DEFINITION is deprecated; kept for backward compatibility.
export { buildSandboxToolDefinition, SANDBOX_TOOL_DEFINITION } from './sandbox-tool';
+// Browser fallback for the server's get_info tool
+export { executeBrowserInfoTool } from './browser-info';
+
// Cryptography utilities
export { uuid } from './uuid';