LLM inference in C/C++
C++ ggml
ui: add opt-in run_javascript frontend tool (#24244)
Pascal483609509d58
diff --git a/tools/ui/eslint.config.js b/tools/ui/eslint.config.jsindex 6b3b1b5c04..b90376b6c1 100644--- a/tools/ui/eslint.config.js+++ b/tools/ui/eslint.config.js@@ -46,7 +46,14 @@ export default ts.config(},{// Exclude generated build output and Storybook files from ESLint- ignores: ['dist/**', 'build/**', '.svelte-kit/**', 'test-results/**', '.storybook/**/*']+ ignores: [+ 'dist/**',+ 'build/**',+ '.svelte-kit/**',+ 'test-results/**',+ '.storybook/**/*',+ 'src/lib/services/sandbox-worker.js'+ ]},storybook.configs['flat/recommended']);diff --git a/tools/ui/src/lib/constants/index.ts b/tools/ui/src/lib/constants/index.tsindex c4334132c9..9ab864cd0e 100644--- a/tools/ui/src/lib/constants/index.ts+++ b/tools/ui/src/lib/constants/index.ts@@ -37,6 +37,7 @@ export * from './model-id';export * from './precision';export * from './processing-info';export * from './routes';+export * from './sandbox';export * from './settings-keys';export * from './settings-registry';export * from './supported-file-types';diff --git a/tools/ui/src/lib/constants/sandbox.ts b/tools/ui/src/lib/constants/sandbox.tsnew file mode 100644index 0000000000..30e769509e--- /dev/null+++ b/tools/ui/src/lib/constants/sandbox.ts@@ -0,0 +1,39 @@+import { JsonSchemaType, ToolCallType } from '$lib/enums';+import type { OpenAIToolDefinition } from '$lib/types';++export const SANDBOX_TOOL_NAME = 'run_javascript';++export const SANDBOX_TIMEOUT_MS_DEFAULT = 10000;++export const SANDBOX_TIMEOUT_MS_MAX = 30000;++export const SANDBOX_OUTPUT_MAX_CHARS = 8192;++export const SANDBOX_EMPTY_OUTPUT = '(no output)';++export const SANDBOX_TRUNCATION_NOTICE = '[output truncated]';++export const SANDBOX_TOOL_DEFINITION: OpenAIToolDefinition = {+ type: ToolCallType.FUNCTION,+ function: {+ name: SANDBOX_TOOL_NAME,+ description:+ 'Execute JavaScript in a sandboxed browser worker (no DOM, no page access). ' ++ 'Top level await is supported. Use console.log to print intermediate values; ' ++ 'a top level return statement is captured as the result.',+ parameters: {+ type: JsonSchemaType.OBJECT,+ properties: {+ code: {+ type: JsonSchemaType.STRING,+ description: 'JavaScript source to execute'+ },+ timeout_ms: {+ type: JsonSchemaType.NUMBER,+ description: `Execution timeout in milliseconds, default ${SANDBOX_TIMEOUT_MS_DEFAULT}, max ${SANDBOX_TIMEOUT_MS_MAX}`+ }+ },+ required: ['code']+ }+ }+};diff --git a/tools/ui/src/lib/constants/settings-keys.ts b/tools/ui/src/lib/constants/settings-keys.tsindex 5fff9f94c2..3e19166dfd 100644--- a/tools/ui/src/lib/constants/settings-keys.ts+++ b/tools/ui/src/lib/constants/settings-keys.ts@@ -69,6 +69,7 @@ export const SETTINGS_KEYS = {ENABLE_THINKING: 'enableThinking',SHOW_RAW_OUTPUT_SWITCH: 'showRawOutputSwitch',// PY_INTERPRETER_ENABLED: 'pyInterpreterEnabled',+ JS_SANDBOX_ENABLED: 'jsSandboxEnabled',CUSTOM_JSON: 'customJson',CUSTOM_CSS: 'customCss'} as const;diff --git a/tools/ui/src/lib/constants/settings-registry.ts b/tools/ui/src/lib/constants/settings-registry.tsindex 9246b97033..910e5c2df0 100644--- a/tools/ui/src/lib/constants/settings-registry.ts+++ b/tools/ui/src/lib/constants/settings-registry.ts@@ -690,6 +690,14 @@ const SETTINGS_REGISTRY: Record<string, SettingsSectionEntry> = {paramType: SyncableParameterType.BOOLEAN}},+ {+ key: SETTINGS_KEYS.JS_SANDBOX_ENABLED,+ label: 'JavaScript sandbox tool',+ help: 'Expose a run_javascript tool to the model. Code runs in a Web Worker inside a sandboxed iframe with an opaque origin, isolated from the WebUI and its API, with a hard timeout.',+ defaultValue: false,+ type: SettingsFieldType.CHECKBOX,+ section: SETTINGS_SECTION_SLUGS.DEVELOPER+ },{key: SETTINGS_KEYS.CUSTOM_JSON,label: 'Custom JSON',diff --git a/tools/ui/src/lib/constants/tools.ts b/tools/ui/src/lib/constants/tools.tsindex efc3476cd7..4d9385f9f5 100644--- a/tools/ui/src/lib/constants/tools.ts+++ b/tools/ui/src/lib/constants/tools.ts@@ -2,10 +2,12 @@ import { ToolSource } from '$lib/enums/tools.enums';export const TOOL_GROUP_LABELS = {[ToolSource.BUILTIN]: 'Built-in',- [ToolSource.CUSTOM]: 'JSON Schema'+ [ToolSource.CUSTOM]: 'JSON Schema',+ [ToolSource.FRONTEND]: 'Browser'} as const;export const TOOL_SERVER_LABELS = {[ToolSource.BUILTIN]: 'Built-in Tools',- [ToolSource.CUSTOM]: 'Custom Tools'+ [ToolSource.CUSTOM]: 'Custom Tools',+ [ToolSource.FRONTEND]: 'Browser Tools'} as const;diff --git a/tools/ui/src/lib/enums/mcp.enums.ts b/tools/ui/src/lib/enums/mcp.enums.tsindex d2c27e1a0c..3d9a2070dc 100644--- a/tools/ui/src/lib/enums/mcp.enums.ts+++ b/tools/ui/src/lib/enums/mcp.enums.ts@@ -54,7 +54,9 @@ export enum MCPContentType {* JSON Schema types used in MCP tool definitions*/export enum JsonSchemaType {- OBJECT = 'object'+ OBJECT = 'object',+ STRING = 'string',+ NUMBER = 'number'}/**diff --git a/tools/ui/src/lib/enums/tools.enums.ts b/tools/ui/src/lib/enums/tools.enums.tsindex 4b2cdab320..120c1d471d 100644--- a/tools/ui/src/lib/enums/tools.enums.ts+++ b/tools/ui/src/lib/enums/tools.enums.ts@@ -1,7 +1,8 @@export enum ToolSource {BUILTIN = 'builtin',MCP = 'mcp',- CUSTOM = 'custom'+ CUSTOM = 'custom',+ FRONTEND = 'frontend'}export enum ToolPermissionDecision {diff --git a/tools/ui/src/lib/services/index.ts b/tools/ui/src/lib/services/index.tsindex 475e6419bb..386f740b8f 100644--- a/tools/ui/src/lib/services/index.ts+++ b/tools/ui/src/lib/services/index.ts@@ -261,6 +261,26 @@ export { ParameterSyncService } from './parameter-sync.service';*/export { MCPService } from './mcp.service';+/**+ * **SandboxService** - Frontend JavaScript execution in a browser sandbox+ *+ * Stateless executor for the run_javascript frontend tool. Model generated+ * code runs in a Web Worker spawned inside a sandboxed iframe with an opaque+ * origin: no access to the app origin, its storage or its API, and outgoing+ * requests carry a null origin. The code never touches a main thread, so the+ * parent enforces the timeout by removing the iframe, which terminates the+ * worker at the browser level.+ *+ * **Architecture & Relationships:**+ * - **SandboxService** (this class): Stateless sandbox execution+ * - **toolsStore**: Exposes the tool definition when the sandbox is enabled+ * - **agenticStore**: Dispatches ToolSource.FRONTEND calls here+ *+ * @see SANDBOX_TOOL_DEFINITION in constants/sandbox.ts - tool schema sent to the LLM+ * @see agenticStore in stores/agentic.svelte.ts - tool dispatch+ */+export { SandboxService } from './sandbox.service';+/*** **RouterService** — Dynamic route URL construction utility*diff --git a/tools/ui/src/lib/services/sandbox-harness.ts b/tools/ui/src/lib/services/sandbox-harness.tsnew file mode 100644index 0000000000..27b05e24b4--- /dev/null+++ b/tools/ui/src/lib/services/sandbox-harness.ts@@ -0,0 +1,25 @@+import WORKER_SHIM from './sandbox-worker.js?raw';++/**+ * Harness loaded as srcdoc into a sandboxed iframe (allow-scripts only).+ * The opaque origin is the security boundary: no access to the app origin,+ * its storage or its API. The harness spawns a worker so model code never+ * runs on a main thread, which makes the parent timeout enforceable by+ * removing the iframe.+ */+export const SANDBOX_HARNESS_HTML = `<!doctype html><script>+const SHIM = ${JSON.stringify(WORKER_SHIM)};+addEventListener('message', (event) => {+ const respond = (payload) => parent.postMessage(payload, '*');+ let worker;+ try {+ worker = new Worker(URL.createObjectURL(new Blob([SHIM], { type: 'text/javascript' })));+ } catch (err) {+ respond({ logs: [], result: null, error: 'Worker creation failed: ' + err });+ return;+ }+ worker.onmessage = (msg) => respond(msg.data);+ worker.onerror = (err) => respond({ logs: [], result: null, error: String(err.message || err) });+ worker.postMessage({ code: event.data.code });+});+</script>`;diff --git a/tools/ui/src/lib/services/sandbox-worker.js b/tools/ui/src/lib/services/sandbox-worker.jsnew file mode 100644index 0000000000..689b9211eb--- /dev/null+++ b/tools/ui/src/lib/services/sandbox-worker.js@@ -0,0 +1,30 @@+const logs = [];+const fmt = (value) => {+ if (typeof value === 'string') return value;+ try {+ return JSON.stringify(value);+ } catch {+ return String(value);+ }+};+const capture =+ (level, prefix) =>+ (...args) => {+ logs.push(prefix + args.map(fmt).join(' '));+ };+console.log = capture('log', '');+console.info = capture('info', '');+console.debug = capture('debug', '');+console.warn = capture('warn', 'warn: ');+console.error = capture('error', 'error: ');+self.onmessage = async (event) => {+ const reply = { logs, result: null, error: null };+ try {+ const AsyncFunction = Object.getPrototypeOf(async function () {}).constructor;+ const value = await new AsyncFunction(event.data.code)();+ if (value !== undefined) reply.result = fmt(value);+ } catch (err) {+ reply.error = err instanceof Error ? err.stack || err.message : String(err);+ }+ self.postMessage(reply);+};diff --git a/tools/ui/src/lib/services/sandbox.service.ts b/tools/ui/src/lib/services/sandbox.service.tsnew file mode 100644index 0000000000..ef0e6cf4eb--- /dev/null+++ b/tools/ui/src/lib/services/sandbox.service.ts@@ -0,0 +1,112 @@+import {+ NEWLINE_SEPARATOR,+ SANDBOX_EMPTY_OUTPUT,+ SANDBOX_OUTPUT_MAX_CHARS,+ SANDBOX_TIMEOUT_MS_DEFAULT,+ SANDBOX_TIMEOUT_MS_MAX,+ SANDBOX_TOOL_NAME,+ SANDBOX_TRUNCATION_NOTICE+} from '$lib/constants';+import { SANDBOX_HARNESS_HTML } from './sandbox-harness';+import type { ToolExecutionResult } from '$lib/types';++interface SandboxReply {+ logs?: unknown;+ result?: unknown;+ error?: unknown;+}++function formatReply(reply: SandboxReply): ToolExecutionResult {+ const lines: string[] = [];++ if (Array.isArray(reply.logs)) {+ for (const line of reply.logs) lines.push(String(line));+ }++ if (reply.error != null) {+ lines.push(`Error: ${String(reply.error)}`);+ } else if (reply.result != null) {+ lines.push(`=> ${String(reply.result)}`);+ }++ let content = lines.join(NEWLINE_SEPARATOR);+ if (!content) content = SANDBOX_EMPTY_OUTPUT;+ if (content.length > SANDBOX_OUTPUT_MAX_CHARS) {+ content = `${content.slice(0, SANDBOX_OUTPUT_MAX_CHARS)}${NEWLINE_SEPARATOR}${SANDBOX_TRUNCATION_NOTICE}`;+ }++ return { content, isError: reply.error != null };+}++export class SandboxService {+ /**+ * Execute a frontend sandbox tool call and return its output.+ * One disposable iframe per execution, removed on completion,+ * timeout or abort. Removing the iframe terminates the worker+ * at the browser level, so runaway code cannot outlive it.+ */+ static executeTool(+ toolName: string,+ params: Record<string, unknown>,+ signal?: AbortSignal+ ): Promise<ToolExecutionResult> {+ if (toolName !== SANDBOX_TOOL_NAME) {+ return Promise.resolve({ content: `Unknown frontend tool: ${toolName}`, isError: true });+ }++ const code = typeof params.code === 'string' ? params.code : '';+ if (!code) {+ return Promise.resolve({ content: 'Missing required parameter: code', isError: true });+ }++ const requested = Number(params.timeout_ms);+ const timeoutMs =+ Number.isFinite(requested) && requested > 0+ ? Math.min(requested, SANDBOX_TIMEOUT_MS_MAX)+ : SANDBOX_TIMEOUT_MS_DEFAULT;++ return new Promise<ToolExecutionResult>((resolve, reject) => {+ const iframe = document.createElement('iframe');+ iframe.setAttribute('sandbox', 'allow-scripts');+ iframe.style.display = 'none';+ iframe.srcdoc = SANDBOX_HARNESS_HTML;++ let settled = false;++ const cleanup = () => {+ settled = true;+ clearTimeout(timer);+ window.removeEventListener('message', onMessage);+ signal?.removeEventListener('abort', onAbort);+ iframe.remove();+ };++ const finish = (result: ToolExecutionResult) => {+ if (settled) return;+ cleanup();+ resolve(result);+ };++ const onAbort = () => {+ if (settled) return;+ cleanup();+ reject(new DOMException('Sandbox execution aborted', 'AbortError'));+ };++ const onMessage = (event: MessageEvent) => {+ if (event.source !== iframe.contentWindow) return;+ finish(formatReply((event.data ?? {}) as SandboxReply));+ };++ const timer = setTimeout(+ () => finish({ content: `Execution timed out after ${timeoutMs} ms`, isError: true }),+ timeoutMs+ );++ window.addEventListener('message', onMessage);+ signal?.addEventListener('abort', onAbort);+ iframe.onload = () => iframe.contentWindow?.postMessage({ code }, '*');+ document.body.appendChild(iframe);+ });+ }+}diff --git a/tools/ui/src/lib/stores/agentic.svelte.ts b/tools/ui/src/lib/stores/agentic.svelte.tsindex 947737d7c1..5579cc1e5a 100644--- a/tools/ui/src/lib/stores/agentic.svelte.ts+++ b/tools/ui/src/lib/stores/agentic.svelte.ts@@ -29,6 +29,7 @@ import { permissionsStore } from '$lib/stores/permissions.svelte';import { ToolSource, ToolPermissionDecision } from '$lib/enums';import { SvelteMap } from 'svelte/reactivity';import { ToolsService } from '$lib/services/tools.service';+import { SandboxService } from '$lib/services/sandbox.service';import { isAbortError } from '$lib/utils';import { DEFAULT_AGENTIC_CONFIG, NEWLINE_SEPARATOR } from '$lib/constants';import {@@ -784,6 +785,13 @@ class AgenticStore {result = executionResult.content;+ if (executionResult.isError) toolSuccess = false;+ } else if (toolSource === ToolSource.FRONTEND) {+ const args = this.parseToolArguments(toolCall.function.arguments);+ const executionResult = await SandboxService.executeTool(toolName, args, signal);++ result = executionResult.content;+if (executionResult.isError) toolSuccess = false;} else {const mcpCall: MCPToolCall = {diff --git a/tools/ui/src/lib/stores/tools.svelte.ts b/tools/ui/src/lib/stores/tools.svelte.tsindex 82e41f0bf5..9f0101a82e 100644--- a/tools/ui/src/lib/stores/tools.svelte.ts+++ b/tools/ui/src/lib/stores/tools.svelte.ts@@ -5,6 +5,7 @@ import { HealthCheckStatus, JsonSchemaType, ToolCallType, ToolSource } from '$liimport { config } from '$lib/stores/settings.svelte';import {DISABLED_TOOL_KEYS_LOCALSTORAGE_KEY,+ SANDBOX_TOOL_DEFINITION,TOOL_GROUP_LABELS,TOOL_SERVER_LABELS} from '$lib/constants';@@ -18,6 +19,8 @@ function toolKey(source: ToolSource, name: string, serverId?: string): string {return serverId ? `mcp-${serverId}:${name}` : `mcp:${name}`;case ToolSource.CUSTOM:return `custom:${name}`;+ case ToolSource.FRONTEND:+ return `frontend:${name}`;default:return `builtin:${name}`;}@@ -82,6 +85,10 @@ class ToolsStore {return mcpStore.getToolDefinitionsForLLM();}+ get frontendTools(): OpenAIToolDefinition[] {+ return config().jsSandboxEnabled ? [SANDBOX_TOOL_DEFINITION] : [];+ }+get customTools(): OpenAIToolDefinition[] {const raw = config().customJson;if (!raw || typeof raw !== 'string') return [];@@ -156,6 +163,15 @@ class ToolsStore {push({ source: ToolSource.BUILTIN, key: toolKey(ToolSource.BUILTIN, name), definition: def });}+ for (const def of this.frontendTools) {+ const name = def.function.name;+ push({+ source: ToolSource.FRONTEND,+ key: toolKey(ToolSource.FRONTEND, name),+ definition: def+ });+ }+for (const { serverId, serverName, definition } of this.mcpEntries()) {const name = definition.function.name;push({@@ -208,6 +224,8 @@ class ToolsStore {return entry.serverName ?? '';case ToolSource.CUSTOM:return TOOL_GROUP_LABELS[ToolSource.CUSTOM];+ case ToolSource.FRONTEND:+ return TOOL_GROUP_LABELS[ToolSource.FRONTEND];default:return TOOL_GROUP_LABELS[ToolSource.BUILTIN];}@@ -237,6 +255,7 @@ class ToolsStore {};for (const def of this._builtinTools) take(def);+ for (const def of this.frontendTools) take(def);for (const def of mcpStore.getToolDefinitionsForLLM()) take(def);for (const def of this.customTools) take(def);@@ -346,6 +365,7 @@ class ToolsStore {if (entry.serverName) return mcpStore.getServerDisplayName(entry.serverName);if (entry.source === ToolSource.BUILTIN) return TOOL_SERVER_LABELS[ToolSource.BUILTIN];if (entry.source === ToolSource.CUSTOM) return TOOL_SERVER_LABELS[ToolSource.CUSTOM];+ if (entry.source === ToolSource.FRONTEND) return TOOL_SERVER_LABELS[ToolSource.FRONTEND];return '';}diff --git a/tools/ui/tsconfig.json b/tools/ui/tsconfig.jsonindex 7c585f4db2..51f55971e9 100644--- a/tools/ui/tsconfig.json+++ b/tools/ui/tsconfig.json@@ -24,7 +24,8 @@"tests/**/*.svelte",".storybook/**/*.ts",".storybook/**/*.svelte"- ]+ ],+ "exclude": ["src/lib/services/sandbox-worker.js"]// Path aliases are handled by https://svelte.dev/docs/kit/configuration#alias// except $lib which is handled by https://svelte.dev/docs/kit/configuration#files//