Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
// UI-driving voice tools: navigate, list interactables, read page text, and the
// DOM-mutating click/fill/select/check actions. `ui_navigate` is intentionally
// always-on (left out of TOOL_GROUPS) so "take me to X" works on any turn; the
// rest gate on UI_INTENT_RE. ui_click runs a destructive-action confirm gate.
import { resolveNavCommand } from '../../../lib/navManifest.js';
import { requiresConfirmation, buildPending } from '../confirmGate.js';
import { UI_KINDS, findUiElement } from './shared.js';
// Loose on purpose — false positives are cheap (one extra tool), false
// negatives are expensive (LLM guesses wrong or can't act).
export const UI_INTENT_RE = /\b(click|press|tap|hit|open|go to|take me|show me|navigate|select|pick|switch|choose|tab|button|dropdown|field|input|fill|enter|type|write|check|uncheck|toggle|link|option|on (?:this|the) page|what(?:'s)? (?:on|here|does (?:this|the page))|read (?:this|the page|me (?:this|the page|what))|read (?:it )?(?:aloud|out))\b/i;
// Strong form-fill signal: when the user is clearly directing content INTO
// a specific field, brain_capture/daily_log_append must be suppressed even
// if capture verbs appear inside the value (e.g. "fill description with
// 'remember to buy milk'"). Matches a UI-fill verb within ~60 chars of a
// form-field word in either order — "fill the description with X", "type X
// in the body", "set the title to X", "put Y in description".
export const UI_FILL_INTENT_RE = /\b(?:fill|type|enter|put|write|set)\b[^.!?\n]{0,60}?\b(?:description|name|title|subject|body|content|field|input|textarea|form|label|placeholder|caption)\b/i;
export const UI_TOOLS = [
{
name: 'ui_navigate',
description:
'Navigate the UI to a page. Use for "take me to X" / "open X" / "go to X" — including the Daily Log when the user just wants to VIEW it without writing. ' +
'Pass `page` as a short name the user would say: tasks, agents, gsd, briefing, calendar, goals, brain, meatspace, memory, messages, settings, shell, instances, wiki, character, health, body, alcohol, daily log, journal, etc. ' +
'Server resolves fuzzy — "chief of staff tasks", "cos tasks", "task page" all map to tasks. If no match, the error lists valid names. ' +
'Only prefer daily_log_open over this tool when the user clearly wants to write/dictate ("start", "new", "entry", "make", "dictate"); plain "open my daily log" or "go to the daily log" should use ui_navigate.',
parameters: {
type: 'object',
properties: {
page: {
type: 'string',
description: 'Short page name the user said (e.g. "tasks", "calendar"). Server fuzzy-matches.',
},
path: {
type: 'string',
description: 'Explicit route path starting with / (e.g. "/cos/tasks"). Only when page doesn\'t fit.',
},
},
},
execute: async ({ page, path } = {}, ctx = {}) => {
let target = null;
let resolvedKey = null;
const currentPath = ctx.state?.ui?.path || '';
const mvMatch = currentPath.match(/^\/music-video\/([^/?#]+)/);
const openProjectId = mvMatch && mvMatch[1] !== 'new' ? mvMatch[1] : null;
if (openProjectId && page && typeof page === 'string') {
const norm = page.trim().toLowerCase().replace(/\s+/g, '-').replace(/[.!?:;,"']+$/, '');
const MV_STAGES = {
setup: 'setup',
'cast-and-sets': 'cast-sets',
'cast-sets': 'cast-sets',
board: 'board',
'the-board': 'board',
'open-the-board': 'board',
plan: 'board',
'the-plan': 'board',
'scene-board': 'produce',
shots: 'produce',
'the-shots': 'produce',
produce: 'produce',
compose: 'compose',
review: 'review',
publish: 'publish',
};
const stage = MV_STAGES[norm] || (norm.startsWith('open-') ? MV_STAGES[norm.replace(/^open-/, '')] : null);
if (stage) {
target = `/music-video/${openProjectId}/${stage}`;
resolvedKey = stage;
}
}
if (!target && page && typeof page === 'string') {
const hit = resolveNavCommand(page);
if (hit) {
target = hit.path;
resolvedKey = hit.matched;
if (openProjectId && target.startsWith('/music-video/') && !target.includes('?')) {
const stage = target.replace(/^\/music-video\//, '').split('/')[0];
target = `/music-video/${openProjectId}/${stage}`;
}
}
}
// A model-chosen raw path must stay in-app: `//host` and `/\\host` are
// protocol-relative and the client would leave the origin.
if (!target && typeof path === 'string' && /^\/(?![/\\])/.test(path)) target = path;
if (!target) {
const suggestions = ['tasks', 'agents', 'gsd', 'briefing', 'calendar', 'goals', 'brain', 'meatspace', 'messages', 'settings', 'shell', 'instances'];
return {
ok: false,
error: `Unknown page "${page || path || ''}"`,
suggestions,
summary: `I don't know that page. Try: ${suggestions.slice(0, 6).join(', ')}.`,
};
}
ctx.sideEffects?.push({ type: 'navigate', path: target });
return { ok: true, path: target, summary: `Opened ${resolvedKey || target}.` };
},
},
{
name: 'ui_list_interactables',
description: 'List interactive elements on the current page. Fallback when the per-turn UI summary isn\'t enough.',
parameters: {
type: 'object',
properties: {
kind: { type: 'string', enum: UI_KINDS, description: 'Optional kind filter.' },
},
},
execute: async ({ kind } = {}, ctx = {}) => {
const ui = ctx.state?.ui;
if (!ui || !Array.isArray(ui.elements)) {
return { ok: false, error: 'No UI index available. The user may not have the voice widget loaded.' };
}
const items = kind ? ui.elements.filter((e) => e.kind === kind) : ui.elements;
return {
ok: true,
path: ui.path,
title: ui.title,
count: items.length,
items: items.slice(0, 100),
summary: `${items.length} interactive element${items.length === 1 ? '' : 's'} on ${ui.title || ui.path || 'this page'}.`,
};
},
},
{
name: 'ui_read',
description:
'Read back the visible text on the current page. Use when the user asks "what does this say?", "read this aloud", "what\'s on the page?", "read me the page". ' +
'Returns the user-visible textual content of the main content area (excluding nav rails, asides, and the voice widget itself). ' +
'Output is capped at ~8 KB; longer pages are tail-trimmed on a word boundary with an ellipsis. ' +
'Default behavior (summarize=false): read the returned `content` verbatim — do NOT summarize. ' +
'When the user asks for a summary instead ("what is this page about?", "summarize this page"), pass summarize=true and produce a short summary of `content` rather than reading it verbatim.',
parameters: {
type: 'object',
properties: {
summarize: {
type: 'boolean',
description: 'If true, the LLM may summarize before reading aloud (use when the user asks "what is this page about?" rather than "read this to me"). Default false — speak `content` verbatim.',
},
},
},
execute: async ({ summarize = false } = {}, ctx = {}) => {
const ui = ctx?.state?.ui;
// The client no longer ships the visible-text blob with every index —
// it sets `textOnDemand` and we fetch it lazily here, only when ui_read
// actually runs. Resolution order:
// 1. Eager/legacy text already on the snapshot (older client, or a
// prior ui_read in this turn cached it) → use it directly.
// 2. textOnDemand client → request it now via ctx.requestUiText().
// 3. Neither (very old / no widget) → "no text available".
let text = typeof ui?.text === 'string' && ui.text.trim() ? ui.text : null;
if (!text && ui?.textOnDemand && typeof ctx?.requestUiText === 'function') {
const fetched = await ctx.requestUiText();
if (typeof fetched === 'string' && fetched.trim()) text = fetched;
}
if (!text) {
return {
ok: false,
error: 'No page text available',
summary: "I can't see the page content right now — make sure the voice widget is loaded and try again.",
};
}
return {
ok: true,
path: ui?.path,
title: ui?.title,
content: text,
chars: text.length,
summarize: !!summarize,
// Keep `summary` short so the LLM message history isn't doubled — the
// full body lives in `content`.
summary: `Read page "${ui?.title || ui?.path || 'current page'}" (${text.length} chars).`,
};
},
},
{
name: 'ui_click',
description: 'Click a tab, button, or link on the current page by visible label. "Select Memory tab" → label="Memory", kind="tab".',
parameters: {
type: 'object',
properties: {
label: { type: 'string', description: 'Visible label.' },
kind: { type: 'string', enum: ['tab', 'button', 'link'], description: 'Optional kind hint.' },
},
required: ['label'],
},
execute: async ({ label, kind } = {}, ctx = {}) => {
const hit = findUiElement(ctx, label, kind);
if (!hit.entry) return hit.err;
// Destructive-action confirmation gate. If the resolved label looks
// destructive (delete/remove/discard/reset/clear) OR the control was
// annotated `data-voice-guard="confirm"` (an outbound effect the label
// alone doesn't name), stash a pending record on the per-session state
// and ask the LLM to prompt the user for spoken confirmation. The next
// user turn is intercepted by pipeline.js → resolvePending() which
// either re-issues the click or cancels. Skip if the caller already
// confirmed (re-issue path).
if (ctx.state && !ctx.confirmed && requiresConfirmation(hit.entry)) {
ctx.state.pendingDestructive = buildPending({
tool: 'ui_click',
args: { label: hit.entry.label, kind: hit.entry.kind },
target: { ref: hit.entry.ref, label: hit.entry.label, kind: hit.entry.kind },
});
return {
ok: true,
confirmation_required: true,
label: hit.entry.label,
kind: hit.entry.kind,
summary: `That looks destructive — confirm by saying "yes" or "confirm" to ${hit.entry.label}, or "cancel" to skip.`,
};
}
ctx.sideEffects?.push({ type: 'ui:click', target: { ref: hit.entry.ref, label: hit.entry.label } });
return { ok: true, label: hit.entry.label, kind: hit.entry.kind, summary: `Clicked ${hit.entry.label}.` };
},
},
{
name: 'ui_fill',
description: 'Type text into an input or textarea by its label. Use ui_select for dropdowns, ui_check for checkboxes.',
parameters: {
type: 'object',
properties: {
label: { type: 'string', description: 'Visible label of the input.' },
value: { type: 'string', description: 'Text to fill in.' },
},
required: ['label', 'value'],
},
execute: async ({ label, value } = {}, ctx = {}) => {
const hit = findUiElement(ctx, label, ['input', 'textarea']);
if (!hit.entry) return hit.err;
// Filling gates ONLY on an explicit `data-voice-guard="confirm"`
// annotation, never on the destructive-label heuristic: text entry is
// reversible and the client already scrolls the field into view, so a
// blanket fill gate would just be noise. An unmarked field behaves
// exactly as before this change.
const filledValue = String(value ?? '');
if (ctx.state && !ctx.confirmed && hit.entry.guard === 'confirm') {
ctx.state.pendingDestructive = buildPending({
tool: 'ui_fill',
args: { label: hit.entry.label, value: filledValue },
target: { ref: hit.entry.ref, label: hit.entry.label, kind: hit.entry.kind },
});
return {
ok: true,
confirmation_required: true,
label: hit.entry.label,
summary: `That field needs confirmation before I fill it — confirm by saying "yes" or "confirm" to fill ${hit.entry.label}, or "cancel" to skip.`,
};
}
ctx.sideEffects?.push({ type: 'ui:fill', target: { ref: hit.entry.ref, label: hit.entry.label }, value: filledValue });
return { ok: true, label: hit.entry.label, summary: `Filled ${hit.entry.label}.` };
},
},
{
name: 'ui_select',
description: 'Pick an option from a <select> dropdown by label. "Set status to Active" → label="Status", option="Active".',
parameters: {
type: 'object',
properties: {
label: { type: 'string', description: 'Visible label of the select.' },
option: { type: 'string', description: 'Option text or value.' },
},
required: ['label', 'option'],
},
execute: async ({ label, option } = {}, ctx = {}) => {
const hit = findUiElement(ctx, label, 'select');
if (!hit.entry) return hit.err;
ctx.sideEffects?.push({ type: 'ui:select', target: { ref: hit.entry.ref, label: hit.entry.label }, option: String(option) });
return { ok: true, label: hit.entry.label, option, summary: `Selected ${option} on ${hit.entry.label}.` };
},
},
{
name: 'ui_check',
description: 'Toggle a checkbox or radio by label. checked=true to check, false to uncheck.',
parameters: {
type: 'object',
properties: {
label: { type: 'string', description: 'Visible label.' },
checked: { type: 'boolean', description: 'Desired state.' },
},
required: ['label', 'checked'],
},
execute: async ({ label, checked } = {}, ctx = {}) => {
const hit = findUiElement(ctx, label, ['checkbox', 'radio']);
if (!hit.entry) return hit.err;
const wantChecked = !!checked;
// Same gate as ui_click: destructive label OR an explicit
// `data-voice-guard="confirm"` annotation.
if (ctx.state && !ctx.confirmed && requiresConfirmation(hit.entry)) {
ctx.state.pendingDestructive = buildPending({
tool: 'ui_check',
args: { label: hit.entry.label, checked: wantChecked },
target: { ref: hit.entry.ref, label: hit.entry.label, kind: hit.entry.kind },
});
return {
ok: true,
confirmation_required: true,
label: hit.entry.label,
checked: wantChecked,
summary: `That looks destructive — confirm by saying "yes" or "confirm" to ${wantChecked ? 'check' : 'uncheck'} ${hit.entry.label}, or "cancel" to skip.`,
};
}
ctx.sideEffects?.push({ type: 'ui:check', target: { ref: hit.entry.ref, label: hit.entry.label }, checked: wantChecked });
return { ok: true, label: hit.entry.label, checked: wantChecked, summary: `${checked ? 'Checked' : 'Unchecked'} ${hit.entry.label}.` };
},
},
];