An agentic browser harness: an AI side panel & orb in your own logged-in Chrome, Edge, Arc, Brave or even Firefox run by the agent/subscription you already use (Claude Code, Codex, Antigravity). Free, open source, no API key. Also an MCP server.
TypeScript agentic-browseragentic-browsingagy-cliaiai-agentai-browser-extensionai-toolsbrave-extensionbrowser-agentbrowser-automationbrowser-extensionbrowser-usebrowserskillchrome-extensionclaude-codecodexfirefox-addonmcp-servernanobrowserweb-automation
feat(intent): add the local grammar and intent scorer
Shahriara97d79dca583
diff --git a/lib/intent/grammar.ts b/lib/intent/grammar.tsnew file mode 100644index 0000000..f7a61cb--- /dev/null+++ b/lib/intent/grammar.ts@@ -0,0 +1,260 @@+import type { z } from 'zod';+import { clickElement } from '@/lib/actions/page/click-element';+import { navigate } from '@/lib/actions/page/navigate';+import { pressKey } from '@/lib/actions/page/press-key';+import { scrollTo } from '@/lib/actions/page/scroll-to';++/**+ * The shapes a quick command can take. Each rule is a fully anchored pattern plus a `build`+ * that fills the action's input from the captured slots — and may reject the match outright+ * when the shape fit but the slot did not ("open the settings menu" looks like navigation+ * until you try to read "settings menu" as a host).+ *+ * Patterns are anchored at both ends by design: a quick command is the *whole* utterance.+ * Anything with a tail ("scroll down and tell me what it says") matches nothing here and+ * goes to the agent, which is the right outcome — so there is no partial-match scoring.+ */+export interface Rule {+ id: string;+ /** Action name from the registry — the same one the agent would have called. */+ action: string;+ /** How certain this *shape* is, before the slot is judged. See `confidence` on Built. */+ certainty: number;+ pattern: RegExp;+ build(groups: Record<string, string | undefined>): Built | null;+}++export interface Built {+ input: unknown;+ /** One line for the timeline, e.g. "Go back" or "Click Sign in". */+ label: string;+ /**+ * How well the captured slot fits, 0–1. Multiplied into the rule's certainty, this is what+ * separates "click Sign in" from "click it" — the same shape, very different odds.+ */+ confidence: number;+ /** Consequential enough that the agent's approval gate should see it instead. */+ risky?: boolean;+}++type NavigateInput = z.input<typeof navigate.input>;+type ScrollInput = z.input<typeof scrollTo.input>;+type ClickInput = z.input<typeof clickElement.input>;+type PressKeyInput = z.input<typeof pressKey.input>;++/**+ * Bare names people say instead of hostnames. A convenience list, not a redirect table: an+ * unlisted name simply fails to resolve and the utterance escalates.+ */+const SITES: Record<string, string> = {+ google: 'google.com',+ 'google maps': 'maps.google.com',+ maps: 'maps.google.com',+ 'google drive': 'drive.google.com',+ drive: 'drive.google.com',+ 'google calendar': 'calendar.google.com',+ calendar: 'calendar.google.com',+ gmail: 'mail.google.com',+ youtube: 'youtube.com',+ github: 'github.com',+ reddit: 'reddit.com',+ wikipedia: 'wikipedia.org',+ twitter: 'twitter.com',+ linkedin: 'linkedin.com',+ facebook: 'facebook.com',+ instagram: 'instagram.com',+ amazon: 'amazon.com',+ netflix: 'netflix.com',+ 'stack overflow': 'stackoverflow.com',+ stackoverflow: 'stackoverflow.com',+ 'hacker news': 'news.ycombinator.com',+ chatgpt: 'chatgpt.com',+ claude: 'claude.ai',+};++/** Spoken key names → `KeyboardEvent.key` values. */+const KEYS: Record<string, string> = {+ enter: 'Enter',+ return: 'Enter',+ escape: 'Escape',+ esc: 'Escape',+ tab: 'Tab',+ space: ' ',+ spacebar: ' ',+ backspace: 'Backspace',+ delete: 'Delete',+ 'page up': 'PageUp',+ 'page down': 'PageDown',+ home: 'Home',+ end: 'End',+ 'up arrow': 'ArrowUp',+ 'down arrow': 'ArrowDown',+ 'left arrow': 'ArrowLeft',+ 'right arrow': 'ArrowRight',+ 'arrow up': 'ArrowUp',+ 'arrow down': 'ArrowDown',+ 'arrow left': 'ArrowLeft',+ 'arrow right': 'ArrowRight',+};++const KEY_NAMES = Object.keys(KEYS).join('|');++/**+ * Labels worth a human's blessing before the click lands. These escalate rather than run+ * locally — not because the agent is safer at clicking, but because an agent run puts the+ * call through the approval gate and onto the timeline, and a local one does neither.+ */+const CONSEQUENTIAL =+ /\b(?:buy|buy now|purchase|pay|payment|checkout|place order|order now|delete|remove|send|submit|transfer|withdraw|deposit|unsubscribe|subscribe|donate|book now|confirm)\b/;++/** Words that name an element only in context; the agent has the context, we do not. */+const DEICTIC = /^(?:it|that|this|there|them|those|these|him|her|thing|one)$/;++/** Beyond this the "label" is prose, not the name of a control. */+const MAX_LABEL_TOKENS = 8;++export const RULES: readonly Rule[] = [+ {+ id: 'history.back',+ action: navigate.name,+ certainty: 0.97,+ pattern: /^(?:go\s+)?back(?:\s+(?:a|one)\s+page)?$/,+ build: () => nav({ action: 'back' }, 'Go back'),+ },+ {+ id: 'history.forward',+ action: navigate.name,+ certainty: 0.97,+ pattern: /^(?:go\s+)?forwards?(?:\s+(?:a|one)\s+page)?$/,+ build: () => nav({ action: 'forward' }, 'Go forward'),+ },+ {+ id: 'history.reload',+ action: navigate.name,+ certainty: 0.97,+ pattern: /^(?:reload|refresh)(?:\s+(?:the|this)\s+page)?$/,+ build: () => nav({ action: 'reload' }, 'Reload the page'),+ },+ {+ // Two tokens at most, so "open the settings menu" cannot be read as a hostname.+ id: 'navigate.url',+ action: navigate.name,+ certainty: 0.95,+ pattern:+ /^(?:go\s+to|goto|open|visit|navigate\s+to|take\s+me\s+to|load|pull\s+up)\s+(?:the\s+)?(?<dest>\S+(?:\s+\S+)?)$/,+ build: ({ dest }) => {+ const site = SITES[dest!];+ if (site) return nav({ url: `https://${site}` }, `Open ${site}`, 1);+ const url = toUrl(dest!);+ // A bare host is a shade less certain than a name we know or a full URL: "open zoom"+ // is a hostname and an app name at once.+ return url ? nav({ url }, `Open ${dest}`, /^https?:\/\//.test(dest!) ? 1 : 0.97) : null;+ },+ },+ {+ id: 'navigate.search',+ action: navigate.name,+ certainty: 0.9,+ pattern: /^(?<verb>google|search\s+(?:the\s+web|google)\s+for|web\s+search\s+for)\s+(?<q>.+)$/,+ build: ({ verb, q }) => {+ const query = q!.trim();+ // "google maps" is a place to go, not a thing to search for; one word after a bare+ // "google" is too likely to be a destination, so let the agent decide.+ if (verb === 'google' && query.split(' ').length < 2) return null;+ return nav(+ { url: `https://www.google.com/search?q=${encodeURIComponent(query)}` },+ `Search the web for "${query}"`,+ verb === 'google' ? 0.95 : 1,+ );+ },+ },+ {+ id: 'scroll.direction',+ action: scrollTo.name,+ certainty: 0.95,+ pattern:+ /^(?:scroll|jump|move|go)(?:\s+(?:to|down\s+to|back\s+to|all\s+the\s+way\s+to))?\s+(?:the\s+)?(?<dir>up|down|top|bottom)(?:\s+of\s+(?:the\s+)?page)?$/,+ build: ({ dir }) => scroll({ direction: dir as ScrollDirection }, `Scroll ${dir}`),+ },+ {+ id: 'scroll.page',+ action: scrollTo.name,+ certainty: 0.95,+ pattern: /^page\s+(?<dir>up|down)$/,+ build: ({ dir }) => scroll({ direction: dir as ScrollDirection }, `Scroll ${dir}`),+ },+ {+ id: 'scroll.element',+ action: scrollTo.name,+ certainty: 0.85,+ pattern: /^scroll\s+(?:down\s+)?to\s+(?:the\s+)?(?<label>.+)$/,+ build: ({ label }) => {+ const text = slot(label!);+ return text && scroll({ target: { text: text.value } }, `Scroll to ${text.value}`, text.confidence);+ },+ },+ {+ id: 'key.press',+ action: pressKey.name,+ certainty: 0.97,+ pattern: new RegExp(`^(?:press|hit|tap|push)(?:\\s+the)?\\s+(?<key>${KEY_NAMES})(?:\\s+key)?$`),+ build: ({ key }) => {+ const dom = KEYS[key!];+ return dom ? built({ key: dom } satisfies PressKeyInput, `Press ${key}`) : null;+ },+ },+ {+ id: 'click.text',+ action: clickElement.name,+ certainty: 0.9,+ pattern: /^(?:click|tap|press|hit|push)(?:\s+on)?(?:\s+the)?\s+(?<label>.+?)(?:\s+(?<role>button|link|tab|checkbox))?$/,+ build: ({ label, role }) => {+ const text = slot(label!);+ if (!text) return null;+ const target = role ? { text: text.value, role } : { text: text.value };+ return {+ input: { target } satisfies ClickInput,+ label: `Click ${text.value}`,+ confidence: text.confidence,+ risky: CONSEQUENTIAL.test(text.value),+ };+ },+ },+];++type ScrollDirection = NonNullable<ScrollInput['direction']>;++const built = (input: unknown, label: string, confidence = 1): Built => ({ input, label, confidence });++const nav = (input: NavigateInput, label: string, confidence = 1): Built => built(input, label, confidence);++const scroll = (input: ScrollInput, label: string, confidence = 1): Built => built(input, label, confidence);++/**+ * Read a captured phrase as the name of a thing on the page, and say how much to trust it.+ * Null means it is not a name at all — prose, or nothing.+ */+function slot(raw: string): { value: string; confidence: number } | null {+ const value = raw.trim();+ if (!value) return null;+ const tokens = value.split(/\s+/);+ if (tokens.length > MAX_LABEL_TOKENS) return null;++ // "it" and friends name something only the conversation knows about.+ if (DEICTIC.test(value)) return { value, confidence: 0.55 };++ let confidence = 1;+ // A control's name is short. The longer the phrase, the more likely it describes a+ // position or a condition ("the blue one at the bottom") that needs the page to resolve.+ if (tokens.length > 4) confidence -= 0.12 * (tokens.length - 4);+ if (value.length < 2) confidence -= 0.4;+ return { value, confidence: Math.max(0, confidence) };+}++/** Read a destination slot as an http(s) URL, or null if it is not one. */+function toUrl(dest: string): string | null {+ if (/^https?:\/\/\S+$/.test(dest)) return dest;+ if (/^localhost(?::\d+)?(?:\/\S*)?$/.test(dest)) return `http://${dest}`;+ if (/^(?:[a-z0-9](?:[a-z0-9-]*[a-z0-9])?\.)+[a-z]{2,}(?::\d+)?(?:\/\S*)?$/.test(dest)) return `https://${dest}`;+ return null;+}diff --git a/lib/intent/index.ts b/lib/intent/index.tsnew file mode 100644index 0000000..550069b--- /dev/null+++ b/lib/intent/index.ts@@ -0,0 +1,16 @@+/**+ * The local intent funnel: a small deterministic grammar that answers "can the extension do+ * this itself, right now?" before an instruction is sent to the daemon and the agent.+ *+ * It is pure — no DOM, no `browser.*`, no network — so it can be exercised from plain Node+ * (`npm run intent:check`) against the fixture table, which is where the threshold is tuned.+ */+export { normalize, type Utterance } from './normalize';+export { RULES, type Rule, type Built } from './grammar';+export {+ ACT_THRESHOLD,+ routeIntent,+ type EscalationReason,+ type LocalIntent,+ type Routing,+} from './route';diff --git a/lib/intent/normalize.ts b/lib/intent/normalize.tsnew file mode 100644index 0000000..118c558--- /dev/null+++ b/lib/intent/normalize.ts@@ -0,0 +1,55 @@+/**+ * Reduce an utterance to the flat form the grammar matches against, so that+ * "Hey VoiceLink, could you scroll down a bit please?" and "scroll down" arrive as the same+ * two words. Everything here is lossy on purpose: the original text is kept alongside, and+ * it — not this — is what gets escalated to the agent.+ */+export interface Utterance {+ /** Lowercased, depunctuated, opener- and trailer-free. The grammar matches this. */+ text: string;+ /** What the user actually said or typed, trimmed. Escalation sends this. */+ raw: string;+ /**+ * Phrased as a question, so it wants an answer rather than an action. A trailing "?" only+ * counts when nothing polite was stripped off: "could you go back please?" is a request+ * wearing a question mark, while a bare "back?" really is one.+ */+ question: boolean;+}++/** Speech-to-text hears a spoken domain as separate words: "github dot com". */+const SPOKEN_TLD = /\b([a-z0-9][a-z0-9-]*)\s+dot\s+(com|net|org|io|dev|co|ai|app|edu|gov|me|uk|xyz)\b/g;++/** Address and politeness ahead of the command. Stripped repeatedly — they stack. */+const OPENER =+ /^(?:hey|hi|ok|okay|yo|um|uh|please|voicelink|assistant|can you|could you|would you|will you|i want you to|i would like you to|id like you to|lets|just|now|go ahead and)\s+/;++/** Softeners that trail a complete command without changing what it asks for. */+const TRAILER =+ /\s+(?:please|thanks|thank you|for me|now|a bit|a little|a lot|some more|more|slightly|real quick|quickly)$/;++/**+ * Characters a URL needs, kept; everything else is sentence punctuation and becomes a space.+ * The question mark goes too, but only after {@link normalize} has noted that it was there.+ */+const PUNCTUATION = /[^a-z0-9\s./:&=@_+#-]/g;++export function normalize(input: string): Utterance {+ const raw = input.trim();++ // Apostrophes are dropped rather than spaced, so "what's" stays one token ("whats").+ let text = raw.toLowerCase().replace(/['‘’]/g, '');+ text = text.replace(SPOKEN_TLD, '$1.$2');+ text = text.replace(PUNCTUATION, ' ').replace(/\s+/g, ' ').trim();++ let softened = false;+ let previous;+ do {+ previous = text;+ text = text.replace(OPENER, '').replace(TRAILER, '').trim();+ softened ||= text !== previous;+ } while (text !== previous);++ // A sentence-ending period survives PUNCTUATION because "github.com" needs dots.+ return { text: text.replace(/\.$/, ''), raw, question: /\?\s*$/.test(raw) && !softened };+}diff --git a/lib/intent/route.ts b/lib/intent/route.tsnew file mode 100644index 0000000..c1ec76f--- /dev/null+++ b/lib/intent/route.ts@@ -0,0 +1,98 @@+import { RULES, type Rule } from './grammar';+import { normalize } from './normalize';++/**+ * Score at which the extension acts on its own instead of waking the agent. Set from the+ * fixture table in `scripts/check-intent.mjs` — raise it and more quick commands take the+ * slow path, lower it and the grammar starts guessing. Every miss is recoverable (a failed+ * local action escalates), so the cost of being wrong is a round trip, not a wrong click.+ */+export const ACT_THRESHOLD = 0.75;++/** Why an utterance went to the agent. Surfaced for debugging, not shown to the user. */+export type EscalationReason =+ | 'no-match'+ | 'below-threshold'+ | 'question'+ | 'multi-step'+ | 'conditional'+ | 'consequential'+ | 'skill-prefix';++export interface LocalIntent {+ /** The rule that matched, e.g. "navigate.url". */+ ruleId: string;+ /** Registry action name — the same call the agent would have made. */+ action: string;+ input: unknown;+ /** One line for the timeline, e.g. "Go back". */+ label: string;+ score: number;+}++export type Routing =+ | { decision: 'act'; intent: LocalIntent }+ | { decision: 'escalate'; reason: EscalationReason; score: number; ruleId?: string };++/** An utterance in question form is asking for an answer, which only the agent can give. */+const QUESTION =+ /^(?:what|whats|why|how|who|when|where|which|is|are|was|were|do|does|did|can|could|should|would|tell|explain|describe|summarize|summarise|read|find|show|check|look|see|list|compare|help|write|draft|fill)\b/;++/** Two commands in one sentence. The grammar handles one thing at a time. */+const MULTI_STEP = /\b(?:and then|then|after that|and also|followed by)\b/;++/** A condition to evaluate, which means looking at the page and deciding — the agent's job. */+const HEDGE = /\b(?:if|unless|whenever|maybe|might|try to|see if|as soon as|in case|otherwise)\b/;++/**+ * Decide whether the extension can carry out an utterance itself.+ *+ * The funnel is deliberately biased toward escalating: a false "act" spends a wrong action on+ * the user's real page, while a false "escalate" only costs the round trip this exists to+ * avoid. Hard stops come first (questions, multiple steps, consequential clicks), then the+ * highest-scoring rule has to clear {@link ACT_THRESHOLD}.+ */+export function routeIntent(input: string): Routing {+ const utterance = normalize(input);+ // "@skill do the thing" is an explicit request for a particular agent skill.+ if (utterance.raw.startsWith('@')) return escalate('skill-prefix', 0);+ if (!utterance.text) return escalate('no-match', 0);+ if (utterance.question || QUESTION.test(utterance.text)) return escalate('question', 0);+ if (MULTI_STEP.test(utterance.text)) return escalate('multi-step', 0);+ if (HEDGE.test(utterance.text)) return escalate('conditional', 0);++ let best: { rule: Rule; score: number; label: string; input: unknown; risky: boolean } | null = null;+ for (const rule of RULES) {+ const match = rule.pattern.exec(utterance.text);+ if (!match) continue;+ const slot = rule.build(match.groups ?? {});+ if (!slot) continue;+ const score = round(rule.certainty * slot.confidence);+ if (!best || score > best.score) {+ best = { rule, score, label: slot.label, input: slot.input, risky: slot.risky === true };+ }+ }++ if (!best) return escalate('no-match', 0);+ if (best.risky) return escalate('consequential', best.score, best.rule.id);+ if (best.score < ACT_THRESHOLD) return escalate('below-threshold', best.score, best.rule.id);+ return {+ decision: 'act',+ intent: {+ ruleId: best.rule.id,+ action: best.rule.action,+ input: best.input,+ label: best.label,+ score: best.score,+ },+ };+}++const escalate = (reason: EscalationReason, score: number, ruleId?: string): Routing => ({+ decision: 'escalate',+ reason,+ score,+ ruleId,+});++const round = (n: number) => Math.round(n * 1000) / 1000;