Skip to content

An agentic browser harness: an AI side panel & orb in your own logged-in Chrome, Edge, Arc, Brave or even Firefox run by the agent/subscription you already use (Claude Code, Codex, Antigravity). Free, open source, no API key. Also an MCP server.

feat(actions): add the pointer actions

Shahriar359c23b6ac8a
diff --git a/lib/actions/page/click-element.ts b/lib/actions/page/click-element.ts
new file mode 100644
index 0000000..3c3b724
--- /dev/null
+++ b/lib/actions/page/click-element.ts
@@ -0,0 +1,28 @@
+import { z } from 'zod';
+import { defineAction } from '../core';
+import { describeElement, resolveTarget, targetSchema } from './dom';
+
+export const clickElement = defineAction({
+ name: 'page.clickElement',
+ description: 'Click an element like a user would, firing the full pointer and mouse event sequence.',
+ input: z.object({
+ target: targetSchema.describe('Element to click'),
+ scrollIntoView: z.boolean().default(true).describe('Bring the element into view before clicking'),
+ }),
+ execute({ target, scrollIntoView }) {
+ const el = resolveTarget(target);
+ if (scrollIntoView) el.scrollIntoView({ block: 'center', behavior: 'instant' });
+ try {
+ el.focus({ preventScroll: true });
+ } catch {
+ // Some elements (e.g. inside closed shadow roots) refuse focus; clicking still works.
+ }
+ const init = { bubbles: true, cancelable: true, composed: true };
+ el.dispatchEvent(new PointerEvent('pointerdown', init));
+ el.dispatchEvent(new MouseEvent('mousedown', init));
+ el.dispatchEvent(new PointerEvent('pointerup', init));
+ el.dispatchEvent(new MouseEvent('mouseup', init));
+ el.click();
+ return describeElement(el);
+ },
+});
diff --git a/lib/actions/page/highlight-element.ts b/lib/actions/page/highlight-element.ts
new file mode 100644
index 0000000..26a26a2
--- /dev/null
+++ b/lib/actions/page/highlight-element.ts
@@ -0,0 +1,57 @@
+import { z } from 'zod';
+import { defineAction } from '../core';
+import { describeElement, resolveTarget, targetSchema } from './dom';
+
+export const highlightElement = defineAction({
+ name: 'page.highlightElement',
+ description: 'Visually highlight an element with a temporary outline overlay and optional caption.',
+ input: z.object({
+ target: targetSchema.describe('Element to highlight'),
+ durationMs: z
+ .number()
+ .int()
+ .positive()
+ .max(30000)
+ .default(2000)
+ .describe('How long the highlight stays visible'),
+ label: z.string().optional().describe('Small caption rendered above the highlight'),
+ }),
+ execute({ target, durationMs, label }) {
+ const element = resolveTarget(target);
+ element.scrollIntoView({ block: 'center', behavior: 'instant' });
+ const rect = element.getBoundingClientRect();
+ const overlay = document.createElement('div');
+ Object.assign(overlay.style, {
+ position: 'fixed',
+ left: `${rect.left}px`,
+ top: `${rect.top}px`,
+ width: `${rect.width}px`,
+ height: `${rect.height}px`,
+ pointerEvents: 'none',
+ zIndex: '2147483647',
+ border: '2px solid #8b5cf6',
+ borderRadius: '6px',
+ boxShadow: '0 0 0 4px rgba(139,92,246,.25)',
+ });
+ if (label) {
+ const chip = document.createElement('span');
+ chip.textContent = label;
+ Object.assign(chip.style, {
+ position: 'absolute',
+ top: '-22px',
+ left: '0',
+ background: '#8b5cf6',
+ color: '#fff',
+ font: '11px/1.4 system-ui',
+ padding: '1px 6px',
+ borderRadius: '4px',
+ whiteSpace: 'nowrap',
+ });
+ overlay.append(chip);
+ }
+ // body can carry CSS transforms that would make position: fixed resolve against it
+ document.documentElement.append(overlay);
+ setTimeout(() => overlay.remove(), durationMs);
+ return { element: describeElement(element), durationMs };
+ },
+});
diff --git a/lib/actions/page/hover-element.ts b/lib/actions/page/hover-element.ts
new file mode 100644
index 0000000..752ff32
--- /dev/null
+++ b/lib/actions/page/hover-element.ts
@@ -0,0 +1,33 @@
+import { z } from 'zod';
+import { defineAction } from '../core';
+import { describeElement, resolveTarget, targetSchema } from './dom';
+
+export const hoverElement = defineAction({
+ name: 'page.hoverElement',
+ description: 'Hover an element to trigger menus, tooltips, and other hover states.',
+ input: z.object({
+ target: targetSchema.describe('Element to hover'),
+ scrollIntoView: z.boolean().default(true).describe('Bring the element into view first'),
+ }),
+ execute({ target, scrollIntoView }) {
+ const el = resolveTarget(target);
+ if (scrollIntoView) el.scrollIntoView({ block: 'center', behavior: 'instant' });
+ const rect = el.getBoundingClientRect();
+ const clientX = rect.x + rect.width / 2;
+ const clientY = rect.y + rect.height / 2;
+ // enter events do not bubble, unlike the rest of the hover sequence
+ const sequence: Array<['pointer' | 'mouse', string, boolean]> = [
+ ['pointer', 'pointerover', true],
+ ['pointer', 'pointerenter', false],
+ ['pointer', 'pointermove', true],
+ ['mouse', 'mouseover', true],
+ ['mouse', 'mouseenter', false],
+ ['mouse', 'mousemove', true],
+ ];
+ for (const [kind, type, bubbles] of sequence) {
+ const Ctor = kind === 'pointer' ? PointerEvent : MouseEvent;
+ el.dispatchEvent(new Ctor(type, { clientX, clientY, bubbles, composed: true }));
+ }
+ return describeElement(el);
+ },
+});
diff --git a/lib/actions/page/scroll-to.ts b/lib/actions/page/scroll-to.ts
new file mode 100644
index 0000000..51fc1aa
--- /dev/null
+++ b/lib/actions/page/scroll-to.ts
@@ -0,0 +1,53 @@
+import { z } from 'zod';
+import { defineAction } from '../core';
+import { resolveTarget, targetSchema } from './dom';
+
+export const scrollTo = defineAction({
+ name: 'page.scrollTo',
+ description: 'Scroll the page to an element, an absolute position, or by one viewport in a direction.',
+ input: z.object({
+ target: targetSchema.optional().describe('Element to bring into view'),
+ position: z
+ .object({
+ x: z.number().default(0).describe('Document x coordinate'),
+ y: z.number().describe('Document y coordinate'),
+ })
+ .optional()
+ .describe('Absolute document coordinates to scroll to'),
+ direction: z
+ .enum(['up', 'down', 'top', 'bottom'])
+ .optional()
+ .describe('Scroll one viewport up or down, or jump to an edge'),
+ behavior: z.enum(['smooth', 'instant']).default('smooth').describe('Animation of the scroll'),
+ }),
+ async execute({ target, position, direction, behavior }) {
+ if (target) {
+ resolveTarget(target).scrollIntoView({ behavior, block: 'center' });
+ } else if (position) {
+ window.scrollTo({ left: position.x, top: position.y, behavior });
+ } else {
+ const destination = {
+ up: window.scrollY - window.innerHeight,
+ down: window.scrollY + window.innerHeight,
+ top: 0,
+ bottom: document.documentElement.scrollHeight,
+ }[direction ?? 'down'];
+ window.scrollTo({ top: destination, behavior });
+ }
+ // Smooth scrolling is async; wait for it to settle so the returned position is the final one.
+ if (behavior === 'smooth') await scrollSettled();
+ return { scrollX: Math.round(window.scrollX), scrollY: Math.round(window.scrollY) };
+ },
+});
+
+function scrollSettled(): Promise<void> {
+ return new Promise((resolve) => {
+ const done = () => {
+ window.removeEventListener('scrollend', done);
+ clearTimeout(timer);
+ resolve();
+ };
+ const timer = setTimeout(done, 1000);
+ window.addEventListener('scrollend', done);
+ });
+}