An agentic browser harness: an AI side panel & orb in your own logged-in Chrome, Edge, Arc, Brave or even Firefox run by the agent/subscription you already use (Claude Code, Codex, Antigravity). Free, open source, no API key. Also an MCP server.
TypeScript agentic-browseragentic-browsingagy-cliaiai-agentai-browser-extensionai-toolsbrave-extensionbrowser-agentbrowser-automationbrowser-extensionbrowser-usebrowserskillchrome-extensionclaude-codecodexfirefox-addonmcp-servernanobrowserweb-automation
fix(actions): let the agent's pointer through Browsentic's own overlays — elementAt looks past them and a trusted click or drag marks them inert while it lands — so the hands-free mic never hides a target or takes the agent's click as Stop
Shahriar0258873594f5
diff --git a/src/lib/actions/page/pointer.test.ts b/src/lib/actions/page/pointer.test.tsnew file mode 100644index 0000000..d46ada4--- /dev/null+++ b/src/lib/actions/page/pointer.test.ts@@ -0,0 +1,41 @@+import { afterEach, describe, expect, it } from 'vitest';+import { OVERLAY_ATTRIBUTE } from '@/lib/overlay';+import { assertUncovered, elementAt } from './pointer';++function stage() {+ const footer = document.createElement('button');+ footer.textContent = 'Accept cookies';+ document.body.append(footer);+ const orb = document.createElement('div');+ orb.setAttribute(OVERLAY_ATTRIBUTE, '');+ document.documentElement.append(orb);+ return { footer, orb };+}++/* happy-dom lays nothing out, so what stands at a point is whatever the test stacks there. */+function stacked(...elements: Element[]) {+ document.elementsFromPoint = () => elements;+}++describe('elementAt', () => {+ afterEach(() => {+ document.body.innerHTML = '';+ for (const host of document.querySelectorAll(`[${OVERLAY_ATTRIBUTE}]`)) host.remove();+ });++ it('looks through the hands-free mic to the page beneath it', () => {+ const { footer, orb } = stage();+ stacked(orb, footer, document.body);+ expect(elementAt({ x: 400, y: 700 })).toBe(footer);+ expect(() => assertUncovered(footer, { x: 400, y: 700 })).not.toThrow();+ });++ it('still reports what the page itself put over an element', () => {+ const { footer } = stage();+ const banner = document.createElement('div');+ banner.id = 'banner';+ document.body.append(banner);+ stacked(banner, footer, document.body);+ expect(() => assertUncovered(footer, { x: 400, y: 700 })).toThrow(/covers \(400, 700\)/);+ });+});diff --git a/src/lib/actions/page/pointer.ts b/src/lib/actions/page/pointer.tsindex 09245c5..b68ef91 100644--- a/src/lib/actions/page/pointer.ts+++ b/src/lib/actions/page/pointer.ts@@ -1,4 +1,5 @@import { z } from 'zod';+import { isOverlay } from '@/lib/overlay';import { ActionError } from '../core';import { cssPath } from './dom';@@ -56,8 +57,9 @@ export function approachPoint(to: Point): Point {};}+/** What the page has at a point, looking through anything Browsentic drew over it. */export function elementAt(point: Point): Element | null {- return document.elementFromPoint(point.x, point.y);+ return document.elementsFromPoint(point.x, point.y).find((element) => !isOverlay(element)) ?? null;}export function assertUncovered(el: HTMLElement, at: Point): void {diff --git a/src/lib/bridge/trusted-input.ts b/src/lib/bridge/trusted-input.tsindex 13716df..57591d2 100644--- a/src/lib/bridge/trusted-input.ts+++ b/src/lib/bridge/trusted-input.ts@@ -1,8 +1,10 @@+import { browser } from 'wxt/browser';import { invokeInTab } from '@/lib/actions/client';import { dragElement } from '@/lib/actions/page/drag-element';import { APPROACH_STEPS, pathBetween, type Point } from '@/lib/actions/page/pointer';import { trustedClick } from '@/lib/actions/page/trusted-click';import { failure, success, type ActionResult } from '@/lib/actions/protocol';+import { OVERLAY_ATTRIBUTE } from '@/lib/overlay';import { send, settle, withDebugger, type DebuggerSession } from './cdp';import { FRAME_GONE_HINT, frameOffset } from './frame-focus';@@ -44,7 +46,9 @@ export function trustedClickInTab(tabId: number, input: unknown): Promise<Actionconst offset = await frameOffset(tabId);if (!offset) return failure('TAB_UNREACHABLE', `The click could not be placed — ${FRAME_GONE_HINT}`);const plan = planned.data as ClickPlan;- await dispatchClick(session, { ...plan, point: shifted(plan.point, offset), from: shifted(plan.from, offset) });+ await throughOverlays(tabId, () =>+ dispatchClick(session, { ...plan, point: shifted(plan.point, offset), from: shifted(plan.from, offset) }),+ );return success({ ...(planned.data as object), trusted: true });});}@@ -56,18 +60,46 @@ export function dragInTab(tabId: number, input: unknown): Promise<ActionResult>const offset = await frameOffset(tabId);if (!offset) return failure('TAB_UNREACHABLE', `The drag could not be placed — ${FRAME_GONE_HINT}`);const plan = planned.data as DragPlan;- await dispatchDrag(session, {- ...plan,- approach: shifted(plan.approach, offset),- grip: shifted(plan.grip, offset),- drop: shifted(plan.drop, offset),- });+ await throughOverlays(tabId, () =>+ dispatchDrag(session, {+ ...plan,+ approach: shifted(plan.approach, offset),+ grip: shifted(plan.grip, offset),+ drop: shifted(plan.drop, offset),+ }),+ );return success({ ...(planned.data as object), trusted: true });});}const shifted = (at: Point, by: Point): Point => ({ x: at.x + by.x, y: at.y + by.y });+/**+ * A real pointer lands on whatever is on top, and the hands-free mic, the rail or a toast may be+ * over the target. Inert for the length of the gesture, they let it through to the page, and an+ * agent's click can never press the mic's own stop.+ */+async function throughOverlays(tabId: number, gesture: () => Promise<void>): Promise<void> {+ await markOverlaysInert(tabId, true);+ try {+ await gesture();+ } finally {+ await markOverlaysInert(tabId, false);+ }+}++async function markOverlaysInert(tabId: number, inert: boolean): Promise<void> {+ await browser.scripting+ .executeScript({+ target: { tabId },+ func: (attribute: string, on: boolean) => {+ for (const host of document.querySelectorAll(`[${attribute}]`)) host.toggleAttribute('inert', on);+ },+ args: [OVERLAY_ATTRIBUTE, inert],+ })+ .catch(() => undefined);+}+export async function dispatchClick(session: DebuggerSession, plan: ClickPlan): Promise<void> {const { point, from, button, clickCount, modifiers, moveSteps, hoverMs, holdMs } = plan;const held = modifiers.reduce((mask, name) => mask | (MODIFIER_BIT[name] ?? 0), 0);