Skip to content

An agentic browser harness: an AI side panel & orb in your own logged-in Chrome, Edge, Arc, Brave or even Firefox run by the agent/subscription you already use (Claude Code, Codex, Antigravity). Free, open source, no API key. Also an MCP server.

fix(actions): let the agent's pointer through Browsentic's own overlays — elementAt looks past them and a trusted click or drag marks them inert while it lands — so the hands-free mic never hides a target or takes the agent's click as Stop

Shahriar0258873594f5
diff --git a/src/lib/actions/page/pointer.test.ts b/src/lib/actions/page/pointer.test.ts
new file mode 100644
index 0000000..d46ada4
--- /dev/null
+++ b/src/lib/actions/page/pointer.test.ts
@@ -0,0 +1,41 @@
+import { afterEach, describe, expect, it } from 'vitest';
+import { OVERLAY_ATTRIBUTE } from '@/lib/overlay';
+import { assertUncovered, elementAt } from './pointer';
+
+function stage() {
+ const footer = document.createElement('button');
+ footer.textContent = 'Accept cookies';
+ document.body.append(footer);
+ const orb = document.createElement('div');
+ orb.setAttribute(OVERLAY_ATTRIBUTE, '');
+ document.documentElement.append(orb);
+ return { footer, orb };
+}
+
+/* happy-dom lays nothing out, so what stands at a point is whatever the test stacks there. */
+function stacked(...elements: Element[]) {
+ document.elementsFromPoint = () => elements;
+}
+
+describe('elementAt', () => {
+ afterEach(() => {
+ document.body.innerHTML = '';
+ for (const host of document.querySelectorAll(`[${OVERLAY_ATTRIBUTE}]`)) host.remove();
+ });
+
+ it('looks through the hands-free mic to the page beneath it', () => {
+ const { footer, orb } = stage();
+ stacked(orb, footer, document.body);
+ expect(elementAt({ x: 400, y: 700 })).toBe(footer);
+ expect(() => assertUncovered(footer, { x: 400, y: 700 })).not.toThrow();
+ });
+
+ it('still reports what the page itself put over an element', () => {
+ const { footer } = stage();
+ const banner = document.createElement('div');
+ banner.id = 'banner';
+ document.body.append(banner);
+ stacked(banner, footer, document.body);
+ expect(() => assertUncovered(footer, { x: 400, y: 700 })).toThrow(/covers \(400, 700\)/);
+ });
+});
diff --git a/src/lib/actions/page/pointer.ts b/src/lib/actions/page/pointer.ts
index 09245c5..b68ef91 100644
--- a/src/lib/actions/page/pointer.ts
+++ b/src/lib/actions/page/pointer.ts
@@ -1,4 +1,5 @@
import { z } from 'zod';
+import { isOverlay } from '@/lib/overlay';
import { ActionError } from '../core';
import { cssPath } from './dom';
@@ -56,8 +57,9 @@ export function approachPoint(to: Point): Point {
};
}
+/** What the page has at a point, looking through anything Browsentic drew over it. */
export function elementAt(point: Point): Element | null {
- return document.elementFromPoint(point.x, point.y);
+ return document.elementsFromPoint(point.x, point.y).find((element) => !isOverlay(element)) ?? null;
}
export function assertUncovered(el: HTMLElement, at: Point): void {
diff --git a/src/lib/bridge/trusted-input.ts b/src/lib/bridge/trusted-input.ts
index 13716df..57591d2 100644
--- a/src/lib/bridge/trusted-input.ts
+++ b/src/lib/bridge/trusted-input.ts
@@ -1,8 +1,10 @@
+import { browser } from 'wxt/browser';
import { invokeInTab } from '@/lib/actions/client';
import { dragElement } from '@/lib/actions/page/drag-element';
import { APPROACH_STEPS, pathBetween, type Point } from '@/lib/actions/page/pointer';
import { trustedClick } from '@/lib/actions/page/trusted-click';
import { failure, success, type ActionResult } from '@/lib/actions/protocol';
+import { OVERLAY_ATTRIBUTE } from '@/lib/overlay';
import { send, settle, withDebugger, type DebuggerSession } from './cdp';
import { FRAME_GONE_HINT, frameOffset } from './frame-focus';
@@ -44,7 +46,9 @@ export function trustedClickInTab(tabId: number, input: unknown): Promise<Action
const offset = await frameOffset(tabId);
if (!offset) return failure('TAB_UNREACHABLE', `The click could not be placed — ${FRAME_GONE_HINT}`);
const plan = planned.data as ClickPlan;
- await dispatchClick(session, { ...plan, point: shifted(plan.point, offset), from: shifted(plan.from, offset) });
+ await throughOverlays(tabId, () =>
+ dispatchClick(session, { ...plan, point: shifted(plan.point, offset), from: shifted(plan.from, offset) }),
+ );
return success({ ...(planned.data as object), trusted: true });
});
}
@@ -56,18 +60,46 @@ export function dragInTab(tabId: number, input: unknown): Promise<ActionResult>
const offset = await frameOffset(tabId);
if (!offset) return failure('TAB_UNREACHABLE', `The drag could not be placed — ${FRAME_GONE_HINT}`);
const plan = planned.data as DragPlan;
- await dispatchDrag(session, {
- ...plan,
- approach: shifted(plan.approach, offset),
- grip: shifted(plan.grip, offset),
- drop: shifted(plan.drop, offset),
- });
+ await throughOverlays(tabId, () =>
+ dispatchDrag(session, {
+ ...plan,
+ approach: shifted(plan.approach, offset),
+ grip: shifted(plan.grip, offset),
+ drop: shifted(plan.drop, offset),
+ }),
+ );
return success({ ...(planned.data as object), trusted: true });
});
}
const shifted = (at: Point, by: Point): Point => ({ x: at.x + by.x, y: at.y + by.y });
+/**
+ * A real pointer lands on whatever is on top, and the hands-free mic, the rail or a toast may be
+ * over the target. Inert for the length of the gesture, they let it through to the page, and an
+ * agent's click can never press the mic's own stop.
+ */
+async function throughOverlays(tabId: number, gesture: () => Promise<void>): Promise<void> {
+ await markOverlaysInert(tabId, true);
+ try {
+ await gesture();
+ } finally {
+ await markOverlaysInert(tabId, false);
+ }
+}
+
+async function markOverlaysInert(tabId: number, inert: boolean): Promise<void> {
+ await browser.scripting
+ .executeScript({
+ target: { tabId },
+ func: (attribute: string, on: boolean) => {
+ for (const host of document.querySelectorAll(`[${attribute}]`)) host.toggleAttribute('inert', on);
+ },
+ args: [OVERLAY_ATTRIBUTE, inert],
+ })
+ .catch(() => undefined);
+}
+
export async function dispatchClick(session: DebuggerSession, plan: ClickPlan): Promise<void> {
const { point, from, button, clickCount, modifiers, moveSteps, hoverMs, holdMs } = plan;
const held = modifiers.reduce((mask, name) => mask | (MODIFIER_BIT[name] ?? 0), 0);